From 6d8818bc1449fe70ba26f7e26c2faf3a40db9b79 Mon Sep 17 00:00:00 2001 From: Baptiste Roux Date: Sat, 19 Feb 2022 11:58:27 +0100 Subject: [PATCH 1/8] Add requirements file --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8fb375c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +trakt==3.4.0 +tinydb==4.6.1 From 9436798e47c779f9ffdc4202018e33ce0539853c Mon Sep 17 00:00:00 2001 From: Baptiste Roux Date: Sat, 19 Feb 2022 11:58:40 +0100 Subject: [PATCH 2/8] Change field selection to named columns instead of indexes. (csv.DictReader) Allow empty menu input, defaulting to choice #1 Change total line count to reuse already opened CSV file Use enumerate() to track row ID --- TimeToTrackt.py | 239 +++++++++++++++++++++++------------------------- 1 file changed, 114 insertions(+), 125 deletions(-) diff --git a/TimeToTrackt.py b/TimeToTrackt.py index 94ec9c5..3edb298 100644 --- a/TimeToTrackt.py +++ b/TimeToTrackt.py @@ -1,17 +1,15 @@ -# main.py -from logging import error -import sys -from trakt import * -import trakt.core -import os +#!/usr/bin/env python3 import csv -from datetime import datetime -import time -from tinydb import TinyDB, Query import json +import os import re import sys +import time +from datetime import datetime +import trakt.core +from tinydb import Query, TinyDB +from trakt import Expando from trakt.tv import TVShow # Adjust this value to increase/decrease your requests between episodes. @@ -58,6 +56,7 @@ def getFollowedShowsPath(): def initTraktAuth(): + return True # Set the method of authentication trakt.core.AUTH_METHOD = trakt.core.OAUTH_AUTH return init(config.TRAKT_USERNAME, store=True, client_id=config.CLIENT_ID, client_secret=config.CLIENT_SECRET) @@ -291,133 +290,122 @@ def processWatchedShows(): # Total amount of rows which have been processed in the CSV file rowsCount = 0 # Total amount of rows in the CSV file - rowsTotal = 0 - # Total amount of errors which have occurred in one streak errorStreak = 0 - - # Get the total amount of rows in the CSV file, - # which is helpful for keeping track of progress. - # However, if you have a VERY large CSV file (e.g above 100,000 rows) - # then it might be a good idea to remove this due to the performance - # overhead. - with open(getWatchedShowsPath()) as f: - rowsTotal = sum(1 for line in f) - # Open the CSV file within the GDPR exported data with open(getWatchedShowsPath(), newline='') as csvfile: # Create the CSV reader, which will break up the fields using the delimiter ',' - showsReader = csv.reader(csvfile, delimiter=',') - + showsReader = csv.DictReader(csvfile, delimiter=',') + # Get the total amount of rows in the CSV file, + rowsTotal = len(list(showsReader)) + # Move position to the beginning of the file + csvfile.seek(0, 0) # Loop through each line/record of the CSV file - for row in showsReader: - # Increment the row counter to keep track of progress completing the - # records during the import process. - rowsCount += 1 + # Ignore the header row + next(showsReader, None) + for rowsCount, row in enumerate(showsReader): # Get the name of the TV show - tvShowName = row[8] + tvShowName = row["tv_show_name"] + # Get the TV Time Episode Id + tvShowEpisodeId = row["episode_id"] + # Get the TV Time Season Number + tvShowSeasonNo = row["episode_season_number"] + # Get the TV Time Episode Number + tvShowEpisodeNo = row["episode_number"] + # Get the date which the show was marked 'watched' in TV Time + tvShowDateWatched = row["updated_at"] + # Parse the watched date value into a Python type + print(tvShowDateWatched) + tvShowDateWatchedConverted = datetime.strptime( + tvShowDateWatched, '%Y-%m-%d %H:%M:%S') - # Ignore the header row - if rowsCount > 1: - # Get the TV Time Episode Id - tvShowEpisodeId = row[4] - # Get the TV Time Season Number - tvShowSeasonNo = row[5] - # Get the TV Time Episode Number - tvShowEpisodeNo = row[6] - # Get the date which the show was marked 'watched' in TV Time - tvShowDateWatched = row[7] - # Parse the watched date value into a Python type - tvShowDateWatchedConverted = datetime.strptime( - tvShowDateWatched, '%Y-%m-%d %H:%M:%S') + # Query the local database for previous entries indicating that + # the episode has already been imported in the past. Which will + # ease pressure on TV Time's API server during a retry of the import + # process, and just save time overall without needing to create network requests + episodeCompletedQuery = Query() + queryResult = syncedEpisodesTable.search( + episodeCompletedQuery.episodeId == tvShowEpisodeId) - # Query the local database for previous entries indicating that - # the episode has already been imported in the past. Which will - # ease pressure on TV Time's API server during a retry of the import - # process, and just save time overall without needing to create network requests - episodeCompletedQuery = Query() - queryResult = syncedEpisodesTable.search( - episodeCompletedQuery.episodeId == tvShowEpisodeId) - - # If the query returned no results, then continue to import it into Trakt - if len(queryResult) == 0: - # Create a repeating loop, which will break on success, but repeats on failures - while True: - # If more than 10 errors occurred in one streak, whilst trying to import the episode - # then give up, and move onto the next episode, but warn the user. - if (errorStreak > 10): - print( - f"WARNING: An error occurred 10 times in a row... skipping episode...") + # If the query returned no results, then continue to import it into Trakt + if len(queryResult) == 0: + # Create a repeating loop, which will break on success, but repeats on failures + while True: + # If more than 10 errors occurred in one streak, whilst trying to import the episode + # then give up, and move onto the next episode, but warn the user. + if (errorStreak > 10): + print( + f"WARNING: An error occurred 10 times in a row... skipping episode...") + break + try: + # Sleep for a second between each process, before going onto the next watched episode. + # This is required to remain within the API rate limit, and use the API server fairly. + # Other developers share the service, for free - so be considerate of your usage. + time.sleep(DELAY_BETWEEN_EPISODES_IN_SECONDS) + # Search Trakt for the TV show matching TV Time's title value + traktShowObj = getShowByName( + tvShowName, tvShowSeasonNo, tvShowEpisodeNo) + # If the method returned 'None', then this is an indication to skip the episode, and + # move onto the next one + if traktShowObj == None: break - try: - # Sleep for a second between each process, before going onto the next watched episode. - # This is required to remain within the API rate limit, and use the API server fairly. - # Other developers share the service, for free - so be considerate of your usage. - time.sleep(DELAY_BETWEEN_EPISODES_IN_SECONDS) - # Search Trakt for the TV show matching TV Time's title value - traktShowObj = getShowByName( - tvShowName, tvShowSeasonNo, tvShowEpisodeNo) - # If the method returned 'None', then this is an indication to skip the episode, and - # move onto the next one - if traktShowObj == None: - break - # Show the progress of the import on-screen - print( - f"({rowsCount}/{rowsTotal}) Processing Show {tvShowName} on Season {tvShowSeasonNo} - Episode {tvShowEpisodeNo}") - # Get the season from the Trakt API - season = traktShowObj.seasons[parseSeasonNo( - tvShowSeasonNo, traktShowObj)] - # Get the episode from the season - episode = season.episodes[int(tvShowEpisodeNo) - 1] - # Mark the episode as watched! - episode.mark_as_seen(tvShowDateWatchedConverted) - # Add the episode to the local database as imported, so it can be skipped, - # if the process is repeated - syncedEpisodesTable.insert( - {'episodeId': tvShowEpisodeId}) - # Clear the error streak on completing the method without errors - errorStreak = 0 - break - # Catch errors which occur because of an incorrect array index. This occurs when - # an incorrect Trakt show has been selected, with season/episodes which don't match TV Time. - # It can also occur due to a bug in Trakt Py, whereby some seasons contain an empty array of episodes. - except IndexError: - print( - f"({rowsCount}/{rowsTotal}) WARNING: {tvShowName} Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo} does not exist (season/episode index) in Trakt!") - break - # Catch any errors which are raised because a show could not be found in Trakt - except trakt.errors.NotFoundException: - print( - f"({rowsCount}/{rowsTotal}) WARNING: {tvShowName} Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo} does not exist (search) in Trakt!") - break - # Catch errors because of the program breaching the Trakt API rate limit - except trakt.errors.RateLimitException: - print( - "WARNING: The program is running too quickly and has hit Trakt's API rate limit! Please increase the delay between " + - "episdoes via the variable 'DELAY_BETWEEN_EPISODES_IN_SECONDS'. The program will now wait 60 seconds before " + - "trying again.") - time.sleep(60) + # Show the progress of the import on-screen + print( + f"({rowsCount}/{rowsTotal}) Processing Show {tvShowName} on Season {tvShowSeasonNo} - Episode {tvShowEpisodeNo}") + # Get the season from the Trakt API + season = traktShowObj.seasons[parseSeasonNo( + tvShowSeasonNo, traktShowObj)] + # Get the episode from the season + episode = season.episodes[int(tvShowEpisodeNo) - 1] + # Mark the episode as watched! + episode.mark_as_seen(tvShowDateWatchedConverted) + # Add the episode to the local database as imported, so it can be skipped, + # if the process is repeated + syncedEpisodesTable.insert( + {'episodeId': tvShowEpisodeId}) + # Clear the error streak on completing the method without errors + errorStreak = 0 + break + # Catch errors which occur because of an incorrect array index. This occurs when + # an incorrect Trakt show has been selected, with season/episodes which don't match TV Time. + # It can also occur due to a bug in Trakt Py, whereby some seasons contain an empty array of episodes. + except IndexError: + print( + f"({rowsCount}/{rowsTotal}) WARNING: {tvShowName} Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo} does not exist (season/episode index) in Trakt!") + break + # Catch any errors which are raised because a show could not be found in Trakt + except trakt.errors.NotFoundException: + print( + f"({rowsCount}/{rowsTotal}) WARNING: {tvShowName} Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo} does not exist (search) in Trakt!") + break + # Catch errors because of the program breaching the Trakt API rate limit + except trakt.errors.RateLimitException: + print( + "WARNING: The program is running too quickly and has hit Trakt's API rate limit! Please increase the delay between " + + "episdoes via the variable 'DELAY_BETWEEN_EPISODES_IN_SECONDS'. The program will now wait 60 seconds before " + + "trying again.") + time.sleep(60) - # Mark the exception in the error streak - errorStreak += 1 - # Catch a JSON decode error - this can be raised when the API server is down and produces a HTML page, instead of JSON - except json.decoder.JSONDecodeError: - print( - f"({rowsCount}/{rowsTotal}) WARNING: A JSON decode error occuring whilst processing {tvShowName} " + - f"Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo}! This might occur when the server is down and has produced " + - "a HTML document instead of JSON. The script will wait 60 seconds before trying again.") + # Mark the exception in the error streak + errorStreak += 1 + # Catch a JSON decode error - this can be raised when the API server is down and produces a HTML page, instead of JSON + except json.decoder.JSONDecodeError: + print( + f"({rowsCount}/{rowsTotal}) WARNING: A JSON decode error occuring whilst processing {tvShowName} " + + f"Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo}! This might occur when the server is down and has produced " + + "a HTML document instead of JSON. The script will wait 60 seconds before trying again.") - # Wait 60 seconds - time.sleep(60) + # Wait 60 seconds + time.sleep(60) - # Mark the exception in the error streak - errorStreak += 1 - # Catch a CTRL + C keyboard input, and exits the program - except KeyboardInterrupt: - sys.exit("Cancel requested...") - # Skip the episode - else: - print( - f"({rowsCount}/{rowsTotal}) Skipping '{tvShowName}' Season {tvShowSeasonNo} Episode {tvShowEpisodeNo}. It's already been imported.") + # Mark the exception in the error streak + errorStreak += 1 + # Catch a CTRL + C keyboard input, and exits the program + except KeyboardInterrupt: + sys.exit("Cancel requested...") + # Skip the episode + else: + print( + f"({rowsCount}/{rowsTotal}) Skipping '{tvShowName}' Season {tvShowSeasonNo} Episode {tvShowEpisodeNo}. It's already been imported.") def start(): @@ -429,7 +417,8 @@ def start(): while True: try: - menuSelection = int(input(f"Enter your menu selection: ")) + menuSelection = input(f"Enter your menu selection: ") + menuSelection = 1 if not menuSelection else int(menuSelection) break except ValueError: print("Invalid input. Please enter a numerical number.") From 8c58b78b8e5c216e894271d3cc87c55c1840f73a Mon Sep 17 00:00:00 2001 From: Baptiste Roux Date: Sat, 19 Feb 2022 12:07:17 +0100 Subject: [PATCH 3/8] Formatting --- TimeToTrackt.py | 125 ++++++++++++++++++++++++++++++------------------ 1 file changed, 79 insertions(+), 46 deletions(-) diff --git a/TimeToTrackt.py b/TimeToTrackt.py index 3edb298..c88f6b4 100644 --- a/TimeToTrackt.py +++ b/TimeToTrackt.py @@ -17,9 +17,9 @@ from trakt.tv import TVShow DELAY_BETWEEN_EPISODES_IN_SECONDS = 5 # Create a database to keep track of completed processes -database = TinyDB('localStorage.json') -syncedEpisodesTable = database.table('SyncedEpisodes') -userMatchedShowsTable = database.table('TvTimeTraktUserMatched') +database = TinyDB("localStorage.json") +syncedEpisodesTable = database.table("SyncedEpisodes") +userMatchedShowsTable = database.table("TvTimeTraktUserMatched") class Expando(object): @@ -29,7 +29,7 @@ class Expando(object): def getConfiguration(): configEx = Expando() - with open('config.json') as f: + with open("config.json") as f: data = json.load(f) configEx.TRAKT_USERNAME = data["TRAKT_USERNAME"] @@ -59,7 +59,13 @@ def initTraktAuth(): return True # Set the method of authentication trakt.core.AUTH_METHOD = trakt.core.OAUTH_AUTH - return init(config.TRAKT_USERNAME, store=True, client_id=config.CLIENT_ID, client_secret=config.CLIENT_SECRET) + return init( + config.TRAKT_USERNAME, + store=True, + client_id=config.CLIENT_ID, + client_secret=config.CLIENT_SECRET, + ) + # With a given title, check if it contains a year (e.g Doctor Who (2005)) # and then return this value, with the title and year removed to improve @@ -74,7 +80,7 @@ def getYearFromTitle(title): yearSearch = re.search(r"\(([A-Za-z0-9_]+)\)", title) yearValue = yearSearch.group(1) # Then, get the title without the year value included - titleValue = title.split('(')[0].strip() + titleValue = title.split("(")[0].strip() # Put this together into an object ex.titleWithoutYear = titleValue ex.yearValue = int(yearValue) @@ -86,6 +92,7 @@ def getYearFromTitle(title): ex.yearValue = -1 return ex + # Shows in TV Time are often different to Trakt.TV - in order to improve results and automation, # calculate how many words are in the title, and return true if more than 50% of the title is a match, # It seems to improve automation, and reduce manual selection.... @@ -115,6 +122,7 @@ def checkTitleNameMatch(tvTimeTitle, traktTitle): # then return the title as a possibility to use return percentage > 50 + # Using TV Time data (Name of Show, Season No and Episode) - find the corresponding show # in Trakt.TV either by automation, or asking the user to confirm. @@ -167,7 +175,7 @@ def getShowByName(name, seasonNo, episodeNo): if nameFromSearch.title == name: completeMatchNames.append(nameFromSearch) - if (len(completeMatchNames) == 1): + if len(completeMatchNames) == 1: showsWithSameName = completeMatchNames # If the search contains multiple results, then we need to confirm with the user which show @@ -177,8 +185,7 @@ def getShowByName(name, seasonNo, episodeNo): # Query the local database for existing selection userMatchedQuery = Query() - queryResult = userMatchedShowsTable.search( - userMatchedQuery.ShowName == name) + queryResult = userMatchedShowsTable.search(userMatchedQuery.ShowName == name) # If the local database already contains an entry for a manual selection # then don't bother prompting the user to select it again! @@ -186,9 +193,9 @@ def getShowByName(name, seasonNo, episodeNo): # Get the first result from the query firstMatch = queryResult[0] # Get the value contains the selection index - firstMatchSelectedIndex = int(firstMatch.get('UserSelectedIndex')) + firstMatchSelectedIndex = int(firstMatch.get("UserSelectedIndex")) # Check if the user previously requested to skip the show - skipShow = firstMatch.get('SkipShow') + skipShow = firstMatch.get("SkipShow") # If the user did not skip, but provided an index selection, get the # matching show if skipShow == False: @@ -201,22 +208,25 @@ def getShowByName(name, seasonNo, episodeNo): # then prompt the user to make a selection else: print( - f"INFO - MANUAL INPUT REQUIRED: The TV Time data for Show '{name}' (Season {seasonNo}, Episode {episodeNo}) has {len(showsWithSameName)} matching Trakt shows with the same name.") + f"INFO - MANUAL INPUT REQUIRED: The TV Time data for Show '{name}' (Season {seasonNo}, Episode {episodeNo}) has {len(showsWithSameName)} matching Trakt shows with the same name." + ) # Output each show for manual selection for idx, item in enumerate(showsWithSameName): # Display the show's title, broadcast year, amount of seasons and a link to the Trakt page. # This will provide the user with enough information to make a selection. print( - f" ({idx + 1}) {item.title} - {item.year} - {len(item.seasons)} Season(s) - More Info: https://trakt.tv/{item.ext}") + f" ({idx + 1}) {item.title} - {item.year} - {len(item.seasons)} Season(s) - More Info: https://trakt.tv/{item.ext}" + ) - while(True): + while True: try: # Get the user's selection, either a numerical input, or a string 'SKIP' value - indexSelected = (input( - f"Please make a selection from above (or enter SKIP):")) + indexSelected = input( + f"Please make a selection from above (or enter SKIP):" + ) - if indexSelected != 'SKIP': + if indexSelected != "SKIP": # Since the value isn't 'skip', check that the result is numerical indexSelected = int(indexSelected) - 1 # Exit the selection loop @@ -230,15 +240,17 @@ def getShowByName(name, seasonNo, episodeNo): # Otherwise, the user has entered an invalid value, warn the user to try again except: print( - f"Sorry! Please select a value between 0 to {len(showsWithSameName)}") + f"Sorry! Please select a value between 0 to {len(showsWithSameName)}" + ) # If the user entered 'SKIP', then exit from the loop with no selection, which # will trigger the program to move onto the next episode - if (indexSelected == 'SKIP'): + if indexSelected == "SKIP": # Record that the user has skipped the TV Show for import, so that # manual input isn't required everytime userMatchedShowsTable.insert( - {'ShowName': name, 'UserSelectedIndex': 0, 'SkipShow': True}) + {"ShowName": name, "UserSelectedIndex": 0, "SkipShow": True} + ) return None # Otherwise, return the selection which the user made from the list @@ -246,18 +258,24 @@ def getShowByName(name, seasonNo, episodeNo): selectedShow = showsWithSameName[int(indexSelected)] userMatchedShowsTable.insert( - {'ShowName': name, 'UserSelectedIndex': indexSelected, 'SkipShow': False}) + { + "ShowName": name, + "UserSelectedIndex": indexSelected, + "SkipShow": False, + } + ) return selectedShow else: - if (len(showsWithSameName) > 0): + if len(showsWithSameName) > 0: # If the search returned only one result, then awesome! # Return the show, so the import automation can continue. return showsWithSameName[0] else: return None + # Since the Trakt.Py starts the indexing of seasons in the array from 0 (e.g Season 1 in Index 0), then # subtract the TV Time numerical value by 1 so it starts from 0 as well. However, when a TV series includes # a 'special' season, Trakt.Py will place this as the first season in the array - so, don't subtract, since @@ -292,9 +310,9 @@ def processWatchedShows(): # Total amount of rows in the CSV file errorStreak = 0 # Open the CSV file within the GDPR exported data - with open(getWatchedShowsPath(), newline='') as csvfile: + with open(getWatchedShowsPath(), newline="") as csvfile: # Create the CSV reader, which will break up the fields using the delimiter ',' - showsReader = csv.DictReader(csvfile, delimiter=',') + showsReader = csv.DictReader(csvfile, delimiter=",") # Get the total amount of rows in the CSV file, rowsTotal = len(list(showsReader)) # Move position to the beginning of the file @@ -316,7 +334,8 @@ def processWatchedShows(): # Parse the watched date value into a Python type print(tvShowDateWatched) tvShowDateWatchedConverted = datetime.strptime( - tvShowDateWatched, '%Y-%m-%d %H:%M:%S') + tvShowDateWatched, "%Y-%m-%d %H:%M:%S" + ) # Query the local database for previous entries indicating that # the episode has already been imported in the past. Which will @@ -324,7 +343,8 @@ def processWatchedShows(): # process, and just save time overall without needing to create network requests episodeCompletedQuery = Query() queryResult = syncedEpisodesTable.search( - episodeCompletedQuery.episodeId == tvShowEpisodeId) + episodeCompletedQuery.episodeId == tvShowEpisodeId + ) # If the query returned no results, then continue to import it into Trakt if len(queryResult) == 0: @@ -332,9 +352,10 @@ def processWatchedShows(): while True: # If more than 10 errors occurred in one streak, whilst trying to import the episode # then give up, and move onto the next episode, but warn the user. - if (errorStreak > 10): + if errorStreak > 10: print( - f"WARNING: An error occurred 10 times in a row... skipping episode...") + f"WARNING: An error occurred 10 times in a row... skipping episode..." + ) break try: # Sleep for a second between each process, before going onto the next watched episode. @@ -343,25 +364,27 @@ def processWatchedShows(): time.sleep(DELAY_BETWEEN_EPISODES_IN_SECONDS) # Search Trakt for the TV show matching TV Time's title value traktShowObj = getShowByName( - tvShowName, tvShowSeasonNo, tvShowEpisodeNo) + tvShowName, tvShowSeasonNo, tvShowEpisodeNo + ) # If the method returned 'None', then this is an indication to skip the episode, and # move onto the next one if traktShowObj == None: break # Show the progress of the import on-screen print( - f"({rowsCount}/{rowsTotal}) Processing Show {tvShowName} on Season {tvShowSeasonNo} - Episode {tvShowEpisodeNo}") + f"({rowsCount+1}/{rowsTotal}) Processing Show {tvShowName} on Season {tvShowSeasonNo} - Episode {tvShowEpisodeNo}" + ) # Get the season from the Trakt API - season = traktShowObj.seasons[parseSeasonNo( - tvShowSeasonNo, traktShowObj)] + season = traktShowObj.seasons[ + parseSeasonNo(tvShowSeasonNo, traktShowObj) + ] # Get the episode from the season episode = season.episodes[int(tvShowEpisodeNo) - 1] # Mark the episode as watched! episode.mark_as_seen(tvShowDateWatchedConverted) # Add the episode to the local database as imported, so it can be skipped, # if the process is repeated - syncedEpisodesTable.insert( - {'episodeId': tvShowEpisodeId}) + syncedEpisodesTable.insert({"episodeId": tvShowEpisodeId}) # Clear the error streak on completing the method without errors errorStreak = 0 break @@ -370,19 +393,22 @@ def processWatchedShows(): # It can also occur due to a bug in Trakt Py, whereby some seasons contain an empty array of episodes. except IndexError: print( - f"({rowsCount}/{rowsTotal}) WARNING: {tvShowName} Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo} does not exist (season/episode index) in Trakt!") + f"({rowsCount}/{rowsTotal}) WARNING: {tvShowName} Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo} does not exist (season/episode index) in Trakt!" + ) break # Catch any errors which are raised because a show could not be found in Trakt except trakt.errors.NotFoundException: print( - f"({rowsCount}/{rowsTotal}) WARNING: {tvShowName} Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo} does not exist (search) in Trakt!") + f"({rowsCount}/{rowsTotal}) WARNING: {tvShowName} Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo} does not exist (search) in Trakt!" + ) break # Catch errors because of the program breaching the Trakt API rate limit except trakt.errors.RateLimitException: print( - "WARNING: The program is running too quickly and has hit Trakt's API rate limit! Please increase the delay between " + - "episdoes via the variable 'DELAY_BETWEEN_EPISODES_IN_SECONDS'. The program will now wait 60 seconds before " + - "trying again.") + "WARNING: The program is running too quickly and has hit Trakt's API rate limit! Please increase the delay between " + + "episdoes via the variable 'DELAY_BETWEEN_EPISODES_IN_SECONDS'. The program will now wait 60 seconds before " + + "trying again." + ) time.sleep(60) # Mark the exception in the error streak @@ -390,9 +416,10 @@ def processWatchedShows(): # Catch a JSON decode error - this can be raised when the API server is down and produces a HTML page, instead of JSON except json.decoder.JSONDecodeError: print( - f"({rowsCount}/{rowsTotal}) WARNING: A JSON decode error occuring whilst processing {tvShowName} " + - f"Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo}! This might occur when the server is down and has produced " + - "a HTML document instead of JSON. The script will wait 60 seconds before trying again.") + f"({rowsCount}/{rowsTotal}) WARNING: A JSON decode error occuring whilst processing {tvShowName} " + + f"Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo}! This might occur when the server is down and has produced " + + "a HTML document instead of JSON. The script will wait 60 seconds before trying again." + ) # Wait 60 seconds time.sleep(60) @@ -405,7 +432,8 @@ def processWatchedShows(): # Skip the episode else: print( - f"({rowsCount}/{rowsTotal}) Skipping '{tvShowName}' Season {tvShowSeasonNo} Episode {tvShowEpisodeNo}. It's already been imported.") + f"({rowsCount}/{rowsTotal}) Skipping '{tvShowName}' Season {tvShowSeasonNo} Episode {tvShowEpisodeNo}. It's already been imported." + ) def start(): @@ -440,7 +468,12 @@ if __name__ == "__main__": if os.path.isdir(config.GDPR_WORKSPACE_PATH): start() else: - print("Oops! The TV Time GDPR folder '" + config.GDPR_WORKSPACE_PATH + - "' does not exist on the local system. Please check it, and try again.") + print( + "Oops! The TV Time GDPR folder '" + + config.GDPR_WORKSPACE_PATH + + "' does not exist on the local system. Please check it, and try again." + ) else: - print(f"ERROR: The 'config.json' file cannot be found - have you created it yet?") + print( + f"ERROR: The 'config.json' file cannot be found - have you created it yet?" + ) From 3cd4d93d7ff805991e4c88483fe4823c52789212 Mon Sep 17 00:00:00 2001 From: Baptiste Roux Date: Sat, 19 Feb 2022 12:34:54 +0100 Subject: [PATCH 4/8] Reduce rate-limit based on API reference; Configure python logging library --- TimeToTrackt.py | 47 +++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/TimeToTrackt.py b/TimeToTrackt.py index c88f6b4..33fce57 100644 --- a/TimeToTrackt.py +++ b/TimeToTrackt.py @@ -2,19 +2,27 @@ import csv import json import os +import logging import re import sys import time from datetime import datetime import trakt.core +from trakt import init from tinydb import Query, TinyDB -from trakt import Expando from trakt.tv import TVShow +# Setup logger +logging.basicConfig( + format='%(asctime)s %(levelname)s :: %(message)s', + level=logging.INFO, + datefmt='%Y-%m-%d %H:%M:%S' +) + # Adjust this value to increase/decrease your requests between episodes. -# Make sure it's above 1 seconds to remain within the rate limit. -DELAY_BETWEEN_EPISODES_IN_SECONDS = 5 +# Make to remain within the rate limit: https://trakt.docs.apiary.io/#introduction/rate-limiting +DELAY_BETWEEN_EPISODES_IN_SECONDS = 0.75 # Create a database to keep track of completed processes database = TinyDB("localStorage.json") @@ -239,7 +247,7 @@ def getShowByName(name, seasonNo, episodeNo): sys.exit("Cancel requested...") # Otherwise, the user has entered an invalid value, warn the user to try again except: - print( + logging.error( f"Sorry! Please select a value between 0 to {len(showsWithSameName)}" ) @@ -332,7 +340,6 @@ def processWatchedShows(): # Get the date which the show was marked 'watched' in TV Time tvShowDateWatched = row["updated_at"] # Parse the watched date value into a Python type - print(tvShowDateWatched) tvShowDateWatchedConverted = datetime.strptime( tvShowDateWatched, "%Y-%m-%d %H:%M:%S" ) @@ -353,7 +360,7 @@ def processWatchedShows(): # If more than 10 errors occurred in one streak, whilst trying to import the episode # then give up, and move onto the next episode, but warn the user. if errorStreak > 10: - print( + logging.warning( f"WARNING: An error occurred 10 times in a row... skipping episode..." ) break @@ -371,8 +378,8 @@ def processWatchedShows(): if traktShowObj == None: break # Show the progress of the import on-screen - print( - f"({rowsCount+1}/{rowsTotal}) Processing Show {tvShowName} on Season {tvShowSeasonNo} - Episode {tvShowEpisodeNo}" + logging.info( + f"({rowsCount+1}/{rowsTotal}) Processing - '{tvShowName}' Season {tvShowSeasonNo} / Episode {tvShowEpisodeNo}" ) # Get the season from the Trakt API season = traktShowObj.seasons[ @@ -392,19 +399,19 @@ def processWatchedShows(): # an incorrect Trakt show has been selected, with season/episodes which don't match TV Time. # It can also occur due to a bug in Trakt Py, whereby some seasons contain an empty array of episodes. except IndexError: - print( + logging.warning( f"({rowsCount}/{rowsTotal}) WARNING: {tvShowName} Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo} does not exist (season/episode index) in Trakt!" ) break # Catch any errors which are raised because a show could not be found in Trakt except trakt.errors.NotFoundException: - print( + logging.warning( f"({rowsCount}/{rowsTotal}) WARNING: {tvShowName} Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo} does not exist (search) in Trakt!" ) break # Catch errors because of the program breaching the Trakt API rate limit except trakt.errors.RateLimitException: - print( + logging.warning( "WARNING: The program is running too quickly and has hit Trakt's API rate limit! Please increase the delay between " + "episdoes via the variable 'DELAY_BETWEEN_EPISODES_IN_SECONDS'. The program will now wait 60 seconds before " + "trying again." @@ -415,7 +422,7 @@ def processWatchedShows(): errorStreak += 1 # Catch a JSON decode error - this can be raised when the API server is down and produces a HTML page, instead of JSON except json.decoder.JSONDecodeError: - print( + logging.warning( f"({rowsCount}/{rowsTotal}) WARNING: A JSON decode error occuring whilst processing {tvShowName} " + f"Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo}! This might occur when the server is down and has produced " + "a HTML document instead of JSON. The script will wait 60 seconds before trying again." @@ -431,8 +438,8 @@ def processWatchedShows(): sys.exit("Cancel requested...") # Skip the episode else: - print( - f"({rowsCount}/{rowsTotal}) Skipping '{tvShowName}' Season {tvShowSeasonNo} Episode {tvShowEpisodeNo}. It's already been imported." + logging.info( + f"({rowsCount}/{rowsTotal}) Already imported, skipping - '{tvShowName}' Season {tvShowSeasonNo} / Episode {tvShowEpisodeNo}." ) @@ -449,16 +456,16 @@ def start(): menuSelection = 1 if not menuSelection else int(menuSelection) break except ValueError: - print("Invalid input. Please enter a numerical number.") + logging.warning("Invalid input. Please enter a numerical number.") # Start the process which is required if menuSelection == 1: # Invoke the method which will import episodes which have been watched # from TV Time into Trakt processWatchedShows() else: - print("Sorry - that's an unknown menu selection") + logging.warning("Sorry - that's an unknown menu selection") else: - print("ERROR: Unable to complete authentication to Trakt - please try again.") + logging.error("ERROR: Unable to complete authentication to Trakt - please try again.") if __name__ == "__main__": @@ -468,12 +475,12 @@ if __name__ == "__main__": if os.path.isdir(config.GDPR_WORKSPACE_PATH): start() else: - print( + logging.error( "Oops! The TV Time GDPR folder '" + config.GDPR_WORKSPACE_PATH + "' does not exist on the local system. Please check it, and try again." ) else: - print( - f"ERROR: The 'config.json' file cannot be found - have you created it yet?" + logging.error( + f"The 'config.json' file cannot be found - have you created it yet?" ) From 7af14f2f8fc12606027814639d8aab631d20d3d0 Mon Sep 17 00:00:00 2001 From: Baptiste Roux Date: Sat, 19 Feb 2022 13:28:12 +0100 Subject: [PATCH 5/8] Debug wrong season returned, fixed by https://github.com/moogar0880/PyTrakt/pull/197 --- TimeToTrackt.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/TimeToTrackt.py b/TimeToTrackt.py index 33fce57..3b6de6d 100644 --- a/TimeToTrackt.py +++ b/TimeToTrackt.py @@ -304,7 +304,7 @@ def parseSeasonNo(seasonNo, traktShowObj): # Otherwise, if the Trakt seasons start with no specials, then return the seasonNo, # but subtracted by one (e.g Season 1 in TV Time, will be 0) else: - # Only subtract is the TV Time season number is greater than 0. + # Only subtract if the TV Time season number is greater than 0. if seasonNo != 0: return seasonNo - 1 # Otherwise, the TV Time season is a special! Then you don't need to change the starting position @@ -399,8 +399,9 @@ def processWatchedShows(): # an incorrect Trakt show has been selected, with season/episodes which don't match TV Time. # It can also occur due to a bug in Trakt Py, whereby some seasons contain an empty array of episodes. except IndexError: + tvShowSlug = traktShowObj.to_json()['shows'][0]['ids']['ids']['slug'] logging.warning( - f"({rowsCount}/{rowsTotal}) WARNING: {tvShowName} Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo} does not exist (season/episode index) in Trakt!" + f"({rowsCount}/{rowsTotal}) WARNING: {tvShowName} Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo} does not exist in Trakt! (https://trakt.tv/shows/{tvShowSlug}/seasons/{tvShowSeasonNo}/episodes/{tvShowEpisodeNo})" ) break # Catch any errors which are raised because a show could not be found in Trakt From 5c3bda15b791275c51e4c976abdd9187f0d44736 Mon Sep 17 00:00:00 2001 From: Baptiste Roux Date: Sat, 19 Feb 2022 14:15:27 +0100 Subject: [PATCH 6/8] Don't prompt for authentication if Oauth token is still valid --- TimeToTrackt.py | 65 +++++++++++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/TimeToTrackt.py b/TimeToTrackt.py index 3b6de6d..01a65d6 100644 --- a/TimeToTrackt.py +++ b/TimeToTrackt.py @@ -1,23 +1,24 @@ #!/usr/bin/env python3 import csv import json -import os import logging +import os import re import sys import time from datetime import datetime +from pathlib import Path import trakt.core -from trakt import init from tinydb import Query, TinyDB +from trakt import init from trakt.tv import TVShow # Setup logger logging.basicConfig( - format='%(asctime)s %(levelname)s :: %(message)s', + format="%(asctime)s [%(levelname)7s] :: %(message)s", level=logging.INFO, - datefmt='%Y-%m-%d %H:%M:%S' + datefmt="%Y-%m-%d %H:%M:%S", ) # Adjust this value to increase/decrease your requests between episodes. @@ -34,6 +35,17 @@ class Expando(object): pass +def isAuthenticated(): + with open(f"{Path.home()}/.pytrakt.json") as f: + data = json.load(f) + daysBeforeExpiration = ( + datetime.fromtimestamp(data["OAUTH_EXPIRES_AT"]) - datetime.now() + ).days + if daysBeforeExpiration < 1: + return False + return True + + def getConfiguration(): configEx = Expando() @@ -64,7 +76,8 @@ def getFollowedShowsPath(): def initTraktAuth(): - return True + if isAuthenticated(): + return True # Set the method of authentication trakt.core.AUTH_METHOD = trakt.core.OAUTH_AUTH return init( @@ -93,7 +106,7 @@ def getYearFromTitle(title): ex.titleWithoutYear = titleValue ex.yearValue = int(yearValue) return ex - except: + except Exception: # If the above failed, then the title doesn't include a year # so return the object as is. ex.titleWithoutYear = title @@ -160,7 +173,7 @@ def getShowByName(name, seasonNo, episodeNo): if checkTitleNameMatch(name, show.title): # If the title included the year of broadcast, then we can be more picky in the results # to look for a show with a broadcast year that matches - if doesTitleIncludeYear == True: + if doesTitleIncludeYear: # If the show title is a 1:1 match, with the same broadcast year, then bingo! if (name == show.title) and (show.year == titleObj.yearValue): # Clear previous results, and only use this one @@ -206,7 +219,7 @@ def getShowByName(name, seasonNo, episodeNo): skipShow = firstMatch.get("SkipShow") # If the user did not skip, but provided an index selection, get the # matching show - if skipShow == False: + if not skipShow: return showsWithSameName[firstMatchSelectedIndex] # Otherwise, return None, which will trigger the script to skip # and move onto the next show @@ -231,7 +244,7 @@ def getShowByName(name, seasonNo, episodeNo): try: # Get the user's selection, either a numerical input, or a string 'SKIP' value indexSelected = input( - f"Please make a selection from above (or enter SKIP):" + "Please make a selection from above (or enter SKIP):" ) if indexSelected != "SKIP": @@ -246,7 +259,7 @@ def getShowByName(name, seasonNo, episodeNo): except KeyboardInterrupt: sys.exit("Cancel requested...") # Otherwise, the user has entered an invalid value, warn the user to try again - except: + except Exception: logging.error( f"Sorry! Please select a value between 0 to {len(showsWithSameName)}" ) @@ -361,7 +374,7 @@ def processWatchedShows(): # then give up, and move onto the next episode, but warn the user. if errorStreak > 10: logging.warning( - f"WARNING: An error occurred 10 times in a row... skipping episode..." + "An error occurred 10 times in a row... skipping episode..." ) break try: @@ -375,11 +388,11 @@ def processWatchedShows(): ) # If the method returned 'None', then this is an indication to skip the episode, and # move onto the next one - if traktShowObj == None: + if traktShowObj is None: break # Show the progress of the import on-screen logging.info( - f"({rowsCount+1}/{rowsTotal}) Processing - '{tvShowName}' Season {tvShowSeasonNo} / Episode {tvShowEpisodeNo}" + f"({rowsCount+1}/{rowsTotal}) - Processing '{tvShowName}' Season {tvShowSeasonNo} / Episode {tvShowEpisodeNo}" ) # Get the season from the Trakt API season = traktShowObj.seasons[ @@ -399,21 +412,23 @@ def processWatchedShows(): # an incorrect Trakt show has been selected, with season/episodes which don't match TV Time. # It can also occur due to a bug in Trakt Py, whereby some seasons contain an empty array of episodes. except IndexError: - tvShowSlug = traktShowObj.to_json()['shows'][0]['ids']['ids']['slug'] + tvShowSlug = traktShowObj.to_json()["shows"][0]["ids"]["ids"][ + "slug" + ] logging.warning( - f"({rowsCount}/{rowsTotal}) WARNING: {tvShowName} Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo} does not exist in Trakt! (https://trakt.tv/shows/{tvShowSlug}/seasons/{tvShowSeasonNo}/episodes/{tvShowEpisodeNo})" + f"({rowsCount}/{rowsTotal}) - {tvShowName} Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo} does not exist in Trakt! (https://trakt.tv/shows/{tvShowSlug}/seasons/{tvShowSeasonNo}/episodes/{tvShowEpisodeNo})" ) break # Catch any errors which are raised because a show could not be found in Trakt except trakt.errors.NotFoundException: logging.warning( - f"({rowsCount}/{rowsTotal}) WARNING: {tvShowName} Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo} does not exist (search) in Trakt!" + f"({rowsCount}/{rowsTotal}) - {tvShowName} Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo} does not exist (search) in Trakt!" ) break # Catch errors because of the program breaching the Trakt API rate limit except trakt.errors.RateLimitException: logging.warning( - "WARNING: The program is running too quickly and has hit Trakt's API rate limit! Please increase the delay between " + "The program is running too quickly and has hit Trakt's API rate limit! Please increase the delay between " + "episdoes via the variable 'DELAY_BETWEEN_EPISODES_IN_SECONDS'. The program will now wait 60 seconds before " + "trying again." ) @@ -424,7 +439,7 @@ def processWatchedShows(): # Catch a JSON decode error - this can be raised when the API server is down and produces a HTML page, instead of JSON except json.decoder.JSONDecodeError: logging.warning( - f"({rowsCount}/{rowsTotal}) WARNING: A JSON decode error occuring whilst processing {tvShowName} " + f"({rowsCount}/{rowsTotal}) - A JSON decode error occuring whilst processing {tvShowName} " + f"Season {tvShowSeasonNo}, Episode {tvShowEpisodeNo}! This might occur when the server is down and has produced " + "a HTML document instead of JSON. The script will wait 60 seconds before trying again." ) @@ -440,7 +455,7 @@ def processWatchedShows(): # Skip the episode else: logging.info( - f"({rowsCount}/{rowsTotal}) Already imported, skipping - '{tvShowName}' Season {tvShowSeasonNo} / Episode {tvShowEpisodeNo}." + f"({rowsCount}/{rowsTotal}) - Already imported, skipping '{tvShowName}' Season {tvShowSeasonNo} / Episode {tvShowEpisodeNo}." ) @@ -448,12 +463,12 @@ def start(): # Create the initial authentication with Trakt, before starting the process if initTraktAuth(): # Display a menu selection - print(f">> What do you want to do?") - print(f" 1) Import Watch History from TV Time") + print(">> What do you want to do?") + print(" 1) Import Watch History from TV Time") while True: try: - menuSelection = input(f"Enter your menu selection: ") + menuSelection = input("Enter your menu selection: ") menuSelection = 1 if not menuSelection else int(menuSelection) break except ValueError: @@ -466,7 +481,9 @@ def start(): else: logging.warning("Sorry - that's an unknown menu selection") else: - logging.error("ERROR: Unable to complete authentication to Trakt - please try again.") + logging.error( + "ERROR: Unable to complete authentication to Trakt - please try again." + ) if __name__ == "__main__": @@ -483,5 +500,5 @@ if __name__ == "__main__": ) else: logging.error( - f"The 'config.json' file cannot be found - have you created it yet?" + "The 'config.json' file cannot be found - have you created it yet?" ) From b4e39be46b1e509be028571bda7a0d92f19d0c72 Mon Sep 17 00:00:00 2001 From: Baptiste Roux Date: Sat, 19 Feb 2022 14:22:01 +0100 Subject: [PATCH 7/8] Install pip modules using requirements.txt; Switch to 'python -m pip' to use the correct version of pip --- README.md | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 94bbc59..3bd20d3 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,23 @@ # TV Time to Trakt - Import Script + ![](https://loch.digital/image_for_external_apps/4342799-01.png) A Python script to import TV Time tracked episode data into Trakt.TV - using data export provided by TV Time through a GDPR request. # Issues + They'll be a few! This was quickly put together within a few hours or so for personal usage. If you come across anything then let me know in the 'Issue' section, and I'll provide support where possible. # Notes + 1. The script is using limited data provided from a GDPR request - so the accuracy isn't 100%. But you will be prompted to manually pick the Trakt show, when it can't be determined automatically. 2. A delay of 5 seconds is added between each episode to ensure fair use of Trakt's API server. You should adjust this for your own import, but make sure it's at least 1 second to remain within the rate limit. 3. Episodes which have been processed will be saved to a TinyDB file `localStorage.json` - when you restart the script, the program will skip those episodes which have been marked 'imported'. # Setup + ## Get your Data + TV Time's API is not open. In order to get access to your personal data, you will have to request it from TV Time's support via a GDPR request - or maybe just ask for it, whatever works, it's your data. 1. Copy the template provided by [www.datarequests.org](https://www.datarequests.org/blog/sample-letter-gdpr-access-request/) into an email @@ -21,6 +26,7 @@ TV Time's API is not open. In order to get access to your personal data, you wil 4. Extract the data somewhere safe on your local system ## Register API Access at Trakt + 1. Go to "Settings" under your profile 2. Select ["Your API Applications"](https://trakt.tv/oauth/applications) 3. Select "New Application" @@ -30,11 +36,17 @@ TV Time's API is not open. In order to get access to your personal data, you wil 7. Make note of your details to be used later. ## Setup Script + ### Install Required Libraries + Install the following frameworks via Pip: -1. `pip install trakt` -2. `pip install tinydb` + +``` +python -m pip install -r requirements.txt +``` + ### Setup Configuration + Create a new file named `config.json` in the same directory of `TimeToTrakt.py`, using the below JSON contents (replace the values with your own). ``` @@ -49,4 +61,5 @@ Create a new file named `config.json` in the same directory of `TimeToTrakt.py`, Once the config is in place, execute the program using `python TimeToTrakt.py`. The process isn't 100% automated - you will need to pop back, especially with large imports, to check if the script requires a manual user input. ##### Credit -City vector created by freepik - www.freepik.com \ No newline at end of file + +City vector created by freepik - www.freepik.com From f8a37b990bc638d65c4ef537b4445c0ab9d52907 Mon Sep 17 00:00:00 2001 From: Baptiste Roux Date: Sat, 19 Feb 2022 14:26:30 +0100 Subject: [PATCH 8/8] Set DELAY_BETWEEN_EPISODES_IN_SECONDS to 1s --- README.md | 2 +- TimeToTrackt.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3bd20d3..17efa16 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ They'll be a few! This was quickly put together within a few hours or so for per # Notes 1. The script is using limited data provided from a GDPR request - so the accuracy isn't 100%. But you will be prompted to manually pick the Trakt show, when it can't be determined automatically. -2. A delay of 5 seconds is added between each episode to ensure fair use of Trakt's API server. You should adjust this for your own import, but make sure it's at least 1 second to remain within the rate limit. +2. A delay of 1 second is added between each episode to ensure fair use of Trakt's API server. You can adjust this for your own import, but make sure it's at least 0.75 second to remain within the rate limit: https://trakt.docs.apiary.io/#introduction/rate-limiting 3. Episodes which have been processed will be saved to a TinyDB file `localStorage.json` - when you restart the script, the program will skip those episodes which have been marked 'imported'. # Setup diff --git a/TimeToTrackt.py b/TimeToTrackt.py index 01a65d6..f8c7cfb 100644 --- a/TimeToTrackt.py +++ b/TimeToTrackt.py @@ -23,7 +23,7 @@ logging.basicConfig( # Adjust this value to increase/decrease your requests between episodes. # Make to remain within the rate limit: https://trakt.docs.apiary.io/#introduction/rate-limiting -DELAY_BETWEEN_EPISODES_IN_SECONDS = 0.75 +DELAY_BETWEEN_EPISODES_IN_SECONDS = 1 # Create a database to keep track of completed processes database = TinyDB("localStorage.json")