From 4ede8bcaa866bb689358b9fe5ffd152e79425d37 Mon Sep 17 00:00:00 2001 From: Markil 3 Date: Tue, 12 Aug 2025 13:38:00 -0600 Subject: [PATCH] Moves the index outside of classes It makes more sense to have a universal index, rather than one per comic. --- comic_download/__init__.py | 2 +- comic_download/__main__.py | 8 +- comic_download/comic_strip.py | 164 +++++++++++++++++++--------------- 3 files changed, 97 insertions(+), 77 deletions(-) diff --git a/comic_download/__init__.py b/comic_download/__init__.py index 8d7e5a9..95148a0 100644 --- a/comic_download/__init__.py +++ b/comic_download/__init__.py @@ -14,4 +14,4 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from comic_download.comic_strip import SequenceException, ImageRepo, Comic, ComicStrip, file_safe_string, get_identifier_string, register_comic_class, find_comic_class +from comic_download.comic_strip import SequenceException, ImageRepo, Comic, ComicStrip, file_safe_string, get_identifier_string, register_comic_class, find_comic_class, read_index, save_index, index_strip, is_strip_present diff --git a/comic_download/__main__.py b/comic_download/__main__.py index 03fe237..7514afd 100644 --- a/comic_download/__main__.py +++ b/comic_download/__main__.py @@ -105,7 +105,7 @@ def __main__(): comic_type = find_comic_class(url.netloc + url.path) comic = comic_type.create_from_url(url) comics.append(comic) - comic.read_index(base_path) + comic_download.read_index(base_path) comic.await_load() if args.latest: r = [comic.get_latest_strip()] @@ -118,19 +118,19 @@ def __main__(): try: for index in reversed(r): - if not comic.is_strip_present(base_path, ".cbz", str(index)): + if not comic_download.is_strip_present(comic, base_path, ".cbz", str(index)): t = threading.Thread(name=str(comic.get_identifier_string()) + "-" + str(index), target=load_strip, args=(base_path, comic, index, args.plain, thread_limit)) threads.append(t) t.start() finally: - comic.save_index(base_path) + comic_download.save_index(base_path) try: for t in threads: t.join() finally: for comic in comics: - comic.save_index(base_path) + comic_download.save_index(base_path) if __name__ == "__main__": __main__() diff --git a/comic_download/comic_strip.py b/comic_download/comic_strip.py index f77bd0b..dea143b 100644 --- a/comic_download/comic_strip.py +++ b/comic_download/comic_strip.py @@ -93,6 +93,97 @@ class SequenceException(Exception): """ pass +strip_index = {} +""" +The strip index is used to map comic strip identifiers to the actual file name (sans stems). This +is needed since sometimes, the information used to name a file (like the title) is loaded from +the webpage. By caching this information, we can more easily check to see if we can skip a download. +""" + +_index_loaded = False +""" +A flag for when the index has been successfully loaded. +""" + +_index_lock = Condition(Lock()) +""" +A lock for enforcing thread safety when loading the comic index file. +""" + +def read_index(base_path: Path): + """ + Saves the strip index to the same base path we download comic strips to. + It is saved to "index.json." + + :param base_path: The folder to save the index to. + """ + global strip_index + global _index_loaded + global _index_lock + with _index_lock: + if not _index_loaded: + index_path = Path(base_path, "index.json") + if index_path.exists(): + try: + with open(Path(base_path, "index.json"), 'r') as index_fp: + strip_index = json.load(index_fp) + except json.JSONDecodeError as e: + logging.exception(e) + strip_index = {} + _index_loaded = True + _index_lock.notify_all() + +def save_index(base_path: Path): + """ + Saves the strip index to the same base path we download comic strips to. + It is saved to "index.json." + + :param base_path: The folder to save the index to. + """ + global strip_index + global _index_loaded + global _index_lock + with _index_lock: + with open(Path(base_path, "index.json"), 'w') as index_fp: + json.dump(strip_index, index_fp) + +def index_strip(strip): + + """ + Registers a comic to the index. This does not save the comic index. + + :param strip: The comic strip to index + """ + global strip_index + global _index_loaded + global _index_lock + with _index_lock: + if not strip.comic.identifier in strip_index: + strip_index[strip.comic.get_identifier_string()] = {} + strip_index[strip.comic.get_identifier_string()][strip.identifier] = strip.get_filename() + +def is_strip_present(comic, base_path: Path, suffix: str, identifier) -> bool: + """ + Checks to see if a given strip has been saved to the file. + + :param comic: The comic object that the strip belongs to. + :param base_path: The folder we expect the strip to show up in. + :param suffix: The type suffix we want to check (i.e. ".cbz") + :param identifier: The identifier to check. + :returns: True if the strip file has been saved. + """ + global strip_index + global _index_loaded + global _index_lock + if not comic.get_identifier_string() in strip_index: + return False + if not identifier in strip_index[comic.get_identifier_string()]: + return False + path = Path(base_path, strip_index[comic.get_identifier_string()][identifier] + suffix) + return path.exists() + + + class ImageRepo(ABC): """ A generic representation of an online repository of data. This class allows for the @@ -284,77 +375,6 @@ class Comic(ImageRepo, ABC): """The title of the comic.""" self.author = None """The author(s) of the comic. Can be a string for a name, a list of names, or a list of objects containing the names under "name" and role under "role".""" - - self.strip_index = {} - """ - The strip index maps comic identifiers to their filenames (sans stems). If we - cache this data, we can find out where each strip lives without having to - load data from the website, thus using less requests for already-downloaded - strips. - """ - self._index_loaded = False - """A flag for when the index has been successfully loaded.""" - self._index_lock = Condition(Lock()) - """A lock for enforcing thread safety when loading the comic index file.""" - - def read_index(self, base_path: Path): - """ - Saves the strip index to the same base path we download comic strips to. - It is saved to "index.json." - - :param base_path: The folder to save the index to. - """ - with self._index_lock: - if not self._index_loaded: - index_path = Path(base_path, "index.json") - if index_path.exists(): - try: - with open(Path(base_path, "index.json"), 'r') as index_fp: - self.strip_index = json.load(index_fp) - except json.JSONDecodeError as e: - logging.exception(e) - self.strip_index = {} - self._index_loaded = True - self._index_lock.notify_all() - - def save_index(self, base_path: Path): - """ - Saves the strip index to the same base path we download comic strips to. - It is saved to "index.json." - - :param base_path: The folder to save the index to. - """ - with self._index_lock: - with open(Path(base_path, "index.json"), 'w') as index_fp: - json.dump(self.strip_index, index_fp) - - def index_strip(self, strip): - """ - Registers a comic to the index. This does not save the comic index. - - :param strip: The comic strip to index - """ - with self._index_lock: - if not self.identifier in self.strip_index: - self.strip_index[self.get_identifier_string()] = {} - self.strip_index[self.get_identifier_string()][strip.identifier] = strip.get_filename() - - def is_strip_present(self, base_path: Path, suffix: str, identifier) -> bool: - """ - Checks to see if a given strip has been saved to the file. - - :param base_path: The folder we expect the strip to show up in. - :param suffix: The type suffix we want to check (i.e. ".cbz") - :param identifier: The identifier to check. - :returns: True if the strip file has been saved. - """ - if not self.get_identifier_string() in self.strip_index: - return False - if not identifier in self.strip_index[self.get_identifier_string()]: - return False - path = Path(base_path, self.strip_index[self.get_identifier_string()][identifier] + suffix) - return path.exists() - #@abstractmethod @classmethod @@ -474,7 +494,7 @@ class ComicStrip(ImageRepo, ABC): def load_data(self): self.comic.load_data() super().load_data() - self.comic.index_strip(self) + index_strip(self) def download_data(self): self.comic.download_data()