From 8357d20af06321560394c1f6fa0db7376224cb89 Mon Sep 17 00:00:00 2001 From: Markil 3 Date: Fri, 28 Nov 2025 12:37:00 -0700 Subject: [PATCH] Overhauls the system to better support single comic books. Not all comics come in serialized issues. --- src/comic_download/__init__.py | 2 +- src/comic_download/__main__.py | 81 +++--- src/comic_download/bluesky.py | 8 +- src/comic_download/comic_strip.py | 309 ++++++++++++++++------- src/comic_download/existential_comics.py | 6 +- src/comic_download/xkcd.py | 6 +- 6 files changed, 276 insertions(+), 136 deletions(-) diff --git a/src/comic_download/__init__.py b/src/comic_download/__init__.py index 95148a0..4ce54ef 100644 --- a/src/comic_download/__init__.py +++ b/src/comic_download/__init__.py @@ -14,4 +14,4 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from comic_download.comic_strip import SequenceException, ImageRepo, Comic, ComicStrip, file_safe_string, get_identifier_string, register_comic_class, find_comic_class, read_index, save_index, index_strip, is_strip_present +from comic_download.comic_strip import SequenceException, ImageRepo, ComicCollection, ComicBook, ComicStrip, file_safe_string, get_identifier_string, register_comic_class, find_comic_class, read_index, save_index, index_strip, is_strip_present diff --git a/src/comic_download/__main__.py b/src/comic_download/__main__.py index 420bc85..efdb731 100644 --- a/src/comic_download/__main__.py +++ b/src/comic_download/__main__.py @@ -27,7 +27,7 @@ from pathlib import Path import importlib import pkgutil import comic_download -from comic_download import Comic, find_comic_class +from comic_download import ComicCollection, ComicBook, ComicStrip, find_comic_class def setup_args(): parser = argparse.ArgumentParser( @@ -71,19 +71,25 @@ def load_plugins(): logging.info("Loading plugin %s" % name) importlib.import_module(name) -def load_strip(base_path, comic, identifier, plain, thread_limit): +def load_book(base_path: Path, comic_type: ComicBook, plain: bool, thread_limit: threading.Semaphore, *args): """ - Threading function for downloading XKCD strips. - - Arguments: - + Threading function to add a book to the loading queue. + + :param base_path: The directory to download all downloads to. + :param comic: The comic book class to load, or the comic book itself. + :param plain: If true, then we skip image transformations. + :param thread_limit: The semaphore that controls how many threads we execute at once. + :param args: Any arguments here will be passed to the contructor of the comic_type class. """ with thread_limit: - strip = comic.create_strip(identifier) - strip.await_load() - if not strip.get_package_path(base_path).exists(): - strip.await_download() - strip.package_data(base_path) + if type(comic_type) == type: + book = comic_type(*args) + else: + book = comic_type + book.await_load() + if not book.get_package_path(base_path).exists(): + book.await_download() + book.package_data(base_path, plain) def __main__(): setup_logging() @@ -105,30 +111,39 @@ def __main__(): for url in args.url: comic_type = find_comic_class(url.netloc + url.path) - comic = comic_type.create_from_url(url) - comics.append(comic) - comic_download.read_index(base_path) - comic.await_load() - if args.latest: - r = [comic.get_latest_strip()] + if issubclass(comic_type, ComicStrip): + collection = comic_type.collection_cls.create_from_url(url) + comic_download.read_index(base_path) + collection.await_load() + + # Figure out which strips to grab + if args.latest: + r = [collection.get_latest_strip()] + else: + r = collection.get_all_strips() + if r and type(r[0]) == int: + start = args.start or collection.get_first_strip() + end = args.end or collection.get_latest_strip() + r = r[r.index(start):r.index(end) + 1] + + # Add the chosen strips to the list of books to add + try: + for index in reversed(r): + comics.append(comic_type(collection, index)) + finally: + logging.info("Saving index") + comic_download.save_index(base_path) else: - r = comic.get_all_strips() - if r and type(r[0]) == int: - start = args.start or comic.get_first_strip() - end = args.end or comic.get_latest_strip() - r = r[r.index(start):r.index(end) + 1] - - try: - for index in reversed(r): - if not comic_download.is_strip_present(comic, base_path, ".cbz", str(index)): - t = threading.Thread(name=str(comic.get_identifier_string()) + "-" + str(index), target=load_strip, args=(base_path, comic, index, args.plain, thread_limit)) - threads.append(t) - t.daemon = True - t.start() - finally: - logging.info("Saving index") - comic_download.save_index(base_path) + comics.append(comic_type.create_from_url(url)) + comic_download.read_index(base_path) + for comic in comics: + # Only make threads for books not accounted for in the index + if not comic_download.is_strip_present(comic, base_path, ".cbz"): + t = threading.Thread(name=str(comic), target=load_book, args=(base_path, comic, args.plain, thread_limit)) + threads.append(t) + t.daemon = True + t.start() try: while any([t.is_alive() for t in threads]): time.sleep(1) diff --git a/src/comic_download/bluesky.py b/src/comic_download/bluesky.py index 3016370..60a5c8c 100644 --- a/src/comic_download/bluesky.py +++ b/src/comic_download/bluesky.py @@ -33,7 +33,7 @@ from PIL import Image, ImageDraw, ImageFont from bs4 import BeautifulSoup from atproto import Client -from comic_download.comic_strip import Comic, ComicStrip, register_comic_class +from comic_download.comic_strip import ComicCollection as Comic, ComicStrip, register_comic_class class BlueskyComic(Comic): """ @@ -185,11 +185,11 @@ class BlueskyComicStrip(ComicStrip): self.captions.append(image.alt) else: self.captions.append("") - self.date = datetime.datetime.fromisoformat(post_data.value.created_at) + self.date = datetime.datetime.fromisoformat(post_data.value.created_at[0:-1]) def _transform_images(self): self.transformed_images = self.images -BlueskyComic.strip_cls = BlueskyComicStrip -register_comic_class("bsky.app", BlueskyComic) +BlueskyComicStrip.collection_cls = BlueskyComic +register_comic_class("bsky.app", BlueskyComicStrip) diff --git a/src/comic_download/comic_strip.py b/src/comic_download/comic_strip.py index 68072bf..a7d5884 100644 --- a/src/comic_download/comic_strip.py +++ b/src/comic_download/comic_strip.py @@ -113,7 +113,7 @@ A lock for enforcing thread safety when loading the comic index file. def read_index(base_path: Path): """ - Saves the strip index to the same base path we download comic strips to. + Saves the strip index to the same base path we download comic books to. It is saved to "index.json." :param base_path: The folder to save the index to. @@ -136,7 +136,7 @@ def read_index(base_path: Path): def save_index(base_path: Path): """ - Saves the strip index to the same base path we download comic strips to. + Saves the strip index to the same base path we download comic books to. It is saved to "index.json." :param base_path: The folder to save the index to. @@ -149,40 +149,34 @@ def save_index(base_path: Path): json.dump(strip_index, index_fp) def index_strip(strip): - """ - Registers a comic to the index. This does not save the comic index. + Registers a book to the index. This does not save the index. - :param strip: The comic strip to index + :param strip: The comic book to index """ global strip_index global _index_loaded global _index_lock with _index_lock: - if not strip.comic.get_identifier_string() in strip_index: - strip_index[strip.comic.get_identifier_string()] = {} - strip_index[strip.comic.get_identifier_string()][strip.get_identifier_string()] = strip.get_filename() + strip_index[str(strip)] = strip.get_filename() -def is_strip_present(comic, base_path: Path, suffix: str, identifier) -> bool: +def is_strip_present(comic, base_path: Path, suffix: str) -> bool: """ - Checks to see if a given strip has been saved to the file. + Checks to see if a given book has been saved to the file. - :param comic: The comic object that the strip belongs to. + :param comic: The comic book object to check. :param base_path: The folder we expect the strip to show up in. :param suffix: The type suffix we want to check (i.e. ".cbz") - :param identifier: The identifier to check. :returns: True if the strip file has been saved. """ global strip_index global _index_loaded global _index_lock - if not comic.get_identifier_string() in strip_index: + if not str(comic) in strip_index: return False - if not get_identifier_string(identifier) in strip_index[comic.get_identifier_string()]: - return False - path = Path(base_path, strip_index[comic.get_identifier_string()][get_identifier_string(identifier)] + suffix) + path = Path(base_path, strip_index[str(comic)] + suffix) return path.exists() - + class ImageRepo(ABC): @@ -224,7 +218,7 @@ class ImageRepo(ABC): def get_identifier_string(self): """ - Obtains a file-safe string representation of the identifer. If the identifier is a list or tuple, the + Obtains a file-safe string representation of the identifier. If the identifier is a list or tuple, the elements will be joined by a dash character. :returns: The identifier in string form. @@ -265,7 +259,7 @@ class ImageRepo(ABC): with self._load_lock: if not self._data_loaded: self._load_data() - logging.info("Completed loading of %s", self.get_identifier_string()) + logging.info("Completed loading of %s", repr(self)) self._data_loaded = True self._load_lock.notify_all() @@ -314,9 +308,9 @@ class ImageRepo(ABC): with self._download_lock: # Checks to see if we need to download and that one is not already in progress if self.image_urls and not self.images: - logging.info("Downloading %s", self.get_identifier_string()) + logging.info("Downloading %s", repr(self)) self._download_data() - logging.info("Completed downloading of %s", self.get_identifier_string()) + logging.info("Completed downloading of %s", repr(self)) self._data_downloaded = True self._download_lock.notify_all() @@ -356,8 +350,8 @@ class ImageRepo(ABC): elif type(self.image_urls) == list: iterator = range(len(self.image_urls)) for name in iterator: - logging.info(" Downloading repository %s resource %s", self.get_identifier_string(), name) - image_path = NamedTemporaryFile(mode='wb', suffix=str(name), prefix=self.get_identifier_string(), delete = False) + logging.info(" Downloading repository %s resource %s", repr(self), name) + image_path = NamedTemporaryFile(mode='wb', suffix=str(name), prefix=str(self), delete = False) result = requests.get(self.image_urls[name], headers=self.download_headers) result.raise_for_status() # Handles necessary transformations for a URL (i.e., redirections) @@ -368,13 +362,19 @@ class ImageRepo(ABC): self.images.append(image_path) else: self.images[name] = image_path + + def __str__(self): + return str(self.__class__.__name__) + "__" + self.get_identifier_string() + + def __repr__(self): + return "<" + str(self.__class__.__name__) + " " + self.get_identifier_string() + ">" -class Comic(ImageRepo, ABC): +class ComicCollection(ImageRepo, ABC): """ A generic representation of a comic collection. Comic-wide resources are stored here. """ - + strip_cls = None """A reference to the class used for this comic's strips.""" @@ -389,12 +389,12 @@ class Comic(ImageRepo, ABC): """The title of the comic.""" self.author = None """The author(s) of the comic. Can be a string for a name, a list of names, or a list of objects containing the names under "name" and role under "role".""" - + #@abstractmethod @classmethod def create_from_url(cls, url): """ - Creates a Comic instance from a given URL + Creates a ComicCollection instance from a given URL """ pass @@ -432,33 +432,33 @@ class Comic(ImageRepo, ABC): """ pass - def create_strip(self, identifier): - """ - Creates a new ComicStrip object. + def __str__(self): + return self.__class__.__name__ + "__" + self.get_identifier_string() - The class variable #strip_cls must be set for this to work. - """ - return self.strip_cls(self, identifier) + def __repr__(self): + if self.title: + return "<" + str(self.__class__.__name__) + " " + self.title + " (" + self.get_identifier_string() + ")>" + else: + return "<" + str(self.__class__.__name__) + " " + self.get_identifier_string() + ">" -class ComicStrip(ImageRepo, ABC): +class ComicBook(ImageRepo, ABC): """ A generic representation of a single comic strip or issue. """ - def __init__(self, comic, identifier): + def __init__(self, identifier): """ Creates a comic strip - :param comic: The `Comic` object that this strip is part of. :param indentifier: The globally unique identifier used to find the strip on the website. Can be a tuple. """ super().__init__(identifier, list) - self.comic = comic - """The comic that this strip is part of.""" self.title = None """The title of the comic. Empty until the comic data is loaded.""" + self.series = None + """The series that this comic is a part of. Empty until the comic data is loaded.""" self.image_urls = [] """A list of URLs we can download the comic panels from, in order. Empty until the comic data is loaded.""" self._image_paths = [] @@ -474,27 +474,34 @@ class ComicStrip(ImageRepo, ABC): self._transformed = False """A flag keeping track of whether the strip transformation took place.""" + @classmethod + def create_from_url(cls, url): + """ + Creates a Comic instance from a given URL + """ + pass + def get_filename(self): """ Obtains the name of the final export (not including the stem). Calling this method requires the data be loaded first. - :returns: A human-friendly file, like "05 - Riddles in the Dark" or "2025-06-01 - Chapter 5." + :returns: A human-friendly filename, like "05 - Riddles in the Dark" or "2025-06-01 - Chapter 5." """ - prefix_num_form = "#{:02}" - if type(self.identifier) == int: - prefix = prefix_num_form.format(self.identifier) - elif (type(self.identifier) == list or type(self.identifier) == tuple) and type(self.identifier[-1]) == int: - prefix = prefix_num_form.format(self.identifier[-1]) - elif type(self.date) == date: - prefix = self.date.strftime("#%Y%m%d") - elif type(self.date) == datetime: - prefix = self.date.date().strftime("#%Y%m%d") - else: - prefix = self.get_identifier_string() if self.title: - return (file_safe_string(self.comic.title) + " " + prefix + " - " + file_safe_string(self.title))[0:250] + return file_safe_string(self.title)[0:250] else: - return prefix + prefix_num_form = "#{:02}" + if type(self.identifier) == int: + prefix = prefix_num_form.format(self.identifier) + elif (type(self.identifier) == list or type(self.identifier) == tuple) and type(self.identifier[-1]) == int: + prefix = prefix_num_form.format(self.identifier[-1]) + elif type(self.date) == date: + prefix = self.date.strftime("#%Y%m%d") + elif type(self.date) == datetime: + prefix = self.date.date().strftime("#%Y%m%d") + else: + prefix = self.get_identifier_string() + return file_safe_string(self.__name__ + " - " + prefix) def get_package_path(self, base_path: Path): """ @@ -503,25 +510,13 @@ class ComicStrip(ImageRepo, ABC): :param base_path: The folder to save comics to. :returns: The path to save to. """ - return Path(base_path, file_safe_string(self.comic.title), f"{self.get_filename()}.cbz") + filename = self.get_filename() + return Path(base_path, filename, f"{filename}.cbz") def load_data(self): - self.comic.load_data() super().load_data() index_strip(self) - def download_data(self): - self.comic.download_data() - super().download_data() - - def await_load(self): - self.comic.await_load() - super().await_load() - - def await_download(self): - self.comic.await_download() - super().await_download() - @abstractmethod def _load_data(self): """ @@ -558,7 +553,7 @@ class ComicStrip(ImageRepo, ABC): soup = BeautifulSoup('', features="xml") info_tag = soup.find("ComicInfo") info_tag.append(soup.new_tag("Title", string=self.title)) - info_tag.append(soup.new_tag("Series", string=self.comic.title)) + info_tag.append(soup.new_tag("Series", string=self.series)) number_el = soup.new_tag("Number") if type(self.identifier) == int: number_el.string = str(self.identifier) @@ -570,13 +565,13 @@ class ComicStrip(ImageRepo, ABC): info_tag.append(soup.new_tag("Year", string=str(self.date.year))) info_tag.append(soup.new_tag("Month", string=str(self.date.month))) info_tag.append(soup.new_tag("Day", string=str(self.date.day))) - if self.comic.author: - if type(self.comic.author) == str: - info_tag.append(soup.new_tag("Writer", string=self.comic.author)) - elif type(self.comic.author) == list: + if self.author: + if type(self.author) == str: + info_tag.append(soup.new_tag("Writer", string=self.author)) + elif type(self.author) == list: authors = {} writer_string = "" - for author in self.comic.author: + for author in self.author: author_class = None if type(author) == str: author_name = author @@ -609,37 +604,45 @@ class ComicStrip(ImageRepo, ABC): info_tag.append(soup.new_tag("PageCount", string=str(len(self.transformed_images)))) return str(soup) - def package_data(self, base_path: Path): + def package_data(self, base_path: Path, plain: bool = False): """ Compresses all the data into a cbz file. + + :param base_path: The path to save all downloads to. + :param plain: If true, we will save the images as downloaded, without any transformations. """ if not self.is_loaded(): - raise SequenceException("Cannot package comic strip before strip data has been fully loaded.") + raise SequenceException("Cannot package comic book before data has been fully loaded.") if not self.is_downloaded(): - raise SequenceException("Cannot package comic strip before strip data has been fully downloaded.") - if not self.comic.is_loaded(): - raise SequenceException("Cannot package comic strip before comic data has been fully loaded.") - if not self.comic.is_downloaded(): - raise SequenceException("Cannot package comic strip before comic data has been fully downloaded.") + raise SequenceException("Cannot package comic book before data has been fully downloaded.") if not self.transformed_images: - if not self._transformed: - if self._transform_lock._lock.locked(): - # Await for the existing transform to complete - self._transform_lock.wait() - else: - # Run the transformation outselves - with self._transform_lock: - self._transform_images() - self._transformed = True - self._transform_lock.notify_all() + if plain: + self.transformed_images = self.images + else: + if not self._transformed: + if self._transform_lock._lock.locked(): + # Await for the existing transform to complete + self._transform_lock.wait() + else: + # Run the transformation outselves + with self._transform_lock: + self._transform_images() + self._transformed = True + self._transform_lock.notify_all() save_path = self.get_package_path(base_path) save_path.parent.mkdir(parents=True, exist_ok=True) with ZipFile(self.get_package_path(base_path), 'w') as comic_zip: for i in range(len(self.transformed_images)): image = self.transformed_images[i] image_path = Path(image.name) + suffix = image_path.suffix + if len(suffix) > 5: + # There is something fishy going on, so let us just use the default + suffix = None + if not suffix: + suffix = ".png" logging.info("Packaging %s" % image_path.suffix) - with open(image.name, 'rb') as image_stream, comic_zip.open(str(i + 1) + (image_path.suffix or '.png'), 'w') as zip_stream: + with open(image.name, 'rb') as image_stream, comic_zip.open(str(i + 1) + (suffix), 'w') as zip_stream: zip_stream.write(image_stream.read()) # Add a ComicInfo file comic_info = self.create_comic_info() @@ -655,6 +658,128 @@ class ComicStrip(ImageRepo, ABC): if image_path.exists(): image_path.unlink() + def __repr__(self): + if self.title: + return "<" + str(self.__class__.__name__) + " " + self.title + " (" + self.get_identifier_string() + ")>" + else: + return "<" + str(self.__class__.__name__) + " " + self.get_identifier_string() + ">" + +class ComicStrip(ComicBook, ABC): + """ + A generic representation of an issue in a comic series. Unlike the base + ComicBook class, these objects are tied to a ComicCollection instance + and derive some data from that class. + """ + def __init__(self, comic, identifier): + """ + Creates a comic strip. + + :param comic: The `ComicCollection` object that this strip is part of. + :param indentifier: The collection-unique identifier used to find the strip on the website. Can be a tuple. + """ + super().__init__(identifier) + + self.comic = comic + + @classmethod + def create_from_url(cls, url, collection): + """ + Creates a Comic instance from a given URL + """ + return collection.create_from_url(url) + + @property + def author(self): + """ + The author(s) of the comic. Can be a string for a name, a list of names, or a list of objects containing the names under "name" and role under "role". + + For Comic Strips, this references back to the main comic series author. + """ + return self.comic.author + + @author.setter + def author(self, x): + pass + + @property + def series(self) -> str: + """The series that this comic is a part of.""" + return self.comic.title + + @series.setter + def series(self, x): + pass + + def get_filename(self): + """ + Obtains the name of the final export (not including the stem). Calling this method requires the data be loaded first. + + :returns: A human-friendly file, like "05 - Riddles in the Dark" or "2025-06-01 - Chapter 5." + """ + prefix_num_form = "#{:02}" + if type(self.identifier) == int: + prefix = prefix_num_form.format(self.identifier) + elif (type(self.identifier) == list or type(self.identifier) == tuple) and type(self.identifier[-1]) == int: + prefix = prefix_num_form.format(self.identifier[-1]) + elif type(self.date) == date: + prefix = self.date.strftime("#%Y%m%d") + elif type(self.date) == datetime: + prefix = self.date.date().strftime("#%Y%m%d") + else: + prefix = self.get_identifier_string() + if self.title: + return (file_safe_string(self.comic.title) + " " + prefix + " - " + file_safe_string(self.title))[0:250] + else: + return prefix + + def get_package_path(self, base_path: Path): + """ + Obtains the path to save the comic archive to. + + :param base_path: The folder to save comics to. + :returns: The path to save to. + """ + return Path(base_path, file_safe_string(self.comic.title), f"{self.get_filename()}.cbz") + + def load_data(self): + self.comic.load_data() + super().load_data() + + def download_data(self): + self.comic.download_data() + super().download_data() + + def await_load(self): + self.comic.await_load() + super().await_load() + + def await_download(self): + self.comic.await_download() + super().await_download() + + def package_data(self, base_path: Path, plain: bool = False): + if not self.comic.is_loaded(): + raise SequenceException("Cannot package comic strip before comic data has been fully loaded.") + if not self.comic.is_downloaded(): + raise SequenceException("Cannot package comic strip before comic data has been fully downloaded.") + super().package_data(base_path, plain) + + def __str__(self): + return self.__class__.__name__ + "__" + self.comic.get_identifier_string() + "__" + self.get_identifier_string() + + def __repr__(self): + representation = "<" + str(self.__class__.__name__) + " " + if self.comic.title: + representation += self.comic.title + " (" + self.comic.get_identifier_string() + "): " + else: + representation += self.comic.get_identifier_string() + ": " + if self.title: + representation += self.title + " (" + self.get_identifier_string() + ")" + else: + representation += self.comic.get_identifier_string() + representation += ">" + return representation + class URLFormat: """An internal class used to help sort URLs based on subdomains and paths""" def __init__(self, url: str): @@ -736,7 +861,7 @@ comic_registry = {} _registry_subdomains = [] """An internal list keeping track of the order we should use to find the right parser (priority is last)""" -def register_comic_class(url: str, comic_cls: type[Comic]): +def register_comic_class(url: str, comic_cls: type[ComicBook]): """ Registers a comic type. @@ -769,7 +894,7 @@ def register_comic_class(url: str, comic_cls: type[Comic]): comic_registry[url_f] = comic_cls bisect.insort(_registry_subdomains, url_f) -def find_comic_class(url: str) -> type[Comic]: +def find_comic_class(url: str) -> type[ComicBook]: """ Finds the appropriate comic for a given URL. diff --git a/src/comic_download/existential_comics.py b/src/comic_download/existential_comics.py index 3b6ffae..8ec7b74 100644 --- a/src/comic_download/existential_comics.py +++ b/src/comic_download/existential_comics.py @@ -31,7 +31,7 @@ import requests from PIL import Image, ImageDraw, ImageFont from bs4 import BeautifulSoup -from comic_download.comic_strip import Comic, ComicStrip, register_comic_class +from comic_download.comic_strip import ComicCollection as Comic, ComicStrip, register_comic_class class ExistentialComic(Comic): """ @@ -233,5 +233,5 @@ class ExistentialComicStrip(ComicStrip): f_img.save(f_image_path.name) self.transformed_images.append(f_image_path) -ExistentialComic.strip_cls = ExistentialComicStrip -register_comic_class("existentialcomics.com", ExistentialComic) +ExistentialComicStrip.collection_cls = ExistentialComic +register_comic_class("existentialcomics.com", ExistentialComicStrip) diff --git a/src/comic_download/xkcd.py b/src/comic_download/xkcd.py index f9a8cae..e14152d 100644 --- a/src/comic_download/xkcd.py +++ b/src/comic_download/xkcd.py @@ -31,7 +31,7 @@ import requests from PIL import Image, ImageDraw, ImageFont from bs4 import BeautifulSoup -from comic_download.comic_strip import Comic, ComicStrip, register_comic_class +from comic_download.comic_strip import ComicCollection as Comic, ComicStrip, register_comic_class class XKCDComic(Comic): """ @@ -152,5 +152,5 @@ class XKCDComicStrip(ComicStrip): self.transformed_images.append(f_image) -XKCDComic.strip_cls = XKCDComicStrip -register_comic_class("xkcd.com", XKCDComic) +XKCDComicStrip.collection_cls = XKCDComic +register_comic_class("xkcd.com", XKCDComicStrip)