Overhauls the system to better support single comic books.

Not all comics come in serialized issues.
This commit is contained in:
2025-11-28 12:37:00 -07:00
parent 73fb7e9a0d
commit 8357d20af0
6 changed files with 276 additions and 136 deletions

View File

@@ -14,4 +14,4 @@
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from comic_download.comic_strip import SequenceException, ImageRepo, Comic, ComicStrip, file_safe_string, get_identifier_string, register_comic_class, find_comic_class, read_index, save_index, index_strip, is_strip_present
from comic_download.comic_strip import SequenceException, ImageRepo, ComicCollection, ComicBook, ComicStrip, file_safe_string, get_identifier_string, register_comic_class, find_comic_class, read_index, save_index, index_strip, is_strip_present

View File

@@ -27,7 +27,7 @@ from pathlib import Path
import importlib
import pkgutil
import comic_download
from comic_download import Comic, find_comic_class
from comic_download import ComicCollection, ComicBook, ComicStrip, find_comic_class
def setup_args():
parser = argparse.ArgumentParser(
@@ -71,19 +71,25 @@ def load_plugins():
logging.info("Loading plugin %s" % name)
importlib.import_module(name)
def load_strip(base_path, comic, identifier, plain, thread_limit):
def load_book(base_path: Path, comic_type: ComicBook, plain: bool, thread_limit: threading.Semaphore, *args):
"""
Threading function for downloading XKCD strips.
Arguments:
Threading function to add a book to the loading queue.
:param base_path: The directory to download all downloads to.
:param comic: The comic book class to load, or the comic book itself.
:param plain: If true, then we skip image transformations.
:param thread_limit: The semaphore that controls how many threads we execute at once.
:param args: Any arguments here will be passed to the contructor of the comic_type class.
"""
with thread_limit:
strip = comic.create_strip(identifier)
strip.await_load()
if not strip.get_package_path(base_path).exists():
strip.await_download()
strip.package_data(base_path)
if type(comic_type) == type:
book = comic_type(*args)
else:
book = comic_type
book.await_load()
if not book.get_package_path(base_path).exists():
book.await_download()
book.package_data(base_path, plain)
def __main__():
setup_logging()
@@ -105,30 +111,39 @@ def __main__():
for url in args.url:
comic_type = find_comic_class(url.netloc + url.path)
comic = comic_type.create_from_url(url)
comics.append(comic)
comic_download.read_index(base_path)
comic.await_load()
if args.latest:
r = [comic.get_latest_strip()]
if issubclass(comic_type, ComicStrip):
collection = comic_type.collection_cls.create_from_url(url)
comic_download.read_index(base_path)
collection.await_load()
# Figure out which strips to grab
if args.latest:
r = [collection.get_latest_strip()]
else:
r = collection.get_all_strips()
if r and type(r[0]) == int:
start = args.start or collection.get_first_strip()
end = args.end or collection.get_latest_strip()
r = r[r.index(start):r.index(end) + 1]
# Add the chosen strips to the list of books to add
try:
for index in reversed(r):
comics.append(comic_type(collection, index))
finally:
logging.info("Saving index")
comic_download.save_index(base_path)
else:
r = comic.get_all_strips()
if r and type(r[0]) == int:
start = args.start or comic.get_first_strip()
end = args.end or comic.get_latest_strip()
r = r[r.index(start):r.index(end) + 1]
try:
for index in reversed(r):
if not comic_download.is_strip_present(comic, base_path, ".cbz", str(index)):
t = threading.Thread(name=str(comic.get_identifier_string()) + "-" + str(index), target=load_strip, args=(base_path, comic, index, args.plain, thread_limit))
threads.append(t)
t.daemon = True
t.start()
finally:
logging.info("Saving index")
comic_download.save_index(base_path)
comics.append(comic_type.create_from_url(url))
comic_download.read_index(base_path)
for comic in comics:
# Only make threads for books not accounted for in the index
if not comic_download.is_strip_present(comic, base_path, ".cbz"):
t = threading.Thread(name=str(comic), target=load_book, args=(base_path, comic, args.plain, thread_limit))
threads.append(t)
t.daemon = True
t.start()
try:
while any([t.is_alive() for t in threads]):
time.sleep(1)

View File

@@ -33,7 +33,7 @@ from PIL import Image, ImageDraw, ImageFont
from bs4 import BeautifulSoup
from atproto import Client
from comic_download.comic_strip import Comic, ComicStrip, register_comic_class
from comic_download.comic_strip import ComicCollection as Comic, ComicStrip, register_comic_class
class BlueskyComic(Comic):
"""
@@ -185,11 +185,11 @@ class BlueskyComicStrip(ComicStrip):
self.captions.append(image.alt)
else:
self.captions.append("")
self.date = datetime.datetime.fromisoformat(post_data.value.created_at)
self.date = datetime.datetime.fromisoformat(post_data.value.created_at[0:-1])
def _transform_images(self):
self.transformed_images = self.images
BlueskyComic.strip_cls = BlueskyComicStrip
register_comic_class("bsky.app", BlueskyComic)
BlueskyComicStrip.collection_cls = BlueskyComic
register_comic_class("bsky.app", BlueskyComicStrip)

View File

@@ -113,7 +113,7 @@ A lock for enforcing thread safety when loading the comic index file.
def read_index(base_path: Path):
"""
Saves the strip index to the same base path we download comic strips to.
Saves the strip index to the same base path we download comic books to.
It is saved to "index.json."
:param base_path: The folder to save the index to.
@@ -136,7 +136,7 @@ def read_index(base_path: Path):
def save_index(base_path: Path):
"""
Saves the strip index to the same base path we download comic strips to.
Saves the strip index to the same base path we download comic books to.
It is saved to "index.json."
:param base_path: The folder to save the index to.
@@ -149,38 +149,32 @@ def save_index(base_path: Path):
json.dump(strip_index, index_fp)
def index_strip(strip):
"""
Registers a comic to the index. This does not save the comic index.
Registers a book to the index. This does not save the index.
:param strip: The comic strip to index
:param strip: The comic book to index
"""
global strip_index
global _index_loaded
global _index_lock
with _index_lock:
if not strip.comic.get_identifier_string() in strip_index:
strip_index[strip.comic.get_identifier_string()] = {}
strip_index[strip.comic.get_identifier_string()][strip.get_identifier_string()] = strip.get_filename()
strip_index[str(strip)] = strip.get_filename()
def is_strip_present(comic, base_path: Path, suffix: str, identifier) -> bool:
def is_strip_present(comic, base_path: Path, suffix: str) -> bool:
"""
Checks to see if a given strip has been saved to the file.
Checks to see if a given book has been saved to the file.
:param comic: The comic object that the strip belongs to.
:param comic: The comic book object to check.
:param base_path: The folder we expect the strip to show up in.
:param suffix: The type suffix we want to check (i.e. ".cbz")
:param identifier: The identifier to check.
:returns: True if the strip file has been saved.
"""
global strip_index
global _index_loaded
global _index_lock
if not comic.get_identifier_string() in strip_index:
if not str(comic) in strip_index:
return False
if not get_identifier_string(identifier) in strip_index[comic.get_identifier_string()]:
return False
path = Path(base_path, strip_index[comic.get_identifier_string()][get_identifier_string(identifier)] + suffix)
path = Path(base_path, strip_index[str(comic)] + suffix)
return path.exists()
@@ -224,7 +218,7 @@ class ImageRepo(ABC):
def get_identifier_string(self):
"""
Obtains a file-safe string representation of the identifer. If the identifier is a list or tuple, the
Obtains a file-safe string representation of the identifier. If the identifier is a list or tuple, the
elements will be joined by a dash character.
:returns: The identifier in string form.
@@ -265,7 +259,7 @@ class ImageRepo(ABC):
with self._load_lock:
if not self._data_loaded:
self._load_data()
logging.info("Completed loading of %s", self.get_identifier_string())
logging.info("Completed loading of %s", repr(self))
self._data_loaded = True
self._load_lock.notify_all()
@@ -314,9 +308,9 @@ class ImageRepo(ABC):
with self._download_lock:
# Checks to see if we need to download and that one is not already in progress
if self.image_urls and not self.images:
logging.info("Downloading %s", self.get_identifier_string())
logging.info("Downloading %s", repr(self))
self._download_data()
logging.info("Completed downloading of %s", self.get_identifier_string())
logging.info("Completed downloading of %s", repr(self))
self._data_downloaded = True
self._download_lock.notify_all()
@@ -356,8 +350,8 @@ class ImageRepo(ABC):
elif type(self.image_urls) == list:
iterator = range(len(self.image_urls))
for name in iterator:
logging.info(" Downloading repository %s resource %s", self.get_identifier_string(), name)
image_path = NamedTemporaryFile(mode='wb', suffix=str(name), prefix=self.get_identifier_string(), delete = False)
logging.info(" Downloading repository %s resource %s", repr(self), name)
image_path = NamedTemporaryFile(mode='wb', suffix=str(name), prefix=str(self), delete = False)
result = requests.get(self.image_urls[name], headers=self.download_headers)
result.raise_for_status()
# Handles necessary transformations for a URL (i.e., redirections)
@@ -369,8 +363,14 @@ class ImageRepo(ABC):
else:
self.images[name] = image_path
def __str__(self):
return str(self.__class__.__name__) + "__" + self.get_identifier_string()
class Comic(ImageRepo, ABC):
def __repr__(self):
return "<" + str(self.__class__.__name__) + " " + self.get_identifier_string() + ">"
class ComicCollection(ImageRepo, ABC):
"""
A generic representation of a comic collection. Comic-wide resources are stored here.
"""
@@ -394,7 +394,7 @@ class Comic(ImageRepo, ABC):
@classmethod
def create_from_url(cls, url):
"""
Creates a Comic instance from a given URL
Creates a ComicCollection instance from a given URL
"""
pass
@@ -432,33 +432,33 @@ class Comic(ImageRepo, ABC):
"""
pass
def create_strip(self, identifier):
"""
Creates a new ComicStrip object.
def __str__(self):
return self.__class__.__name__ + "__" + self.get_identifier_string()
The class variable #strip_cls must be set for this to work.
"""
return self.strip_cls(self, identifier)
def __repr__(self):
if self.title:
return "<" + str(self.__class__.__name__) + " " + self.title + " (" + self.get_identifier_string() + ")>"
else:
return "<" + str(self.__class__.__name__) + " " + self.get_identifier_string() + ">"
class ComicStrip(ImageRepo, ABC):
class ComicBook(ImageRepo, ABC):
"""
A generic representation of a single comic strip or issue.
"""
def __init__(self, comic, identifier):
def __init__(self, identifier):
"""
Creates a comic strip
:param comic: The `Comic` object that this strip is part of.
:param indentifier: The globally unique identifier used to find the strip on the website. Can be a tuple.
"""
super().__init__(identifier, list)
self.comic = comic
"""The comic that this strip is part of."""
self.title = None
"""The title of the comic. Empty until the comic data is loaded."""
self.series = None
"""The series that this comic is a part of. Empty until the comic data is loaded."""
self.image_urls = []
"""A list of URLs we can download the comic panels from, in order. Empty until the comic data is loaded."""
self._image_paths = []
@@ -474,27 +474,34 @@ class ComicStrip(ImageRepo, ABC):
self._transformed = False
"""A flag keeping track of whether the strip transformation took place."""
@classmethod
def create_from_url(cls, url):
"""
Creates a Comic instance from a given URL
"""
pass
def get_filename(self):
"""
Obtains the name of the final export (not including the stem). Calling this method requires the data be loaded first.
:returns: A human-friendly file, like "05 - Riddles in the Dark" or "2025-06-01 - Chapter 5."
:returns: A human-friendly filename, like "05 - Riddles in the Dark" or "2025-06-01 - Chapter 5."
"""
prefix_num_form = "#{:02}"
if type(self.identifier) == int:
prefix = prefix_num_form.format(self.identifier)
elif (type(self.identifier) == list or type(self.identifier) == tuple) and type(self.identifier[-1]) == int:
prefix = prefix_num_form.format(self.identifier[-1])
elif type(self.date) == date:
prefix = self.date.strftime("#%Y%m%d")
elif type(self.date) == datetime:
prefix = self.date.date().strftime("#%Y%m%d")
else:
prefix = self.get_identifier_string()
if self.title:
return (file_safe_string(self.comic.title) + " " + prefix + " - " + file_safe_string(self.title))[0:250]
return file_safe_string(self.title)[0:250]
else:
return prefix
prefix_num_form = "#{:02}"
if type(self.identifier) == int:
prefix = prefix_num_form.format(self.identifier)
elif (type(self.identifier) == list or type(self.identifier) == tuple) and type(self.identifier[-1]) == int:
prefix = prefix_num_form.format(self.identifier[-1])
elif type(self.date) == date:
prefix = self.date.strftime("#%Y%m%d")
elif type(self.date) == datetime:
prefix = self.date.date().strftime("#%Y%m%d")
else:
prefix = self.get_identifier_string()
return file_safe_string(self.__name__ + " - " + prefix)
def get_package_path(self, base_path: Path):
"""
@@ -503,25 +510,13 @@ class ComicStrip(ImageRepo, ABC):
:param base_path: The folder to save comics to.
:returns: The path to save to.
"""
return Path(base_path, file_safe_string(self.comic.title), f"{self.get_filename()}.cbz")
filename = self.get_filename()
return Path(base_path, filename, f"{filename}.cbz")
def load_data(self):
self.comic.load_data()
super().load_data()
index_strip(self)
def download_data(self):
self.comic.download_data()
super().download_data()
def await_load(self):
self.comic.await_load()
super().await_load()
def await_download(self):
self.comic.await_download()
super().await_download()
@abstractmethod
def _load_data(self):
"""
@@ -558,7 +553,7 @@ class ComicStrip(ImageRepo, ABC):
soup = BeautifulSoup('<?xml version="1.0" encoding="utf-8"?><ComicInfo></ComicInfo>', features="xml")
info_tag = soup.find("ComicInfo")
info_tag.append(soup.new_tag("Title", string=self.title))
info_tag.append(soup.new_tag("Series", string=self.comic.title))
info_tag.append(soup.new_tag("Series", string=self.series))
number_el = soup.new_tag("Number")
if type(self.identifier) == int:
number_el.string = str(self.identifier)
@@ -570,13 +565,13 @@ class ComicStrip(ImageRepo, ABC):
info_tag.append(soup.new_tag("Year", string=str(self.date.year)))
info_tag.append(soup.new_tag("Month", string=str(self.date.month)))
info_tag.append(soup.new_tag("Day", string=str(self.date.day)))
if self.comic.author:
if type(self.comic.author) == str:
info_tag.append(soup.new_tag("Writer", string=self.comic.author))
elif type(self.comic.author) == list:
if self.author:
if type(self.author) == str:
info_tag.append(soup.new_tag("Writer", string=self.author))
elif type(self.author) == list:
authors = {}
writer_string = ""
for author in self.comic.author:
for author in self.author:
author_class = None
if type(author) == str:
author_name = author
@@ -609,37 +604,45 @@ class ComicStrip(ImageRepo, ABC):
info_tag.append(soup.new_tag("PageCount", string=str(len(self.transformed_images))))
return str(soup)
def package_data(self, base_path: Path):
def package_data(self, base_path: Path, plain: bool = False):
"""
Compresses all the data into a cbz file.
:param base_path: The path to save all downloads to.
:param plain: If true, we will save the images as downloaded, without any transformations.
"""
if not self.is_loaded():
raise SequenceException("Cannot package comic strip before strip data has been fully loaded.")
raise SequenceException("Cannot package comic book before data has been fully loaded.")
if not self.is_downloaded():
raise SequenceException("Cannot package comic strip before strip data has been fully downloaded.")
if not self.comic.is_loaded():
raise SequenceException("Cannot package comic strip before comic data has been fully loaded.")
if not self.comic.is_downloaded():
raise SequenceException("Cannot package comic strip before comic data has been fully downloaded.")
raise SequenceException("Cannot package comic book before data has been fully downloaded.")
if not self.transformed_images:
if not self._transformed:
if self._transform_lock._lock.locked():
# Await for the existing transform to complete
self._transform_lock.wait()
else:
# Run the transformation outselves
with self._transform_lock:
self._transform_images()
self._transformed = True
self._transform_lock.notify_all()
if plain:
self.transformed_images = self.images
else:
if not self._transformed:
if self._transform_lock._lock.locked():
# Await for the existing transform to complete
self._transform_lock.wait()
else:
# Run the transformation outselves
with self._transform_lock:
self._transform_images()
self._transformed = True
self._transform_lock.notify_all()
save_path = self.get_package_path(base_path)
save_path.parent.mkdir(parents=True, exist_ok=True)
with ZipFile(self.get_package_path(base_path), 'w') as comic_zip:
for i in range(len(self.transformed_images)):
image = self.transformed_images[i]
image_path = Path(image.name)
suffix = image_path.suffix
if len(suffix) > 5:
# There is something fishy going on, so let us just use the default
suffix = None
if not suffix:
suffix = ".png"
logging.info("Packaging %s" % image_path.suffix)
with open(image.name, 'rb') as image_stream, comic_zip.open(str(i + 1) + (image_path.suffix or '.png'), 'w') as zip_stream:
with open(image.name, 'rb') as image_stream, comic_zip.open(str(i + 1) + (suffix), 'w') as zip_stream:
zip_stream.write(image_stream.read())
# Add a ComicInfo file
comic_info = self.create_comic_info()
@@ -655,6 +658,128 @@ class ComicStrip(ImageRepo, ABC):
if image_path.exists():
image_path.unlink()
def __repr__(self):
if self.title:
return "<" + str(self.__class__.__name__) + " " + self.title + " (" + self.get_identifier_string() + ")>"
else:
return "<" + str(self.__class__.__name__) + " " + self.get_identifier_string() + ">"
class ComicStrip(ComicBook, ABC):
"""
A generic representation of an issue in a comic series. Unlike the base
ComicBook class, these objects are tied to a ComicCollection instance
and derive some data from that class.
"""
def __init__(self, comic, identifier):
"""
Creates a comic strip.
:param comic: The `ComicCollection` object that this strip is part of.
:param indentifier: The collection-unique identifier used to find the strip on the website. Can be a tuple.
"""
super().__init__(identifier)
self.comic = comic
@classmethod
def create_from_url(cls, url, collection):
"""
Creates a Comic instance from a given URL
"""
return collection.create_from_url(url)
@property
def author(self):
"""
The author(s) of the comic. Can be a string for a name, a list of names, or a list of objects containing the names under "name" and role under "role".
For Comic Strips, this references back to the main comic series author.
"""
return self.comic.author
@author.setter
def author(self, x):
pass
@property
def series(self) -> str:
"""The series that this comic is a part of."""
return self.comic.title
@series.setter
def series(self, x):
pass
def get_filename(self):
"""
Obtains the name of the final export (not including the stem). Calling this method requires the data be loaded first.
:returns: A human-friendly file, like "05 - Riddles in the Dark" or "2025-06-01 - Chapter 5."
"""
prefix_num_form = "#{:02}"
if type(self.identifier) == int:
prefix = prefix_num_form.format(self.identifier)
elif (type(self.identifier) == list or type(self.identifier) == tuple) and type(self.identifier[-1]) == int:
prefix = prefix_num_form.format(self.identifier[-1])
elif type(self.date) == date:
prefix = self.date.strftime("#%Y%m%d")
elif type(self.date) == datetime:
prefix = self.date.date().strftime("#%Y%m%d")
else:
prefix = self.get_identifier_string()
if self.title:
return (file_safe_string(self.comic.title) + " " + prefix + " - " + file_safe_string(self.title))[0:250]
else:
return prefix
def get_package_path(self, base_path: Path):
"""
Obtains the path to save the comic archive to.
:param base_path: The folder to save comics to.
:returns: The path to save to.
"""
return Path(base_path, file_safe_string(self.comic.title), f"{self.get_filename()}.cbz")
def load_data(self):
self.comic.load_data()
super().load_data()
def download_data(self):
self.comic.download_data()
super().download_data()
def await_load(self):
self.comic.await_load()
super().await_load()
def await_download(self):
self.comic.await_download()
super().await_download()
def package_data(self, base_path: Path, plain: bool = False):
if not self.comic.is_loaded():
raise SequenceException("Cannot package comic strip before comic data has been fully loaded.")
if not self.comic.is_downloaded():
raise SequenceException("Cannot package comic strip before comic data has been fully downloaded.")
super().package_data(base_path, plain)
def __str__(self):
return self.__class__.__name__ + "__" + self.comic.get_identifier_string() + "__" + self.get_identifier_string()
def __repr__(self):
representation = "<" + str(self.__class__.__name__) + " "
if self.comic.title:
representation += self.comic.title + " (" + self.comic.get_identifier_string() + "): "
else:
representation += self.comic.get_identifier_string() + ": "
if self.title:
representation += self.title + " (" + self.get_identifier_string() + ")"
else:
representation += self.comic.get_identifier_string()
representation += ">"
return representation
class URLFormat:
"""An internal class used to help sort URLs based on subdomains and paths"""
def __init__(self, url: str):
@@ -736,7 +861,7 @@ comic_registry = {}
_registry_subdomains = []
"""An internal list keeping track of the order we should use to find the right parser (priority is last)"""
def register_comic_class(url: str, comic_cls: type[Comic]):
def register_comic_class(url: str, comic_cls: type[ComicBook]):
"""
Registers a comic type.
@@ -769,7 +894,7 @@ def register_comic_class(url: str, comic_cls: type[Comic]):
comic_registry[url_f] = comic_cls
bisect.insort(_registry_subdomains, url_f)
def find_comic_class(url: str) -> type[Comic]:
def find_comic_class(url: str) -> type[ComicBook]:
"""
Finds the appropriate comic for a given URL.

View File

@@ -31,7 +31,7 @@ import requests
from PIL import Image, ImageDraw, ImageFont
from bs4 import BeautifulSoup
from comic_download.comic_strip import Comic, ComicStrip, register_comic_class
from comic_download.comic_strip import ComicCollection as Comic, ComicStrip, register_comic_class
class ExistentialComic(Comic):
"""
@@ -233,5 +233,5 @@ class ExistentialComicStrip(ComicStrip):
f_img.save(f_image_path.name)
self.transformed_images.append(f_image_path)
ExistentialComic.strip_cls = ExistentialComicStrip
register_comic_class("existentialcomics.com", ExistentialComic)
ExistentialComicStrip.collection_cls = ExistentialComic
register_comic_class("existentialcomics.com", ExistentialComicStrip)

View File

@@ -31,7 +31,7 @@ import requests
from PIL import Image, ImageDraw, ImageFont
from bs4 import BeautifulSoup
from comic_download.comic_strip import Comic, ComicStrip, register_comic_class
from comic_download.comic_strip import ComicCollection as Comic, ComicStrip, register_comic_class
class XKCDComic(Comic):
"""
@@ -152,5 +152,5 @@ class XKCDComicStrip(ComicStrip):
self.transformed_images.append(f_image)
XKCDComic.strip_cls = XKCDComicStrip
register_comic_class("xkcd.com", XKCDComic)
XKCDComicStrip.collection_cls = XKCDComic
register_comic_class("xkcd.com", XKCDComicStrip)