diff --git a/__main__.py b/__main__.py new file mode 100644 index 0000000..bac32d4 --- /dev/null +++ b/__main__.py @@ -0,0 +1,87 @@ +# Copyright (c) 2025 Markil 3 +import logging +import logging.handlers +import argparse +import datetime +import threading +import time +from pathlib import Path +from comic_download.xkcd import XKCDComic, XKCDComicStrip + +def setup_args(): + parser = argparse.ArgumentParser( + prog='xkcd', + description='Downloads the entire XKCD collection') + + parser.add_argument('-o', '--output', type=Path, default=Path(), help="The directory to dump the files to.") + parser.add_argument('-w', '--wait', type=int, default=0, help="How many seconds to wait between each request") + parser.add_argument('-t', '--threads', type=int, default=10, help="How many download threads will run at once") + parser.add_argument('-s', '--start', type=int, default=0, help="The comic index to start at. A zero will be interpreted as using up to the first comic.") + parser.add_argument('-e', '--end', type=int, default=0, help="The comic index to end at. A zero will be interpreted as using up to the last comic.") + parser.add_argument('-l', '--latest', action='store_true', help="If set, only the latest comic will be downloaded") + parser.add_argument('-p', '--plain', action='store_true', help="If set, only the raw image will be downloaded, and titles, caption, etc. will not be added.") + + return parser + +def setup_logging(): + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + + ch = logging.StreamHandler() + ch.setLevel(logging.INFO) + formatter = logging.Formatter('%(levelname)s - %(message)s') + ch.setFormatter(formatter) + logger.addHandler(ch) + + ch = logging.handlers.RotatingFileHandler("gutenberg_download.log", encoding='utf-8') + ch.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + ch.setFormatter(formatter) + ch.doRollover() + logger.addHandler(ch) + +def load_strip(base_path, index, plain, thread_limit): + """ + Threading function for downloading XKCD strips. + + Arguments: + + """ + with thread_limit: + strip = XKCDComicStrip(index) + strip.await_load() + if not strip.get_package_path(base_path).exists(): + strip.await_download() + strip.package_data(base_path) + +if __name__ == "__main__": + setup_logging() + parser = setup_args() + + args = parser.parse_args() + base_path = args.output + + if not base_path.is_dir(): + base_path.mkdir(parents=True) + + logging.info("Beginning parsing") + comic = XKCDComic() + comic.await_load() + logging.info("There are %d comics", comic.latest_identifier) + if args.latest: + r = range(comic.latest_identifier, comic.latest_identifier + 1) + else: + start = args.start or 1 + end = args.end or comic.latest_identifier + r = range(end, start - 1, -1) + + threads = [] + thread_limit = threading.BoundedSemaphore(value=args.threads) + + for index in r: + t = threading.Thread(name=str(index), target=load_strip, args=(base_path, index, args.plain, thread_limit)) + threads.append(t) + t.start() + + for t in threads: + t.join() diff --git a/Lucida Sans Bold.ttf b/comic_download/Lucida Sans Bold.ttf similarity index 100% rename from Lucida Sans Bold.ttf rename to comic_download/Lucida Sans Bold.ttf diff --git a/Lucida Sans.ttf b/comic_download/Lucida Sans.ttf similarity index 100% rename from Lucida Sans.ttf rename to comic_download/Lucida Sans.ttf diff --git a/comic_download/__init__.py b/comic_download/__init__.py new file mode 100644 index 0000000..e499275 --- /dev/null +++ b/comic_download/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2025 Markil 3 +from comic_download.comic_strip import SequenceException, ImageRepo, Comic, ComicStrip, file_safe_string, get_identifier_string diff --git a/comic_download/comic_strip.py b/comic_download/comic_strip.py new file mode 100644 index 0000000..192942b --- /dev/null +++ b/comic_download/comic_strip.py @@ -0,0 +1,429 @@ +#!/usr/bin/env python3 +# Copyright (c) 2025 Markil 3 +import logging +import logging.handlers +import argparse +from abc import ABC, abstractmethod +from datetime import datetime +from threading import Lock +import time +import re +import textwrap +import json +from zipfile import ZipFile +from urllib.parse import urlparse + +from pathlib import Path +from tempfile import NamedTemporaryFile + +import requests +from PIL import Image, ImageDraw, ImageFont + +def file_safe_string(string): + """ + Makes a string safe for saving to a file by replacing characters. + + The characters "/", "&", "\\", "|", and "=" are replaced by " - ". + + The ":" character is replaced by " -". + + The "{" and "<" characters are replaced by "(", along with the corresponding closed brackets. + + All quote-like characters are replaced by a single quote. + + The "?", "*", "%", and "$" characters are removed entirely. + + Note that file lengths are not considered here. + + :param string: The string to convert. + :returns: The file-safe form of the string. + """ + new_string = re.sub(r'[/&\\|=]', " - ", string) + new_string = re.sub(r':', " -", new_string) + new_string = re.sub(r'[{<]', '(', new_string) + new_string = re.sub(r'[}>]', ')', new_string) + new_string = re.sub(r'["“‘`]', '\'', new_string) + new_string = re.sub(r'[?*%$]', '', new_string) + new_string = re.sub(r'\s+', ' ', new_string) + + return new_string + + + +def get_identifier_string(identifier): + """ + Obtains a default file-safe string representation of a comic or strip identifer. If the identifier is a list or tuple, the + elements will be joined by a dash character. + + :param identifier: The identifier of the comic or strip. + + :returns: The identifier in string form. + """ + if type(identifier) == str: + return identifier + elif type(identifier) == tuple or type(identifier) == list: + # Check for spaces. Their presence influenced the delimiter string. + if any(re.search(r'\s', id) for id in identifier): + delimiter = ' - ' + else: + delimiter = '-' + return file_safe_string(delimiter.join([str(id) for id in identifier])) + else: + return str(identifier) + +class SequenceException(Exception): + """ + This exception is thrown when a repository object is processed out of order (i.e. downloading images before we have loaded the images). + """ + pass + +class ImageRepo(ABC): + """ + A generic representation of an online repository of data. This class allows for the + setup of downloading the main information in one pass, and secondary information (images) + seperately. + """ + + def __init__(self, identifier, image_type): + """ + Creates an information repository + + :param identifier: The globally-unique identifier used to find this repository. + :param image_type: The type of collection used for images (i.e. "dict" or "list"). This needs to be an indexed collection. + """ + self.identifier = identifier + self.image_urls = image_type() + """A named collection of image URLs. The value is overridden by implementing classes. This can be any sort of indexed collection (named dictionary, ordered list, etc.) + + Note that despite the name, this does not have to necessarily link to images. Any data that is accessible with GET HTTP requests are valid.""" + self.images = image_type() + """A collection of temporary files for the relevent images in #image_urls. This should be the same collection type as #image_urls.""" + + self._data_loaded = False + """A flag for when the data has been successfully loaded.""" + self._load_lock = Lock() + """A lock for enforcing thread safety when loading this repository.""" + self._download_lock = Lock() + """A lock for enforcing thread safety when downloading secondary resources.""" + + def get_identifier_string(self): + """ + Obtains a file-safe string representation of the identifer. If the identifier is a list or tuple, the + elements will be joined by a dash character. + + :returns: The identifier in string form. + """ + return get_identifier_string(self.identifier) + + def is_loaded(self): + """ + Checks to see if the repostiory data is fully loaded. + + :returns: True if the data has been fully loaded, false otherwise. + """ + return not self._load_lock.locked() and self._data_loaded + + def is_downloaded(self): + """ + Checks to see if all images (if any are present) are downloaded. + + :returns: True if the images have been fully downloaded, false otherwise. + """ + return not self.image_urls or (not self._download_lock.locked() and self.images) + + def load_data(self): + """ + Queries information on the comic in a thread-safe way. + + This load is only performed once, and subsequent calls + will be ignored. Calls made while an existing load is + being performed are also ignored. If you need to obtain + the data directly after triggering a load, #await_load + should be called. + + Implementing classes should overload the #_load_data method. + + :see: #await_load + :see: #_load_data + """ + if not self._load_lock.locked() and not self._data_loaded: + if self._load_lock.acquire(): + self._load_data() + logging.info("Completed loading of %s", self.get_identifier_string()) + self._data_loaded = True + self._load_lock.release() + + def await_load(self): + """ + Locks the current thread until all data has been loaded, starting + a load as needed. + + :see: #load_data + """ + + if self._load_lock.locked(): + self._load_lock.acquire() + self._load_lock.release() + else: + # Loads this ourselves + self.load_data() + + + @abstractmethod + def _load_data(self): + """ + Queries information on the repository. To be implemented by successor classes. + + Successor classes should not attempt any thread safety or be concerned about + checking for previous/concurrent calls to this method, as this logic is handled + by #load_data. + """ + pass + + def download_data(self): + """ + Downloads all images for the comic. + + This download is only performed once, and subsequent calls + will be ignored. Calls made while an existing download is + being performed are also ignored. If you need to obtain + the images directly after triggering a download, #await_download + should be called. + + Implementing classes should overload the #_download_data method. + + :see: #await_download + :see: #_download_data + """ + + if not self.is_loaded(): + raise SequenceException("Cannot download images before first loading performed.") + + # Checks to see if we need to download and that one is not already in progress + if not self._download_lock.locked() and self.image_urls and not self.images: + if self._download_lock.acquire(): + self._download_data() + logging.info("Completed downloading of %s", self.get_identifier_string()) + self._download_lock.release() + + def await_download(self): + """ + Locks the current thread until all images have been downloaded, starting + the download as needed. + + :see: #download_data + """ + + if self._download_lock.locked(): + self._download_lock.acquire() + self._download_lock.release() + else: + # Downloads this ourselves + self.download_data() + + def _download_data(self): + """ + Downloads all images and saves them to temporary files in the #images field. + + The default implementation should be satisfactory for most use cases, but + can be overridden if necessary. Overriders should not attempt any thread + safety or be concerned about checking for previous/concurrent calls to + this method, as this logic is handled by #download_data. + """ + if self.image_urls: + if type(self.image_urls) == dict: + iterator = self.image_urls.keys() + elif type(self.image_urls) == list: + iterator = range(len(self.image_urls)) + for name in iterator: + logging.info(" Downloading repository %s resource %s", self.get_identifier_string(), name) + image_path = NamedTemporaryFile(mode='wb', suffix=str(name), prefix=self.get_identifier_string(), delete = False) + result = requests.get(self.image_urls[name]) + result.raise_for_status() + image_path.write(result.content) + image_path.close() + if type(self.images) == list: + self.images.append(image_path) + else: + self.images[name] = image_path + + +class Comic(ImageRepo, ABC): + """ + A generic representation of a comic collection. Comic-wide resources are stored here. + """ + + def __init__(self, identifier): + """ + Creates a comic collection. + + :param identifier: The globally-unique identifier used to find the comic. + """ + super().__init__(identifier, dict) + self.title = None + """The title of the comic.""" + self.author = None + """The author(s) of the comic. Can be a string for a name, a list of names, or a list of objects containing the names under "name" and role under "role".""" + + @abstractmethod + def _load_data(self): + """ + Queries the source to obtain information on the comic. + + Implementing methods need to find the following information: + * The title of the comic, loaded to `title`. + * The author of the comic, loaded to `author`. + * A collection of banners, author avatars, and other + relevant images, loaded into `image_urls`. + """ + pass + + + +class ComicStrip(ImageRepo, ABC): + """ + A generic representation of a single comic strip or issue. + """ + + def __init__(self, comic, identifier): + """ + Creates a comic strip + + :param comic: The `Comic` object that this strip is part of. + :param indentifier: The globally unique identifier used to find the strip on the website. Can be a tuple. + """ + super().__init__(identifier, list) + + self.comic = comic + """The comic that this strip is part of.""" + self.title = None + """The title of the comic. Empty until the comic data is loaded.""" + self.image_urls = [] + """A list of URLs we can download the comic panels from, in order. Empty until the comic data is loaded.""" + self._image_paths = [] + """A list of temporary files for the raw downloaded panels. Used internally.""" + self.captions = [] + """A list of captions for each panel.""" + self.date = None + """The date that the comic was published.""" + self._transform_lock = Lock() + """A lock for enforcing thread safety when transforming resources.""" + self.transformed_images = [] + """A list of NamedTemporaryFiles linking to the transformed versions of the image downloads.""" + self._transformed = False + """A flag keeping track of whether the strip transformation took place.""" + + def get_filename(self): + """ + Obtains the name of the final export (not including the stem). Calling this method requires the data be loaded first. + + :returns: A human-friendly file, like "05 - Riddles in the Dark" or "2025-06-01 - Chapter 5." + """ + prefix_num_form = "{:02}" + if type(self.identifier) == int: + prefix = prefix_num_form.format(self.identifier) + elif (type(self.identifier) == list or type(self.identifier) == tuple) and type(self.identifier[-1]) == int: + prefix = prefix_num_form.format(self.identifier[-1]) + elif type(self.date) == datetime.date: + prefix = self.date.isoformat() + elif type(self.date) == datetime.datetime: + prefix = self.date.date().isoformat() + else: + prefix = self.get_identifier_string() + if self.title: + return (prefix + " - " + file_safe_string(self.title))[0:250] + else: + return prefix + + def get_package_path(self, base_path: Path): + """ + Obtains the path to save the comic archive to. + + :param base_path: The folder to save comics to. + :returns: The path to save to. + """ + return Path(base_path, f"{self.get_filename()}.cbz") + + def load_data(self): + super().load_data() + self.comic.load_data() + + def download_data(self): + super().download_data() + self.comic.download_data() + + def await_load(self): + super().await_load() + self.comic.await_load() + + def await_download(self): + super().await_download() + self.comic.await_download() + + @abstractmethod + def _load_data(self): + """ + Queries the source to obtain information on the comic. + + Implementing methods need to find the following information: + The title of the comic, loaded to `title`. + The publish date (or datetime) of the comic, loaded to `date`. + A list of panel images, loaded into `image_urls`. + A list of captions, loaded into `captions`. Any captions beyond the list of images can be added to new pages without comics. + """ + pass + + @abstractmethod + def _transform_images(self): + """ + Called by the packaging method to create a series of locally-transformed + images and returning the ordered list of them. + + This method is meant to do things like add alt text, descriptions, title pages, etc. + to the final comic. Simpler implementations can simply return a list of the + unmodified images with the comic headers from #comic at the beginning. + + All implementations should assign the ordered list to #transformed_images. If this + field is not set, then packaging will default to just package the comic strip + images. + """ + self.transformed_images = self.images + + def package_data(self, base_path: Path): + """ + Compresses all the data into a cbz file. + """ + if not self.is_loaded(): + raise SequenceException("Cannot package comic strip before strip data has been fully loaded.") + if not self.is_downloaded(): + raise SequenceException("Cannot package comic strip before strip data has been fully downloaded.") + if not self.comic.is_loaded(): + raise SequenceException("Cannot package comic before data has been fully loaded.") + if not self.comic.is_downloaded(): + raise SequenceException("Cannot package comic before data has been fully downloaded.") + if not self.transformed_images: + if not self._transformed: + if self._transform_lock.locked(): + # Await for the existing transform to complete + self._transform_lock.acquire() + self._transform_lock.release() + else: + # Run the transformation outselves + self._transform_lock.acquire() + self._transform_images() + self._transformed = True + self._transform_lock.release() + with ZipFile(self.get_package_path(base_path), 'w') as comic_zip: + for i in range(len(self.transformed_images)): + image = self.transformed_images[i] + image_path = Path(image.name) + with open(image.name, 'rb') as image_stream, comic_zip.open(str(i + 1) + image_path.suffix, 'w') as zip_stream: + zip_stream.write(image_stream.read()) + # Delete the base images, as they are no longer needed. + # We cannot touch the transformed images, since some + # of them may come directly from the base comic, and + # we have no way of telling. + for image in self.images: + image_path = Path(image.name) + if image_path.exists(): + image_path.unlink() diff --git a/comic_download/xkcd.py b/comic_download/xkcd.py new file mode 100644 index 0000000..08419a1 --- /dev/null +++ b/comic_download/xkcd.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2025 Markil 3 + +import logging +import datetime +import threading +import textwrap +from zipfile import ZipFile +from urllib.parse import urlparse + +from pathlib import Path +from tempfile import NamedTemporaryFile + +import requests +from PIL import Image, ImageDraw, ImageFont +from bs4 import BeautifulSoup + +from comic_download.comic_strip import Comic, ComicStrip + +class XKCDComic(Comic): + """ + The XKCD Comic. + """ + def __init__(self): + super().__init__("xkcd") + + def _load_data(self): + self.title = "XKCD" + self.author = "Randal Munroe" + self.image_urls = { + "banner": "https://xkcd.com/s/0b7742.png" + } + self.description = "A webcomic of romance, sarcasm, math, and language." + result = requests.get("https://xkcd.com/info.0.json") + result.raise_for_status() + self.data = result.json() + self.latest_identifier = self.data["num"] + + def __new__(cls): + if not hasattr(cls, 'instance'): + cls.instance = super(XKCDComic, cls).__new__(cls) + return cls.instance + +class XKCDComicStrip(ComicStrip): + """ + A comic strip object represents a single XKCD strip. + """ + def __init__(self, index: int): + """ + Creates a comic strip object. + + :param index: The index of the strip to load. + """ + super().__init__(XKCDComic(), index) + self.url = f"https://xkcd.com/{index}" + self.data_url = f"{self.url}/info.0.json" + self.transcript = None + + @property + def index(self): + # Alias + return self.identifier + + def _load_data(self): + logging.info("Loading data for comic %s from %s", self.index, self.url) + result = requests.get(self.data_url) + result.raise_for_status() + self.data = result.json() + self.date = datetime.date(int(self.data["year"]), int(self.data["month"]), int(self.data["day"])) + self.title = self.data["title"] + self.image_urls.append(self.data["img"]) + self.captions.append(self.data["alt"]) + self.transcript = self.data["transcript"] + + def _transform_images(self): + """ + Takes the raw image data from #download_data and transforms it to add a title, captions, etc. This function is not idempotent. + """ + f_width = 780 + title_size = 21 + caption_size = 12 + title_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), title_size, layout_engine=ImageFont.Layout.RAQM) + caption_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans.ttf"), caption_size) + + title_box = title_font.getbbox(self.title) + title_box = (title_box[2] - title_box[0], title_box[3] - title_box[1]) + + url_box = caption_font.getbbox(self.url) + url_box = (url_box[2] - url_box[0], url_box[3] - url_box[1]) + date_box = caption_font.getbbox(self.date.isoformat()) + date_box = (date_box[2] - date_box[0], date_box[3] - date_box[1]) + + self.transformed_images = [] + for i in range(len(self.images)): + image = self.images[i] + image_path = Path(image.name) + f_image = NamedTemporaryFile(mode='wb', suffix=f"f-{i}.png", prefix=self.get_identifier_string(), delete = False) + if image_path.exists(): + caption = self.captions[i] + try: + caption = textwrap.wrap(caption, width=(f_width - 20) / caption_size) + cap_max = max([cap_box[2] - cap_box[0] for cap_box in [caption_font.getbbox(cap_line) for cap_line in caption]]) + caption_box = (cap_max, caption_size * len(caption)) + caption = "\n".join(caption) + except Exception as e: + logging.exception("Unable to get caption bounding box for panel %d", i, exc_info=e) + caption_box = caption_font.getbbox(caption) + caption_box = (caption_box[2] - caption_box[0], caption_box[3] - caption_box[1]) + img = Image.open(image_path) + f_img = Image.new("RGBA", (f_width, img.size[1] + caption_box[1] + title_box[1] + 40), (255, 255, 255, 255)) + f_img.paste(img, ((f_width - img.size[0]) // 2, title_box[1] + 20)) + + draw = ImageDraw.Draw(f_img) + + draw.text(((f_width - title_box[0]) / 2, 2), self.title.upper(), (0, 0, 0), font=title_font, features=["c2sc", "smcp"]) + draw.multiline_text(((f_width - caption_box[0]) / 2, img.size[1] + title_box[1] + 22), caption, fill=(0, 0, 0), align="center", font=caption_font) + draw.text((10, f_img.size[1] - url_box[1] - 10), self.url, (0, 0, 0), align="left", font=caption_font) + draw.text((f_width - date_box[0] - 10, f_img.size[1] - date_box[1] - 10), self.date.isoformat(), (0, 0, 0), align="right", font=caption_font) + f_img.save(f_image) + f_image.close() + self.transformed_images.append(f_image) diff --git a/xkcd.py b/xkcd.py deleted file mode 100755 index fe6e227..0000000 --- a/xkcd.py +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) 2025 Markil 3 - -import logging -import logging.handlers -import argparse -import datetime -import threading -import time -import textwrap -from zipfile import ZipFile -from urllib.parse import urlparse - -from pathlib import Path - -import requests -from PIL import Image, ImageDraw, ImageFont -from bs4 import BeautifulSoup - -class ComicStrip: - """ - A comic strip object represents a single XKCD strip. - """ - def __init__(self, index): - """ - Creates a comic strop object. - """ - self.index = index - self.url = f"https://xkcd.com/{index}" - self.data_url = f"{self.url}/info.0.json" - self.title = None - self.image_url = None - self.caption = None - self.transcript = None - - def get_metadata_path(self, base_path: Path): - return Path(base_path, f"{self.index}.json") - - def get_image_path(self, base_path: Path): - return Path(base_path, f"{self.index}.png") - - def get_package_path(self, base_path: Path): - return Path(base_path, f"{self.index}.cbz") - - def load_data(self): - logging.info("Loading data for comic %s from %s", self.index, self.url) - result = requests.get(self.data_url) - result.raise_for_status() - self.data = result.json() - self.date = datetime.date(int(self.data["year"]), int(self.data["month"]), int(self.data["day"])) - self.image_url = self.data["img"] - self.title = self.data["title"] - self.caption = self.data["alt"] - self.transcript = self.data["transcript"] - - - def download_data(self, base_path: Path): - """ - Downloads the raw image data. - """ - if self.image_url: - image_path = self.get_image_path(base_path) - if not image_path.exists(): - with open(image_path, 'wb') as image_stream: - result = requests.get(self.image_url) - result.raise_for_status() - image_stream.write(result.content) - metadata_path = self.get_metadata_path(base_path) - if not metadata_path.exists(): - with open(metadata_path, 'wb') as meta_stream: - result = requests.get(self.data_url) - result.raise_for_status() - meta_stream.write(result.content) - - def transform_data(self, base_path: Path): - """ - Takes the raw image data from #download_data and transforms it to add a title, captions, etc. This function is not idempotent. - """ - image_path = self.get_image_path(base_path) - if image_path.exists(): - f_width = 780 - title_size = 21 - caption_size = 12 - title_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), title_size, layout_engine=ImageFont.Layout.RAQM) - title_box = title_font.getbbox(self.title) - title_box = (title_box[2] - title_box[0], title_box[3] - title_box[1]) - - caption_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans.ttf"), caption_size) - caption = self.caption - try: - caption = textwrap.wrap(caption, width=(f_width - 20) / caption_size) - cap_max = max([cap_box[2] - cap_box[0] for cap_box in [caption_font.getbbox(cap_line) for cap_line in caption]]) - caption_box = (cap_max, caption_size * len(caption)) - caption = "\n".join(caption) - except Exception as e: - logging.exception("Unable to get caption bounding box", exc_info=e) - caption_box = caption_font.getbbox(caption) - caption_box = (caption_box[2] - caption_box[0], caption_box[3] - caption_box[1]) - url_box = caption_font.getbbox(self.url) - url_box = (url_box[2] - url_box[0], url_box[3] - url_box[1]) - date_box = caption_font.getbbox(self.date.isoformat()) - date_box = (date_box[2] - date_box[0], date_box[3] - date_box[1]) - img = Image.open(image_path) - f_img = Image.new("RGBA", (f_width, img.size[1] + caption_box[1] + title_box[1] + 40), (255, 255, 255, 255)) - f_img.paste(img, ((f_width - img.size[0]) // 2, title_box[1] + 20)) - - draw = ImageDraw.Draw(f_img) - - draw.text(((f_width - title_box[0]) / 2, 2), self.title.upper(), (0, 0, 0), font=title_font, features=["c2sc", "smcp"]) - draw.multiline_text(((f_width - caption_box[0]) / 2, img.size[1] + title_box[1] + 22), caption, fill=(0, 0, 0), align="center", font=caption_font) - draw.text((10, f_img.size[1] - url_box[1] - 10), self.url, (0, 0, 0), align="left", font=caption_font) - draw.text((f_width - date_box[0] - 10, f_img.size[1] - date_box[1] - 10), self.date.isoformat(), (0, 0, 0), align="right", font=caption_font) - f_img.save(image_path) - - def package_data(self, base_path: Path): - """ - Compresses all the data into a cbz file and removes the source files - """ - with ZipFile(self.get_package_path(base_path), 'w') as comic_zip: - img_url = urlparse(self.image_url) - comic_zip.write(self.get_image_path(base_path), img_url.path.split('/')[-1]) - if self.get_metadata_path(base_path).exists(): - comic_zip.write(self.get_metadata_path(base_path), "info.0.json") - self.get_image_path(base_path).unlink() - self.get_metadata_path(base_path).unlink(missing_ok=True) - - -def setup_args(): - parser = argparse.ArgumentParser( - prog='xkcd', - description='Downloads the entire XKCD collection') - - parser.add_argument('-o', '--output', type=Path, default=Path(), help="The directory to dump the files to.") - parser.add_argument('-w', '--wait', type=int, default=0, help="How many seconds to wait between each request") - parser.add_argument('-t', '--threads', type=int, default=10, help="How many download threads will run at once") - parser.add_argument('-s', '--start', type=int, default=0, help="The comic index to start at. A zero will be interpreted as using up to the first comic.") - parser.add_argument('-e', '--end', type=int, default=0, help="The comic index to end at. A zero will be interpreted as using up to the last comic.") - parser.add_argument('-l', '--latest', action='store_true', help="If set, only the latest comic will be downloaded") - parser.add_argument('-p', '--plain', action='store_true', help="If set, only the raw image will be downloaded, and titles, caption, etc. will not be added.") - - return parser - -def setup_logging(): - logger = logging.getLogger() - logger.setLevel(logging.DEBUG) - - ch = logging.StreamHandler() - ch.setLevel(logging.INFO) - formatter = logging.Formatter('%(levelname)s - %(message)s') - ch.setFormatter(formatter) - logger.addHandler(ch) - - ch = logging.handlers.RotatingFileHandler("gutenberg_download.log", encoding='utf-8') - ch.setLevel(logging.DEBUG) - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - ch.setFormatter(formatter) - ch.doRollover() - logger.addHandler(ch) - -def load_strip(base_path, index, plain, thread_limit): - """ - Threading function for downloading XKCD strips. - - Arguments: - - """ - with thread_limit: - strip = ComicStrip(index) - if not strip.get_image_path(base_path).exists() and not strip.get_package_path(base_path).exists(): - strip.load_data() - strip.download_data(base_path) - if not plain: - strip.transform_data(base_path) - strip.package_data(base_path) - -if __name__ == "__main__": - setup_logging() - parser = setup_args() - - args = parser.parse_args() - base_path = args.output - - if not base_path.is_dir(): - base_path.mkdir(parents=True) - - logging.info("Beginning parsing") - latest = ComicStrip("") - latest.load_data() - logging.info("There are %d comics", latest.data["num"]) - if args.latest: - r = range(latest.data["num"], latest.data["num"] + 1) - else: - start = args.start or 1 - end = args.end or latest.data["num"] - r = range(end, start - 1, -1) - - threads = [] - thread_limit = threading.BoundedSemaphore(value=args.threads) - - for index in r: - t = threading.Thread(name=str(index), target=load_strip, args=(base_path, index, args.plain, thread_limit)) - threads.append(t) - t.start() - - for t in threads: - t.join()