diff --git a/__main__.py b/__main__.py index 15f4baf..a02571d 100644 --- a/__main__.py +++ b/__main__.py @@ -9,6 +9,7 @@ from urllib.parse import urlparse from pathlib import Path from comic_download import Comic, find_comic_class import comic_download.xkcd +import comic_download.existential_comics def setup_args(): parser = argparse.ArgumentParser( @@ -81,9 +82,9 @@ if __name__ == "__main__": r = [comic.get_latest_strip()] else: r = comic.get_all_strips() - start = args.start or 0 - end = args.end or (len(r) - 1) - r = r[start:end + 1] + start = args.start or comic.get_first_strip() + end = args.end or comic.get_latest_strip() + r = r[start - 1:end] for index in reversed(r): t = threading.Thread(name=str(comic) + str(index), target=load_strip, args=(base_path, comic, index, args.plain, thread_limit)) diff --git a/comic_download/existential_comics.py b/comic_download/existential_comics.py index f703ccb..408c066 100644 --- a/comic_download/existential_comics.py +++ b/comic_download/existential_comics.py @@ -3,22 +3,21 @@ # Copyright (c) 2025 Markil 3 import logging -import logging.handlers -import argparse import datetime -import threading -import time import textwrap import json -from zipfile import ZipFile +import re from urllib.parse import urlparse from pathlib import Path +from tempfile import NamedTemporaryFile import requests from PIL import Image, ImageDraw, ImageFont from bs4 import BeautifulSoup +from comic_download.comic_strip import Comic, ComicStrip, register_comic_class + class ExistentialComic(Comic): """ The Existential Comics strip @@ -43,10 +42,12 @@ class ExistentialComic(Comic): result = requests.get("https://existentialcomics.com/") result.raise_for_status() - content = BeautifulSoup(result.content) + content = BeautifulSoup(result.content, features="lxml") + # The home page automatically fetches the latest comic, and we + # can grab the regular URL (and thus the index) from there. index = content.find("meta", property="og:url") if index and index["content"]: - self.latest_identifier = int(self.url.split('/')[-1]) + self.latest_identifier = int(index["content"].split('/')[-1]) else: raise ValueError("Invalid comic data") @@ -54,19 +55,18 @@ class ExistentialComic(Comic): return 1 def get_latest_strip(self): - return self.latest_identfier + return self.latest_identifier def get_all_strips(self) -> list: return list(range(self.get_first_strip(), self.get_latest_strip() + 1)) - @classmethod def __new__(cls): if not hasattr(cls, 'instance'): cls.instance = super(ExistentialComic, cls).__new__(cls) return cls.instance -class ExistentialComicStrip: +class ExistentialComicStrip(ComicStrip): """ A comic strip object that represents a single Existential Comic strip. """ @@ -86,13 +86,12 @@ class ExistentialComicStrip: result = requests.get(self.url) result.raise_for_status() - content = BeautifulSoup(result.content) - #print(content.prettify()) + content = BeautifulSoup(result.content, features="lxml") title = content.find("meta", property="og:title") if title and title["content"]: self.title = title["content"] for img in content.find_all("img", class_="comicImg"): - self.image_url.append(f'http:{img["src"]}') + self.image_urls.append(f'http:{img["src"]}') if img.get("title"): self.captions.append(img["title"]) else: @@ -101,174 +100,123 @@ class ExistentialComicStrip: final_caption = "" explanation = content.find(id="explanation") if explanation: - final_caption += explanation.get_text() + self.captions.extend(explanation.get_text().split("\n")) philosophers = content.find(id="philosophers-comic") if philosophers: - if final_caption: - final_caption += "\n\n" - final_caption += philosophers.get_text() + self.captions.extend(philosophers.get_text().split("\n")) if final_caption: self.captions.append(final_caption) self.date = datetime.date(2013, 11, 11) + datetime.timedelta(weeks=self.index - 2) - - def _transform_data(self, base_path: Path): + if self.date >= datetime.date(2023, 11, 13): + self.safe = (self.date - datetime.date(2023, 11, 13)).days + else: + self.safe = (self.date - datetime.date(2013, 11, 12)).days + + + def _transform_images(self, base_path: Path): """ Takes the raw image data from #download_data and transforms it to add a title, captions, etc. This function is not idempotent. """ - title_size = 24 + + # Marks out various bounding boxes of strip-global texts (i.e. how much space the title takes up) + title_size = 18 + safety_size = 24 caption_size = 15 f_size = (1000, 1500) title_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), title_size) + safety_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), safety_size) caption_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans.ttf"), caption_size) title_box = title_font.getbbox(self.title) title_box = (title_box[2] - title_box[0], title_box[3] - title_box[1]) + caption_spacing = 4 - if self.date >= datetime.date(2023, 11, 13): - safe = (self.date - datetime.date(2023, 11, 12)).days - else: - safe = (self.date - datetime.date(2013, 11, 11)).days - safe = str(safe) - safe_box = title_font.getbbox(safe) + safe_box = safety_font.getbbox(str(self.safe)) safe_box = (safe_box[2] - safe_box[0], safe_box[3] - safe_box[1]) url_box = caption_font.getbbox(self.url) url_box = (url_box[2] - url_box[0], url_box[3] - url_box[1]) date_box = caption_font.getbbox(self.date.isoformat()) date_box = (date_box[2] - date_box[0], date_box[3] - date_box[1]) + text_margin = 10 - if len(self.captions) > len(self.image_urls): - # We have a final description to add - f_caption = "\n\n".join(self.caption[len(self.image_urls):]) + img_size = (0, 0) + + self.transformed_images = [] + header_img = Image.open(self.comic.images["header"].name) + safety_img = Image.open(self.comic.images["safety"].name) + for i in range(len(self.images)): + img = Image.open(self.images[i].name) + if self.captions[i]: + # Take measurements of the page caption, if one is present. + # Pillow's bbox functions do not handle newline characters properly, so we have to + # split the text, find the widest line, and use that. + caption = textwrap.wrap(self.captions[i], width = img.width // caption_size) + caption_lines = len(caption) + caption_box = caption_font.getbbox(max(caption, key=len)) + caption_box = (caption_box[2] - caption_box[0], (caption_box[3] - caption_box[1] + caption_spacing) * len(caption)) + caption = "\n".join(caption) + else: + caption_box = (0, 0) + # The first page requires extra vertical space for the headers + if i == 0: + f_offset = header_img.height + safety_img.height + max(date_box[1], url_box[1]) + text_margin + # Grab the image size while we are at it, for use outside of the loop + img_size = img.size + else: + f_offset = 0 + f_img_size = (img.width, f_offset + img.height + caption_box[1] + text_margin) + f_img = Image.new("RGBA", f_img_size, (255, 255, 255, 255)) + # Draw the headers as needed + if i == 0: + f_img.paste(header_img, (0, 0)) + f_img.paste(safety_img, (0, header_img.height)) + + # Draw the actual comic + f_img.paste(img, (0, f_offset)) + + draw = ImageDraw.Draw(f_img) + if i == 0: + # Draw the title + draw.text(((img.width - title_box[0]) // 2, header_img.height + 20), self.title, (0, 0, 0), font=title_font) + # Draw the safety text + draw.text(((safety_img.width - safe_box[0]) // 2, header_img.height + 20), str(self.safe), (0, 0, 0), font=safety_font) + # Draw the URL + draw.text((text_margin, header_img.height + safety_img.height + text_margin), self.url, (0, 0, 0), font=caption_font) + # Draw the date + draw.text((img.width - date_box[0] - text_margin, header_img.height + safety_img.height + text_margin), self.date.isoformat(), (0, 0, 0), align="right", font=caption_font) + # Draw the caption + if self.captions[i]: + draw.multiline_text(((img.width - caption_box[0]) // 2, f_offset + img.height), caption, (0, 0, 0), spacing=caption_spacing, font=caption_font, align="center") + # Save the image + f_image_path = NamedTemporaryFile(mode='wb', suffix=f"f_{i}.png", prefix=self.get_identifier_string(), delete = False) + f_image_path.close() + f_img.save(f_image_path.name) + self.transformed_images.append(f_image_path) + if len(self.captions) > len(self.image_urls): + # Add the explanation to a new page + explanation = [] + # Pillow's bbox functions do not handle newline characters properly, so we have to + # split the text, find the widest line, and use that. + for paragraph in self.captions[len(self.image_urls):]: + explanation.extend(textwrap.wrap(paragraph.strip(), width = img_size[0] // caption_size)) + explanation_box = [caption_font.getbbox(ex) for ex in explanation] + explanation_box = [(box[2] - box[0], box[3] - box[1]) for box in explanation_box] + # Using a comparison function (i.e. max) on a tuple just compares the first value of each + explanation_box = [max(explanation_box)[0], (explanation_box[0][1] + caption_spacing) * len(explanation_box)] + explanation = "\n".join(explanation) + + f_img = Image.new("RGBA", (img_size[0], explanation_box[1] + text_margin * 4), (255, 255, 255, 255)) + draw = ImageDraw.Draw(f_img) + # Adds a background rectangle, like on the website + draw.rounded_rectangle(((img_size[0] - explanation_box[0] - text_margin * 2) // 2, text_margin, (img_size[0] + explanation_box[0] + text_margin * 2) // 2, explanation_box[1] + text_margin * 3), 20, (200, 200, 200), (0, 0, 0), 2) + draw.multiline_text(((img_size[0] - explanation_box[0]) // 2, text_margin * 2), explanation, (0, 0, 0), align="center", font=caption_font, spacing=caption_spacing) - caption = "\n".join(self.caption).split("\n") - try: - wrapped_caption = [] - for line in caption: - if line: - wrapped_caption.extend(textwrap.wrap(line.strip(), width=(f_size[0] - 20) / caption_size)) - else: - wrapped_caption.append(line.strip()) - caption = wrapped_caption - cap_max = max([cap_box[2] - cap_box[0] for cap_box in [caption_font.getbbox(cap_line) for cap_line in caption]]) - caption_box = (cap_max, caption_size * len(caption)) - caption = "\n".join(caption) - except Exception as e: - logging.exception("Unable to get caption bounding box", exc_info=e) - caption_box = caption_font.getbbox(caption) - caption_box = (caption_box[2] - caption_box[0], caption_box[3] - caption_box[1]) - - header_path = self.get_header_path(base_path) - header = Image.open(header_path) - f_img = Image.new("RGBA", f_size, (255, 255, 255, 255)) - f_img.paste(header, (0, 20)) - - safety_path = self.get_safety_path(base_path) - safety = Image.open(safety_path) - f_img.paste(safety, (20, 230)) + # Save the image + f_image_path = NamedTemporaryFile(mode='wb', suffix=f"f_ex.png", prefix=self.get_identifier_string(), delete = False) + f_image_path.close() + f_img.save(f_image_path.name) + self.transformed_images.append(f_image_path) - draw = ImageDraw.Draw(f_img) - draw.text(((f_size[0] - title_box[0]) // 2, 250), self.title, (0, 0, 0), font=title_font) - draw.text((70 + safe_box[0] // 2, 240 + safe_box[1] // 2), safe, (0, 0, 0), font=title_font) - draw.text(((f_size[0] - url_box[0]) // 2, 300), self.url, (0, 0, 0), align="center", font=caption_font) - draw.text((f_size[0] - date_box[0] - 10, 250), self.date.isoformat(), (0, 0, 0), align="right", font=caption_font) - f_img.save(self.get_image_path(base_path, 0)) - - l_img = Image.new("RGBA", f_size, (255, 255, 255, 255)) - draw = ImageDraw.Draw(l_img) - draw.rounded_rectangle(((f_size[0] - caption_box[0]) // 2 - 10, 10, (f_size[0] + caption_box[0]) // 2 + 10, 100 + caption_box[1]), 20, (200, 200, 200), (0, 0, 0), 2) - draw.multiline_text(((f_size[0] - caption_box[0]) / 2, 20), caption, fill=(0, 0, 0), align="center", font=caption_font) - l_img.save(self.get_image_path(base_path, len(self.image_url) + 1)) - - def package_data(self, base_path: Path): - """ - Compresses all the data into a cbz file and removes the source files - """ - with ZipFile(self.get_package_path(base_path), 'w') as comic_zip: - img_url = urlparse(self.image_url[0]) - img_name = Path(img_url.path.split('/')[-1]) - image_path = self.get_image_path(base_path, 0) - for i in range(len(self.image_url) + 2): - image_path = self.get_image_path(base_path, i) - comic_zip.write(image_path, img_name.stem + "-" + str(i) + img_name.suffix) - if self.get_metadata_path(base_path).exists(): - comic_zip.write(self.get_metadata_path(base_path), "info.0.json") - for i in range(len(self.image_url) + 2): - self.get_image_path(base_path, i).unlink() - self.get_metadata_path(base_path).unlink(missing_ok=True) - - -def setup_args(): - parser = argparse.ArgumentParser( - prog='existential_comics', - description='Downloads the entire Existential Comics collection') - - parser.add_argument('-o', '--output', type=Path, default=Path(), help="The directory to dump the files to.") - parser.add_argument('-w', '--wait', type=int, default=0, help="How many seconds to wait between each request") - parser.add_argument('-t', '--threads', type=int, default=10, help="How many download threads will run at once") - parser.add_argument('-s', '--start', type=int, default=0, help="The comic index to start at. A zero will be interpreted as using up to the first comic.") - parser.add_argument('-e', '--end', type=int, default=0, help="The comic index to end at. A zero will be interpreted as using up to the last comic.") - parser.add_argument('-l', '--latest', action='store_true', help="If set, only the latest comic will be downloaded") - parser.add_argument('-p', '--plain', action='store_true', help="If set, only the raw image will be downloaded, and titles, caption, etc. will not be added.") - - return parser - -def setup_logging(): - logger = logging.getLogger() - logger.setLevel(logging.DEBUG) - - ch = logging.StreamHandler() - ch.setLevel(logging.INFO) - formatter = logging.Formatter('%(levelname)s - %(message)s') - ch.setFormatter(formatter) - logger.addHandler(ch) - - ch = logging.handlers.RotatingFileHandler("gutenberg_download.log", encoding='utf-8') - ch.setLevel(logging.DEBUG) - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - ch.setFormatter(formatter) - ch.doRollover() - logger.addHandler(ch) - -def load_strip(base_path, index, plain, thread_limit): - - with thread_limit: - strip = ComicStrip(index) - if not strip.get_image_path(base_path).exists() and not strip.get_package_path(base_path).exists(): - strip.load_data() - strip.download_data(base_path) - if not plain: - strip.transform_data(base_path) - strip.package_data(base_path) - -if __name__ == "__main__": - setup_logging() - parser = setup_args() - - args = parser.parse_args() - base_path = args.output - - if not base_path.is_dir(): - base_path.mkdir(parents=True) - - logging.info("Beginning parsing") - latest = ComicStrip("") - latest.load_data() - logging.info("There are %d comics", latest.index) - if args.latest: - r = range(latest.index, latest.index + 1) - else: - start = args.start or 1 - end = args.end or latest.index - r = range(end, start - 1, -1) - - threads = [] - thread_limit = threading.BoundedSemaphore(value=args.threads) - - for index in r: - t = threading.Thread(name=str(index), target=load_strip, args=(base_path, index, args.plain, thread_limit)) - threads.append(t) - t.start() - - for t in threads: - t.join() +ExistentialComic.strip_cls = ExistentialComicStrip +register_comic_class("existentialcomics.com", ExistentialComic) diff --git a/comic_download/xkcd.py b/comic_download/xkcd.py index aea4164..dc49365 100644 --- a/comic_download/xkcd.py +++ b/comic_download/xkcd.py @@ -51,7 +51,6 @@ class XKCDComic(Comic): def get_all_strips(self) -> list: return list(range(self.get_first_strip(), self.get_latest_strip() + 1)) - @classmethod def __new__(cls): if not hasattr(cls, 'instance'): cls.instance = super(XKCDComic, cls).__new__(cls)