Formalizes the existential comics process

__main__.py

@@ -9,6 +9,7 @@ from urllib.parse import urlparse
 from pathlib import Path
 from comic_download import Comic, find_comic_class
 import comic_download.xkcd
+import comic_download.existential_comics
 
 def setup_args():
     parser = argparse.ArgumentParser(
@@ -81,9 +82,9 @@ if __name__ == "__main__":
             r = [comic.get_latest_strip()]
         else:
             r = comic.get_all_strips()
-            start = args.start or 0
+            start = args.start or comic.get_first_strip()
-            end = args.end or (len(r) - 1)
+            end = args.end or comic.get_latest_strip()
-            r = r[start:end + 1]
+            r = r[start - 1:end]
     
         for index in reversed(r):
             t = threading.Thread(name=str(comic) + str(index), target=load_strip, args=(base_path, comic, index, args.plain, thread_limit))

comic_download/existential_comics.py

@@ -3,22 +3,21 @@
 # Copyright (c) 2025 Markil 3
 
 import logging
-import logging.handlers
-import argparse
 import datetime
-import threading
-import time
 import textwrap
 import json
-from zipfile import ZipFile
+import re
 from urllib.parse import urlparse
 
 from pathlib import Path
+from tempfile import NamedTemporaryFile
 
 import requests
 from PIL import Image, ImageDraw, ImageFont
 from bs4 import BeautifulSoup
 
+from comic_download.comic_strip import Comic, ComicStrip, register_comic_class
+
 class ExistentialComic(Comic):
     """
     The Existential Comics strip
@@ -43,10 +42,12 @@ class ExistentialComic(Comic):
         result = requests.get("https://existentialcomics.com/")
         result.raise_for_status()
 
-        content = BeautifulSoup(result.content)
+        content = BeautifulSoup(result.content, features="lxml")
+        # The home page automatically fetches the latest comic, and we
+        # can grab the regular URL (and thus the index) from there.
         index = content.find("meta", property="og:url")
         if index and index["content"]:
-            self.latest_identifier = int(self.url.split('/')[-1])
+            self.latest_identifier = int(index["content"].split('/')[-1])
         else:
             raise ValueError("Invalid comic data")
 
@@ -54,19 +55,18 @@ class ExistentialComic(Comic):
         return 1
 
     def get_latest_strip(self):
-        return self.latest_identfier
+        return self.latest_identifier
 
     def get_all_strips(self) -> list:
         return list(range(self.get_first_strip(), self.get_latest_strip() + 1))
 
-    @classmethod
     def __new__(cls):
         if not hasattr(cls, 'instance'):
             cls.instance = super(ExistentialComic, cls).__new__(cls)
         return cls.instance
         
 
-class ExistentialComicStrip:
+class ExistentialComicStrip(ComicStrip):
     """
     A comic strip object that represents a single Existential Comic strip.
     """
@@ -86,13 +86,12 @@ class ExistentialComicStrip:
         result = requests.get(self.url)
         result.raise_for_status()
 
-        content = BeautifulSoup(result.content)
+        content = BeautifulSoup(result.content, features="lxml")
-        #print(content.prettify())
         title = content.find("meta", property="og:title")
         if title and title["content"]:
             self.title = title["content"]
         for img in content.find_all("img", class_="comicImg"):
-            self.image_url.append(f'http:{img["src"]}')
+            self.image_urls.append(f'http:{img["src"]}')
             if img.get("title"):
                 self.captions.append(img["title"])
             else:
@@ -101,174 +100,123 @@ class ExistentialComicStrip:
         final_caption = ""
         explanation = content.find(id="explanation")
         if explanation:
-            final_caption += explanation.get_text()
+            self.captions.extend(explanation.get_text().split("\n"))
         philosophers = content.find(id="philosophers-comic")
         if philosophers:
-            if final_caption:
+            self.captions.extend(philosophers.get_text().split("\n"))
-                final_caption += "\n\n"
-            final_caption += philosophers.get_text()
         if final_caption:
             self.captions.append(final_caption)
         self.date = datetime.date(2013, 11, 11) + datetime.timedelta(weeks=self.index - 2)
 
+        if self.date >= datetime.date(2023, 11, 13):
+            self.safe = (self.date - datetime.date(2023, 11, 13)).days
+        else:
+            self.safe = (self.date - datetime.date(2013, 11, 12)).days
 
-    def _transform_data(self, base_path: Path):
+            
+    def _transform_images(self, base_path: Path):
         """
         Takes the raw image data from #download_data and transforms it to add a title, captions, etc. This function is not idempotent.
         """
-        title_size = 24
+        
+        # Marks out various bounding boxes of strip-global texts (i.e. how much space the title takes up)
+        title_size = 18
+        safety_size = 24
         caption_size = 15
         f_size = (1000, 1500)
         title_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), title_size)
+        safety_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), safety_size)
         caption_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans.ttf"), caption_size)
         title_box = title_font.getbbox(self.title)
         title_box = (title_box[2] - title_box[0], title_box[3] - title_box[1])
+        caption_spacing = 4
 
-        if self.date >= datetime.date(2023, 11, 13):
+        safe_box = safety_font.getbbox(str(self.safe))
-            safe = (self.date - datetime.date(2023, 11, 12)).days
-        else:
-            safe = (self.date - datetime.date(2013, 11, 11)).days
-        safe = str(safe)
-        safe_box = title_font.getbbox(safe)
         safe_box = (safe_box[2] - safe_box[0], safe_box[3] - safe_box[1])
         url_box = caption_font.getbbox(self.url)
         url_box = (url_box[2] - url_box[0], url_box[3] - url_box[1])
         date_box = caption_font.getbbox(self.date.isoformat())
         date_box = (date_box[2] - date_box[0], date_box[3] - date_box[1])
+        text_margin = 10
+
+        img_size = (0, 0)
+
+        self.transformed_images = []
+        header_img = Image.open(self.comic.images["header"].name)
+        safety_img = Image.open(self.comic.images["safety"].name)
+        for i in range(len(self.images)):
+            img = Image.open(self.images[i].name)
+            if self.captions[i]:
+                # Take measurements of the page caption, if one is present. 
+                # Pillow's bbox functions do not handle newline characters properly, so we have to
+                # split the text, find the widest line, and use that.
+                caption = textwrap.wrap(self.captions[i], width = img.width // caption_size)
+                caption_lines = len(caption)
+                caption_box = caption_font.getbbox(max(caption, key=len))
+                caption_box = (caption_box[2] - caption_box[0], (caption_box[3] - caption_box[1] + caption_spacing) * len(caption))
+                caption = "\n".join(caption)
+            else:
+                caption_box = (0, 0)
+            # The first page requires extra vertical space for the headers
+            if i == 0:
+                f_offset = header_img.height + safety_img.height + max(date_box[1], url_box[1]) + text_margin
+                # Grab the image size while we are at it, for use outside of the loop
+                img_size = img.size
+            else:
+                f_offset = 0
+            f_img_size = (img.width, f_offset + img.height + caption_box[1] + text_margin)
+            f_img = Image.new("RGBA", f_img_size, (255, 255, 255, 255))
+            # Draw the headers as needed
+            if i == 0:
+                f_img.paste(header_img, (0, 0))
+                f_img.paste(safety_img, (0, header_img.height))
+
+            # Draw the actual comic
+            f_img.paste(img, (0, f_offset))
+
+            draw = ImageDraw.Draw(f_img)
+            if i == 0:
+                # Draw the title
+                draw.text(((img.width - title_box[0]) // 2, header_img.height + 20), self.title, (0, 0, 0), font=title_font)
+                # Draw the safety text
+                draw.text(((safety_img.width - safe_box[0]) // 2, header_img.height + 20), str(self.safe), (0, 0, 0), font=safety_font)
+                # Draw the URL
+                draw.text((text_margin, header_img.height + safety_img.height + text_margin), self.url, (0, 0, 0), font=caption_font)
+                # Draw the date
+                draw.text((img.width - date_box[0] - text_margin, header_img.height + safety_img.height + text_margin), self.date.isoformat(), (0, 0, 0), align="right", font=caption_font)
+            # Draw the caption
+            if self.captions[i]:
+                draw.multiline_text(((img.width - caption_box[0]) // 2, f_offset + img.height), caption, (0, 0, 0), spacing=caption_spacing, font=caption_font, align="center")
+            # Save the image
+            f_image_path = NamedTemporaryFile(mode='wb', suffix=f"f_{i}.png", prefix=self.get_identifier_string(), delete = False)
+            f_image_path.close()
+            f_img.save(f_image_path.name)
+            self.transformed_images.append(f_image_path)
             
         if len(self.captions) > len(self.image_urls):
-            # We have a final description to add
+            # Add the explanation to a new page
-            f_caption = "\n\n".join(self.caption[len(self.image_urls):])
+            explanation = []
+            # Pillow's bbox functions do not handle newline characters properly, so we have to
+            # split the text, find the widest line, and use that.
+            for paragraph in self.captions[len(self.image_urls):]:
+                explanation.extend(textwrap.wrap(paragraph.strip(), width = img_size[0] // caption_size))
+            explanation_box = [caption_font.getbbox(ex) for ex in explanation]
+            explanation_box = [(box[2] - box[0], box[3] - box[1]) for box in explanation_box]
+            # Using a comparison function (i.e. max) on a tuple just compares the first value of each
+            explanation_box = [max(explanation_box)[0], (explanation_box[0][1] + caption_spacing) * len(explanation_box)]
+            explanation = "\n".join(explanation)
             
+            f_img = Image.new("RGBA", (img_size[0], explanation_box[1] + text_margin * 4), (255, 255, 255, 255))
+            draw = ImageDraw.Draw(f_img)
+            # Adds a background rectangle, like on the website
+            draw.rounded_rectangle(((img_size[0] - explanation_box[0] - text_margin * 2) // 2, text_margin, (img_size[0] + explanation_box[0] + text_margin * 2) // 2, explanation_box[1] + text_margin * 3), 20, (200, 200, 200), (0, 0, 0), 2)
+            draw.multiline_text(((img_size[0] - explanation_box[0]) // 2, text_margin * 2), explanation, (0, 0, 0), align="center", font=caption_font, spacing=caption_spacing)
 
-        caption = "\n".join(self.caption).split("\n")
+            # Save the image
-        try:
+            f_image_path = NamedTemporaryFile(mode='wb', suffix=f"f_ex.png", prefix=self.get_identifier_string(), delete = False)
-            wrapped_caption = []
+            f_image_path.close()
-            for line in caption:
+            f_img.save(f_image_path.name)
-                if line:
+            self.transformed_images.append(f_image_path)
-                    wrapped_caption.extend(textwrap.wrap(line.strip(), width=(f_size[0] - 20) / caption_size))
-                else:
-                    wrapped_caption.append(line.strip())
-            caption = wrapped_caption
-            cap_max = max([cap_box[2] - cap_box[0] for cap_box in [caption_font.getbbox(cap_line) for cap_line in caption]])
-            caption_box = (cap_max, caption_size * len(caption))
-            caption = "\n".join(caption)
-        except Exception as e:
-            logging.exception("Unable to get caption bounding box", exc_info=e)
-            caption_box = caption_font.getbbox(caption)
-            caption_box = (caption_box[2] - caption_box[0], caption_box[3] - caption_box[1])
 
-        header_path = self.get_header_path(base_path)
+ExistentialComic.strip_cls = ExistentialComicStrip
-        header = Image.open(header_path)
+register_comic_class("existentialcomics.com", ExistentialComic)
-        f_img = Image.new("RGBA", f_size, (255, 255, 255, 255))
-        f_img.paste(header, (0, 20))
-        
-        safety_path = self.get_safety_path(base_path)
-        safety = Image.open(safety_path)
-        f_img.paste(safety, (20, 230))
-
-        draw = ImageDraw.Draw(f_img)
-        draw.text(((f_size[0] - title_box[0]) // 2, 250), self.title, (0, 0, 0), font=title_font)
-        draw.text((70 + safe_box[0] // 2, 240 + safe_box[1] // 2), safe, (0, 0, 0), font=title_font)
-        draw.text(((f_size[0] - url_box[0]) // 2, 300), self.url, (0, 0, 0), align="center", font=caption_font)
-        draw.text((f_size[0] - date_box[0] - 10, 250), self.date.isoformat(), (0, 0, 0), align="right", font=caption_font)
-        f_img.save(self.get_image_path(base_path, 0))
-
-        l_img = Image.new("RGBA", f_size, (255, 255, 255, 255))
-        draw = ImageDraw.Draw(l_img)
-        draw.rounded_rectangle(((f_size[0] - caption_box[0]) // 2 - 10, 10, (f_size[0] + caption_box[0]) // 2 + 10, 100 + caption_box[1]), 20, (200, 200, 200), (0, 0, 0), 2)
-        draw.multiline_text(((f_size[0] - caption_box[0]) / 2, 20), caption, fill=(0, 0, 0), align="center", font=caption_font)
-        l_img.save(self.get_image_path(base_path, len(self.image_url) + 1))
-
-    def package_data(self, base_path: Path):
-        """
-        Compresses all the data into a cbz file and removes the source files
-        """
-        with ZipFile(self.get_package_path(base_path), 'w') as comic_zip:
-            img_url = urlparse(self.image_url[0])
-            img_name = Path(img_url.path.split('/')[-1])
-            image_path = self.get_image_path(base_path, 0)
-            for i in range(len(self.image_url) + 2):
-                image_path = self.get_image_path(base_path, i)
-                comic_zip.write(image_path, img_name.stem + "-" + str(i) + img_name.suffix)
-            if self.get_metadata_path(base_path).exists():
-                comic_zip.write(self.get_metadata_path(base_path), "info.0.json")
-        for i in range(len(self.image_url) + 2):
-            self.get_image_path(base_path, i).unlink()
-        self.get_metadata_path(base_path).unlink(missing_ok=True)
-
-
-def setup_args():
-    parser = argparse.ArgumentParser(
-                    prog='existential_comics',
-                    description='Downloads the entire Existential Comics collection')
-
-    parser.add_argument('-o', '--output', type=Path, default=Path(), help="The directory to dump the files to.")
-    parser.add_argument('-w', '--wait', type=int, default=0, help="How many seconds to wait between each request")
-    parser.add_argument('-t', '--threads', type=int, default=10, help="How many download threads will run at once")
-    parser.add_argument('-s', '--start', type=int, default=0, help="The comic index to start at. A zero will be interpreted as using up to the first comic.")
-    parser.add_argument('-e', '--end', type=int, default=0, help="The comic index to end at. A zero will be interpreted as using up to the last comic.")
-    parser.add_argument('-l', '--latest', action='store_true', help="If set, only the latest comic will be downloaded")
-    parser.add_argument('-p', '--plain', action='store_true', help="If set, only the raw image will be downloaded, and titles, caption, etc. will not be added.")
-    
-    return parser
-
-def setup_logging():
-    logger = logging.getLogger()
-    logger.setLevel(logging.DEBUG)
-    
-    ch = logging.StreamHandler()
-    ch.setLevel(logging.INFO)
-    formatter = logging.Formatter('%(levelname)s - %(message)s')
-    ch.setFormatter(formatter)
-    logger.addHandler(ch)
-
-    ch = logging.handlers.RotatingFileHandler("gutenberg_download.log", encoding='utf-8')
-    ch.setLevel(logging.DEBUG)
-    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-    ch.setFormatter(formatter)
-    ch.doRollover()
-    logger.addHandler(ch)
-
-def load_strip(base_path, index, plain, thread_limit):
-    
-    with thread_limit:
-        strip = ComicStrip(index)
-        if not strip.get_image_path(base_path).exists() and not strip.get_package_path(base_path).exists():
-            strip.load_data()
-            strip.download_data(base_path)
-            if not plain:
-                strip.transform_data(base_path)
-            strip.package_data(base_path)
-
-if __name__ == "__main__":
-    setup_logging()
-    parser = setup_args()
-
-    args = parser.parse_args()
-    base_path = args.output
-
-    if not base_path.is_dir():
-        base_path.mkdir(parents=True)
-
-    logging.info("Beginning parsing")
-    latest = ComicStrip("")
-    latest.load_data()
-    logging.info("There are %d comics", latest.index)
-    if args.latest:
-        r = range(latest.index, latest.index + 1)
-    else:
-        start = args.start or 1
-        end = args.end or latest.index
-        r = range(end, start - 1, -1)
-
-    threads = []
-    thread_limit = threading.BoundedSemaphore(value=args.threads)
-    
-    for index in r:
-        t = threading.Thread(name=str(index), target=load_strip, args=(base_path, index, args.plain, thread_limit))
-        threads.append(t)
-        t.start()
-
-    for t in threads:
-        t.join()

comic_download/xkcd.py

@@ -51,7 +51,6 @@ class XKCDComic(Comic):
     def get_all_strips(self) -> list:
         return list(range(self.get_first_strip(), self.get_latest_strip() + 1))
 
-    @classmethod
     def __new__(cls):
         if not hasattr(cls, 'instance'):
             cls.instance = super(XKCDComic, cls).__new__(cls)