markil3/ComicDownload
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Formalizes the existential comics process

Browse Source
This commit is contained in:
Markil 3
2025-08-02 13:15:00 -06:00
parent f1bb6fce14
commit 83ad521fea
3 changed files with 111 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
__main__.py
View File

@@ -9,6 +9,7 @@ from urllib.parse import urlparse
from pathlib import Path
from comic_download import Comic, find_comic_class
import comic_download.xkcd
import comic_download.existential_comics
def setup_args():
parser = argparse.ArgumentParser(
@@ -81,9 +82,9 @@ if __name__ == "__main__":
r = [comic.get_latest_strip()]
else:
r = comic.get_all_strips()
start = args.start or 0
end = args.end or (len(r) - 1)
r = r[start:end + 1]
start = args.start or comic.get_first_strip()
end = args.end or comic.get_latest_strip()
r = r[start - 1:end]
for index in reversed(r):
t = threading.Thread(name=str(comic) + str(index), target=load_strip, args=(base_path, comic, index, args.plain, thread_limit))

266
comic_download/existential_comics.py
View File

@@ -3,22 +3,21 @@
# Copyright (c) 2025 Markil 3
import logging
import logging.handlers
import argparse
import datetime
import threading
import time
import textwrap
import json
from zipfile import ZipFile
import re
from urllib.parse import urlparse
from pathlib import Path
from tempfile import NamedTemporaryFile
import requests
from PIL import Image, ImageDraw, ImageFont
from bs4 import BeautifulSoup
from comic_download.comic_strip import Comic, ComicStrip, register_comic_class
class ExistentialComic(Comic):
"""
The Existential Comics strip
@@ -43,10 +42,12 @@ class ExistentialComic(Comic):
result = requests.get("https://existentialcomics.com/")
result.raise_for_status()
content = BeautifulSoup(result.content)
content = BeautifulSoup(result.content, features="lxml")
# The home page automatically fetches the latest comic, and we
# can grab the regular URL (and thus the index) from there.
index = content.find("meta", property="og:url")
if index and index["content"]:
self.latest_identifier = int(self.url.split('/')[-1])
self.latest_identifier = int(index["content"].split('/')[-1])
else:
raise ValueError("Invalid comic data")
@@ -54,19 +55,18 @@ class ExistentialComic(Comic):
return 1
def get_latest_strip(self):
return self.latest_identfier
return self.latest_identifier
def get_all_strips(self) -> list:
return list(range(self.get_first_strip(), self.get_latest_strip() + 1))
@classmethod
def __new__(cls):
if not hasattr(cls, 'instance'):
cls.instance = super(ExistentialComic, cls).__new__(cls)
return cls.instance
class ExistentialComicStrip:
class ExistentialComicStrip(ComicStrip):
"""
A comic strip object that represents a single Existential Comic strip.
"""
@@ -86,13 +86,12 @@ class ExistentialComicStrip:
result = requests.get(self.url)
result.raise_for_status()
content = BeautifulSoup(result.content)
#print(content.prettify())
content = BeautifulSoup(result.content, features="lxml")
title = content.find("meta", property="og:title")
if title and title["content"]:
self.title = title["content"]
for img in content.find_all("img", class_="comicImg"):
self.image_url.append(f'http:{img["src"]}')
self.image_urls.append(f'http:{img["src"]}')
if img.get("title"):
self.captions.append(img["title"])
else:
@@ -101,174 +100,123 @@ class ExistentialComicStrip:
final_caption = ""
explanation = content.find(id="explanation")
if explanation:
final_caption += explanation.get_text()
self.captions.extend(explanation.get_text().split("\n"))
philosophers = content.find(id="philosophers-comic")
if philosophers:
if final_caption:
final_caption += "\n\n"
final_caption += philosophers.get_text()
self.captions.extend(philosophers.get_text().split("\n"))
if final_caption:
self.captions.append(final_caption)
self.date = datetime.date(2013, 11, 11) + datetime.timedelta(weeks=self.index - 2)
def _transform_data(self, base_path: Path):
if self.date >= datetime.date(2023, 11, 13):
self.safe = (self.date - datetime.date(2023, 11, 13)).days
else:
self.safe = (self.date - datetime.date(2013, 11, 12)).days
def _transform_images(self, base_path: Path):
"""
Takes the raw image data from #download_data and transforms it to add a title, captions, etc. This function is not idempotent.
"""
title_size = 24
# Marks out various bounding boxes of strip-global texts (i.e. how much space the title takes up)
title_size = 18
safety_size = 24
caption_size = 15
f_size = (1000, 1500)
title_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), title_size)
safety_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), safety_size)
caption_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans.ttf"), caption_size)
title_box = title_font.getbbox(self.title)
title_box = (title_box[2] - title_box[0], title_box[3] - title_box[1])
caption_spacing = 4
if self.date >= datetime.date(2023, 11, 13):
safe = (self.date - datetime.date(2023, 11, 12)).days
else:
safe = (self.date - datetime.date(2013, 11, 11)).days
safe = str(safe)
safe_box = title_font.getbbox(safe)
safe_box = safety_font.getbbox(str(self.safe))
safe_box = (safe_box[2] - safe_box[0], safe_box[3] - safe_box[1])
url_box = caption_font.getbbox(self.url)
url_box = (url_box[2] - url_box[0], url_box[3] - url_box[1])
date_box = caption_font.getbbox(self.date.isoformat())
date_box = (date_box[2] - date_box[0], date_box[3] - date_box[1])
text_margin = 10
if len(self.captions) > len(self.image_urls):
# We have a final description to add
f_caption = "\n\n".join(self.caption[len(self.image_urls):])
img_size = (0, 0)
self.transformed_images = []
header_img = Image.open(self.comic.images["header"].name)
safety_img = Image.open(self.comic.images["safety"].name)
for i in range(len(self.images)):
img = Image.open(self.images[i].name)
if self.captions[i]:
# Take measurements of the page caption, if one is present.
# Pillow's bbox functions do not handle newline characters properly, so we have to
# split the text, find the widest line, and use that.
caption = textwrap.wrap(self.captions[i], width = img.width // caption_size)
caption_lines = len(caption)
caption_box = caption_font.getbbox(max(caption, key=len))
caption_box = (caption_box[2] - caption_box[0], (caption_box[3] - caption_box[1] + caption_spacing) * len(caption))
caption = "\n".join(caption)
else:
caption_box = (0, 0)
# The first page requires extra vertical space for the headers
if i == 0:
f_offset = header_img.height + safety_img.height + max(date_box[1], url_box[1]) + text_margin
# Grab the image size while we are at it, for use outside of the loop
img_size = img.size
else:
f_offset = 0
f_img_size = (img.width, f_offset + img.height + caption_box[1] + text_margin)
f_img = Image.new("RGBA", f_img_size, (255, 255, 255, 255))
# Draw the headers as needed
if i == 0:
f_img.paste(header_img, (0, 0))
f_img.paste(safety_img, (0, header_img.height))
# Draw the actual comic
f_img.paste(img, (0, f_offset))
draw = ImageDraw.Draw(f_img)
if i == 0:
# Draw the title
draw.text(((img.width - title_box[0]) // 2, header_img.height + 20), self.title, (0, 0, 0), font=title_font)
# Draw the safety text
draw.text(((safety_img.width - safe_box[0]) // 2, header_img.height + 20), str(self.safe), (0, 0, 0), font=safety_font)
# Draw the URL
draw.text((text_margin, header_img.height + safety_img.height + text_margin), self.url, (0, 0, 0), font=caption_font)
# Draw the date
draw.text((img.width - date_box[0] - text_margin, header_img.height + safety_img.height + text_margin), self.date.isoformat(), (0, 0, 0), align="right", font=caption_font)
# Draw the caption
if self.captions[i]:
draw.multiline_text(((img.width - caption_box[0]) // 2, f_offset + img.height), caption, (0, 0, 0), spacing=caption_spacing, font=caption_font, align="center")
# Save the image
f_image_path = NamedTemporaryFile(mode='wb', suffix=f"f_{i}.png", prefix=self.get_identifier_string(), delete = False)
f_image_path.close()
f_img.save(f_image_path.name)
self.transformed_images.append(f_image_path)
if len(self.captions) > len(self.image_urls):
# Add the explanation to a new page
explanation = []
# Pillow's bbox functions do not handle newline characters properly, so we have to
# split the text, find the widest line, and use that.
for paragraph in self.captions[len(self.image_urls):]:
explanation.extend(textwrap.wrap(paragraph.strip(), width = img_size[0] // caption_size))
explanation_box = [caption_font.getbbox(ex) for ex in explanation]
explanation_box = [(box[2] - box[0], box[3] - box[1]) for box in explanation_box]
# Using a comparison function (i.e. max) on a tuple just compares the first value of each
explanation_box = [max(explanation_box)[0], (explanation_box[0][1] + caption_spacing) * len(explanation_box)]
explanation = "\n".join(explanation)
f_img = Image.new("RGBA", (img_size[0], explanation_box[1] + text_margin * 4), (255, 255, 255, 255))
draw = ImageDraw.Draw(f_img)
# Adds a background rectangle, like on the website
draw.rounded_rectangle(((img_size[0] - explanation_box[0] - text_margin * 2) // 2, text_margin, (img_size[0] + explanation_box[0] + text_margin * 2) // 2, explanation_box[1] + text_margin * 3), 20, (200, 200, 200), (0, 0, 0), 2)
draw.multiline_text(((img_size[0] - explanation_box[0]) // 2, text_margin * 2), explanation, (0, 0, 0), align="center", font=caption_font, spacing=caption_spacing)
caption = "\n".join(self.caption).split("\n")
try:
wrapped_caption = []
for line in caption:
if line:
wrapped_caption.extend(textwrap.wrap(line.strip(), width=(f_size[0] - 20) / caption_size))
else:
wrapped_caption.append(line.strip())
caption = wrapped_caption
cap_max = max([cap_box[2] - cap_box[0] for cap_box in [caption_font.getbbox(cap_line) for cap_line in caption]])
caption_box = (cap_max, caption_size * len(caption))
caption = "\n".join(caption)
except Exception as e:
logging.exception("Unable to get caption bounding box", exc_info=e)
caption_box = caption_font.getbbox(caption)
caption_box = (caption_box[2] - caption_box[0], caption_box[3] - caption_box[1])
header_path = self.get_header_path(base_path)
header = Image.open(header_path)
f_img = Image.new("RGBA", f_size, (255, 255, 255, 255))
f_img.paste(header, (0, 20))
safety_path = self.get_safety_path(base_path)
safety = Image.open(safety_path)
f_img.paste(safety, (20, 230))
# Save the image
f_image_path = NamedTemporaryFile(mode='wb', suffix=f"f_ex.png", prefix=self.get_identifier_string(), delete = False)
f_image_path.close()
f_img.save(f_image_path.name)
self.transformed_images.append(f_image_path)
draw = ImageDraw.Draw(f_img)
draw.text(((f_size[0] - title_box[0]) // 2, 250), self.title, (0, 0, 0), font=title_font)
draw.text((70 + safe_box[0] // 2, 240 + safe_box[1] // 2), safe, (0, 0, 0), font=title_font)
draw.text(((f_size[0] - url_box[0]) // 2, 300), self.url, (0, 0, 0), align="center", font=caption_font)
draw.text((f_size[0] - date_box[0] - 10, 250), self.date.isoformat(), (0, 0, 0), align="right", font=caption_font)
f_img.save(self.get_image_path(base_path, 0))
l_img = Image.new("RGBA", f_size, (255, 255, 255, 255))
draw = ImageDraw.Draw(l_img)
draw.rounded_rectangle(((f_size[0] - caption_box[0]) // 2 - 10, 10, (f_size[0] + caption_box[0]) // 2 + 10, 100 + caption_box[1]), 20, (200, 200, 200), (0, 0, 0), 2)
draw.multiline_text(((f_size[0] - caption_box[0]) / 2, 20), caption, fill=(0, 0, 0), align="center", font=caption_font)
l_img.save(self.get_image_path(base_path, len(self.image_url) + 1))
def package_data(self, base_path: Path):
"""
Compresses all the data into a cbz file and removes the source files
"""
with ZipFile(self.get_package_path(base_path), 'w') as comic_zip:
img_url = urlparse(self.image_url[0])
img_name = Path(img_url.path.split('/')[-1])
image_path = self.get_image_path(base_path, 0)
for i in range(len(self.image_url) + 2):
image_path = self.get_image_path(base_path, i)
comic_zip.write(image_path, img_name.stem + "-" + str(i) + img_name.suffix)
if self.get_metadata_path(base_path).exists():
comic_zip.write(self.get_metadata_path(base_path), "info.0.json")
for i in range(len(self.image_url) + 2):
self.get_image_path(base_path, i).unlink()
self.get_metadata_path(base_path).unlink(missing_ok=True)
def setup_args():
parser = argparse.ArgumentParser(
prog='existential_comics',
description='Downloads the entire Existential Comics collection')
parser.add_argument('-o', '--output', type=Path, default=Path(), help="The directory to dump the files to.")
parser.add_argument('-w', '--wait', type=int, default=0, help="How many seconds to wait between each request")
parser.add_argument('-t', '--threads', type=int, default=10, help="How many download threads will run at once")
parser.add_argument('-s', '--start', type=int, default=0, help="The comic index to start at. A zero will be interpreted as using up to the first comic.")
parser.add_argument('-e', '--end', type=int, default=0, help="The comic index to end at. A zero will be interpreted as using up to the last comic.")
parser.add_argument('-l', '--latest', action='store_true', help="If set, only the latest comic will be downloaded")
parser.add_argument('-p', '--plain', action='store_true', help="If set, only the raw image will be downloaded, and titles, caption, etc. will not be added.")
return parser
def setup_logging():
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
ch = logging.handlers.RotatingFileHandler("gutenberg_download.log", encoding='utf-8')
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
ch.doRollover()
logger.addHandler(ch)
def load_strip(base_path, index, plain, thread_limit):
with thread_limit:
strip = ComicStrip(index)
if not strip.get_image_path(base_path).exists() and not strip.get_package_path(base_path).exists():
strip.load_data()
strip.download_data(base_path)
if not plain:
strip.transform_data(base_path)
strip.package_data(base_path)
if __name__ == "__main__":
setup_logging()
parser = setup_args()
args = parser.parse_args()
base_path = args.output
if not base_path.is_dir():
base_path.mkdir(parents=True)
logging.info("Beginning parsing")
latest = ComicStrip("")
latest.load_data()
logging.info("There are %d comics", latest.index)
if args.latest:
r = range(latest.index, latest.index + 1)
else:
start = args.start or 1
end = args.end or latest.index
r = range(end, start - 1, -1)
threads = []
thread_limit = threading.BoundedSemaphore(value=args.threads)
for index in r:
t = threading.Thread(name=str(index), target=load_strip, args=(base_path, index, args.plain, thread_limit))
threads.append(t)
t.start()
for t in threads:
t.join()
ExistentialComic.strip_cls = ExistentialComicStrip
register_comic_class("existentialcomics.com", ExistentialComic)

1
comic_download/xkcd.py
View File

@@ -51,7 +51,6 @@ class XKCDComic(Comic):
def get_all_strips(self) -> list:
return list(range(self.get_first_strip(), self.get_latest_strip() + 1))
@classmethod
def __new__(cls):
if not hasattr(cls, 'instance'):
cls.instance = super(XKCDComic, cls).__new__(cls)