275 lines
11 KiB
Python
275 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
|
|
# Copyright (c) 2025 Markil 3
|
|
|
|
import logging
|
|
import logging.handlers
|
|
import argparse
|
|
import datetime
|
|
import threading
|
|
import time
|
|
import textwrap
|
|
import json
|
|
from zipfile import ZipFile
|
|
from urllib.parse import urlparse
|
|
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
from PIL import Image, ImageDraw, ImageFont
|
|
from bs4 import BeautifulSoup
|
|
|
|
class ExistentialComic(Comic):
|
|
"""
|
|
The Existential Comics strip
|
|
"""
|
|
|
|
def __init__(self):
|
|
super().__init__("existential_comics")
|
|
|
|
@classmethod
|
|
def create_from_url(cls, url):
|
|
return cls()
|
|
|
|
def _load_data(self):
|
|
self.title = "Existential Comics"
|
|
self.author = "Corey Mohler"
|
|
self.image_urls = {
|
|
"header": "https://static.existentialcomics.com/title.jpg",
|
|
"safety": "https://static.existentialcomics.com/safety.png"
|
|
}
|
|
self.description = "A webcomic of romance, sarcasm, math, and language."
|
|
|
|
result = requests.get("https://existentialcomics.com/")
|
|
result.raise_for_status()
|
|
|
|
content = BeautifulSoup(result.content)
|
|
index = content.find("meta", property="og:url")
|
|
if index and index["content"]:
|
|
self.latest_identifier = int(self.url.split('/')[-1])
|
|
else:
|
|
raise ValueError("Invalid comic data")
|
|
|
|
def get_first_strip(self):
|
|
return 1
|
|
|
|
def get_latest_strip(self):
|
|
return self.latest_identfier
|
|
|
|
def get_all_strips(self) -> list:
|
|
return list(range(self.get_first_strip(), self.get_latest_strip() + 1))
|
|
|
|
@classmethod
|
|
def __new__(cls):
|
|
if not hasattr(cls, 'instance'):
|
|
cls.instance = super(ExistentialComic, cls).__new__(cls)
|
|
return cls.instance
|
|
|
|
|
|
class ExistentialComicStrip:
|
|
"""
|
|
A comic strip object that represents a single Existential Comic strip.
|
|
"""
|
|
|
|
def __init__(self, comic, index: int):
|
|
super().__init__(comic, index)
|
|
self.url = f"https://existentialcomics.com/comic/{index}"
|
|
self.title = None
|
|
|
|
@property
|
|
def index(self):
|
|
# Alias
|
|
return self.identifier
|
|
|
|
def _load_data(self):
|
|
logging.info("Loading data for comic %s from %s", self.index, self.url)
|
|
result = requests.get(self.url)
|
|
result.raise_for_status()
|
|
|
|
content = BeautifulSoup(result.content)
|
|
#print(content.prettify())
|
|
title = content.find("meta", property="og:title")
|
|
if title and title["content"]:
|
|
self.title = title["content"]
|
|
for img in content.find_all("img", class_="comicImg"):
|
|
self.image_url.append(f'http:{img["src"]}')
|
|
if img.get("title"):
|
|
self.captions.append(img["title"])
|
|
else:
|
|
self.captions.append("")
|
|
|
|
final_caption = ""
|
|
explanation = content.find(id="explanation")
|
|
if explanation:
|
|
final_caption += explanation.get_text()
|
|
philosophers = content.find(id="philosophers-comic")
|
|
if philosophers:
|
|
if final_caption:
|
|
final_caption += "\n\n"
|
|
final_caption += philosophers.get_text()
|
|
if final_caption:
|
|
self.captions.append(final_caption)
|
|
self.date = datetime.date(2013, 11, 11) + datetime.timedelta(weeks=self.index - 2)
|
|
|
|
|
|
def _transform_data(self, base_path: Path):
|
|
"""
|
|
Takes the raw image data from #download_data and transforms it to add a title, captions, etc. This function is not idempotent.
|
|
"""
|
|
title_size = 24
|
|
caption_size = 15
|
|
f_size = (1000, 1500)
|
|
title_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), title_size)
|
|
caption_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans.ttf"), caption_size)
|
|
title_box = title_font.getbbox(self.title)
|
|
title_box = (title_box[2] - title_box[0], title_box[3] - title_box[1])
|
|
|
|
if self.date >= datetime.date(2023, 11, 13):
|
|
safe = (self.date - datetime.date(2023, 11, 12)).days
|
|
else:
|
|
safe = (self.date - datetime.date(2013, 11, 11)).days
|
|
safe = str(safe)
|
|
safe_box = title_font.getbbox(safe)
|
|
safe_box = (safe_box[2] - safe_box[0], safe_box[3] - safe_box[1])
|
|
url_box = caption_font.getbbox(self.url)
|
|
url_box = (url_box[2] - url_box[0], url_box[3] - url_box[1])
|
|
date_box = caption_font.getbbox(self.date.isoformat())
|
|
date_box = (date_box[2] - date_box[0], date_box[3] - date_box[1])
|
|
|
|
if len(self.captions) > len(self.image_urls):
|
|
# We have a final description to add
|
|
f_caption = "\n\n".join(self.caption[len(self.image_urls):])
|
|
|
|
|
|
caption = "\n".join(self.caption).split("\n")
|
|
try:
|
|
wrapped_caption = []
|
|
for line in caption:
|
|
if line:
|
|
wrapped_caption.extend(textwrap.wrap(line.strip(), width=(f_size[0] - 20) / caption_size))
|
|
else:
|
|
wrapped_caption.append(line.strip())
|
|
caption = wrapped_caption
|
|
cap_max = max([cap_box[2] - cap_box[0] for cap_box in [caption_font.getbbox(cap_line) for cap_line in caption]])
|
|
caption_box = (cap_max, caption_size * len(caption))
|
|
caption = "\n".join(caption)
|
|
except Exception as e:
|
|
logging.exception("Unable to get caption bounding box", exc_info=e)
|
|
caption_box = caption_font.getbbox(caption)
|
|
caption_box = (caption_box[2] - caption_box[0], caption_box[3] - caption_box[1])
|
|
|
|
header_path = self.get_header_path(base_path)
|
|
header = Image.open(header_path)
|
|
f_img = Image.new("RGBA", f_size, (255, 255, 255, 255))
|
|
f_img.paste(header, (0, 20))
|
|
|
|
safety_path = self.get_safety_path(base_path)
|
|
safety = Image.open(safety_path)
|
|
f_img.paste(safety, (20, 230))
|
|
|
|
draw = ImageDraw.Draw(f_img)
|
|
draw.text(((f_size[0] - title_box[0]) // 2, 250), self.title, (0, 0, 0), font=title_font)
|
|
draw.text((70 + safe_box[0] // 2, 240 + safe_box[1] // 2), safe, (0, 0, 0), font=title_font)
|
|
draw.text(((f_size[0] - url_box[0]) // 2, 300), self.url, (0, 0, 0), align="center", font=caption_font)
|
|
draw.text((f_size[0] - date_box[0] - 10, 250), self.date.isoformat(), (0, 0, 0), align="right", font=caption_font)
|
|
f_img.save(self.get_image_path(base_path, 0))
|
|
|
|
l_img = Image.new("RGBA", f_size, (255, 255, 255, 255))
|
|
draw = ImageDraw.Draw(l_img)
|
|
draw.rounded_rectangle(((f_size[0] - caption_box[0]) // 2 - 10, 10, (f_size[0] + caption_box[0]) // 2 + 10, 100 + caption_box[1]), 20, (200, 200, 200), (0, 0, 0), 2)
|
|
draw.multiline_text(((f_size[0] - caption_box[0]) / 2, 20), caption, fill=(0, 0, 0), align="center", font=caption_font)
|
|
l_img.save(self.get_image_path(base_path, len(self.image_url) + 1))
|
|
|
|
def package_data(self, base_path: Path):
|
|
"""
|
|
Compresses all the data into a cbz file and removes the source files
|
|
"""
|
|
with ZipFile(self.get_package_path(base_path), 'w') as comic_zip:
|
|
img_url = urlparse(self.image_url[0])
|
|
img_name = Path(img_url.path.split('/')[-1])
|
|
image_path = self.get_image_path(base_path, 0)
|
|
for i in range(len(self.image_url) + 2):
|
|
image_path = self.get_image_path(base_path, i)
|
|
comic_zip.write(image_path, img_name.stem + "-" + str(i) + img_name.suffix)
|
|
if self.get_metadata_path(base_path).exists():
|
|
comic_zip.write(self.get_metadata_path(base_path), "info.0.json")
|
|
for i in range(len(self.image_url) + 2):
|
|
self.get_image_path(base_path, i).unlink()
|
|
self.get_metadata_path(base_path).unlink(missing_ok=True)
|
|
|
|
|
|
def setup_args():
|
|
parser = argparse.ArgumentParser(
|
|
prog='existential_comics',
|
|
description='Downloads the entire Existential Comics collection')
|
|
|
|
parser.add_argument('-o', '--output', type=Path, default=Path(), help="The directory to dump the files to.")
|
|
parser.add_argument('-w', '--wait', type=int, default=0, help="How many seconds to wait between each request")
|
|
parser.add_argument('-t', '--threads', type=int, default=10, help="How many download threads will run at once")
|
|
parser.add_argument('-s', '--start', type=int, default=0, help="The comic index to start at. A zero will be interpreted as using up to the first comic.")
|
|
parser.add_argument('-e', '--end', type=int, default=0, help="The comic index to end at. A zero will be interpreted as using up to the last comic.")
|
|
parser.add_argument('-l', '--latest', action='store_true', help="If set, only the latest comic will be downloaded")
|
|
parser.add_argument('-p', '--plain', action='store_true', help="If set, only the raw image will be downloaded, and titles, caption, etc. will not be added.")
|
|
|
|
return parser
|
|
|
|
def setup_logging():
|
|
logger = logging.getLogger()
|
|
logger.setLevel(logging.DEBUG)
|
|
|
|
ch = logging.StreamHandler()
|
|
ch.setLevel(logging.INFO)
|
|
formatter = logging.Formatter('%(levelname)s - %(message)s')
|
|
ch.setFormatter(formatter)
|
|
logger.addHandler(ch)
|
|
|
|
ch = logging.handlers.RotatingFileHandler("gutenberg_download.log", encoding='utf-8')
|
|
ch.setLevel(logging.DEBUG)
|
|
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
ch.setFormatter(formatter)
|
|
ch.doRollover()
|
|
logger.addHandler(ch)
|
|
|
|
def load_strip(base_path, index, plain, thread_limit):
|
|
|
|
with thread_limit:
|
|
strip = ComicStrip(index)
|
|
if not strip.get_image_path(base_path).exists() and not strip.get_package_path(base_path).exists():
|
|
strip.load_data()
|
|
strip.download_data(base_path)
|
|
if not plain:
|
|
strip.transform_data(base_path)
|
|
strip.package_data(base_path)
|
|
|
|
if __name__ == "__main__":
|
|
setup_logging()
|
|
parser = setup_args()
|
|
|
|
args = parser.parse_args()
|
|
base_path = args.output
|
|
|
|
if not base_path.is_dir():
|
|
base_path.mkdir(parents=True)
|
|
|
|
logging.info("Beginning parsing")
|
|
latest = ComicStrip("")
|
|
latest.load_data()
|
|
logging.info("There are %d comics", latest.index)
|
|
if args.latest:
|
|
r = range(latest.index, latest.index + 1)
|
|
else:
|
|
start = args.start or 1
|
|
end = args.end or latest.index
|
|
r = range(end, start - 1, -1)
|
|
|
|
threads = []
|
|
thread_limit = threading.BoundedSemaphore(value=args.threads)
|
|
|
|
for index in r:
|
|
t = threading.Thread(name=str(index), target=load_strip, args=(base_path, index, args.plain, thread_limit))
|
|
threads.append(t)
|
|
t.start()
|
|
|
|
for t in threads:
|
|
t.join()
|