Migrates to a new packaging scheme
This is easier to release.
This commit is contained in:
BIN
src/comic_download/Lucida Sans Bold.ttf
Normal file
BIN
src/comic_download/Lucida Sans Bold.ttf
Normal file
Binary file not shown.
BIN
src/comic_download/Lucida Sans.ttf
Normal file
BIN
src/comic_download/Lucida Sans.ttf
Normal file
Binary file not shown.
17
src/comic_download/__init__.py
Normal file
17
src/comic_download/__init__.py
Normal file
@@ -0,0 +1,17 @@
|
||||
# This CLI tool allows for the download and packaging of various internet comics into .cbz files.
|
||||
# Copyright (C) 2025 Markil 3
|
||||
# http://www.singlepilot.net
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
from comic_download.comic_strip import SequenceException, ImageRepo, Comic, ComicStrip, file_safe_string, get_identifier_string, register_comic_class, find_comic_class, read_index, save_index, index_strip, is_strip_present
|
||||
140
src/comic_download/__main__.py
Normal file
140
src/comic_download/__main__.py
Normal file
@@ -0,0 +1,140 @@
|
||||
# This CLI tool allows for the download and packaging of various internet comics into .cbz files.
|
||||
# Copyright (C) 2025 Markil 3
|
||||
# http://www.singlepilot.net
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
import logging
|
||||
import logging.handlers
|
||||
import argparse
|
||||
from signal import signal, SIGINT
|
||||
from sys import exit
|
||||
import datetime
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import urlparse
|
||||
from pathlib import Path
|
||||
import importlib
|
||||
import pkgutil
|
||||
import comic_download
|
||||
from comic_download import Comic, find_comic_class
|
||||
|
||||
def setup_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='xkcd',
|
||||
description='Downloads the entire XKCD collection')
|
||||
|
||||
parser.add_argument('url', action='extend', nargs='+', type=urlparse, help="The URLs to load")
|
||||
parser.add_argument('-o', '--output', type=Path, default=Path(), help="The directory to dump the files to.")
|
||||
parser.add_argument('-w', '--wait', type=int, default=0, help="How many seconds to wait between each request")
|
||||
parser.add_argument('-t', '--threads', type=int, default=10, help="How many download threads will run at once")
|
||||
parser.add_argument('-s', '--start', type=int, default=0, help="The comic index to start at. A zero will be interpreted as using up to the first comic. This option is only valid if there is a single URL.")
|
||||
parser.add_argument('-e', '--end', type=int, default=0, help="The comic index to end at. A zero will be interpreted as using up to the last comic. This option is only valid if there is a single URL.")
|
||||
parser.add_argument('-l', '--latest', action='store_true', help="If set, only the latest comic will be downloaded")
|
||||
parser.add_argument('-p', '--plain', action='store_true', help="If set, only the raw image will be downloaded, and titles, caption, etc. will not be added.")
|
||||
|
||||
return parser
|
||||
|
||||
def setup_logging():
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
ch = logging.StreamHandler()
|
||||
ch.setLevel(logging.INFO)
|
||||
formatter = logging.Formatter('%(levelname)s - %(message)s')
|
||||
ch.setFormatter(formatter)
|
||||
logger.addHandler(ch)
|
||||
|
||||
ch = logging.handlers.RotatingFileHandler("gutenberg_download.log", encoding='utf-8')
|
||||
ch.setLevel(logging.DEBUG)
|
||||
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
ch.setFormatter(formatter)
|
||||
ch.doRollover()
|
||||
logger.addHandler(ch)
|
||||
|
||||
def load_plugins():
|
||||
"""
|
||||
Loads all registered extentions for Comic and ComicStrip. It is expected that they are all modules within the "comic_download" module.
|
||||
"""
|
||||
for finder, name, ispkg in pkgutil.iter_modules(comic_download.__path__, comic_download.__name__ + "."):
|
||||
if name not in ("comic_download.comic_strip", "comic_download.__main__"):
|
||||
logging.info("Loading plugin %s" % name)
|
||||
importlib.import_module(name)
|
||||
|
||||
def load_strip(base_path, comic, identifier, plain, thread_limit):
|
||||
"""
|
||||
Threading function for downloading XKCD strips.
|
||||
|
||||
Arguments:
|
||||
|
||||
"""
|
||||
with thread_limit:
|
||||
strip = comic.create_strip(identifier)
|
||||
strip.await_load()
|
||||
if not strip.get_package_path(base_path).exists():
|
||||
strip.await_download()
|
||||
strip.package_data(base_path)
|
||||
|
||||
def __main__():
|
||||
setup_logging()
|
||||
parser = setup_args()
|
||||
|
||||
load_plugins()
|
||||
|
||||
args = parser.parse_args()
|
||||
base_path = args.output
|
||||
|
||||
if not base_path.is_dir():
|
||||
base_path.mkdir(parents=True)
|
||||
|
||||
logging.info("Beginning parsing")
|
||||
|
||||
threads = []
|
||||
comics = []
|
||||
thread_limit = threading.BoundedSemaphore(value=args.threads)
|
||||
|
||||
for url in args.url:
|
||||
comic_type = find_comic_class(url.netloc + url.path)
|
||||
comic = comic_type.create_from_url(url)
|
||||
comics.append(comic)
|
||||
comic_download.read_index(base_path)
|
||||
comic.await_load()
|
||||
if args.latest:
|
||||
r = [comic.get_latest_strip()]
|
||||
else:
|
||||
r = comic.get_all_strips()
|
||||
if r and type(r[0]) == int:
|
||||
start = args.start or comic.get_first_strip()
|
||||
end = args.end or comic.get_latest_strip()
|
||||
r = r[r.index(start):r.index(end) + 1]
|
||||
|
||||
try:
|
||||
for index in reversed(r):
|
||||
if not comic_download.is_strip_present(comic, base_path, ".cbz", str(index)):
|
||||
t = threading.Thread(name=str(comic.get_identifier_string()) + "-" + str(index), target=load_strip, args=(base_path, comic, index, args.plain, thread_limit))
|
||||
threads.append(t)
|
||||
t.daemon = True
|
||||
t.start()
|
||||
finally:
|
||||
logging.info("Saving index")
|
||||
comic_download.save_index(base_path)
|
||||
|
||||
try:
|
||||
while any([t.is_alive() for t in threads]):
|
||||
time.sleep(1)
|
||||
finally:
|
||||
logging.info("Saving index")
|
||||
comic_download.save_index(base_path)
|
||||
|
||||
if __name__ == "__main__":
|
||||
__main__()
|
||||
195
src/comic_download/bluesky.py
Normal file
195
src/comic_download/bluesky.py
Normal file
@@ -0,0 +1,195 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# This CLI tool allows for the download and packaging of various internet comics into .cbz files.
|
||||
# Copyright (C) 2025 Markil 3
|
||||
# http://www.singlepilot.net
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
import logging
|
||||
import datetime
|
||||
import os
|
||||
import time
|
||||
import re
|
||||
import textwrap
|
||||
import json
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
from bs4 import BeautifulSoup
|
||||
from atproto import Client
|
||||
|
||||
from comic_download.comic_strip import Comic, ComicStrip, register_comic_class
|
||||
|
||||
class BlueskyComic(Comic):
|
||||
"""
|
||||
A comic class for any Bluesky-posted comics.
|
||||
"""
|
||||
client = None
|
||||
"""The single atproto client"""
|
||||
logged_in = False
|
||||
"""A flag for keeping track of whether we logged in."""
|
||||
|
||||
def __init__(self, identifier: tuple):
|
||||
"""
|
||||
Creates a BlueSky comic.
|
||||
|
||||
:param identifier: The comic identifier. This is a tuple containing
|
||||
the Bluesky instance ID (the base domain), and the bluesky user
|
||||
handle.
|
||||
"""
|
||||
super().__init__(identifier)
|
||||
|
||||
@classmethod
|
||||
def create_from_url(cls, url):
|
||||
if type(url) == str:
|
||||
if not re.match(r'^\w+://', url):
|
||||
url = "https://" + url
|
||||
url = urlparse(url)
|
||||
instance = url.netloc
|
||||
handle = url.path.split('/')[2]
|
||||
return cls((instance, handle))
|
||||
|
||||
@property
|
||||
def instance(self):
|
||||
"""The bluesky instance this comic is published on (i.e. "bsky.app.")"""
|
||||
return self.identifier[0]
|
||||
|
||||
@property
|
||||
def handle(self):
|
||||
"""The handle of the user that posts this comic."""
|
||||
return self.identifier[1]
|
||||
|
||||
@classmethod
|
||||
def get_client(cls):
|
||||
"""
|
||||
Obtains the ATProto client, creating it if needed.
|
||||
"""
|
||||
if not cls.client:
|
||||
logging.info("Creating AT client")
|
||||
cls.client = Client()
|
||||
# Handles logins
|
||||
if os.environ.get("BSKY_USER_PATH", None):
|
||||
with open(os.environ.get("BSKY_USER_PATH"), 'r') as env_file:
|
||||
os.environ["BSKY_USER"] = env_file.read().strip()
|
||||
if os.environ.get("BSKY_PASSWORD_PATH", None):
|
||||
with open(os.environ.get("BSKY_PASSWORD_PATH"), 'r') as env_file:
|
||||
os.environ["BSKY_PASSWORD"] = env_file.read().strip()
|
||||
if os.environ.get("BSKY_SESSION_PATH", None):
|
||||
with open(os.environ.get("BSKY_SESSION_PATH"), 'r') as env_file:
|
||||
os.environ["BSKY_SESSION"] = env_file.read().strip()
|
||||
if os.environ.get("BSKY_AUTH_TOKEN_PATH", None):
|
||||
with open(os.environ.get("BSKY_AUTH_TOKEN_PATH"), 'r') as env_file:
|
||||
os.environ["BSKY_AUTH_TOKEN"] = env_file.read().strip()
|
||||
|
||||
if "BSKY_USER" in os.environ or "BSKY_PASSWORD" in os.environ or "BSKY_SESSION" in os.environ or "BSKY_AUTH_TOKEN" in os.environ:
|
||||
cls.client.login(login=os.environ.get("BSKY_USER", None), password=os.environ.get("BSKY_PASSWORD", None), session_string=os.environ.get("BSKY_SESSION", None), auth_factor_token=os.environ.get("BSKY_AUTH_TOKEN", None))
|
||||
cls.logged_in = True
|
||||
else:
|
||||
logging.info("No Bluesky login details were provided. Going forward with the more limited, unauthenticated APIs.")
|
||||
|
||||
return cls.client
|
||||
|
||||
def _load_data(self):
|
||||
if self.logged_in:
|
||||
profile = self.get_client().get_profile(self.handle)
|
||||
self.author = profile.display_name
|
||||
self.title = profile.display_name
|
||||
self.image_urls = {
|
||||
"avatar": profile.avatar,
|
||||
"banner": profile.banner
|
||||
}
|
||||
self.description = profile.description
|
||||
self.posts = []
|
||||
# TODO - Find post IDs with the API
|
||||
else:
|
||||
rss_url = f"https://{self.instance}/profile/{self.handle}/rss"
|
||||
result = requests.get(rss_url)
|
||||
result.raise_for_status()
|
||||
rss = BeautifulSoup(result.content, features="xml")
|
||||
|
||||
self.author = rss.channel.title.string
|
||||
self.author = self.author[self.author.index(" - ") + 3:]
|
||||
self.title = self.author
|
||||
self.image_urls = {}
|
||||
self.description = rss.channel.description.string
|
||||
|
||||
self.posts = []
|
||||
for item in rss.channel.find_all("item"):
|
||||
post_url = item.link.string.strip()
|
||||
post_url = urlparse(post_url)
|
||||
post_id = post_url.path.split("/")[-1]
|
||||
self.posts.insert(0, post_id)
|
||||
self.did = self.get_client().resolve_handle(self.handle).did
|
||||
|
||||
def get_first_strip(self):
|
||||
return self.posts[0]
|
||||
|
||||
def get_latest_strip(self):
|
||||
return self.posts[-1]
|
||||
|
||||
def get_all_strips(self) -> list:
|
||||
return self.posts
|
||||
|
||||
|
||||
class BlueskyComicStrip(ComicStrip):
|
||||
"""
|
||||
The Bluesky Comics Strip
|
||||
"""
|
||||
client = None
|
||||
|
||||
def __init__(self, comic, post: str):
|
||||
"""
|
||||
Creates a Bluesky comic instance
|
||||
|
||||
:param comic: The BlueSky comic instance that this strip belongs to.
|
||||
:param post: The ID of the post.
|
||||
"""
|
||||
super().__init__(comic, post)
|
||||
self.url = f"https://{self.comic.instance}/profile/{self.comic.handle}/post/{post}"
|
||||
self.title = None
|
||||
|
||||
@property
|
||||
def post(self):
|
||||
# Alias
|
||||
return self.identifier
|
||||
|
||||
def _load_data(self):
|
||||
logging.info("Loading data for post %s by %s", self.post, self.comic.handle)
|
||||
post_data = self.comic.get_client().get_post(self.post, self.comic.handle)
|
||||
|
||||
url = urlparse(post_data.uri)
|
||||
self.title = post_data.value.text
|
||||
if post_data.value.embed and post_data.value.embed.py_type == "app.bsky.embed.images":
|
||||
for i in range(len(post_data.value.embed.images)):
|
||||
image = post_data.value.embed.images[i]
|
||||
if image.py_type == "app.bsky.embed.images#image":
|
||||
image_link = image.image.ref.link
|
||||
image_link = f"https://cdn.{self.comic.instance}/img/feed_fullsize/plain/{self.comic.did}/{image_link}@jpeg"
|
||||
self.image_urls.append(image_link)
|
||||
if image.alt:
|
||||
self.captions.append(image.alt)
|
||||
else:
|
||||
self.captions.append("")
|
||||
self.date = datetime.datetime.fromisoformat(post_data.value.created_at)
|
||||
|
||||
def _transform_images(self):
|
||||
self.transformed_images = self.images
|
||||
|
||||
|
||||
BlueskyComic.strip_cls = BlueskyComicStrip
|
||||
register_comic_class("bsky.app", BlueskyComic)
|
||||
788
src/comic_download/comic_strip.py
Normal file
788
src/comic_download/comic_strip.py
Normal file
@@ -0,0 +1,788 @@
|
||||
#!/usr/bin/env python3
|
||||
# This CLI tool allows for the download and packaging of various internet comics into .cbz files.
|
||||
# Copyright (C) 2025 Markil 3
|
||||
# http://www.singlepilot.net
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
import logging
|
||||
import logging.handlers
|
||||
import argparse
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import datetime, date
|
||||
from threading import Lock, Condition
|
||||
import bisect
|
||||
import time
|
||||
import re
|
||||
import textwrap
|
||||
import json
|
||||
from zipfile import ZipFile
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from pathlib import Path
|
||||
from tempfile import NamedTemporaryFile
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
def file_safe_string(string):
|
||||
"""
|
||||
Makes a string safe for saving to a file by replacing characters.
|
||||
|
||||
The characters "/", "&", "\\", "|", and "=" are replaced by " - ".
|
||||
|
||||
The ":" character is replaced by " -".
|
||||
|
||||
The "{" and "<" characters are replaced by "(", along with the corresponding closed brackets.
|
||||
|
||||
All quote-like characters are replaced by a single quote.
|
||||
|
||||
The "?", "*", "%", and "$" characters are removed entirely.
|
||||
|
||||
Note that file lengths are not considered here.
|
||||
|
||||
:param string: The string to convert.
|
||||
:returns: The file-safe form of the string.
|
||||
"""
|
||||
new_string = re.sub(r'[/&\\|=]', " - ", string)
|
||||
new_string = re.sub(r':', " -", new_string)
|
||||
new_string = re.sub(r'[{<]', '(', new_string)
|
||||
new_string = re.sub(r'[}>]', ')', new_string)
|
||||
new_string = re.sub(r'["“‘`]', '\'', new_string)
|
||||
new_string = re.sub(r'[?*%$]', '', new_string)
|
||||
new_string = re.sub(r'\s+', ' ', new_string)
|
||||
|
||||
return new_string
|
||||
|
||||
|
||||
|
||||
def get_identifier_string(identifier):
|
||||
"""
|
||||
Obtains a default file-safe string representation of a comic or strip identifer. If the identifier is a list or tuple, the
|
||||
elements will be joined by a dash character.
|
||||
|
||||
:param identifier: The identifier of the comic or strip.
|
||||
|
||||
:returns: The identifier in string form.
|
||||
"""
|
||||
if type(identifier) == str:
|
||||
return identifier
|
||||
elif type(identifier) == tuple or type(identifier) == list:
|
||||
# Check for spaces. Their presence influenced the delimiter string.
|
||||
if any(re.search(r'\s', str(id)) for id in identifier):
|
||||
delimiter = ' - '
|
||||
else:
|
||||
delimiter = '-'
|
||||
return file_safe_string(delimiter.join([str(id) for id in identifier]))
|
||||
else:
|
||||
return str(identifier)
|
||||
|
||||
class SequenceException(Exception):
|
||||
"""
|
||||
This exception is thrown when a repository object is processed out of order (i.e. downloading images before we have loaded the images).
|
||||
"""
|
||||
pass
|
||||
|
||||
strip_index = {}
|
||||
"""
|
||||
The strip index is used to map comic strip identifiers to the actual file name (sans stems). This
|
||||
is needed since sometimes, the information used to name a file (like the title) is loaded from
|
||||
the webpage. By caching this information, we can more easily check to see if we can skip a download.
|
||||
"""
|
||||
|
||||
_index_loaded = False
|
||||
"""
|
||||
A flag for when the index has been successfully loaded.
|
||||
"""
|
||||
|
||||
_index_lock = Condition(Lock())
|
||||
"""
|
||||
A lock for enforcing thread safety when loading the comic index file.
|
||||
"""
|
||||
|
||||
def read_index(base_path: Path):
|
||||
"""
|
||||
Saves the strip index to the same base path we download comic strips to.
|
||||
It is saved to "index.json."
|
||||
|
||||
:param base_path: The folder to save the index to.
|
||||
"""
|
||||
global strip_index
|
||||
global _index_loaded
|
||||
global _index_lock
|
||||
with _index_lock:
|
||||
if not _index_loaded:
|
||||
index_path = Path(base_path, "index.json")
|
||||
if index_path.exists():
|
||||
try:
|
||||
with open(Path(base_path, "index.json"), 'r') as index_fp:
|
||||
strip_index = json.load(index_fp)
|
||||
except json.JSONDecodeError as e:
|
||||
logging.exception(e)
|
||||
strip_index = {}
|
||||
_index_loaded = True
|
||||
_index_lock.notify_all()
|
||||
|
||||
def save_index(base_path: Path):
|
||||
"""
|
||||
Saves the strip index to the same base path we download comic strips to.
|
||||
It is saved to "index.json."
|
||||
|
||||
:param base_path: The folder to save the index to.
|
||||
"""
|
||||
global strip_index
|
||||
global _index_loaded
|
||||
global _index_lock
|
||||
with _index_lock:
|
||||
with open(Path(base_path, "index.json"), 'w') as index_fp:
|
||||
json.dump(strip_index, index_fp)
|
||||
|
||||
def index_strip(strip):
|
||||
|
||||
"""
|
||||
Registers a comic to the index. This does not save the comic index.
|
||||
|
||||
:param strip: The comic strip to index
|
||||
"""
|
||||
global strip_index
|
||||
global _index_loaded
|
||||
global _index_lock
|
||||
with _index_lock:
|
||||
if not strip.comic.get_identifier_string() in strip_index:
|
||||
strip_index[strip.comic.get_identifier_string()] = {}
|
||||
strip_index[strip.comic.get_identifier_string()][strip.get_identifier_string()] = strip.get_filename()
|
||||
|
||||
def is_strip_present(comic, base_path: Path, suffix: str, identifier) -> bool:
|
||||
"""
|
||||
Checks to see if a given strip has been saved to the file.
|
||||
|
||||
:param comic: The comic object that the strip belongs to.
|
||||
:param base_path: The folder we expect the strip to show up in.
|
||||
:param suffix: The type suffix we want to check (i.e. ".cbz")
|
||||
:param identifier: The identifier to check.
|
||||
:returns: True if the strip file has been saved.
|
||||
"""
|
||||
global strip_index
|
||||
global _index_loaded
|
||||
global _index_lock
|
||||
if not comic.get_identifier_string() in strip_index:
|
||||
return False
|
||||
if not get_identifier_string(identifier) in strip_index[comic.get_identifier_string()]:
|
||||
return False
|
||||
path = Path(base_path, strip_index[comic.get_identifier_string()][get_identifier_string(identifier)] + suffix)
|
||||
return path.exists()
|
||||
|
||||
|
||||
|
||||
class ImageRepo(ABC):
|
||||
"""
|
||||
A generic representation of an online repository of data. This class allows for the
|
||||
setup of downloading the main information in one pass, and secondary information (images)
|
||||
seperately.
|
||||
"""
|
||||
|
||||
download_headers = {
|
||||
"accept-language": "en-US,en;q=0.9",
|
||||
"dnt": "1",
|
||||
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
}
|
||||
|
||||
def __init__(self, identifier, image_type):
|
||||
"""
|
||||
Creates an information repository
|
||||
|
||||
:param identifier: The globally-unique identifier used to find this repository.
|
||||
:param image_type: The type of collection used for images (i.e. "dict" or "list"). This needs to be an indexed collection.
|
||||
"""
|
||||
self.identifier = identifier
|
||||
self.image_urls = image_type()
|
||||
"""A named collection of image URLs. The value is overridden by implementing classes. This can be any sort of indexed collection (named dictionary, ordered list, etc.)
|
||||
|
||||
Note that despite the name, this does not have to necessarily link to images. Any data that is accessible with GET HTTP requests are valid."""
|
||||
self.images = image_type()
|
||||
"""A collection of temporary files for the relevent images in #image_urls. This should be the same collection type as #image_urls."""
|
||||
|
||||
self._data_loaded = False
|
||||
"""A flag for when the data has been successfully loaded."""
|
||||
self._data_downloaded = False
|
||||
"""A flag for when the data has been successfully downloaded."""
|
||||
self._load_lock = Condition(Lock())
|
||||
"""A lock for enforcing thread safety when loading this repository."""
|
||||
self._download_lock = Condition(Lock())
|
||||
"""A lock for enforcing thread safety when downloading secondary resources."""
|
||||
|
||||
def get_identifier_string(self):
|
||||
"""
|
||||
Obtains a file-safe string representation of the identifer. If the identifier is a list or tuple, the
|
||||
elements will be joined by a dash character.
|
||||
|
||||
:returns: The identifier in string form.
|
||||
"""
|
||||
return get_identifier_string(self.identifier)
|
||||
|
||||
def is_loaded(self):
|
||||
"""
|
||||
Checks to see if the repostiory data is fully loaded.
|
||||
|
||||
:returns: True if the data has been fully loaded, false otherwise.
|
||||
"""
|
||||
return self._data_loaded
|
||||
|
||||
def is_downloaded(self):
|
||||
"""
|
||||
Checks to see if all images (if any are present) are downloaded.
|
||||
|
||||
:returns: True if the images have been fully downloaded, false otherwise.
|
||||
"""
|
||||
return self._data_downloaded
|
||||
|
||||
def load_data(self):
|
||||
"""
|
||||
Queries information on the comic in a thread-safe way.
|
||||
|
||||
This load is only performed once, and subsequent calls
|
||||
will be ignored. Calls made while an existing load is
|
||||
being performed are also ignored. If you need to obtain
|
||||
the data directly after triggering a load, #await_load
|
||||
should be called.
|
||||
|
||||
Implementing classes should overload the #_load_data method.
|
||||
|
||||
:see: #await_load
|
||||
:see: #_load_data
|
||||
"""
|
||||
with self._load_lock:
|
||||
if not self._data_loaded:
|
||||
self._load_data()
|
||||
logging.info("Completed loading of %s", self.get_identifier_string())
|
||||
self._data_loaded = True
|
||||
self._load_lock.notify_all()
|
||||
|
||||
def await_load(self):
|
||||
"""
|
||||
Locks the current thread until all data has been loaded, starting
|
||||
a load as needed.
|
||||
|
||||
:see: #load_data
|
||||
"""
|
||||
|
||||
# Loads this ourselves
|
||||
self.load_data()
|
||||
|
||||
|
||||
@abstractmethod
|
||||
def _load_data(self):
|
||||
"""
|
||||
Queries information on the repository. To be implemented by successor classes.
|
||||
|
||||
Successor classes should not attempt any thread safety or be concerned about
|
||||
checking for previous/concurrent calls to this method, as this logic is handled
|
||||
by #load_data.
|
||||
"""
|
||||
pass
|
||||
|
||||
def download_data(self):
|
||||
"""
|
||||
Downloads all images for the comic.
|
||||
|
||||
This download is only performed once, and subsequent calls
|
||||
will be ignored. Calls made while an existing download is
|
||||
being performed are also ignored. If you need to obtain
|
||||
the images directly after triggering a download, #await_download
|
||||
should be called.
|
||||
|
||||
Implementing classes should overload the #_download_data method.
|
||||
|
||||
:see: #await_download
|
||||
:see: #_download_data
|
||||
"""
|
||||
|
||||
if not self.is_loaded():
|
||||
raise SequenceException("Cannot download images before first loading performed.")
|
||||
|
||||
with self._download_lock:
|
||||
# Checks to see if we need to download and that one is not already in progress
|
||||
if self.image_urls and not self.images:
|
||||
logging.info("Downloading %s", self.get_identifier_string())
|
||||
self._download_data()
|
||||
logging.info("Completed downloading of %s", self.get_identifier_string())
|
||||
self._data_downloaded = True
|
||||
self._download_lock.notify_all()
|
||||
|
||||
def await_download(self):
|
||||
"""
|
||||
Locks the current thread until all images have been downloaded, starting
|
||||
the download as needed.
|
||||
|
||||
:see: #download_data
|
||||
"""
|
||||
|
||||
self.download_data()
|
||||
|
||||
def _download_data(self):
|
||||
"""
|
||||
Downloads all images and saves them to temporary files in the #images field.
|
||||
|
||||
The default implementation should be satisfactory for most use cases, but
|
||||
can be overridden if necessary. Overriders should not attempt any thread
|
||||
safety or be concerned about checking for previous/concurrent calls to
|
||||
this method, as this logic is handled by #download_data.
|
||||
"""
|
||||
if self.image_urls:
|
||||
if type(self.image_urls) == dict:
|
||||
iterator = self.image_urls.keys()
|
||||
elif type(self.image_urls) == list:
|
||||
iterator = range(len(self.image_urls))
|
||||
for name in iterator:
|
||||
logging.info(" Downloading repository %s resource %s", self.get_identifier_string(), name)
|
||||
image_path = NamedTemporaryFile(mode='wb', suffix=str(name), prefix=self.get_identifier_string(), delete = False)
|
||||
result = requests.get(self.image_urls[name], headers=self.download_headers)
|
||||
result.raise_for_status()
|
||||
image_path.write(result.content)
|
||||
image_path.close()
|
||||
if type(self.images) == list:
|
||||
self.images.append(image_path)
|
||||
else:
|
||||
self.images[name] = image_path
|
||||
|
||||
|
||||
class Comic(ImageRepo, ABC):
|
||||
"""
|
||||
A generic representation of a comic collection. Comic-wide resources are stored here.
|
||||
"""
|
||||
|
||||
strip_cls = None
|
||||
"""A reference to the class used for this comic's strips."""
|
||||
|
||||
def __init__(self, identifier):
|
||||
"""
|
||||
Creates a comic collection.
|
||||
|
||||
:param identifier: The globally-unique identifier used to find the comic.
|
||||
"""
|
||||
super().__init__(identifier, dict)
|
||||
self.title = None
|
||||
"""The title of the comic."""
|
||||
self.author = None
|
||||
"""The author(s) of the comic. Can be a string for a name, a list of names, or a list of objects containing the names under "name" and role under "role"."""
|
||||
|
||||
#@abstractmethod
|
||||
@classmethod
|
||||
def create_from_url(cls, url):
|
||||
"""
|
||||
Creates a Comic instance from a given URL
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def _load_data(self):
|
||||
"""
|
||||
Queries the source to obtain information on the comic.
|
||||
|
||||
Implementing methods need to find the following information:
|
||||
* The title of the comic, loaded to `title`.
|
||||
* The author of the comic, loaded to `author`.
|
||||
* A collection of banners, author avatars, and other
|
||||
relevant images, loaded into `image_urls`.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_first_strip(self):
|
||||
"""
|
||||
Obtains the identifier of the first strip.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_latest_strip(self):
|
||||
"""
|
||||
Obtains the identifier of the latest strip.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_all_strips(self) -> list:
|
||||
"""
|
||||
Obtains a list of all strips.
|
||||
"""
|
||||
pass
|
||||
|
||||
def create_strip(self, identifier):
|
||||
"""
|
||||
Creates a new ComicStrip object.
|
||||
|
||||
The class variable #strip_cls must be set for this to work.
|
||||
"""
|
||||
return self.strip_cls(self, identifier)
|
||||
|
||||
|
||||
class ComicStrip(ImageRepo, ABC):
|
||||
"""
|
||||
A generic representation of a single comic strip or issue.
|
||||
"""
|
||||
|
||||
def __init__(self, comic, identifier):
|
||||
"""
|
||||
Creates a comic strip
|
||||
|
||||
:param comic: The `Comic` object that this strip is part of.
|
||||
:param indentifier: The globally unique identifier used to find the strip on the website. Can be a tuple.
|
||||
"""
|
||||
super().__init__(identifier, list)
|
||||
|
||||
self.comic = comic
|
||||
"""The comic that this strip is part of."""
|
||||
self.title = None
|
||||
"""The title of the comic. Empty until the comic data is loaded."""
|
||||
self.image_urls = []
|
||||
"""A list of URLs we can download the comic panels from, in order. Empty until the comic data is loaded."""
|
||||
self._image_paths = []
|
||||
"""A list of temporary files for the raw downloaded panels. Used internally."""
|
||||
self.captions = []
|
||||
"""A list of captions for each panel."""
|
||||
self.date = None
|
||||
"""The date that the comic was published."""
|
||||
self._transform_lock = Condition(Lock())
|
||||
"""A lock for enforcing thread safety when transforming resources."""
|
||||
self.transformed_images = []
|
||||
"""A list of NamedTemporaryFiles linking to the transformed versions of the image downloads."""
|
||||
self._transformed = False
|
||||
"""A flag keeping track of whether the strip transformation took place."""
|
||||
|
||||
def get_filename(self):
|
||||
"""
|
||||
Obtains the name of the final export (not including the stem). Calling this method requires the data be loaded first.
|
||||
|
||||
:returns: A human-friendly file, like "05 - Riddles in the Dark" or "2025-06-01 - Chapter 5."
|
||||
"""
|
||||
prefix_num_form = "#{:02}"
|
||||
if type(self.identifier) == int:
|
||||
prefix = prefix_num_form.format(self.identifier)
|
||||
elif (type(self.identifier) == list or type(self.identifier) == tuple) and type(self.identifier[-1]) == int:
|
||||
prefix = prefix_num_form.format(self.identifier[-1])
|
||||
elif type(self.date) == date:
|
||||
prefix = self.date.strftime("#%Y%m%d")
|
||||
elif type(self.date) == datetime:
|
||||
prefix = self.date.date().strftime("#%Y%m%d")
|
||||
else:
|
||||
prefix = self.get_identifier_string()
|
||||
if self.title:
|
||||
return (file_safe_string(self.comic.title) + " " + prefix + " - " + file_safe_string(self.title))[0:250]
|
||||
else:
|
||||
return prefix
|
||||
|
||||
def get_package_path(self, base_path: Path):
|
||||
"""
|
||||
Obtains the path to save the comic archive to.
|
||||
|
||||
:param base_path: The folder to save comics to.
|
||||
:returns: The path to save to.
|
||||
"""
|
||||
return Path(base_path, file_safe_string(self.comic.title), f"{self.get_filename()}.cbz")
|
||||
|
||||
def load_data(self):
|
||||
self.comic.load_data()
|
||||
super().load_data()
|
||||
index_strip(self)
|
||||
|
||||
def download_data(self):
|
||||
self.comic.download_data()
|
||||
super().download_data()
|
||||
|
||||
def await_load(self):
|
||||
self.comic.await_load()
|
||||
super().await_load()
|
||||
|
||||
def await_download(self):
|
||||
self.comic.await_download()
|
||||
super().await_download()
|
||||
|
||||
@abstractmethod
|
||||
def _load_data(self):
|
||||
"""
|
||||
Queries the source to obtain information on the comic.
|
||||
|
||||
Implementing methods need to find the following information:
|
||||
The title of the comic, loaded to `title`.
|
||||
The publish date (or datetime) of the comic, loaded to `date`.
|
||||
A list of panel images, loaded into `image_urls`.
|
||||
A list of captions, loaded into `captions`. Any captions beyond the list of images can be added to new pages without comics.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def _transform_images(self):
|
||||
"""
|
||||
Called by the packaging method to create a series of locally-transformed
|
||||
images and returning the ordered list of them.
|
||||
|
||||
This method is meant to do things like add alt text, descriptions, title pages, etc.
|
||||
to the final comic. Simpler implementations can simply return a list of the
|
||||
unmodified images with the comic headers from #comic at the beginning.
|
||||
|
||||
All implementations should assign the ordered list to #transformed_images. If this
|
||||
field is not set, then packaging will default to just package the comic strip
|
||||
images.
|
||||
"""
|
||||
self.transformed_images = self.images
|
||||
|
||||
def create_comic_info(self) -> str:
|
||||
"""
|
||||
Creates the contents of a ComicInfo.xml file.
|
||||
"""
|
||||
soup = BeautifulSoup('<?xml version="1.0" encoding="utf-8"?><ComicInfo></ComicInfo>', features="xml")
|
||||
info_tag = soup.find("ComicInfo")
|
||||
info_tag.append(soup.new_tag("Title", string=self.title))
|
||||
info_tag.append(soup.new_tag("Series", string=self.comic.title))
|
||||
number_el = soup.new_tag("Number")
|
||||
if type(self.identifier) == int:
|
||||
number_el.string = str(self.identifier)
|
||||
elif (type(self.identifier) == list or type(self.identifier) == tuple) and type(self.identifier[-1]) == int:
|
||||
number_el.string = str(self.identifier[-1])
|
||||
if number_el.string:
|
||||
info_tag.append(number_el)
|
||||
if self.date:
|
||||
info_tag.append(soup.new_tag("Year", string=str(self.date.year)))
|
||||
info_tag.append(soup.new_tag("Month", string=str(self.date.month)))
|
||||
info_tag.append(soup.new_tag("Day", string=str(self.date.day)))
|
||||
if self.comic.author:
|
||||
if type(self.comic.author) == str:
|
||||
info_tag.append(soup.new_tag("Writer", string=self.comic.author))
|
||||
elif type(self.comic.author) == list:
|
||||
authors = {}
|
||||
writer_string = ""
|
||||
for author in self.comic.author:
|
||||
author_class = None
|
||||
if type(author) == str:
|
||||
author_name = author
|
||||
author_class = "Writer"
|
||||
elif type(author) == dict:
|
||||
author_name = author["name"]
|
||||
if author["role"].lower() in ("writer", "author", "creator"):
|
||||
author_class = "Writer"
|
||||
elif author["role"].lower() in ("penciller"):
|
||||
author_class = "Penciller"
|
||||
elif author["role"].lower() in ("inker"):
|
||||
author_class = "Inker"
|
||||
elif author["role"].lower() in ("colorist"):
|
||||
author_class = "Colorist"
|
||||
elif author["role"].lower() in ("letterer"):
|
||||
author_class = "Letterer"
|
||||
elif author["role"].lower() in ("coverartist", "conver_artist"):
|
||||
author_class = "CoverArtist"
|
||||
elif author["role"].lower() in ("editor"):
|
||||
author_class = "Editor"
|
||||
elif author["role"].lower() in ("translator"):
|
||||
author_class = "Translator"
|
||||
if author_class not in authors:
|
||||
authors[author_class] = author_name
|
||||
else:
|
||||
authors[author_class] += "," + author_name
|
||||
for author_class, author_list in authors.items():
|
||||
info_tag.append(soup.new_tag(author_class, string=author_list))
|
||||
info_tag.append(soup.new_tag("Format", string="Web"))
|
||||
info_tag.append(soup.new_tag("PageCount", string=str(len(self.transformed_images))))
|
||||
return str(soup)
|
||||
|
||||
def package_data(self, base_path: Path):
|
||||
"""
|
||||
Compresses all the data into a cbz file.
|
||||
"""
|
||||
if not self.is_loaded():
|
||||
raise SequenceException("Cannot package comic strip before strip data has been fully loaded.")
|
||||
if not self.is_downloaded():
|
||||
raise SequenceException("Cannot package comic strip before strip data has been fully downloaded.")
|
||||
if not self.comic.is_loaded():
|
||||
raise SequenceException("Cannot package comic strip before comic data has been fully loaded.")
|
||||
if not self.comic.is_downloaded():
|
||||
raise SequenceException("Cannot package comic strip before comic data has been fully downloaded.")
|
||||
if not self.transformed_images:
|
||||
if not self._transformed:
|
||||
if self._transform_lock._lock.locked():
|
||||
# Await for the existing transform to complete
|
||||
self._transform_lock.wait()
|
||||
else:
|
||||
# Run the transformation outselves
|
||||
with self._transform_lock:
|
||||
self._transform_images()
|
||||
self._transformed = True
|
||||
self._transform_lock.notify_all()
|
||||
save_path = self.get_package_path(base_path)
|
||||
save_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with ZipFile(self.get_package_path(base_path), 'w') as comic_zip:
|
||||
for i in range(len(self.transformed_images)):
|
||||
image = self.transformed_images[i]
|
||||
image_path = Path(image.name)
|
||||
logging.info("Packaging %s" % image_path.suffix)
|
||||
with open(image.name, 'rb') as image_stream, comic_zip.open(str(i + 1) + (image_path.suffix or '.png'), 'w') as zip_stream:
|
||||
zip_stream.write(image_stream.read())
|
||||
# Add a ComicInfo file
|
||||
comic_info = self.create_comic_info()
|
||||
if comic_info:
|
||||
with comic_zip.open("ComicInfo.xml", 'w') as zip_stream:
|
||||
zip_stream.write(comic_info.encode("utf-8"));
|
||||
# Delete the base images, as they are no longer needed.
|
||||
# We cannot touch the transformed images, since some
|
||||
# of them may come directly from the base comic, and
|
||||
# we have no way of telling.
|
||||
for image in self.images:
|
||||
image_path = Path(image.name)
|
||||
if image_path.exists():
|
||||
image_path.unlink()
|
||||
|
||||
class URLFormat:
|
||||
"""An internal class used to help sort URLs based on subdomains and paths"""
|
||||
def __init__(self, url: str):
|
||||
self.raw_url = url
|
||||
if not re.match(r'^\w+://', url):
|
||||
url = 'http://' + url
|
||||
self.url = urlparse(url)
|
||||
self.domains = self.url.netloc.split('.')
|
||||
self.paths = self.url.path.split('/')
|
||||
|
||||
def __lt__(self, other):
|
||||
if type(other) == str:
|
||||
other = URLFormat(other)
|
||||
|
||||
if len(self.domains) != len(other.domains):
|
||||
return len(self.domains) < len(other.domains)
|
||||
if self.domains != other.domains:
|
||||
return self.domains < other.domains
|
||||
if len(self.paths) != len(other.paths):
|
||||
return len(self.paths) < len(other.paths)
|
||||
if self.paths != other.paths:
|
||||
return self.paths < other.paths
|
||||
return False
|
||||
|
||||
def __le__(self, other):
|
||||
if type(other) == str:
|
||||
other = URLFormat(other)
|
||||
|
||||
if self == other:
|
||||
return True
|
||||
return self < other
|
||||
|
||||
def __gt__(self, other):
|
||||
if type(other) == str:
|
||||
other = URLFormat(other)
|
||||
|
||||
if len(self.domains) != len(other.domains):
|
||||
return len(self.domains) > len(other.domains)
|
||||
if self.domains != other.domains:
|
||||
return self.domains > other.domains
|
||||
if len(self.paths) != len(other.paths):
|
||||
return len(self.paths) > len(other.paths)
|
||||
if self.paths != other.paths:
|
||||
return self.paths > other.paths
|
||||
return False
|
||||
|
||||
def __ge__(self, other):
|
||||
if type(other) == str:
|
||||
other = URLFormat(other)
|
||||
|
||||
if self == other:
|
||||
return True
|
||||
return self > other
|
||||
|
||||
def __eq__(self, other):
|
||||
if type(other) == str:
|
||||
other = URLFormat(other)
|
||||
|
||||
return self.domains == other.domains and self.paths == other.paths
|
||||
|
||||
def __ne__(self, other):
|
||||
if type(other) == str:
|
||||
other = URLFormat(other)
|
||||
|
||||
return self.domains != other.domains or self.paths != other.paths
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.url.netloc}{self.url.path}"
|
||||
|
||||
def __repr__(self):
|
||||
return f"\"{self.__str__()}\""
|
||||
|
||||
def __hash__(self):
|
||||
return hash(tuple(self.domains + self.paths))
|
||||
|
||||
comic_registry = {}
|
||||
"""A registry that maps URL Domains to Comic classes."""
|
||||
|
||||
_registry_subdomains = []
|
||||
"""An internal list keeping track of the order we should use to find the right parser (priority is last)"""
|
||||
|
||||
def register_comic_class(url: str, comic_cls: type[Comic]):
|
||||
"""
|
||||
Registers a comic type.
|
||||
|
||||
The URL format provided here is used to help determine
|
||||
which class should be used for a given strip. There are
|
||||
a few rules for the URL parameter provided here:
|
||||
|
||||
1. The url should not include the schema (i.e. "http://").
|
||||
For example, "xkcd.com" is okay, but "https://xkcd.com" is
|
||||
not.
|
||||
2. The URL provided here should usually just express the
|
||||
domain name. Paths should only be expressed if absolutely
|
||||
necessary, such as in cases where different comic formats
|
||||
are published to the same site (i.e. Tumblr). For example,
|
||||
"bsky.app" works, but "bsky.app/profile" is discourages
|
||||
(although "bsky.app/profile/strange_user.bsky.social"
|
||||
is acceptable if strange_user does something unusual
|
||||
with the way he publishes his comics and needs a custom
|
||||
parser).
|
||||
3. If the subdomain is "www", it should be ommited.
|
||||
Use "tumblr.com" instead of "www.tumblr.com" (although
|
||||
"john_doe.tumblr.com" is valid).
|
||||
|
||||
:param url: The URL format to register.
|
||||
:param comic_cls: The class to use for parsing this comic.
|
||||
"""
|
||||
url_f = URLFormat(url)
|
||||
if url_f in comic_registry:
|
||||
raise ValueError("URL %s is already registered" % url)
|
||||
comic_registry[url_f] = comic_cls
|
||||
bisect.insort(_registry_subdomains, url_f)
|
||||
|
||||
def find_comic_class(url: str) -> type[Comic]:
|
||||
"""
|
||||
Finds the appropriate comic for a given URL.
|
||||
|
||||
:param url: The URL of the comic to parse. This must be a proper URL (including schema, etc.)
|
||||
:returns: A parser
|
||||
"""
|
||||
url_f = URLFormat(url)
|
||||
while len(url_f.domains) > 0:
|
||||
i = bisect.bisect(_registry_subdomains, url_f)
|
||||
if i == 0:
|
||||
raise ValueError("Could not find a parser for url \"%s\"" % url)
|
||||
if url_f.domains == _registry_subdomains[i - 1].domains:
|
||||
# We have found a series of parsers with a matching subdomain.
|
||||
# Now we check the paths.
|
||||
while len(url_f.paths) > 0:
|
||||
i = bisect.bisect(_registry_subdomains, url_f)
|
||||
if i == 0:
|
||||
raise ValueError("Could not find a parser for url \"%s\"" % url)
|
||||
if url_f.paths == _registry_subdomains[i - 1].paths:
|
||||
# We have found a registered URL pattern that matches the URL
|
||||
# provided.
|
||||
return comic_registry[url_f]
|
||||
else:
|
||||
# Drill down to find an exact matching path
|
||||
url_f = URLFormat('.'.join(url_f.domains) + '/'.join(url_f.paths[:-1]))
|
||||
else:
|
||||
# The result we found has a different subdomain. Look for the root subdomain
|
||||
url_f = URLFormat('.'.join(url_f.domains[1:]) + '/'.join(url_f.paths))
|
||||
raise ValueError("Could not find a parse for url \"%s\"" % url)
|
||||
237
src/comic_download/existential_comics.py
Normal file
237
src/comic_download/existential_comics.py
Normal file
@@ -0,0 +1,237 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# This CLI tool allows for the download and packaging of various internet comics into .cbz files.
|
||||
# Copyright (C) 2025 Markil 3
|
||||
# http://www.singlepilot.net
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
import logging
|
||||
import datetime
|
||||
import textwrap
|
||||
import json
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from pathlib import Path
|
||||
from tempfile import NamedTemporaryFile
|
||||
|
||||
import requests
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from comic_download.comic_strip import Comic, ComicStrip, register_comic_class
|
||||
|
||||
class ExistentialComic(Comic):
|
||||
"""
|
||||
The Existential Comics strip
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("existential_comics")
|
||||
|
||||
@classmethod
|
||||
def create_from_url(cls, url):
|
||||
return cls()
|
||||
|
||||
def _load_data(self):
|
||||
self.title = "Existential Comics"
|
||||
self.author = "Corey Mohler"
|
||||
self.image_urls = {
|
||||
"header": "https://static.existentialcomics.com/title.jpg",
|
||||
"safety": "https://static.existentialcomics.com/safety.png"
|
||||
}
|
||||
self.description = "A webcomic of romance, sarcasm, math, and language."
|
||||
|
||||
result = requests.get("https://existentialcomics.com/")
|
||||
result.raise_for_status()
|
||||
|
||||
content = BeautifulSoup(result.content, features="lxml")
|
||||
# The home page automatically fetches the latest comic, and we
|
||||
# can grab the regular URL (and thus the index) from there.
|
||||
index = content.find("meta", property="og:url")
|
||||
if index and index["content"]:
|
||||
self.latest_identifier = int(index["content"].split('/')[-1])
|
||||
else:
|
||||
raise ValueError("Invalid comic data")
|
||||
|
||||
def get_first_strip(self):
|
||||
return 1
|
||||
|
||||
def get_latest_strip(self):
|
||||
return self.latest_identifier
|
||||
|
||||
def get_all_strips(self) -> list:
|
||||
return list(range(self.get_first_strip(), self.get_latest_strip() + 1))
|
||||
|
||||
def __new__(cls):
|
||||
if not hasattr(cls, 'instance'):
|
||||
cls.instance = super(ExistentialComic, cls).__new__(cls)
|
||||
return cls.instance
|
||||
|
||||
|
||||
class ExistentialComicStrip(ComicStrip):
|
||||
"""
|
||||
A comic strip object that represents a single Existential Comic strip.
|
||||
"""
|
||||
|
||||
def __init__(self, comic, index: int):
|
||||
super().__init__(comic, index)
|
||||
self.url = f"https://existentialcomics.com/comic/{index}"
|
||||
self.title = None
|
||||
|
||||
@property
|
||||
def index(self):
|
||||
# Alias
|
||||
return self.identifier
|
||||
|
||||
def _load_data(self):
|
||||
logging.info("Loading data for comic %s from %s", self.index, self.url)
|
||||
result = requests.get(self.url)
|
||||
result.raise_for_status()
|
||||
|
||||
content = BeautifulSoup(result.content, features="lxml")
|
||||
title = content.find("meta", property="og:title")
|
||||
if title and title["content"]:
|
||||
self.title = title["content"]
|
||||
for img in content.find_all("img", class_="comicImg"):
|
||||
self.image_urls.append(f'http:{img["src"]}')
|
||||
if img.get("title"):
|
||||
self.captions.append(img["title"])
|
||||
else:
|
||||
self.captions.append("")
|
||||
|
||||
final_caption = ""
|
||||
explanation = content.find(id="explanation")
|
||||
if explanation:
|
||||
self.captions.extend(explanation.get_text().split("\n"))
|
||||
philosophers = content.find(id="philosophers-comic")
|
||||
if philosophers:
|
||||
self.captions.extend(philosophers.get_text().split("\n"))
|
||||
if final_caption:
|
||||
self.captions.append(final_caption)
|
||||
self.date = max(datetime.date(2013, 11, 11) + datetime.timedelta(weeks=self.index - 2), datetime.date(2013, 11, 12))
|
||||
|
||||
if self.date >= datetime.date(2023, 11, 13):
|
||||
self.safe = (self.date - datetime.date(2023, 11, 13)).days
|
||||
else:
|
||||
self.safe = (self.date - datetime.date(2013, 11, 12)).days
|
||||
|
||||
|
||||
def _transform_images(self, base_path: Path):
|
||||
"""
|
||||
Takes the raw image data from #download_data and transforms it to add a title, captions, etc. This function is not idempotent.
|
||||
"""
|
||||
|
||||
# Marks out various bounding boxes of strip-global texts (i.e. how much space the title takes up)
|
||||
title_size = 18
|
||||
safety_size = 24
|
||||
caption_size = 15
|
||||
f_size = (1000, 1500)
|
||||
title_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), title_size)
|
||||
safety_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), safety_size)
|
||||
caption_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans.ttf"), caption_size)
|
||||
title_box = title_font.getbbox(self.title)
|
||||
title_box = (title_box[2] - title_box[0], title_box[3] - title_box[1])
|
||||
caption_spacing = 4
|
||||
|
||||
safe_box = safety_font.getbbox(str(self.safe))
|
||||
safe_box = (safe_box[2] - safe_box[0], safe_box[3] - safe_box[1])
|
||||
url_box = caption_font.getbbox(self.url)
|
||||
url_box = (url_box[2] - url_box[0], url_box[3] - url_box[1])
|
||||
date_box = caption_font.getbbox(self.date.isoformat())
|
||||
date_box = (date_box[2] - date_box[0], date_box[3] - date_box[1])
|
||||
text_margin = 10
|
||||
|
||||
img_size = (0, 0)
|
||||
|
||||
self.transformed_images = []
|
||||
header_img = Image.open(self.comic.images["header"].name)
|
||||
safety_img = Image.open(self.comic.images["safety"].name)
|
||||
for i in range(len(self.images)):
|
||||
img = Image.open(self.images[i].name)
|
||||
if self.captions[i]:
|
||||
# Take measurements of the page caption, if one is present.
|
||||
# Pillow's bbox functions do not handle newline characters properly, so we have to
|
||||
# split the text, find the widest line, and use that.
|
||||
caption = textwrap.wrap(self.captions[i], width = img.width // caption_size)
|
||||
caption_lines = len(caption)
|
||||
caption_box = caption_font.getbbox(max(caption, key=len))
|
||||
caption_box = (caption_box[2] - caption_box[0], (caption_box[3] - caption_box[1] + caption_spacing) * len(caption))
|
||||
caption = "\n".join(caption)
|
||||
else:
|
||||
caption_box = (0, 0)
|
||||
# The first page requires extra vertical space for the headers
|
||||
if i == 0:
|
||||
f_offset = header_img.height + safety_img.height + max(date_box[1], url_box[1]) + text_margin
|
||||
# Grab the image size while we are at it, for use outside of the loop
|
||||
img_size = img.size
|
||||
else:
|
||||
f_offset = 0
|
||||
f_img_size = (img.width, f_offset + img.height + caption_box[1] + text_margin)
|
||||
f_img = Image.new("RGBA", f_img_size, (255, 255, 255, 255))
|
||||
# Draw the headers as needed
|
||||
if i == 0:
|
||||
f_img.paste(header_img, (0, 0))
|
||||
f_img.paste(safety_img, (0, header_img.height))
|
||||
|
||||
# Draw the actual comic
|
||||
f_img.paste(img, (0, f_offset))
|
||||
|
||||
draw = ImageDraw.Draw(f_img)
|
||||
if i == 0:
|
||||
# Draw the title
|
||||
draw.text(((img.width - title_box[0]) // 2, header_img.height + 20), self.title, (0, 0, 0), font=title_font)
|
||||
# Draw the safety text
|
||||
draw.text(((safety_img.width - safe_box[0]) // 2, header_img.height + 20), str(self.safe), (0, 0, 0), font=safety_font)
|
||||
# Draw the URL
|
||||
draw.text((text_margin, header_img.height + safety_img.height + text_margin), self.url, (0, 0, 0), font=caption_font)
|
||||
# Draw the date
|
||||
draw.text((img.width - date_box[0] - text_margin, header_img.height + safety_img.height + text_margin), self.date.isoformat(), (0, 0, 0), align="right", font=caption_font)
|
||||
# Draw the caption
|
||||
if self.captions[i]:
|
||||
draw.multiline_text(((img.width - caption_box[0]) // 2, f_offset + img.height), caption, (0, 0, 0), spacing=caption_spacing, font=caption_font, align="center")
|
||||
# Save the image
|
||||
f_image_path = NamedTemporaryFile(mode='wb', suffix=f"f_{i}.png", prefix=self.get_identifier_string(), delete = False)
|
||||
f_image_path.close()
|
||||
f_img.save(f_image_path.name)
|
||||
self.transformed_images.append(f_image_path)
|
||||
|
||||
if len(self.captions) > len(self.image_urls):
|
||||
# Add the explanation to a new page
|
||||
explanation = []
|
||||
# Pillow's bbox functions do not handle newline characters properly, so we have to
|
||||
# split the text, find the widest line, and use that.
|
||||
for paragraph in self.captions[len(self.image_urls):]:
|
||||
explanation.extend(textwrap.wrap(paragraph.strip(), width = img_size[0] // caption_size))
|
||||
explanation_box = [caption_font.getbbox(ex) for ex in explanation]
|
||||
explanation_box = [(box[2] - box[0], box[3] - box[1]) for box in explanation_box]
|
||||
# Using a comparison function (i.e. max) on a tuple just compares the first value of each
|
||||
explanation_box = [max(explanation_box)[0], (explanation_box[0][1] + caption_spacing) * len(explanation_box)]
|
||||
explanation = "\n".join(explanation)
|
||||
|
||||
f_img = Image.new("RGBA", (img_size[0], explanation_box[1] + text_margin * 4), (255, 255, 255, 255))
|
||||
draw = ImageDraw.Draw(f_img)
|
||||
# Adds a background rectangle, like on the website
|
||||
draw.rounded_rectangle(((img_size[0] - explanation_box[0] - text_margin * 2) // 2, text_margin, (img_size[0] + explanation_box[0] + text_margin * 2) // 2, explanation_box[1] + text_margin * 3), 20, (200, 200, 200), (0, 0, 0), 2)
|
||||
draw.multiline_text(((img_size[0] - explanation_box[0]) // 2, text_margin * 2), explanation, (0, 0, 0), align="center", font=caption_font, spacing=caption_spacing)
|
||||
|
||||
# Save the image
|
||||
f_image_path = NamedTemporaryFile(mode='wb', suffix=f"f_ex.png", prefix=self.get_identifier_string(), delete = False)
|
||||
f_image_path.close()
|
||||
f_img.save(f_image_path.name)
|
||||
self.transformed_images.append(f_image_path)
|
||||
|
||||
ExistentialComic.strip_cls = ExistentialComicStrip
|
||||
register_comic_class("existentialcomics.com", ExistentialComic)
|
||||
156
src/comic_download/xkcd.py
Normal file
156
src/comic_download/xkcd.py
Normal file
@@ -0,0 +1,156 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# This CLI tool allows for the download and packaging of various internet comics into .cbz files.
|
||||
# Copyright (C) 2025 Markil 3
|
||||
# http://www.singlepilot.net
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
import logging
|
||||
import datetime
|
||||
import threading
|
||||
import textwrap
|
||||
from zipfile import ZipFile
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from pathlib import Path
|
||||
from tempfile import NamedTemporaryFile
|
||||
|
||||
import requests
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from comic_download.comic_strip import Comic, ComicStrip, register_comic_class
|
||||
|
||||
class XKCDComic(Comic):
|
||||
"""
|
||||
The XKCD Comic.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("xkcd")
|
||||
|
||||
@classmethod
|
||||
def create_from_url(cls, url):
|
||||
return cls()
|
||||
|
||||
def _load_data(self):
|
||||
self.title = "XKCD"
|
||||
self.author = "Randal Munroe"
|
||||
self.image_urls = {
|
||||
"banner": "https://xkcd.com/s/0b7742.png"
|
||||
}
|
||||
self.description = "A webcomic of romance, sarcasm, math, and language."
|
||||
result = requests.get("https://xkcd.com/info.0.json")
|
||||
result.raise_for_status()
|
||||
self.data = result.json()
|
||||
self.latest_identifier = self.data["num"]
|
||||
|
||||
def get_first_strip(self):
|
||||
return 1
|
||||
|
||||
def get_latest_strip(self):
|
||||
return self.latest_identifier
|
||||
|
||||
def get_all_strips(self) -> list:
|
||||
return list(range(self.get_first_strip(), self.get_latest_strip() + 1))
|
||||
|
||||
def __new__(cls):
|
||||
if not hasattr(cls, 'instance'):
|
||||
cls.instance = super(XKCDComic, cls).__new__(cls)
|
||||
return cls.instance
|
||||
|
||||
class XKCDComicStrip(ComicStrip):
|
||||
"""
|
||||
A comic strip object represents a single XKCD strip.
|
||||
"""
|
||||
def __init__(self, comic, index: int):
|
||||
"""
|
||||
Creates a comic strip object.
|
||||
|
||||
:param comic: The XKCD comic.
|
||||
:param index: The index of the strip to load.
|
||||
"""
|
||||
super().__init__(comic, index)
|
||||
self.url = f"https://xkcd.com/{index}"
|
||||
self.data_url = f"{self.url}/info.0.json"
|
||||
self.transcript = None
|
||||
|
||||
@property
|
||||
def index(self):
|
||||
# Alias
|
||||
return self.identifier
|
||||
|
||||
def _load_data(self):
|
||||
logging.info("Loading data for comic %s from %s", self.index, self.url)
|
||||
result = requests.get(self.data_url)
|
||||
result.raise_for_status()
|
||||
self.data = result.json()
|
||||
self.date = datetime.date(int(self.data["year"]), int(self.data["month"]), int(self.data["day"]))
|
||||
self.title = self.data["title"]
|
||||
self.image_urls.append(self.data["img"])
|
||||
self.captions.append(self.data["alt"])
|
||||
self.transcript = self.data["transcript"]
|
||||
|
||||
def _transform_images(self):
|
||||
"""
|
||||
Takes the raw image data from #download_data and transforms it to add a title, captions, etc. This function is not idempotent.
|
||||
"""
|
||||
f_width = 780
|
||||
title_size = 21
|
||||
caption_size = 12
|
||||
title_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), title_size, layout_engine=ImageFont.Layout.RAQM)
|
||||
caption_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans.ttf"), caption_size)
|
||||
|
||||
title_box = title_font.getbbox(self.title)
|
||||
title_box = (title_box[2] - title_box[0], title_box[3] - title_box[1])
|
||||
|
||||
url_box = caption_font.getbbox(self.url)
|
||||
url_box = (url_box[2] - url_box[0], url_box[3] - url_box[1])
|
||||
date_box = caption_font.getbbox(self.date.isoformat())
|
||||
date_box = (date_box[2] - date_box[0], date_box[3] - date_box[1])
|
||||
|
||||
self.transformed_images = []
|
||||
for i in range(len(self.images)):
|
||||
image = self.images[i]
|
||||
image_path = Path(image.name)
|
||||
f_image = NamedTemporaryFile(mode='wb', suffix=f"f-{i}.png", prefix=self.get_identifier_string(), delete = False)
|
||||
if image_path.exists():
|
||||
caption = self.captions[i]
|
||||
try:
|
||||
caption = textwrap.wrap(caption, width=(f_width - 20) / caption_size)
|
||||
cap_max = max([cap_box[2] - cap_box[0] for cap_box in [caption_font.getbbox(cap_line) for cap_line in caption]])
|
||||
caption_box = (cap_max, caption_size * len(caption))
|
||||
caption = "\n".join(caption)
|
||||
except Exception as e:
|
||||
logging.exception("Unable to get caption bounding box for panel %d", i, exc_info=e)
|
||||
caption_box = caption_font.getbbox(caption)
|
||||
caption_box = (caption_box[2] - caption_box[0], caption_box[3] - caption_box[1])
|
||||
img = Image.open(image_path)
|
||||
f_img = Image.new("RGBA", (f_width, img.size[1] + caption_box[1] + title_box[1] + 40), (255, 255, 255, 255))
|
||||
f_img.paste(img, ((f_width - img.size[0]) // 2, title_box[1] + 20))
|
||||
|
||||
draw = ImageDraw.Draw(f_img)
|
||||
|
||||
draw.text(((f_width - title_box[0]) / 2, 2), self.title.upper(), (0, 0, 0), font=title_font, features=["c2sc", "smcp"])
|
||||
draw.multiline_text(((f_width - caption_box[0]) / 2, img.size[1] + title_box[1] + 22), caption, fill=(0, 0, 0), align="center", font=caption_font)
|
||||
draw.text((10, f_img.size[1] - url_box[1] - 10), self.url, (0, 0, 0), align="left", font=caption_font)
|
||||
draw.text((f_width - date_box[0] - 10, f_img.size[1] - date_box[1] - 10), self.date.isoformat(), (0, 0, 0), align="right", font=caption_font)
|
||||
f_img.save(f_image)
|
||||
f_image.close()
|
||||
self.transformed_images.append(f_image)
|
||||
|
||||
|
||||
XKCDComic.strip_cls = XKCDComicStrip
|
||||
register_comic_class("xkcd.com", XKCDComic)
|
||||
Reference in New Issue
Block a user