Compare commits
10 Commits
afeeeedf15
...
7bc0f353f3
| Author | SHA1 | Date | |
|---|---|---|---|
|
7bc0f353f3
|
|||
|
9e6b9c35c2
|
|||
|
936c24fd79
|
|||
|
19108e9035
|
|||
|
73e0476bc6
|
|||
|
d7cdd28f96
|
|||
|
703a2b781e
|
|||
|
8357d20af0
|
|||
|
73fb7e9a0d
|
|||
|
8f9aab27c2
|
2
.gitignore
vendored
2
.gitignore
vendored
@@ -2,4 +2,6 @@
|
||||
*.log
|
||||
**/__pycache__
|
||||
*~
|
||||
src/*.egg-info
|
||||
dist/
|
||||
index.json
|
||||
|
||||
@@ -6,3 +6,8 @@ The following comics are nativly supported:
|
||||
* XKCD
|
||||
* Existential Comics
|
||||
* Bluesky
|
||||
|
||||
# Installing
|
||||
This project requires the following dependencies:
|
||||
|
||||
* `libraqm`
|
||||
|
||||
20
__main__.py
20
__main__.py
@@ -1,20 +0,0 @@
|
||||
# This CLI tool allows for the download and packaging of various internet comics into .cbz files.
|
||||
# Copyright (C) 2025 Markil 3
|
||||
# http://www.singlepilot.net
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
from comic_download.__main__ import __main__
|
||||
|
||||
if __name__ == "__main__":
|
||||
__main__()
|
||||
@@ -1,5 +1,26 @@
|
||||
[build-system]
|
||||
requires = [
|
||||
"setuptools"
|
||||
"setuptools >= 80.9.0"
|
||||
]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "comic_download"
|
||||
version = "0.2.2"
|
||||
authors = [
|
||||
{name = "Markil 3", email = "markil3@singlepilot.net"}
|
||||
]
|
||||
description = "An application to download comics from various sources to the local hard drive."
|
||||
readme = "README.md"
|
||||
keywords = ["comic", "download", "cbz", "ebook"]
|
||||
dependencies = [
|
||||
"requests",
|
||||
"beautifulsoup4",
|
||||
"lxml",
|
||||
"pillow",
|
||||
"atproto",
|
||||
"pycryptodome",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
comic_download = "comic_download.__main__:__main__"
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
requests
|
||||
beautifulsoup4
|
||||
lxml
|
||||
pillow
|
||||
atproto
|
||||
28
setup.cfg
Normal file
28
setup.cfg
Normal file
@@ -0,0 +1,28 @@
|
||||
[metadata]
|
||||
name = comic_download
|
||||
version = 0.2.2
|
||||
|
||||
[options]
|
||||
package_dir=
|
||||
=src
|
||||
zip_safe = False
|
||||
packages = find:
|
||||
include_package_data = True
|
||||
install_requires =
|
||||
requests
|
||||
beautifulsoup4
|
||||
lxml
|
||||
pillow
|
||||
atproto
|
||||
pycryptodome
|
||||
|
||||
[options.packages.find]
|
||||
where=src
|
||||
|
||||
[options.package_data]
|
||||
comic_download = *.ttf
|
||||
* = README.md, LICENSE
|
||||
|
||||
[options.entry_points]
|
||||
console_scripts =
|
||||
executable-name = comic_download.__main__:__main__
|
||||
29
setup.py
29
setup.py
@@ -1,29 +0,0 @@
|
||||
import os
|
||||
from setuptools import find_packages, setup
|
||||
|
||||
def read(fname):
|
||||
return open(os.path.join(os.path.dirname(__file__), fname)).read()
|
||||
|
||||
setup(
|
||||
name='comic_download',
|
||||
packages=find_packages(include=['comic_download']),
|
||||
version='0.0.1',
|
||||
description='An application to download comics from various sources to the local hard drive.',
|
||||
long_description=read('README.md'),
|
||||
long_description_content_type='text/markdown',
|
||||
author='Copyrate',
|
||||
license='gpl-3.0',
|
||||
url='https://git.singlepilot.net/copyrate/ComicDownload',
|
||||
install_requires=[
|
||||
'requests',
|
||||
'beautifulsoup4',
|
||||
'lxml',
|
||||
'pillow',
|
||||
'atproto'
|
||||
],
|
||||
entry_points={
|
||||
'console_scripts': [
|
||||
'comic-download = comic_download.__main__:__main__'
|
||||
]
|
||||
}
|
||||
)
|
||||
@@ -14,4 +14,4 @@
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
from comic_download.comic_strip import SequenceException, ImageRepo, Comic, ComicStrip, file_safe_string, get_identifier_string, register_comic_class, find_comic_class, read_index, save_index, index_strip, is_strip_present
|
||||
from comic_download.comic_strip import SequenceException, AssetRepo, ComicCollection, ComicBook, ComicStrip, file_safe_string, get_identifier_string, register_comic_class, find_comic_class, read_index, save_index, index_strip, is_strip_present
|
||||
@@ -27,7 +27,7 @@ from pathlib import Path
|
||||
import importlib
|
||||
import pkgutil
|
||||
import comic_download
|
||||
from comic_download import Comic, find_comic_class
|
||||
from comic_download import ComicCollection, ComicBook, ComicStrip, find_comic_class
|
||||
|
||||
def setup_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
@@ -71,19 +71,25 @@ def load_plugins():
|
||||
logging.info("Loading plugin %s" % name)
|
||||
importlib.import_module(name)
|
||||
|
||||
def load_strip(base_path, comic, identifier, plain, thread_limit):
|
||||
def load_book(base_path: Path, comic_type: ComicBook, plain: bool, thread_limit: threading.Semaphore, *args):
|
||||
"""
|
||||
Threading function for downloading XKCD strips.
|
||||
|
||||
Arguments:
|
||||
Threading function to add a book to the loading queue.
|
||||
|
||||
:param base_path: The directory to download all downloads to.
|
||||
:param comic: The comic book class to load, or the comic book itself.
|
||||
:param plain: If true, then we skip image transformations.
|
||||
:param thread_limit: The semaphore that controls how many threads we execute at once.
|
||||
:param args: Any arguments here will be passed to the contructor of the comic_type class.
|
||||
"""
|
||||
with thread_limit:
|
||||
strip = comic.create_strip(identifier)
|
||||
strip.await_load()
|
||||
if not strip.get_package_path(base_path).exists():
|
||||
strip.await_download()
|
||||
strip.package_data(base_path)
|
||||
if type(comic_type) == type:
|
||||
book = comic_type(*args)
|
||||
else:
|
||||
book = comic_type
|
||||
book.await_load()
|
||||
if not book.get_package_path(base_path).exists():
|
||||
book.await_download()
|
||||
book.package_data(base_path, plain)
|
||||
|
||||
def __main__():
|
||||
setup_logging()
|
||||
@@ -105,30 +111,39 @@ def __main__():
|
||||
|
||||
for url in args.url:
|
||||
comic_type = find_comic_class(url.netloc + url.path)
|
||||
comic = comic_type.create_from_url(url)
|
||||
comics.append(comic)
|
||||
comic_download.read_index(base_path)
|
||||
comic.await_load()
|
||||
if args.latest:
|
||||
r = [comic.get_latest_strip()]
|
||||
if issubclass(comic_type, ComicStrip):
|
||||
collection = comic_type.collection_cls.create_from_url(url)
|
||||
comic_download.read_index(base_path)
|
||||
collection.await_load()
|
||||
|
||||
# Figure out which strips to grab
|
||||
if args.latest:
|
||||
r = [collection.get_latest_strip()]
|
||||
else:
|
||||
r = collection.get_all_strips()
|
||||
if r and type(r[0]) == int:
|
||||
start = args.start or collection.get_first_strip()
|
||||
end = args.end or collection.get_latest_strip()
|
||||
r = r[r.index(start):r.index(end) + 1]
|
||||
|
||||
# Add the chosen strips to the list of books to add
|
||||
try:
|
||||
for index in reversed(r):
|
||||
comics.append(comic_type(collection, index))
|
||||
finally:
|
||||
logging.info("Saving index")
|
||||
comic_download.save_index(base_path)
|
||||
else:
|
||||
r = comic.get_all_strips()
|
||||
if r and type(r[0]) == int:
|
||||
start = args.start or comic.get_first_strip()
|
||||
end = args.end or comic.get_latest_strip()
|
||||
r = r[r.index(start):r.index(end) + 1]
|
||||
|
||||
try:
|
||||
for index in reversed(r):
|
||||
if not comic_download.is_strip_present(comic, base_path, ".cbz", str(index)):
|
||||
t = threading.Thread(name=str(comic.get_identifier_string()) + "-" + str(index), target=load_strip, args=(base_path, comic, index, args.plain, thread_limit))
|
||||
threads.append(t)
|
||||
t.daemon = True
|
||||
t.start()
|
||||
finally:
|
||||
logging.info("Saving index")
|
||||
comic_download.save_index(base_path)
|
||||
comics.append(comic_type.create_from_url(url))
|
||||
|
||||
comic_download.read_index(base_path)
|
||||
for comic in comics:
|
||||
# Only make threads for books not accounted for in the index
|
||||
if not comic_download.is_strip_present(comic, base_path, ".cbz"):
|
||||
t = threading.Thread(name=str(comic), target=load_book, args=(base_path, comic, args.plain, thread_limit))
|
||||
threads.append(t)
|
||||
t.daemon = True
|
||||
t.start()
|
||||
try:
|
||||
while any([t.is_alive() for t in threads]):
|
||||
time.sleep(1)
|
||||
@@ -33,7 +33,7 @@ from PIL import Image, ImageDraw, ImageFont
|
||||
from bs4 import BeautifulSoup
|
||||
from atproto import Client
|
||||
|
||||
from comic_download.comic_strip import Comic, ComicStrip, register_comic_class
|
||||
from comic_download.comic_strip import ComicCollection as Comic, ComicStrip, register_comic_class
|
||||
|
||||
class BlueskyComic(Comic):
|
||||
"""
|
||||
@@ -185,11 +185,11 @@ class BlueskyComicStrip(ComicStrip):
|
||||
self.captions.append(image.alt)
|
||||
else:
|
||||
self.captions.append("")
|
||||
self.date = datetime.datetime.fromisoformat(post_data.value.created_at)
|
||||
self.date = datetime.datetime.fromisoformat(post_data.value.created_at[0:-1])
|
||||
|
||||
def _transform_images(self):
|
||||
self.transformed_images = self.images
|
||||
|
||||
|
||||
BlueskyComic.strip_cls = BlueskyComicStrip
|
||||
register_comic_class("bsky.app", BlueskyComic)
|
||||
BlueskyComicStrip.collection_cls = BlueskyComic
|
||||
register_comic_class("bsky.app", BlueskyComicStrip)
|
||||
@@ -21,6 +21,7 @@ import argparse
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import datetime, date
|
||||
from threading import Lock, Condition
|
||||
import io
|
||||
import bisect
|
||||
import time
|
||||
import re
|
||||
@@ -113,7 +114,7 @@ A lock for enforcing thread safety when loading the comic index file.
|
||||
|
||||
def read_index(base_path: Path):
|
||||
"""
|
||||
Saves the strip index to the same base path we download comic strips to.
|
||||
Saves the strip index to the same base path we download comic books to.
|
||||
It is saved to "index.json."
|
||||
|
||||
:param base_path: The folder to save the index to.
|
||||
@@ -136,7 +137,7 @@ def read_index(base_path: Path):
|
||||
|
||||
def save_index(base_path: Path):
|
||||
"""
|
||||
Saves the strip index to the same base path we download comic strips to.
|
||||
Saves the strip index to the same base path we download comic books to.
|
||||
It is saved to "index.json."
|
||||
|
||||
:param base_path: The folder to save the index to.
|
||||
@@ -149,43 +150,37 @@ def save_index(base_path: Path):
|
||||
json.dump(strip_index, index_fp)
|
||||
|
||||
def index_strip(strip):
|
||||
|
||||
"""
|
||||
Registers a comic to the index. This does not save the comic index.
|
||||
Registers a book to the index. This does not save the index.
|
||||
|
||||
:param strip: The comic strip to index
|
||||
:param strip: The comic book to index
|
||||
"""
|
||||
global strip_index
|
||||
global _index_loaded
|
||||
global _index_lock
|
||||
with _index_lock:
|
||||
if not strip.comic.get_identifier_string() in strip_index:
|
||||
strip_index[strip.comic.get_identifier_string()] = {}
|
||||
strip_index[strip.comic.get_identifier_string()][strip.get_identifier_string()] = strip.get_filename()
|
||||
strip_index[str(strip)] = strip.get_filename()
|
||||
|
||||
def is_strip_present(comic, base_path: Path, suffix: str, identifier) -> bool:
|
||||
def is_strip_present(comic, base_path: Path, suffix: str) -> bool:
|
||||
"""
|
||||
Checks to see if a given strip has been saved to the file.
|
||||
Checks to see if a given book has been saved to the file.
|
||||
|
||||
:param comic: The comic object that the strip belongs to.
|
||||
:param comic: The comic book object to check.
|
||||
:param base_path: The folder we expect the strip to show up in.
|
||||
:param suffix: The type suffix we want to check (i.e. ".cbz")
|
||||
:param identifier: The identifier to check.
|
||||
:returns: True if the strip file has been saved.
|
||||
"""
|
||||
global strip_index
|
||||
global _index_loaded
|
||||
global _index_lock
|
||||
if not comic.get_identifier_string() in strip_index:
|
||||
if not str(comic) in strip_index:
|
||||
return False
|
||||
if not get_identifier_string(identifier) in strip_index[comic.get_identifier_string()]:
|
||||
return False
|
||||
path = Path(base_path, strip_index[comic.get_identifier_string()][get_identifier_string(identifier)] + suffix)
|
||||
path = Path(base_path, strip_index[str(comic)] + suffix)
|
||||
return path.exists()
|
||||
|
||||
|
||||
|
||||
class ImageRepo(ABC):
|
||||
class AssetRepo(ABC):
|
||||
"""
|
||||
A generic representation of an online repository of data. This class allows for the
|
||||
setup of downloading the main information in one pass, and secondary information (images)
|
||||
@@ -224,7 +219,7 @@ class ImageRepo(ABC):
|
||||
|
||||
def get_identifier_string(self):
|
||||
"""
|
||||
Obtains a file-safe string representation of the identifer. If the identifier is a list or tuple, the
|
||||
Obtains a file-safe string representation of the identifier. If the identifier is a list or tuple, the
|
||||
elements will be joined by a dash character.
|
||||
|
||||
:returns: The identifier in string form.
|
||||
@@ -265,7 +260,7 @@ class ImageRepo(ABC):
|
||||
with self._load_lock:
|
||||
if not self._data_loaded:
|
||||
self._load_data()
|
||||
logging.info("Completed loading of %s", self.get_identifier_string())
|
||||
logging.info("Completed loading of %s", repr(self))
|
||||
self._data_loaded = True
|
||||
self._load_lock.notify_all()
|
||||
|
||||
@@ -314,9 +309,9 @@ class ImageRepo(ABC):
|
||||
with self._download_lock:
|
||||
# Checks to see if we need to download and that one is not already in progress
|
||||
if self.image_urls and not self.images:
|
||||
logging.info("Downloading %s", self.get_identifier_string())
|
||||
logging.info("Downloading %s", repr(self))
|
||||
self._download_data()
|
||||
logging.info("Completed downloading of %s", self.get_identifier_string())
|
||||
logging.info("Completed downloading of %s", repr(self))
|
||||
self._data_downloaded = True
|
||||
self._download_lock.notify_all()
|
||||
|
||||
@@ -330,6 +325,17 @@ class ImageRepo(ABC):
|
||||
|
||||
self.download_data()
|
||||
|
||||
def transform_download_url(self, result):
|
||||
"""
|
||||
If we need to run extra operations on a request (i.e. a redirect), we
|
||||
can override this method to handle that before we attempt to extract
|
||||
image data. If not overridden, this simply returns the original result.
|
||||
|
||||
:param result: The original request result.
|
||||
:returns: The new request result to use.
|
||||
"""
|
||||
return result
|
||||
|
||||
def _download_data(self):
|
||||
"""
|
||||
Downloads all images and saves them to temporary files in the #images field.
|
||||
@@ -345,10 +351,12 @@ class ImageRepo(ABC):
|
||||
elif type(self.image_urls) == list:
|
||||
iterator = range(len(self.image_urls))
|
||||
for name in iterator:
|
||||
logging.info(" Downloading repository %s resource %s", self.get_identifier_string(), name)
|
||||
image_path = NamedTemporaryFile(mode='wb', suffix=str(name), prefix=self.get_identifier_string(), delete = False)
|
||||
logging.info(" Downloading repository %s resource %s", repr(self), name)
|
||||
image_path = NamedTemporaryFile(mode='wb', suffix=str(name).replace('/', '-'), prefix=str(self), delete = False)
|
||||
result = requests.get(self.image_urls[name], headers=self.download_headers)
|
||||
result.raise_for_status()
|
||||
# Handles necessary transformations for a URL (i.e., redirections)
|
||||
result = self.transform_download_url(result)
|
||||
image_path.write(result.content)
|
||||
image_path.close()
|
||||
if type(self.images) == list:
|
||||
@@ -356,8 +364,14 @@ class ImageRepo(ABC):
|
||||
else:
|
||||
self.images[name] = image_path
|
||||
|
||||
def __str__(self):
|
||||
return str(self.__class__.__name__) + "__" + self.get_identifier_string()
|
||||
|
||||
class Comic(ImageRepo, ABC):
|
||||
def __repr__(self):
|
||||
return "<" + str(self.__class__.__name__) + " " + self.get_identifier_string() + ">"
|
||||
|
||||
|
||||
class ComicCollection(AssetRepo, ABC):
|
||||
"""
|
||||
A generic representation of a comic collection. Comic-wide resources are stored here.
|
||||
"""
|
||||
@@ -381,7 +395,7 @@ class Comic(ImageRepo, ABC):
|
||||
@classmethod
|
||||
def create_from_url(cls, url):
|
||||
"""
|
||||
Creates a Comic instance from a given URL
|
||||
Creates a ComicCollection instance from a given URL
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -419,39 +433,34 @@ class Comic(ImageRepo, ABC):
|
||||
"""
|
||||
pass
|
||||
|
||||
def create_strip(self, identifier):
|
||||
"""
|
||||
Creates a new ComicStrip object.
|
||||
def __str__(self):
|
||||
return self.__class__.__name__ + "__" + self.get_identifier_string()
|
||||
|
||||
The class variable #strip_cls must be set for this to work.
|
||||
"""
|
||||
return self.strip_cls(self, identifier)
|
||||
def __repr__(self):
|
||||
if self.title:
|
||||
return "<" + str(self.__class__.__name__) + " " + self.title + " (" + self.get_identifier_string() + ")>"
|
||||
else:
|
||||
return "<" + str(self.__class__.__name__) + " " + self.get_identifier_string() + ">"
|
||||
|
||||
|
||||
class ComicStrip(ImageRepo, ABC):
|
||||
class Book(AssetRepo, ABC):
|
||||
"""
|
||||
A generic representation of a single comic strip or issue.
|
||||
An abstract class for downloading ebook resources.
|
||||
"""
|
||||
|
||||
def __init__(self, comic, identifier):
|
||||
def __init__(self, identifier, asset_type = list):
|
||||
"""
|
||||
Creates a comic strip
|
||||
Creates a book.
|
||||
|
||||
:param comic: The `Comic` object that this strip is part of.
|
||||
:param indentifier: The globally unique identifier used to find the strip on the website. Can be a tuple.
|
||||
:param indentifier: The globally unique identifier used to find the book on the website. Coan be a tuple.
|
||||
"""
|
||||
super().__init__(identifier, list)
|
||||
super().__init__(identifier, asset_type)
|
||||
|
||||
self.comic = comic
|
||||
"""The comic that this strip is part of."""
|
||||
self.title = None
|
||||
"""The title of the comic. Empty until the comic data is loaded."""
|
||||
self.image_urls = []
|
||||
"""A list of URLs we can download the comic panels from, in order. Empty until the comic data is loaded."""
|
||||
self._image_paths = []
|
||||
"""The title of the book. Empty until the book is loaded."""
|
||||
self.series = None
|
||||
"""The series that this book is a part of, if applicable. Empty until the book is loaded."""
|
||||
self._image_paths = asset_type()
|
||||
"""A list of temporary files for the raw downloaded panels. Used internally."""
|
||||
self.captions = []
|
||||
"""A list of captions for each panel."""
|
||||
self.date = None
|
||||
"""The date that the comic was published."""
|
||||
self._transform_lock = Condition(Lock())
|
||||
@@ -461,6 +470,411 @@ class ComicStrip(ImageRepo, ABC):
|
||||
self._transformed = False
|
||||
"""A flag keeping track of whether the strip transformation took place."""
|
||||
|
||||
@classmethod
|
||||
def create_from_url(cls, url):
|
||||
"""
|
||||
Creates a book instance from a given URL
|
||||
"""
|
||||
pass
|
||||
|
||||
def get_filename(self):
|
||||
"""
|
||||
Obtains the name of the final export (not including the stem). Calling this method requires the data be loaded first.
|
||||
|
||||
:returns: A human-friendly filename, like "05 - Riddles in the Dark" or "2025-06-01 - Chapter 5."
|
||||
"""
|
||||
if self.title:
|
||||
return file_safe_string(self.title)[0:250]
|
||||
else:
|
||||
prefix_num_form = "#{:02}"
|
||||
if type(self.identifier) == int:
|
||||
prefix = prefix_num_form.format(self.identifier)
|
||||
elif (type(self.identifier) == list or type(self.identifier) == tuple) and type(self.identifier[-1]) == int:
|
||||
prefix = prefix_num_form.format(self.identifier[-1])
|
||||
elif type(self.date) == date:
|
||||
prefix = self.date.strftime("#%Y%m%d")
|
||||
elif type(self.date) == datetime:
|
||||
prefix = self.date.date().strftime("#%Y%m%d")
|
||||
else:
|
||||
prefix = self.get_identifier_string()
|
||||
return file_safe_string(self.__name__ + " - " + prefix)
|
||||
|
||||
@abstractmethod
|
||||
def get_package_path(self, base_path: Path):
|
||||
"""
|
||||
Obtains the path to save the book to.
|
||||
|
||||
:param base_path: The folder to save books to.
|
||||
:returns: The path to save to.
|
||||
"""
|
||||
pass
|
||||
|
||||
def load_data(self):
|
||||
super().load_data()
|
||||
index_strip(self)
|
||||
|
||||
@abstractmethod
|
||||
def _load_data(self):
|
||||
"""
|
||||
Queries the source to obtain information on the comic.
|
||||
|
||||
Implementing methods need to find the following information:
|
||||
The title of the comic, loaded to `title`.
|
||||
The publish date (or datetime) of the comic, loaded to `date`.
|
||||
A list of panel images, loaded into `image_urls`.
|
||||
A list of captions, loaded into `captions`. Any captions beyond the list of images can be added to new pages without comics.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def _transform_images(self):
|
||||
"""
|
||||
Called by the packaging method to create a series of locally-transformed
|
||||
images and returning the ordered list of them.
|
||||
|
||||
This method is meant to do things like add alt text, descriptions, title pages, etc.
|
||||
to the final comic. Simpler implementations can simply return a list of the
|
||||
unmodified images with the comic headers from #comic at the beginning.
|
||||
|
||||
All implementations should assign the ordered list to #transformed_images. If this
|
||||
field is not set, then packaging will default to just package the comic strip
|
||||
images.
|
||||
"""
|
||||
self.transformed_images = self.images
|
||||
|
||||
@abstractmethod
|
||||
def package_data(self, base_path: Path, plain: bool = False):
|
||||
"""
|
||||
Compresses all of the raw downloaded data into a single archive, saving it to a location in the base path.
|
||||
|
||||
:param base_path: The path to save the archive to.
|
||||
:param plain: If true, we will save the images as downloaded, without any transformations.
|
||||
"""
|
||||
pass
|
||||
|
||||
def __repr__(self):
|
||||
if self.title:
|
||||
return "<" + str(self.__class__.__name__) + " " + self.title + " (" + self.get_identifier_string() + ")>"
|
||||
else:
|
||||
return "<" + str(self.__class__.__name__) + " " + self.get_identifier_string() + ">"
|
||||
|
||||
class EPUBBook(Book, ABC):
|
||||
"""
|
||||
A generic representation of a single EPUB book.
|
||||
"""
|
||||
|
||||
def __init__(self, identifier):
|
||||
"""
|
||||
Creates an ebook.
|
||||
|
||||
:param indentifier: The globally unique identifier used to find the strip on the website. Can be a tuple.
|
||||
"""
|
||||
super().__init__(identifier, dict)
|
||||
|
||||
@classmethod
|
||||
def create_from_url(cls, url):
|
||||
"""
|
||||
Creates a Comic instance from a given URL
|
||||
"""
|
||||
pass
|
||||
|
||||
def get_package_path(self, base_path: Path):
|
||||
"""
|
||||
Obtains the path to save the comic archive to.
|
||||
|
||||
:param base_path: The folder to save comics to.
|
||||
:returns: The path to save to.
|
||||
"""
|
||||
filename = self.get_filename()
|
||||
return Path(base_path, filename, f"{filename}.epub")
|
||||
|
||||
def package_data(self, base_path: Path, plain: bool = False):
|
||||
if not self.is_loaded():
|
||||
raise SequenceException("Cannot package book before data has been fully loaded.")
|
||||
if not self.is_downloaded():
|
||||
raise SequenceException("Cannot package book before data has been fully downloaded.")
|
||||
if not self.transformed_images:
|
||||
if plain:
|
||||
self.transformed_images = self.images
|
||||
else:
|
||||
if not self._transformed:
|
||||
if self._transform_lock._lock.locked():
|
||||
# Await for the existing transform to complete
|
||||
self._transform_lock.wait()
|
||||
else:
|
||||
# Run the transformation outselves
|
||||
with self._transform_lock:
|
||||
self._transform_images()
|
||||
self._transformed = True
|
||||
self._transform_lock.notify_all()
|
||||
save_path = self.get_package_path(base_path)
|
||||
save_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if len(self.transformed_images.keys()) == 1:
|
||||
file = list(self.transformed_images.values)[0]
|
||||
if hasattr(file, "name"):
|
||||
with open(file.name, 'rb') as download_stream:
|
||||
download_cont = download_stream.read()
|
||||
elif isinstance(file, io.IOBase):
|
||||
download_cont = file.read()
|
||||
download_cont.close()
|
||||
elif isinstance(file, Path) or type(file) == str and len(file) < 250 and Path(file).exists():
|
||||
with open(Path(file), 'rb') as download_stream:
|
||||
download_cont = download_stream.read()
|
||||
else:
|
||||
download_cont = file
|
||||
if type(download_cont) == str:
|
||||
download_cont = bytes(download_cont, 'utf-8')
|
||||
with open(save_path, 'wb') as save_stream, open(file.name, 'rb') as download_stream:
|
||||
logging.info("Packaging single download of %s" % retr(self))
|
||||
save_stream.write(download_cont)
|
||||
else:
|
||||
with ZipFile(self.get_package_path(base_path), 'w') as book_zip:
|
||||
with book_zip.open("mimetype", 'w') as zip_stream:
|
||||
zip_stream.write(bytes('application/epub+zip', 'ascii'))
|
||||
for path, file in self.transformed_images.items():
|
||||
if hasattr(file, "name"):
|
||||
logging.info("Packaging temporary file %s", path)
|
||||
with open(file.name, 'rb') as download_stream:
|
||||
download_cont = download_stream.read()
|
||||
elif isinstance(file, io.IOBase):
|
||||
logging.info("Packaging stream %s", path)
|
||||
download_cont = file.read()
|
||||
download_cont.close()
|
||||
elif isinstance(file, Path) or type(file) == str and len(file) < 250 and Path(file).exists():
|
||||
logging.info("Packaging path %s", path)
|
||||
with open(Path(file), 'rb') as download_stream:
|
||||
download_cont = download_stream.read()
|
||||
else:
|
||||
logging.info("Packaging string %s", path)
|
||||
download_cont = file
|
||||
if type(download_cont) == str:
|
||||
download_cont = bytes(download_cont, 'utf-8')
|
||||
with book_zip.open(path, 'w') as zip_stream:
|
||||
zip_stream.write(download_cont)
|
||||
# Delete the base images, as they are no longer needed.
|
||||
# We cannot touch the transformed images, since some
|
||||
# of them may come directly from the base comic, and
|
||||
# we have no way of telling.
|
||||
for file in self.images.values():
|
||||
file_path = Path(file.name)
|
||||
if file_path.exists():
|
||||
file_path.unlink()
|
||||
|
||||
|
||||
|
||||
|
||||
class ComicBook(Book, ABC):
|
||||
"""
|
||||
A generic representation of a single comic strip or issue.
|
||||
"""
|
||||
|
||||
def __init__(self, identifier):
|
||||
"""
|
||||
Creates a comic strip
|
||||
|
||||
:param indentifier: The globally unique identifier used to find the strip on the website. Can be a tuple.
|
||||
"""
|
||||
super().__init__(identifier)
|
||||
|
||||
self.captions = []
|
||||
"""A list of captions for each panel."""
|
||||
|
||||
@classmethod
|
||||
def create_from_url(cls, url):
|
||||
"""
|
||||
Creates a Comic instance from a given URL
|
||||
"""
|
||||
pass
|
||||
|
||||
def get_package_path(self, base_path: Path):
|
||||
"""
|
||||
Obtains the path to save the comic archive to.
|
||||
|
||||
:param base_path: The folder to save comics to.
|
||||
:returns: The path to save to.
|
||||
"""
|
||||
filename = self.get_filename()
|
||||
return Path(base_path, filename, f"{filename}.cbz")
|
||||
|
||||
def create_comic_info(self) -> str:
|
||||
"""
|
||||
Creates the contents of a ComicInfo.xml file.
|
||||
"""
|
||||
soup = BeautifulSoup('<?xml version="1.0" encoding="utf-8"?><ComicInfo></ComicInfo>', features="xml")
|
||||
info_tag = soup.find("ComicInfo")
|
||||
info_tag.append(soup.new_tag("Title", string=self.title))
|
||||
info_tag.append(soup.new_tag("Series", string=self.series))
|
||||
number_el = soup.new_tag("Number")
|
||||
if type(self.identifier) == int:
|
||||
number_el.string = str(self.identifier)
|
||||
elif (type(self.identifier) == list or type(self.identifier) == tuple) and type(self.identifier[-1]) == int:
|
||||
number_el.string = str(self.identifier[-1])
|
||||
if number_el.string:
|
||||
info_tag.append(number_el)
|
||||
if self.date:
|
||||
info_tag.append(soup.new_tag("Year", string=str(self.date.year)))
|
||||
info_tag.append(soup.new_tag("Month", string=str(self.date.month)))
|
||||
info_tag.append(soup.new_tag("Day", string=str(self.date.day)))
|
||||
if self.author:
|
||||
if type(self.author) == str:
|
||||
info_tag.append(soup.new_tag("Writer", string=self.author))
|
||||
elif type(self.author) == list:
|
||||
authors = {}
|
||||
writer_string = ""
|
||||
for author in self.author:
|
||||
author_class = None
|
||||
if type(author) == str:
|
||||
author_name = author
|
||||
author_class = "Writer"
|
||||
elif type(author) == dict:
|
||||
author_name = author["name"]
|
||||
if author["role"].lower() in ("writer", "author", "creator"):
|
||||
author_class = "Writer"
|
||||
elif author["role"].lower() in ("penciller"):
|
||||
author_class = "Penciller"
|
||||
elif author["role"].lower() in ("inker"):
|
||||
author_class = "Inker"
|
||||
elif author["role"].lower() in ("colorist"):
|
||||
author_class = "Colorist"
|
||||
elif author["role"].lower() in ("letterer"):
|
||||
author_class = "Letterer"
|
||||
elif author["role"].lower() in ("coverartist", "conver_artist"):
|
||||
author_class = "CoverArtist"
|
||||
elif author["role"].lower() in ("editor"):
|
||||
author_class = "Editor"
|
||||
elif author["role"].lower() in ("translator"):
|
||||
author_class = "Translator"
|
||||
if author_class not in authors:
|
||||
authors[author_class] = author_name
|
||||
else:
|
||||
authors[author_class] += "," + author_name
|
||||
for author_class, author_list in authors.items():
|
||||
info_tag.append(soup.new_tag(author_class, string=author_list))
|
||||
info_tag.append(soup.new_tag("Format", string="Web"))
|
||||
info_tag.append(soup.new_tag("PageCount", string=str(len(self.transformed_images))))
|
||||
return str(soup)
|
||||
|
||||
def package_data(self, base_path: Path, plain: bool = False):
|
||||
"""
|
||||
Compresses all the data into a cbz file.
|
||||
|
||||
:param base_path: The path to save all downloads to.
|
||||
:param plain: If true, we will save the images as downloaded, without any transformations.
|
||||
"""
|
||||
if not self.is_loaded():
|
||||
raise SequenceException("Cannot package comic book before data has been fully loaded.")
|
||||
if not self.is_downloaded():
|
||||
raise SequenceException("Cannot package comic book before data has been fully downloaded.")
|
||||
if not self.transformed_images:
|
||||
if plain:
|
||||
self.transformed_images = self.images
|
||||
else:
|
||||
if not self._transformed:
|
||||
if self._transform_lock._lock.locked():
|
||||
# Await for the existing transform to complete
|
||||
self._transform_lock.wait()
|
||||
else:
|
||||
# Run the transformation outselves
|
||||
with self._transform_lock:
|
||||
self._transform_images()
|
||||
self._transformed = True
|
||||
self._transform_lock.notify_all()
|
||||
save_path = self.get_package_path(base_path)
|
||||
save_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with ZipFile(self.get_package_path(base_path), 'w') as comic_zip:
|
||||
for i in range(len(self.transformed_images)):
|
||||
image = self.transformed_images[i]
|
||||
if hasattr(image, "name"):
|
||||
with open(image.name, 'rb') as download_stream:
|
||||
download_cont = download_stream.read()
|
||||
image_path = Path(image.name)
|
||||
suffix = image_path.suffix
|
||||
if len(suffix) > 5:
|
||||
# There is something fishy going on, so let us just use the default
|
||||
suffix = None
|
||||
if not suffix:
|
||||
suffix = ".png"
|
||||
elif isinstance(image, io.IOBase):
|
||||
download_cont = image.read()
|
||||
download_cont.close()
|
||||
suffix = ".png"
|
||||
elif isinstance(image, Path) or type(image) == str and len(image) < 250 and Path(image).exists():
|
||||
with open(Path(image), 'rb') as download_stream:
|
||||
download_cont = download_stream.read()
|
||||
suffix = image.suffix
|
||||
if len(suffix) > 5:
|
||||
# There is something fishy going on, so let us just use the default
|
||||
suffix = None
|
||||
if not suffix:
|
||||
suffix = ".png"
|
||||
else:
|
||||
download_cont = image
|
||||
suffix = ".png"
|
||||
if type(download_cont) == str:
|
||||
download_cont = bytes(download_cont, 'utf-8')
|
||||
logging.info("Packaging resource %d" % i)
|
||||
with comic_zip.open(str(i + 1) + (suffix), 'w') as zip_stream:
|
||||
zip_stream.write(download_cont)
|
||||
# Add a ComicInfo file
|
||||
comic_info = self.create_comic_info()
|
||||
if comic_info:
|
||||
with comic_zip.open("ComicInfo.xml", 'w') as zip_stream:
|
||||
zip_stream.write(comic_info.encode("utf-8"));
|
||||
# Delete the base images, as they are no longer needed.
|
||||
# We cannot touch the transformed images, since some
|
||||
# of them may come directly from the base comic, and
|
||||
# we have no way of telling.
|
||||
for image in self.images:
|
||||
image_path = Path(image.name)
|
||||
if image_path.exists():
|
||||
image_path.unlink()
|
||||
|
||||
class ComicStrip(ComicBook, ABC):
|
||||
"""
|
||||
A generic representation of an issue in a comic series. Unlike the base
|
||||
ComicBook class, these objects are tied to a ComicCollection instance
|
||||
and derive some data from that class.
|
||||
"""
|
||||
def __init__(self, comic, identifier):
|
||||
"""
|
||||
Creates a comic strip.
|
||||
|
||||
:param comic: The `ComicCollection` object that this strip is part of.
|
||||
:param indentifier: The collection-unique identifier used to find the strip on the website. Can be a tuple.
|
||||
"""
|
||||
super().__init__(identifier)
|
||||
|
||||
self.comic = comic
|
||||
|
||||
@classmethod
|
||||
def create_from_url(cls, url, collection):
|
||||
"""
|
||||
Creates a Comic instance from a given URL
|
||||
"""
|
||||
return collection.create_from_url(url)
|
||||
|
||||
@property
|
||||
def author(self):
|
||||
"""
|
||||
The author(s) of the comic. Can be a string for a name, a list of names, or a list of objects containing the names under "name" and role under "role".
|
||||
|
||||
For Comic Strips, this references back to the main comic series author.
|
||||
"""
|
||||
return self.comic.author
|
||||
|
||||
@author.setter
|
||||
def author(self, x):
|
||||
pass
|
||||
|
||||
@property
|
||||
def series(self) -> str:
|
||||
"""The series that this comic is a part of."""
|
||||
return self.comic.title
|
||||
|
||||
@series.setter
|
||||
def series(self, x):
|
||||
pass
|
||||
|
||||
def get_filename(self):
|
||||
"""
|
||||
Obtains the name of the final export (not including the stem). Calling this method requires the data be loaded first.
|
||||
@@ -495,7 +909,6 @@ class ComicStrip(ImageRepo, ABC):
|
||||
def load_data(self):
|
||||
self.comic.load_data()
|
||||
super().load_data()
|
||||
index_strip(self)
|
||||
|
||||
def download_data(self):
|
||||
self.comic.download_data()
|
||||
@@ -509,138 +922,28 @@ class ComicStrip(ImageRepo, ABC):
|
||||
self.comic.await_download()
|
||||
super().await_download()
|
||||
|
||||
@abstractmethod
|
||||
def _load_data(self):
|
||||
"""
|
||||
Queries the source to obtain information on the comic.
|
||||
|
||||
Implementing methods need to find the following information:
|
||||
The title of the comic, loaded to `title`.
|
||||
The publish date (or datetime) of the comic, loaded to `date`.
|
||||
A list of panel images, loaded into `image_urls`.
|
||||
A list of captions, loaded into `captions`. Any captions beyond the list of images can be added to new pages without comics.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def _transform_images(self):
|
||||
"""
|
||||
Called by the packaging method to create a series of locally-transformed
|
||||
images and returning the ordered list of them.
|
||||
|
||||
This method is meant to do things like add alt text, descriptions, title pages, etc.
|
||||
to the final comic. Simpler implementations can simply return a list of the
|
||||
unmodified images with the comic headers from #comic at the beginning.
|
||||
|
||||
All implementations should assign the ordered list to #transformed_images. If this
|
||||
field is not set, then packaging will default to just package the comic strip
|
||||
images.
|
||||
"""
|
||||
self.transformed_images = self.images
|
||||
|
||||
def create_comic_info(self) -> str:
|
||||
"""
|
||||
Creates the contents of a ComicInfo.xml file.
|
||||
"""
|
||||
soup = BeautifulSoup('<?xml version="1.0" encoding="utf-8"?><ComicInfo></ComicInfo>', features="xml")
|
||||
info_tag = soup.find("ComicInfo")
|
||||
info_tag.append(soup.new_tag("Title", string=self.title))
|
||||
info_tag.append(soup.new_tag("Series", string=self.comic.title))
|
||||
number_el = soup.new_tag("Number")
|
||||
if type(self.identifier) == int:
|
||||
number_el.string = str(self.identifier)
|
||||
elif (type(self.identifier) == list or type(self.identifier) == tuple) and type(self.identifier[-1]) == int:
|
||||
number_el.string = str(self.identifier[-1])
|
||||
if number_el.string:
|
||||
info_tag.append(number_el)
|
||||
if self.date:
|
||||
info_tag.append(soup.new_tag("Year", string=str(self.date.year)))
|
||||
info_tag.append(soup.new_tag("Month", string=str(self.date.month)))
|
||||
info_tag.append(soup.new_tag("Day", string=str(self.date.day)))
|
||||
if self.comic.author:
|
||||
if type(self.comic.author) == str:
|
||||
info_tag.append(soup.new_tag("Writer", string=self.comic.author))
|
||||
elif type(self.comic.author) == list:
|
||||
authors = {}
|
||||
writer_string = ""
|
||||
for author in self.comic.author:
|
||||
author_class = None
|
||||
if type(author) == str:
|
||||
author_name = author
|
||||
author_class = "Writer"
|
||||
elif type(author) == dict:
|
||||
author_name = author["name"]
|
||||
if author["role"].lower() in ("writer", "author", "creator"):
|
||||
author_class = "Writer"
|
||||
elif author["role"].lower() in ("penciller"):
|
||||
author_class = "Penciller"
|
||||
elif author["role"].lower() in ("inker"):
|
||||
author_class = "Inker"
|
||||
elif author["role"].lower() in ("colorist"):
|
||||
author_class = "Colorist"
|
||||
elif author["role"].lower() in ("letterer"):
|
||||
author_class = "Letterer"
|
||||
elif author["role"].lower() in ("coverartist", "conver_artist"):
|
||||
author_class = "CoverArtist"
|
||||
elif author["role"].lower() in ("editor"):
|
||||
author_class = "Editor"
|
||||
elif author["role"].lower() in ("translator"):
|
||||
author_class = "Translator"
|
||||
if author_class not in authors:
|
||||
authors[author_class] = author_name
|
||||
else:
|
||||
authors[author_class] += "," + author_name
|
||||
for author_class, author_list in authors.items():
|
||||
info_tag.append(soup.new_tag(author_class, string=author_list))
|
||||
info_tag.append(soup.new_tag("Format", string="Web"))
|
||||
info_tag.append(soup.new_tag("PageCount", string=str(len(self.transformed_images))))
|
||||
return str(soup)
|
||||
|
||||
def package_data(self, base_path: Path):
|
||||
"""
|
||||
Compresses all the data into a cbz file.
|
||||
"""
|
||||
if not self.is_loaded():
|
||||
raise SequenceException("Cannot package comic strip before strip data has been fully loaded.")
|
||||
if not self.is_downloaded():
|
||||
raise SequenceException("Cannot package comic strip before strip data has been fully downloaded.")
|
||||
def package_data(self, base_path: Path, plain: bool = False):
|
||||
if not self.comic.is_loaded():
|
||||
raise SequenceException("Cannot package comic strip before comic data has been fully loaded.")
|
||||
if not self.comic.is_downloaded():
|
||||
raise SequenceException("Cannot package comic strip before comic data has been fully downloaded.")
|
||||
if not self.transformed_images:
|
||||
if not self._transformed:
|
||||
if self._transform_lock._lock.locked():
|
||||
# Await for the existing transform to complete
|
||||
self._transform_lock.wait()
|
||||
else:
|
||||
# Run the transformation outselves
|
||||
with self._transform_lock:
|
||||
self._transform_images()
|
||||
self._transformed = True
|
||||
self._transform_lock.notify_all()
|
||||
save_path = self.get_package_path(base_path)
|
||||
save_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with ZipFile(self.get_package_path(base_path), 'w') as comic_zip:
|
||||
for i in range(len(self.transformed_images)):
|
||||
image = self.transformed_images[i]
|
||||
image_path = Path(image.name)
|
||||
logging.info("Packaging %s" % image_path.suffix)
|
||||
with open(image.name, 'rb') as image_stream, comic_zip.open(str(i + 1) + (image_path.suffix or '.png'), 'w') as zip_stream:
|
||||
zip_stream.write(image_stream.read())
|
||||
# Add a ComicInfo file
|
||||
comic_info = self.create_comic_info()
|
||||
if comic_info:
|
||||
with comic_zip.open("ComicInfo.xml", 'w') as zip_stream:
|
||||
zip_stream.write(comic_info.encode("utf-8"));
|
||||
# Delete the base images, as they are no longer needed.
|
||||
# We cannot touch the transformed images, since some
|
||||
# of them may come directly from the base comic, and
|
||||
# we have no way of telling.
|
||||
for image in self.images:
|
||||
image_path = Path(image.name)
|
||||
if image_path.exists():
|
||||
image_path.unlink()
|
||||
super().package_data(base_path, plain)
|
||||
|
||||
def __str__(self):
|
||||
return self.__class__.__name__ + "__" + self.comic.get_identifier_string() + "__" + self.get_identifier_string()
|
||||
|
||||
def __repr__(self):
|
||||
representation = "<" + str(self.__class__.__name__) + " "
|
||||
if self.comic.title:
|
||||
representation += self.comic.title + " (" + self.comic.get_identifier_string() + "): "
|
||||
else:
|
||||
representation += self.comic.get_identifier_string() + ": "
|
||||
if self.title:
|
||||
representation += self.title + " (" + self.get_identifier_string() + ")"
|
||||
else:
|
||||
representation += self.comic.get_identifier_string()
|
||||
representation += ">"
|
||||
return representation
|
||||
|
||||
class URLFormat:
|
||||
"""An internal class used to help sort URLs based on subdomains and paths"""
|
||||
@@ -723,7 +1026,7 @@ comic_registry = {}
|
||||
_registry_subdomains = []
|
||||
"""An internal list keeping track of the order we should use to find the right parser (priority is last)"""
|
||||
|
||||
def register_comic_class(url: str, comic_cls: type[Comic]):
|
||||
def register_comic_class(url: str, comic_cls: type[ComicBook]):
|
||||
"""
|
||||
Registers a comic type.
|
||||
|
||||
@@ -756,7 +1059,7 @@ def register_comic_class(url: str, comic_cls: type[Comic]):
|
||||
comic_registry[url_f] = comic_cls
|
||||
bisect.insort(_registry_subdomains, url_f)
|
||||
|
||||
def find_comic_class(url: str) -> type[Comic]:
|
||||
def find_comic_class(url: str) -> type[ComicBook]:
|
||||
"""
|
||||
Finds the appropriate comic for a given URL.
|
||||
|
||||
219
src/comic_download/dinosaur_comics.py
Normal file
219
src/comic_download/dinosaur_comics.py
Normal file
@@ -0,0 +1,219 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# This CLI tool allows for the download and packaging of various internet comics into .cbz files.
|
||||
# Copyright (C) 2025 Markil 3
|
||||
# http://www.singlepilot.net
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
import logging
|
||||
import datetime
|
||||
import textwrap
|
||||
import json
|
||||
import re
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
|
||||
from importlib import resources
|
||||
from pathlib import Path
|
||||
from tempfile import NamedTemporaryFile
|
||||
|
||||
import requests
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from comic_download.comic_strip import ComicCollection as Comic, ComicStrip, register_comic_class
|
||||
|
||||
class DinosaurComic(Comic):
|
||||
"""
|
||||
The Dinosaur Comics strip
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("dinosaur_comics")
|
||||
|
||||
@classmethod
|
||||
def create_from_url(cls, url):
|
||||
return cls()
|
||||
|
||||
def _load_data(self):
|
||||
self.title = "Dinosaur Comics!"
|
||||
self.author = "Ryan North"
|
||||
self.image_urls = {
|
||||
"header": "https://www.qwantz.com/logo.png",
|
||||
"background": "https://www.qwantz.com/sky.png"
|
||||
}
|
||||
self.description = "this comic... might be the best comic?"
|
||||
|
||||
result = requests.get("https://www.qwantz.com/archive.php")
|
||||
result.raise_for_status()
|
||||
|
||||
content = BeautifulSoup(result.content, features="lxml")
|
||||
archives = content.find_all("ul", class_="archive")
|
||||
self.strips = []
|
||||
self.strip_index = {}
|
||||
date_pat = re.compile(r'(\d+)(st|nd|rd|th)')
|
||||
for archive_month in archives:
|
||||
for line_item in archive_month.find_all("li"):
|
||||
link_el = line_item.find("a")
|
||||
if link_el:
|
||||
parsed_index = urlparse(link_el["href"])
|
||||
identifier = int(parse_qs(parsed_index.query)["comic"][0])
|
||||
date_str = link_el.get_text()
|
||||
date = datetime.datetime.strptime(date_pat.sub(r'\1', date_str), '%B %d, %Y').date()
|
||||
self.strips.append({
|
||||
"identifier": identifier,
|
||||
"date": date,
|
||||
"description": link_el.next_sibling.get_text()[2:]
|
||||
})
|
||||
self.strips.sort(key=lambda x: x["date"])
|
||||
for i in range(len(self.strips)):
|
||||
self.strip_index[self.strips[i]["identifier"]] = i
|
||||
|
||||
def get_first_strip(self):
|
||||
return self.strips[0]["identifier"]
|
||||
|
||||
def get_latest_strip(self):
|
||||
return self.strips[-1]["identifier"]
|
||||
|
||||
def get_all_strips(self) -> list:
|
||||
return [strip["identifier"] for strip in self.strips]
|
||||
|
||||
def __new__(cls):
|
||||
if not hasattr(cls, 'instance'):
|
||||
cls.instance = super(DinosaurComic, cls).__new__(cls)
|
||||
return cls.instance
|
||||
|
||||
|
||||
class DinosaurComicStrip(ComicStrip):
|
||||
"""
|
||||
A comic strip object that represents a single Existential Comic strip.
|
||||
"""
|
||||
|
||||
def __init__(self, comic, index: int):
|
||||
super().__init__(comic, index)
|
||||
self.url = f"https://qwantz.com/index.php?comic={index}"
|
||||
self.title = None
|
||||
strip_data = comic.strips[comic.strip_index[index]]
|
||||
self.date = strip_data["date"]
|
||||
self.description = strip_data["description"]
|
||||
|
||||
@property
|
||||
def index(self):
|
||||
# Alias
|
||||
return self.identifier
|
||||
|
||||
def _load_data(self):
|
||||
logging.info("Loading data for comic %s from %s", self.index, self.url)
|
||||
result = requests.get(self.url)
|
||||
result.raise_for_status()
|
||||
|
||||
content = BeautifulSoup(result.content, features="lxml")
|
||||
title = content.find("meta", property="og:title")
|
||||
if title and title["content"]:
|
||||
self.title = title["content"]
|
||||
for img in content.find_all("img", class_="comic"):
|
||||
self.image_urls.append(f'https://qwantz.com/{img["src"]}')
|
||||
if img.get("title"):
|
||||
self.captions.append(img["title"])
|
||||
else:
|
||||
self.captions.append("")
|
||||
|
||||
|
||||
def _transform_images(self):
|
||||
"""
|
||||
Takes the raw image data from #download_data and transforms it to add a title, captions, etc. This function is not idempotent.
|
||||
"""
|
||||
f_width = 780
|
||||
title_size = 21
|
||||
description_spacing = 10
|
||||
description_size = 18
|
||||
caption_size = 12
|
||||
|
||||
with resources.path("comic_download", "Lucida Sans Bold.ttf") as luc_sans_bold, resources.path("comic_download", "Lucida Sans.ttf") as luc_sans:
|
||||
title_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), title_size, layout_engine=ImageFont.Layout.RAQM)
|
||||
description_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), description_size, layout_engine=ImageFont.Layout.RAQM)
|
||||
caption_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans.ttf"), caption_size)
|
||||
|
||||
title_box = title_font.getbbox(self.title)
|
||||
title_box = (title_box[2] - title_box[0], title_box[3] - title_box[1])
|
||||
caption_spacing = 4
|
||||
|
||||
url_box = caption_font.getbbox(self.url)
|
||||
url_box = (url_box[2] - url_box[0], url_box[3] - url_box[1])
|
||||
date_box = caption_font.getbbox(self.date.isoformat())
|
||||
date_box = (date_box[2] - date_box[0], date_box[3] - date_box[1])
|
||||
text_margin = 20
|
||||
|
||||
self.transformed_images = []
|
||||
header_img = Image.open(self.comic.images["header"].name)
|
||||
background_img = Image.open(self.comic.images["background"].name)
|
||||
for i in range(len(self.images)):
|
||||
img = Image.open(self.images[i].name)
|
||||
comic_width = max(img.width, header_img.width)
|
||||
if self.captions[i]:
|
||||
# Take measurements of the page caption, if one is present.
|
||||
# Pillow's bbox functions do not handle newline characters properly, so we have to
|
||||
# split the text, find the widest line, and use that.
|
||||
caption = textwrap.wrap(self.captions[i], width = comic_width // caption_size)
|
||||
caption_lines = len(caption)
|
||||
caption_box = caption_font.getbbox(max(caption, key=len))
|
||||
caption_box = (caption_box[2] - caption_box[0], (caption_box[3] - caption_box[1] + caption_spacing) * len(caption))
|
||||
caption = "\n".join(caption)
|
||||
else:
|
||||
caption_box = (0, 0)
|
||||
# The first page requires extra vertical space for the headers
|
||||
if i == 0:
|
||||
description = textwrap.wrap(self.description, width=comic_width // description_size)
|
||||
description_lines = len(description)
|
||||
description_box = description_font.getbbox(max(description, key=len))
|
||||
description_box = (description_box[2] - description_box[0], (description_box[3] - description_box[1] + description_spacing) * (len(description) + 1))
|
||||
description = "\n".join(description)
|
||||
|
||||
f_offset = header_img.height + description_box[1] + max(date_box[1], url_box[1]) + text_margin * 2
|
||||
# Grab the image size while we are at it, for use outside of the loop
|
||||
img_size = img.size
|
||||
else:
|
||||
f_offset = 0
|
||||
f_img_size = (comic_width, f_offset + img.height + caption_box[1] + text_margin)
|
||||
f_img = Image.new("RGBA", f_img_size, (255, 255, 255, 255))
|
||||
# Draw the headers as needed
|
||||
if i == 0:
|
||||
#f_img.paste(background_img, (0, 0))
|
||||
f_img.paste(header_img, ((comic_width - header_img.width) // 2, 0))
|
||||
|
||||
# Draw the actual comic
|
||||
f_img.paste(img, ((comic_width - img.width) // 2, f_offset))
|
||||
|
||||
draw = ImageDraw.Draw(f_img)
|
||||
if i == 0:
|
||||
# Draw the description
|
||||
draw.multiline_text(((comic_width - description_box[0]) // 2, header_img.height + 20), description, (0, 0, 0), spacing=description_spacing, font=description_font, align="center")
|
||||
# Draw the URL
|
||||
draw.text((text_margin, header_img.height + description_box[1] + text_margin), self.url, (0, 0, 0), font=caption_font)
|
||||
# Draw the date
|
||||
draw.text((comic_width - date_box[0] - text_margin, header_img.height + description_box[1] + text_margin), self.date.isoformat(), (0, 0, 0), align="right", font=caption_font)
|
||||
# Draw the caption
|
||||
if self.captions[i]:
|
||||
draw.multiline_text(((comic_width - caption_box[0]) // 2, f_offset + img.height), caption, (0, 0, 0), spacing=caption_spacing, font=caption_font, align="center")
|
||||
# Save the image
|
||||
f_image_path = NamedTemporaryFile(mode='wb', suffix=f"f_{i}.png", prefix=self.get_identifier_string(), delete = False)
|
||||
f_image_path.close()
|
||||
f_img.save(f_image_path.name)
|
||||
self.transformed_images.append(f_image_path)
|
||||
|
||||
DinosaurComicStrip.collection_cls = DinosaurComic
|
||||
register_comic_class("qwantz.com", DinosaurComicStrip)
|
||||
register_comic_class("dinosaurcomics.com", DinosaurComicStrip)
|
||||
register_comic_class("exotica.ca", DinosaurComicStrip)
|
||||
register_comic_class("chewbac.ca", DinosaurComicStrip)
|
||||
@@ -24,6 +24,7 @@ import json
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from importlib import resources
|
||||
from pathlib import Path
|
||||
from tempfile import NamedTemporaryFile
|
||||
|
||||
@@ -31,7 +32,7 @@ import requests
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from comic_download.comic_strip import Comic, ComicStrip, register_comic_class
|
||||
from comic_download.comic_strip import ComicCollection as Comic, ComicStrip, register_comic_class
|
||||
|
||||
class ExistentialComic(Comic):
|
||||
"""
|
||||
@@ -139,9 +140,10 @@ class ExistentialComicStrip(ComicStrip):
|
||||
safety_size = 24
|
||||
caption_size = 15
|
||||
f_size = (1000, 1500)
|
||||
title_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), title_size)
|
||||
safety_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), safety_size)
|
||||
caption_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans.ttf"), caption_size)
|
||||
with resources.path("comic_download", "Lucida Sans Bold.ttf") as luc_sans_bold, resources.path("comic_download", "Lucida Sans.ttf") as luc_sans:
|
||||
title_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), title_size)
|
||||
safety_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), safety_size)
|
||||
caption_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans.ttf"), caption_size)
|
||||
title_box = title_font.getbbox(self.title)
|
||||
title_box = (title_box[2] - title_box[0], title_box[3] - title_box[1])
|
||||
caption_spacing = 4
|
||||
@@ -233,5 +235,5 @@ class ExistentialComicStrip(ComicStrip):
|
||||
f_img.save(f_image_path.name)
|
||||
self.transformed_images.append(f_image_path)
|
||||
|
||||
ExistentialComic.strip_cls = ExistentialComicStrip
|
||||
register_comic_class("existentialcomics.com", ExistentialComic)
|
||||
ExistentialComicStrip.collection_cls = ExistentialComic
|
||||
register_comic_class("existentialcomics.com", ExistentialComicStrip)
|
||||
@@ -24,6 +24,7 @@ import textwrap
|
||||
from zipfile import ZipFile
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from importlib import resources
|
||||
from pathlib import Path
|
||||
from tempfile import NamedTemporaryFile
|
||||
|
||||
@@ -31,7 +32,7 @@ import requests
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from comic_download.comic_strip import Comic, ComicStrip, register_comic_class
|
||||
from comic_download.comic_strip import ComicCollection as Comic, ComicStrip, register_comic_class
|
||||
|
||||
class XKCDComic(Comic):
|
||||
"""
|
||||
@@ -110,8 +111,10 @@ class XKCDComicStrip(ComicStrip):
|
||||
f_width = 780
|
||||
title_size = 21
|
||||
caption_size = 12
|
||||
title_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), title_size, layout_engine=ImageFont.Layout.RAQM)
|
||||
caption_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans.ttf"), caption_size)
|
||||
|
||||
with resources.path("comic_download", "Lucida Sans Bold.ttf") as luc_sans_bold, resources.path("comic_download", "Lucida Sans.ttf") as luc_sans:
|
||||
title_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans Bold.ttf"), title_size, layout_engine=ImageFont.Layout.RAQM)
|
||||
caption_font = ImageFont.truetype(Path(Path(__file__).parent, "Lucida Sans.ttf"), caption_size)
|
||||
|
||||
title_box = title_font.getbbox(self.title)
|
||||
title_box = (title_box[2] - title_box[0], title_box[3] - title_box[1])
|
||||
@@ -152,5 +155,5 @@ class XKCDComicStrip(ComicStrip):
|
||||
self.transformed_images.append(f_image)
|
||||
|
||||
|
||||
XKCDComic.strip_cls = XKCDComicStrip
|
||||
register_comic_class("xkcd.com", XKCDComic)
|
||||
XKCDComicStrip.collection_cls = XKCDComic
|
||||
register_comic_class("xkcd.com", XKCDComicStrip)
|
||||
Reference in New Issue
Block a user