diff --git a/src/comic_download/__init__.py b/src/comic_download/__init__.py index 4ce54ef..b126fc3 100644 --- a/src/comic_download/__init__.py +++ b/src/comic_download/__init__.py @@ -14,4 +14,4 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from comic_download.comic_strip import SequenceException, ImageRepo, ComicCollection, ComicBook, ComicStrip, file_safe_string, get_identifier_string, register_comic_class, find_comic_class, read_index, save_index, index_strip, is_strip_present +from comic_download.comic_strip import SequenceException, AssetRepo, ComicCollection, ComicBook, ComicStrip, file_safe_string, get_identifier_string, register_comic_class, find_comic_class, read_index, save_index, index_strip, is_strip_present diff --git a/src/comic_download/comic_strip.py b/src/comic_download/comic_strip.py index a7d5884..c4072a0 100644 --- a/src/comic_download/comic_strip.py +++ b/src/comic_download/comic_strip.py @@ -21,6 +21,7 @@ import argparse from abc import ABC, abstractmethod from datetime import datetime, date from threading import Lock, Condition +import io import bisect import time import re @@ -179,7 +180,7 @@ def is_strip_present(comic, base_path: Path, suffix: str) -> bool: -class ImageRepo(ABC): +class AssetRepo(ABC): """ A generic representation of an online repository of data. This class allows for the setup of downloading the main information in one pass, and secondary information (images) @@ -351,7 +352,7 @@ class ImageRepo(ABC): iterator = range(len(self.image_urls)) for name in iterator: logging.info(" Downloading repository %s resource %s", repr(self), name) - image_path = NamedTemporaryFile(mode='wb', suffix=str(name), prefix=str(self), delete = False) + image_path = NamedTemporaryFile(mode='wb', suffix=str(name.replace('/', '-')), prefix=str(self), delete = False) result = requests.get(self.image_urls[name], headers=self.download_headers) result.raise_for_status() # Handles necessary transformations for a URL (i.e., redirections) @@ -370,7 +371,7 @@ class ImageRepo(ABC): return "<" + str(self.__class__.__name__) + " " + self.get_identifier_string() + ">" -class ComicCollection(ImageRepo, ABC): +class ComicCollection(AssetRepo, ABC): """ A generic representation of a comic collection. Comic-wide resources are stored here. """ @@ -441,30 +442,25 @@ class ComicCollection(ImageRepo, ABC): else: return "<" + str(self.__class__.__name__) + " " + self.get_identifier_string() + ">" - -class ComicBook(ImageRepo, ABC): +class Book(AssetRepo, ABC): """ - A generic representation of a single comic strip or issue. + An abstract class for downloading ebook resources. """ - def __init__(self, identifier): + def __init__(self, identifier, asset_type = list): """ - Creates a comic strip + Creates a book. - :param indentifier: The globally unique identifier used to find the strip on the website. Can be a tuple. + :param indentifier: The globally unique identifier used to find the book on the website. Coan be a tuple. """ - super().__init__(identifier, list) + super().__init__(identifier, asset_type) self.title = None - """The title of the comic. Empty until the comic data is loaded.""" + """The title of the book. Empty until the book is loaded.""" self.series = None - """The series that this comic is a part of. Empty until the comic data is loaded.""" - self.image_urls = [] - """A list of URLs we can download the comic panels from, in order. Empty until the comic data is loaded.""" - self._image_paths = [] + """The series that this book is a part of, if applicable. Empty until the book is loaded.""" + self._image_paths = asset_type() """A list of temporary files for the raw downloaded panels. Used internally.""" - self.captions = [] - """A list of captions for each panel.""" self.date = None """The date that the comic was published.""" self._transform_lock = Condition(Lock()) @@ -477,7 +473,7 @@ class ComicBook(ImageRepo, ABC): @classmethod def create_from_url(cls, url): """ - Creates a Comic instance from a given URL + Creates a book instance from a given URL """ pass @@ -503,15 +499,15 @@ class ComicBook(ImageRepo, ABC): prefix = self.get_identifier_string() return file_safe_string(self.__name__ + " - " + prefix) + @abstractmethod def get_package_path(self, base_path: Path): """ - Obtains the path to save the comic archive to. + Obtains the path to save the book to. - :param base_path: The folder to save comics to. + :param base_path: The folder to save books to. :returns: The path to save to. """ - filename = self.get_filename() - return Path(base_path, filename, f"{filename}.cbz") + pass def load_data(self): super().load_data() @@ -546,6 +542,160 @@ class ComicBook(ImageRepo, ABC): """ self.transformed_images = self.images + @abstractmethod + def package_data(self, base_path: Path, plain: bool = False): + """ + Compresses all of the raw downloaded data into a single archive, saving it to a location in the base path. + + :param base_path: The path to save the archive to. + :param plain: If true, we will save the images as downloaded, without any transformations. + """ + pass + + def __repr__(self): + if self.title: + return "<" + str(self.__class__.__name__) + " " + self.title + " (" + self.get_identifier_string() + ")>" + else: + return "<" + str(self.__class__.__name__) + " " + self.get_identifier_string() + ">" + +class EPUBBook(Book, ABC): + """ + A generic representation of a single EPUB book. + """ + + def __init__(self, identifier): + """ + Creates an ebook. + + :param indentifier: The globally unique identifier used to find the strip on the website. Can be a tuple. + """ + super().__init__(identifier, dict) + + @classmethod + def create_from_url(cls, url): + """ + Creates a Comic instance from a given URL + """ + pass + + def get_package_path(self, base_path: Path): + """ + Obtains the path to save the comic archive to. + + :param base_path: The folder to save comics to. + :returns: The path to save to. + """ + filename = self.get_filename() + return Path(base_path, filename, f"{filename}.epub") + + def package_data(self, base_path: Path, plain: bool = False): + if not self.is_loaded(): + raise SequenceException("Cannot package book before data has been fully loaded.") + if not self.is_downloaded(): + raise SequenceException("Cannot package book before data has been fully downloaded.") + if not self.transformed_images: + if plain: + self.transformed_images = self.images + else: + if not self._transformed: + if self._transform_lock._lock.locked(): + # Await for the existing transform to complete + self._transform_lock.wait() + else: + # Run the transformation outselves + with self._transform_lock: + self._transform_images() + self._transformed = True + self._transform_lock.notify_all() + save_path = self.get_package_path(base_path) + save_path.parent.mkdir(parents=True, exist_ok=True) + if len(self.transformed_images.keys()) == 1: + file = list(self.transformed_images.values)[0] + if hasattr(file, "name"): + with open(file.name, 'rb') as download_stream: + download_cont = download_stream.read() + elif isinstance(file, io.IOBase): + download_cont = file.read() + download_cont.close() + elif isinstance(file, Path) or type(file) == str and len(file) < 250 and Path(file).exists(): + with open(Path(file), 'rb') as download_stream: + download_cont = download_stream.read() + else: + download_cont = file + if type(download_cont) == str: + download_cont = bytes(download_cont, 'utf-8') + with open(save_path, 'wb') as save_stream, open(file.name, 'rb') as download_stream: + logging.info("Packaging single download of %s" % retr(self)) + save_stream.write(download_cont) + else: + with ZipFile(self.get_package_path(base_path), 'w') as book_zip: + with book_zip.open("mimetype", 'w') as zip_stream: + zip_stream.write(bytes('application/epub+zip', 'ascii')) + for path, file in self.transformed_images.items(): + if hasattr(file, "name"): + logging.info("Packaging temporary file %s", path) + with open(file.name, 'rb') as download_stream: + download_cont = download_stream.read() + elif isinstance(file, io.IOBase): + logging.info("Packaging stream %s", path) + download_cont = file.read() + download_cont.close() + elif isinstance(file, Path) or type(file) == str and len(file) < 250 and Path(file).exists(): + logging.info("Packaging path %s", path) + with open(Path(file), 'rb') as download_stream: + download_cont = download_stream.read() + else: + logging.info("Packaging string %s", path) + download_cont = file + if type(download_cont) == str: + download_cont = bytes(download_cont, 'utf-8') + with book_zip.open(path, 'w') as zip_stream: + zip_stream.write(download_cont) + # Delete the base images, as they are no longer needed. + # We cannot touch the transformed images, since some + # of them may come directly from the base comic, and + # we have no way of telling. + for file in self.images.values(): + file_path = Path(file.name) + if file_path.exists(): + file_path.unlink() + + + + +class ComicBook(Book, ABC): + """ + A generic representation of a single comic strip or issue. + """ + + def __init__(self, identifier): + """ + Creates a comic strip + + :param indentifier: The globally unique identifier used to find the strip on the website. Can be a tuple. + """ + super().__init__(identifier) + + self.captions = [] + """A list of captions for each panel.""" + + @classmethod + def create_from_url(cls, url): + """ + Creates a Comic instance from a given URL + """ + pass + + def get_package_path(self, base_path: Path): + """ + Obtains the path to save the comic archive to. + + :param base_path: The folder to save comics to. + :returns: The path to save to. + """ + filename = self.get_filename() + return Path(base_path, filename, f"{filename}.cbz") + def create_comic_info(self) -> str: """ Creates the contents of a ComicInfo.xml file. @@ -634,16 +784,37 @@ class ComicBook(ImageRepo, ABC): with ZipFile(self.get_package_path(base_path), 'w') as comic_zip: for i in range(len(self.transformed_images)): image = self.transformed_images[i] - image_path = Path(image.name) - suffix = image_path.suffix - if len(suffix) > 5: - # There is something fishy going on, so let us just use the default - suffix = None - if not suffix: + if hasattr(image, "name"): + with open(image.name, 'rb') as download_stream: + download_cont = download_stream.read() + image_path = Path(image.name) + suffix = image_path.suffix + if len(suffix) > 5: + # There is something fishy going on, so let us just use the default + suffix = None + if not suffix: + suffix = ".png" + elif isinstance(image, io.IOBase): + download_cont = image.read() + download_cont.close() suffix = ".png" - logging.info("Packaging %s" % image_path.suffix) - with open(image.name, 'rb') as image_stream, comic_zip.open(str(i + 1) + (suffix), 'w') as zip_stream: - zip_stream.write(image_stream.read()) + elif isinstance(image, Path) or type(image) == str and len(image) < 250 and Path(image).exists(): + with open(Path(image), 'rb') as download_stream: + download_cont = download_stream.read() + suffix = image.suffix + if len(suffix) > 5: + # There is something fishy going on, so let us just use the default + suffix = None + if not suffix: + suffix = ".png" + else: + download_cont = image + suffix = ".png" + if type(download_cont) == str: + download_cont = bytes(download_cont, 'utf-8') + logging.info("Packaging resource %d" % i) + with comic_zip.open(str(i + 1) + (suffix), 'w') as zip_stream: + zip_stream.write(download_cont) # Add a ComicInfo file comic_info = self.create_comic_info() if comic_info: @@ -658,12 +829,6 @@ class ComicBook(ImageRepo, ABC): if image_path.exists(): image_path.unlink() - def __repr__(self): - if self.title: - return "<" + str(self.__class__.__name__) + " " + self.title + " (" + self.get_identifier_string() + ")>" - else: - return "<" + str(self.__class__.__name__) + " " + self.get_identifier_string() + ">" - class ComicStrip(ComicBook, ABC): """ A generic representation of an issue in a comic series. Unlike the base