Preliminary work in moving existential comics to the new system
This commit is contained in:
140
existential_comics.py → comic_download/existential_comics.py
Executable file → Normal file
140
existential_comics.py → comic_download/existential_comics.py
Executable file → Normal file
@@ -19,93 +19,100 @@ import requests
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
class ComicStrip:
|
||||
def __init__(self, index):
|
||||
self.index = index or 0
|
||||
if index:
|
||||
self.url = f"https://existentialcomics.com/comic/{index}"
|
||||
class ExistentialComic(Comic):
|
||||
"""
|
||||
The Existential Comics strip
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__("existential_comics")
|
||||
|
||||
@classmethod
|
||||
def create_from_url(cls, url):
|
||||
return cls()
|
||||
|
||||
def _load_data(self):
|
||||
self.title = "Existential Comics"
|
||||
self.author = "Corey Mohler"
|
||||
self.image_urls = {
|
||||
"header": "https://static.existentialcomics.com/title.jpg",
|
||||
"safety": "https://static.existentialcomics.com/safety.png"
|
||||
}
|
||||
self.description = "A webcomic of romance, sarcasm, math, and language."
|
||||
|
||||
result = requests.get("https://existentialcomics.com/")
|
||||
result.raise_for_status()
|
||||
|
||||
content = BeautifulSoup(result.content)
|
||||
index = content.find("meta", property="og:url")
|
||||
if index and index["content"]:
|
||||
self.latest_identifier = int(self.url.split('/')[-1])
|
||||
else:
|
||||
self.url = f"https://existentialcomics.com"
|
||||
raise ValueError("Invalid comic data")
|
||||
|
||||
def get_first_strip(self):
|
||||
return 1
|
||||
|
||||
def get_latest_strip(self):
|
||||
return self.latest_identfier
|
||||
|
||||
def get_all_strips(self) -> list:
|
||||
return list(range(self.get_first_strip(), self.get_latest_strip() + 1))
|
||||
|
||||
@classmethod
|
||||
def __new__(cls):
|
||||
if not hasattr(cls, 'instance'):
|
||||
cls.instance = super(ExistentialComic, cls).__new__(cls)
|
||||
return cls.instance
|
||||
|
||||
|
||||
class ExistentialComicStrip:
|
||||
"""
|
||||
A comic strip object that represents a single Existential Comic strip.
|
||||
"""
|
||||
|
||||
def __init__(self, comic, index: int):
|
||||
super().__init__(comic, index)
|
||||
self.url = f"https://existentialcomics.com/comic/{index}"
|
||||
self.title = None
|
||||
self.image_url = []
|
||||
self.caption = []
|
||||
|
||||
def get_metadata_path(self, base_path: Path):
|
||||
return Path(base_path, f"{self.index}.txt")
|
||||
@property
|
||||
def index(self):
|
||||
# Alias
|
||||
return self.identifier
|
||||
|
||||
def get_image_path(self, base_path: Path, image_num: int = 1):
|
||||
return Path(base_path, f"{self.index}-{image_num}.png")
|
||||
|
||||
def get_package_path(self, base_path: Path):
|
||||
return Path(base_path, f"{self.index}.cbz")
|
||||
|
||||
def get_header_path(self, base_path: Path):
|
||||
return Path(base_path, "header.jpg")
|
||||
|
||||
def get_safety_path(self, base_path: Path):
|
||||
return Path(base_path, "safety.png")
|
||||
|
||||
def load_data(self):
|
||||
def _load_data(self):
|
||||
logging.info("Loading data for comic %s from %s", self.index, self.url)
|
||||
result = requests.get(self.url)
|
||||
result.raise_for_status()
|
||||
|
||||
content = BeautifulSoup(result.content)
|
||||
#print(content.prettify())
|
||||
if not self.index:
|
||||
index = content.find("meta", property="og:url")
|
||||
if index and index["content"]:
|
||||
self.url = index["content"]
|
||||
self.index = int(self.url.split('/')[-1])
|
||||
logging.info("Url of %s and index of %d", self.url, self.index)
|
||||
title = content.find("meta", property="og:title")
|
||||
if title and title["content"]:
|
||||
self.title = title["content"]
|
||||
for img in content.find_all("img", class_="comicImg"):
|
||||
self.image_url.append(f'http:{img["src"]}')
|
||||
if img.get("title"):
|
||||
self.caption.append(img["title"])
|
||||
self.captions.append(img["title"])
|
||||
else:
|
||||
self.captions.append("")
|
||||
|
||||
final_caption = ""
|
||||
explanation = content.find(id="explanation")
|
||||
if explanation:
|
||||
self.caption.append(explanation.get_text())
|
||||
final_caption += explanation.get_text()
|
||||
philosophers = content.find(id="philosophers-comic")
|
||||
if philosophers:
|
||||
self.caption.append(philosophers.get_text())
|
||||
if final_caption:
|
||||
final_caption += "\n\n"
|
||||
final_caption += philosophers.get_text()
|
||||
if final_caption:
|
||||
self.captions.append(final_caption)
|
||||
self.date = datetime.date(2013, 11, 11) + datetime.timedelta(weeks=self.index - 2)
|
||||
|
||||
|
||||
def download_data(self, base_path: Path):
|
||||
if self.image_url:
|
||||
meta_path = self.get_metadata_path(base_path)
|
||||
if not meta_path.exists():
|
||||
with open(meta_path, 'w') as meta_stream:
|
||||
json.dump({
|
||||
"title": self.title,
|
||||
"image_url": self.image_url,
|
||||
"caption": self.caption,
|
||||
}, meta_stream)
|
||||
header_path = self.get_header_path(base_path)
|
||||
if not header_path.exists():
|
||||
result = requests.get("https://static.existentialcomics.com/title.jpg")
|
||||
result.raise_for_status()
|
||||
with open(header_path, 'wb') as image_stream:
|
||||
image_stream.write(result.content)
|
||||
safety_path = self.get_safety_path(base_path)
|
||||
if not safety_path.exists():
|
||||
result = requests.get("https://static.existentialcomics.com/safety.png")
|
||||
result.raise_for_status()
|
||||
with open(safety_path, 'wb') as image_stream:
|
||||
image_stream.write(result.content)
|
||||
for i in range(len(self.image_url)):
|
||||
logging.info(" Downloading %d-%d", self.index, i + 1)
|
||||
image_path = self.get_image_path(base_path, i + 1)
|
||||
if not image_path.exists():
|
||||
result = requests.get(self.image_url[i])
|
||||
result.raise_for_status()
|
||||
with open(image_path, 'wb') as image_stream:
|
||||
image_stream.write(result.content)
|
||||
|
||||
def transform_data(self, base_path: Path):
|
||||
def _transform_data(self, base_path: Path):
|
||||
"""
|
||||
Takes the raw image data from #download_data and transforms it to add a title, captions, etc. This function is not idempotent.
|
||||
"""
|
||||
@@ -129,6 +136,11 @@ class ComicStrip:
|
||||
date_box = caption_font.getbbox(self.date.isoformat())
|
||||
date_box = (date_box[2] - date_box[0], date_box[3] - date_box[1])
|
||||
|
||||
if len(self.captions) > len(self.image_urls):
|
||||
# We have a final description to add
|
||||
f_caption = "\n\n".join(self.caption[len(self.image_urls):])
|
||||
|
||||
|
||||
caption = "\n".join(self.caption).split("\n")
|
||||
try:
|
||||
wrapped_caption = []
|
||||
Reference in New Issue
Block a user