diff --git a/core/download.py b/core/download.py index 889ed56..3d708dd 100644 --- a/core/download.py +++ b/core/download.py @@ -1,10 +1,10 @@ import asyncio from typing import List -import cfscrape import os import re -import requests from concurrent.futures import ThreadPoolExecutor +import requests +import cfscrape from models import Chapter, Extension @@ -25,7 +25,7 @@ def download_chapters(ext_active: Extension, valid_chapters: List[Chapter]): for chapter in valid_chapters: # runs the pre_download for the extension if needed # most likely used to retrieve page_urls for the chapter - if chapter.pre_download == True: + if chapter.pre_download is True: chapter = ext_active.pre_download(chapter) loop = asyncio.get_event_loop() @@ -52,7 +52,7 @@ async def download_chapter_async(chapter: Chapter): loop = asyncio.get_event_loop() tasks = [] - for i in range(len(chapter.page_urls)): + for i, page_url in enumerate(chapter.page_urls): dl_print = f"{chapter.manga_title}: Chapter {chapter.number} page {i} download complete" if not os.path.exists(chapter_path): @@ -63,7 +63,7 @@ async def download_chapter_async(chapter: Chapter): executor, download_page, *( - chapter.page_urls[i], + page_url, chapter_path, i + 1, dl_print, @@ -81,7 +81,7 @@ def download_page( page_num: int, dl_print: str = "", cloudflare: bool = False, - headers: dict = {}, + headers: dict = None, ): """Downloads manga image from URL @@ -90,7 +90,8 @@ def download_page( chapter_path (str): Directory path to save chapter page_num (int): Page number of image dl_print (str): String to print upon download completion - cloudflare (bool, optional): Boolean flag to indicate whether cloudflare bypass is required. Defaults to False. + cloudflare (bool, optional): Boolean flag to indicate whether cloudflare bypass is required. + Defaults to False. headers (dict, optional): Dict of http headers for cloudflare bypass. Defaults to {}. """ @@ -98,6 +99,9 @@ def download_page( if not os.path.exists(chapter_path): os.makedirs(chapter_path) + # in case url has spaces + url = re.sub(r"\s", "", url) + # Use either cloudflare bypass or regular http requests depending on extension requirements if cloudflare: res = cf_scraper.get(url, headers=headers) diff --git a/core/manga_info.py b/core/manga_info.py index e403fca..5e29c90 100644 --- a/core/manga_info.py +++ b/core/manga_info.py @@ -20,7 +20,7 @@ def get_manga_info(ext_active: Extension, manga: Manga) -> List[Chapter]: # if no available chapters, exit if len(manga_info.chapters) == 0: print("There are no chapters for this manga in english") - return + return None # if only 1 available chapter, no need to process if len(manga_info.chapters) == 1: @@ -31,7 +31,8 @@ def get_manga_info(ext_active: Extension, manga: Manga) -> List[Chapter]: for chapter in manga_info.chapters: scanlator = f"[{chapter.scanlator}] " if chapter.scanlator else "" - foldername = f"{scanlator}Ch.{chapter.number}{'' if chapter.title == '' else ' - '}{chapter.title}" + foldername = f"{scanlator}Ch.{chapter.number}{'' if chapter.title == '' else ' - '}\ + {chapter.title}" chapter_float = round(float(chapter.number), 3) chapter.foldername = foldername @@ -49,12 +50,10 @@ def get_manga_info(ext_active: Extension, manga: Manga) -> List[Chapter]: to_download = input(query_str).lower().strip() or "q" if to_download == "q": - return + return None # keeps asking for chapters until a valid input is given - if is_valid_download_range(to_download): - break - else: + if not is_valid_download_range(to_download): print("Error with input, please try again.") to_download = parse_to_download(to_download, valid_chapters) @@ -73,7 +72,8 @@ def parse_to_download(to_download: str, valid_chapters: List[Chapter]) -> List[C valid_chapters (List[Chapter]): List of models.Chapter objects Returns: - List[Chapter]: List of models.Chapter objects with chapter numbers in range of 'to_download' parameter + List[Chapter]: List of models.Chapter objects with chapter numbers + in range of 'to_download' parameter """ new_to_download = [] diff --git a/core/misc.py b/core/misc.py index 7c7f964..9a53add 100644 --- a/core/misc.py +++ b/core/misc.py @@ -14,9 +14,9 @@ def check_pickle(ext: str): # creates pickle file if it doesn't exist and stores an empty dict in it if not os.path.exists(DATA_FILE): - f = open(DATA_FILE, "wb") - pickle.dump({}, f) - f.close() + with open(DATA_FILE, "wb") as f: + pickle.dump({}, f) + f.close() with open(DATA_FILE, "rb") as f: data = pickle.load(f) @@ -52,7 +52,7 @@ def write_pickle(ext: str, key: str, value): f.close() -def read_pickle(ext: str, key: str): +def read_pickle(ext: str, key: str) -> str: """Reads pickle data for extension Args: @@ -72,8 +72,7 @@ def read_pickle(ext: str, key: str): if key in data[ext]: return data[ext][key] - else: - return None + return None def delete_pickle(ext: str, key: str = "") -> bool: @@ -100,7 +99,7 @@ def delete_pickle(ext: str, key: str = "") -> bool: else: del data[ext][key] - except: + except KeyError: return False with open(DATA_FILE, "wb") as f: @@ -130,7 +129,7 @@ def is_url(url: str) -> bool: re.IGNORECASE, ) - return not re.match(regex, url) == None + return re.match(regex, url) is not None def is_valid_download_range(to_download: str) -> bool: @@ -143,12 +142,12 @@ def is_valid_download_range(to_download: str) -> bool: bool: _description_ """ res = True - pattern = re.compile("^\d+(\.?\d+)?$") + pattern = re.compile(r"^\d+(\.?\d+)?$") dl_spl = to_download.split(",") num_list = [] - for i in range(len(dl_spl)): - num_list += dl_spl[i].split("-") + for i, spl in enumerate(dl_spl): + num_list += spl[i].split("-") for number in num_list: if not bool(re.search(pattern, number.strip())): diff --git a/core/random_manga.py b/core/random_manga.py index 93ad223..6a4001d 100644 --- a/core/random_manga.py +++ b/core/random_manga.py @@ -20,9 +20,9 @@ def random_manga(ext_active: Extension) -> Manga: dl = input(query) or "Y" if dl.upper() == "N": - return + return None - elif dl.upper() == "Y": + if dl.upper() == "Y": break return manga diff --git a/core/search.py b/core/search.py index 8231147..0f2a19f 100644 --- a/core/search.py +++ b/core/search.py @@ -17,13 +17,13 @@ def search(ext_active: Extension, query: str) -> Manga: search_page = 1 page_index_in = -1 - while manga == None: + while manga is None: # first retrieves list of manga from search query search_res = ext_active.search(query, search_page) if len(search_res.manga_list) == 0: print("There are 0 results for your search query. Exiting...") - return + return None print(f"Search results for '{query}' page {search_page}:") @@ -35,11 +35,12 @@ def search(ext_active: Extension, query: str) -> Manga: # keep asking until a manga selection is made, flagged by reinput while reinput: max_num = len(search_res.manga_list) - query_str = f"Which manga do you wish to download (1-{max_num}, < or > to move search page, q to quit): " + query_str = f"Which manga do you wish to download (1-{max_num}, \ + < or > to move search page, q to quit): " page_index_in = (input(query_str) or "q").strip() if page_index_in == "q": - return + return None print("") @@ -52,15 +53,14 @@ def search(ext_active: Extension, query: str) -> Manga: continue # incrementing page when last page is reached - elif search_res.last_page and page_index_in == ">": + if search_res.last_page and page_index_in == ">": page_index_in = -1 print("You can't go to a next page") continue # increment/decrement as per usual - else: - search_page += -1 if page_index_in == "<" else 1 - reinput = False + search_page += -1 if page_index_in == "<" else 1 + reinput = False page_index_in = -1 @@ -78,7 +78,7 @@ def search(ext_active: Extension, query: str) -> Manga: print("Invalid input") # only choose manga if valid index is input via user - if page_index_in >= 0 and page_index_in < len(search_res.manga_list): + if 0 <= page_index_in < len(search_res.manga_list): manga = search_res.manga_list[page_index_in] print(f"You chose: '{manga.title}'") diff --git a/extensions/hitomi/ext.py b/extensions/hitomi/ext.py index 76f1783..efb8c51 100644 --- a/extensions/hitomi/ext.py +++ b/extensions/hitomi/ext.py @@ -1,35 +1,49 @@ import re + + from typing import List + + import requests + # local files + + from models import Chapter, Extension, Manga, Tag, SearchResult, ParseResult import core + import extensions.hitomi.parse as parse + NAME = "hitomi" class Hitomi(Extension): session = requests.Session() + always_webp = True def __init__(self): super().__init__() + res = self.session.get("https://ltn.hitomi.la/gg.js") res.close() + self.gg = res.text - def parse_url(self, query: str) -> dict: + def parse_url(self, url: str) -> dict: pattern = r"https:\/\/hitomi.la\/(gamecg|cg|manga|doujinshi){1}\/" - matches = re.search(pattern, query) + + matches = re.search(pattern, url) if matches == None: return None - chapter = parse.parse_gallery(self, query) - return ParseResult(ParseResult._CHAPTER, chapter) + chapter = parse.parse_gallery(self, url) + + return ParseResult(ParseResult.CHAPTER, chapter) def search(self, query: str, page: int, cover: bool = False) -> SearchResult: return SearchResult(None, None) diff --git a/extensions/mangadex/account.py b/extensions/mangadex/account.py index bcabd10..ecdd29f 100644 --- a/extensions/mangadex/account.py +++ b/extensions/mangadex/account.py @@ -1,8 +1,7 @@ from datetime import datetime import json import requests - -import core +from core.misc import read_pickle, write_pickle API_URL = "https://api.mangadex.org" @@ -43,7 +42,7 @@ def login( if data["result"] == "ok": update_login_session(session, data) print("Successfully logged in as", username) - mark_on_dl_store = False + mark_on_dl_store = mark_on_dl if mark_on_dl == "": mark_on_dl = input( @@ -58,7 +57,7 @@ def login( print("Invalid input, defaulting to no") mark_on_dl_store = False - core.write_pickle("mangadex", "mark_on_dl", str(mark_on_dl_store)) + write_pickle("mangadex", "mark_on_dl", str(mark_on_dl_store)) def check_login_session(session: requests.Session): @@ -101,14 +100,14 @@ def update_login_session(session: requests.Session, data: dict): session.headers.update({"Authorization": f"Bearer {data['token']['session']}"}) # saving current session into a pickle - core.write_pickle("mangadex", "session", session) + write_pickle("mangadex", "session", session) def toggle_data_saver(): """Toggles data setting for MangaDex""" - data_saver = core.read_pickle("mangadex", "data_saver") + data_saver = read_pickle("mangadex", "data_saver") data_saver = not data_saver - core.write_pickle("mangadex", "data_saver", data_saver) + write_pickle("mangadex", "data_saver", data_saver) print(f"Data saver set to: {data_saver}") @@ -120,7 +119,7 @@ def set_language(language: str): language (str): Language code to set to """ - core.write_pickle("mangadex", "language", language) + write_pickle("mangadex", "language", language) print(f"Language set to: {language}") diff --git a/extensions/mangadex/ext.py b/extensions/mangadex/ext.py index d448e2a..fb0f30b 100644 --- a/extensions/mangadex/ext.py +++ b/extensions/mangadex/ext.py @@ -1,14 +1,12 @@ # standard libraries +from datetime import datetime import json from typing import List import requests -from datetime import datetime # local files from models import Chapter, Extension, Manga, Tag, ParseResult -import extensions.mangadex.account as account -import extensions.mangadex.search as search -import extensions.mangadex.parse as parse +from extensions.mangadex import account, search, parse import core NAME = "mangadex" @@ -16,6 +14,8 @@ class Mangadex(Extension): + """Extension for https://mangadex.org""" + def __init__(self): super().__init__() @@ -24,12 +24,12 @@ def __init__(self): # initialises pickled variables self.language = core.read_pickle("mangadex", "language") - if self.language == None: + if self.language is None: self.language = "en" core.write_pickle("mangadex", "language", self.language) self.data_saver = core.read_pickle("mangadex", "data_saver") - if self.data_saver == None: + if self.data_saver is None: self.data_saver = True core.write_pickle("mangadex", "data_saver", self.data_saver) @@ -42,11 +42,16 @@ def __init__(self): self.mark_on_dl = stored_mark if stored_mark else False def parse_url(self, url: str) -> ParseResult: + """Parses URL, MangaDex URL should be passed as argument + + Returns: + ParseResult: Parsed result of manga or chapter + """ return parse.parse_url(self, url) def search(self, query: str, page: int, cover: bool = False, prompt_tag=True): # if tag search, ask for tags only once and save it locally - if prompt_tag and self.tags == None: + if prompt_tag and self.tags is None: self.tags = search.query_tags(self.session) return search.search(self, query, page, cover, self.tags) @@ -67,8 +72,8 @@ def get_manga_info(self, manga: Manga) -> Manga: for tag in data["attributes"]["tags"]: name = tag["attributes"]["name"][self.language] - id = tag["id"] - manga.tags.append(Tag(name, id)) + tag_id = tag["id"] + manga.tags.append(Tag(name, tag_id)) # retrieves list of chapters for the current manga res = self.session.get( @@ -90,6 +95,11 @@ def get_manga_info(self, manga: Manga) -> Manga: return manga def set_tags(self, tags: List[str]): + """Sets to query with + + Args: + tags (List[str]): Tags to use in search query + """ self.tags = tags def get_chapter(self, res_results: dict) -> Chapter: @@ -104,7 +114,8 @@ def get_chapter(self, res_results: dict) -> Chapter: # only return page_urls as filenames for now as it takes a GET request for the full URL # which would take too much time and probably exceed the API limit if done prior to download - # full url will be handled in pre_download, "hash" is also a non-mandatory key for retrieving the full url + # full url will be handled in pre_download, "hash" is also a non-mandatory + # key for retrieving the full url chapter = Chapter(pre_download=True) chapter.number = res_results["attributes"]["chapter"] @@ -142,6 +153,7 @@ def get_scanlator(self, relationships: dict) -> str: ] return self.scanlators[scanlator_id] + return "" # gets the full list of image urls before download @@ -161,7 +173,8 @@ def pre_download(self, chapter: Chapter) -> Chapter: tmp_data = res.json() # image url without the filename - base_url = f"{tmp_data['baseUrl']}/{'data-saver' if self.data_saver else 'data'}/{tmp_data['chapter']['hash']}" + base_url = f"{tmp_data['baseUrl']}/{'data-saver' if self.data_saver else 'data'}/\ + {tmp_data['chapter']['hash']}" # constructs full image url chapter.page_urls = [ @@ -172,7 +185,7 @@ def pre_download(self, chapter: Chapter) -> Chapter: ] if self.mark_on_dl and self.session.cookies.get("Login"): - account.mark_chapter_read(self.session, chapter.id) + account.mark_chapter_read(self.session, chapter.manga_id, chapter.id) return chapter diff --git a/extensions/mangadex/parse.py b/extensions/mangadex/parse.py index a8f33ea..b10de10 100644 --- a/extensions/mangadex/parse.py +++ b/extensions/mangadex/parse.py @@ -3,6 +3,8 @@ from models import Manga, Chapter, ParseResult API_URL = "https://api.mangadex.org" +MANGA_TEMPLATE = "https://mangadex.org/title/" +CHAPTER_TEMPLATE = "https://mangadex.org/chapter/" def parse_url(self, url: str) -> ParseResult: @@ -14,8 +16,6 @@ def parse_url(self, url: str) -> ParseResult: Returns: ParseResult: ParseResult object containing information on parsed webpage """ - MANGA_TEMPLATE = "https://mangadex.org/title/" - CHAPTER_TEMPLATE = "https://mangadex.org/chapter/" if MANGA_TEMPLATE in url and url.index(MANGA_TEMPLATE) == 0: manga_id = url.replace(MANGA_TEMPLATE, "") @@ -24,10 +24,12 @@ def parse_url(self, url: str) -> ParseResult: manga_id = manga_id[:-1] return parse_url_manga(self, manga_id) - elif CHAPTER_TEMPLATE in url and url.index(CHAPTER_TEMPLATE) == 0: + if CHAPTER_TEMPLATE in url and url.index(CHAPTER_TEMPLATE) == 0: chapter_id = url.replace(CHAPTER_TEMPLATE, "").split("/")[0] return parse_url_chapter(self, chapter_id) + return None + def parse_url_manga(self, manga_id: str) -> ParseResult: """Parses manga url string and returns ParseResult object for type manga @@ -49,7 +51,7 @@ def parse_url_manga(self, manga_id: str) -> ParseResult: manga.title = data["data"]["attributes"]["title"][self.language] manga.id = manga_id - return ParseResult(ParseResult._MANGA, manga) + return ParseResult(ParseResult.MANGA, manga) def parse_url_chapter(self, chapter_id: str) -> ParseResult: @@ -85,6 +87,7 @@ def parse_url_chapter(self, chapter_id: str) -> ParseResult: data = res.json()["data"] chapter.manga_title = data["attributes"]["title"][chapter_lang] - chapter.foldername = f"[{chapter.scanlator}] Ch.{chapter.number}{'' if chapter.title == '' else ' - '}{chapter.title}" + chapter.foldername = f"[{chapter.scanlator}] Ch.{chapter.number}\ + {'' if chapter.title == '' else ' - '}{chapter.title}" - return ParseResult(ParseResult._CHAPTER, chapter) + return ParseResult(ParseResult.CHAPTER, chapter) diff --git a/extensions/mangadex/search.py b/extensions/mangadex/search.py index 0ea8ae4..c7508fd 100644 --- a/extensions/mangadex/search.py +++ b/extensions/mangadex/search.py @@ -1,6 +1,6 @@ +from typing import List import json import requests -from typing import Dict, List from models import Manga, Tag from models.results import SearchResult @@ -9,8 +9,20 @@ def search( - self, query: str, page: int, cover: bool = False, search_tags: List[str] = [] + self, query: str, page: int, cover: bool = False, search_tags: List[str] = None ) -> SearchResult: + """Searches for manga in MangaDex + + Args: + query (str): Text query to search with + page (int): Current page of results + cover (bool, optional): Flag whether to show cover. Defaults to False. + search_tags (List[str], optional): Manga tags to include with search. Defaults to []. + + Returns: + SearchResult + """ + # only show 10 manga in search results at a time search_len = 10 search_url = f"{API_URL}/manga" @@ -28,7 +40,8 @@ def search( res.close() data = json.loads(res.text) - # only reaches last page of search result when chapter offset + chapter displayed is greater of equals total search results + # only reaches last page of search result when chapter offset + chapter + # displayed is greater of equals total search results last_page = (len(data["data"]) + data["offset"]) >= data["total"] manga_list = [] @@ -53,7 +66,8 @@ def search( cover_data = json.loads(res.text) cover_filename = cover_data["data"]["attributes"]["fileName"] - manga.cover_url = f"https://uploads.mangadex.org/covers/{manga.id}/{cover_filename}" + manga.cover_url = f"https://uploads.mangadex.org/covers/\ + {manga.id}/{cover_filename}" manga_list.append(manga) @@ -71,7 +85,7 @@ def query_tags(session: requests.Session) -> List[str]: """ tag_query = "" - while not tag_query.lower() == "y": + while tag_query.lower() != "y": tag_query = input("Do you wish to search with tags? (y/N): ") or "N" if tag_query.lower() == "n": @@ -90,7 +104,7 @@ def query_tags(session: requests.Session) -> List[str]: tag_query = "" tags_used = [] - while not tag_query.lower() == "s": + while tag_query.lower() != "s": query = f"Enter ID of tag to include (1-{len(tags)}, s to stop): " tag_query = input(query) or "" @@ -100,18 +114,19 @@ def query_tags(session: requests.Session) -> List[str]: i = int(tag_query) - 1 - if i >= 0 and i < len(tags): + if 0 <= i < len(tags): if tags[i] in tags_used: tags_used.remove(tags[i]) else: tags_used.append(tags[i]) print("Currently using: ", end="" if len(tags_used) > 0 else "\n") - for i in range(len(tags_used)): + + for i, tag in enumerate(tags_used): if i < len(tags_used) - 1: - print(f"{tags_used[i].name}", end=", ") + print(f"{tag.name}", end=", ") else: - print(tags_used[i].name) + print(tag.name) # we only want the ids of the tags for querying tag_ids_used = list(map(lambda x: x.id, tags_used)) @@ -136,8 +151,8 @@ def get_tags(session: requests.Session) -> List[Tag]: for tag in data["data"]: name = tag["attributes"]["name"]["en"] - id = tag["id"] - tags.append(Tag(name, id)) + tag_id = tag["id"] + tags.append(Tag(name, tag_id)) # sort tags in alphabetical order tags.sort(key=lambda x: x.name) diff --git a/extensions/mangakakalot/ext.py b/extensions/mangakakalot/ext.py index 640b4f8..45e6567 100644 --- a/extensions/mangakakalot/ext.py +++ b/extensions/mangakakalot/ext.py @@ -1,10 +1,10 @@ # standard libraries from typing import List -import bs4 +from urllib.parse import urlparse import re import requests +import bs4 from bs4 import BeautifulSoup -from urllib.parse import urlparse # local files from models import Chapter, Extension, Manga, Tag, ParseResult, SearchResult @@ -13,14 +13,21 @@ class Mangakakalot(Extension): + """Extension for https://mangakakalot.com and https://chapmanganato.com""" + session = requests.Session() - def parse_url(self, query: str) -> ParseResult: - return super().parse_url(query) + def parse_url(self, url: str) -> ParseResult: + """Parses URL, Mangakakalot URL should be passed as argument + + Returns: + ParseResult: Parsed result of manga or chapter + """ + print("Mangakakalot parse_url") # search_len is unapplicable here def search(self, query: str, page: int, cover: bool = False): - # mangakakalot replaces spaces with underscores + # Mangakakalot replaces spaces with underscores query = query.replace(" ", "_") search_url = f"https://mangakakalot.com/search/story/{query}?page={page}" @@ -32,7 +39,7 @@ def search(self, query: str, page: int, cover: bool = False): paging_elem = soup.find("a", "page_last") # if paging cannot be found, there's only 1 page of search results - if paging_elem == None: + if paging_elem is None: last_page = True else: paging_string = paging_elem.string.strip() @@ -69,11 +76,13 @@ def get_manga_info(self, manga: Manga): if re.fullmatch(r"https:\/\/mangakakalot\.com\/[\w\/-]+", res.url): manga = self.get_manga_info_mangakakalot(soup, manga) elif re.fullmatch(r"https:\/\/chapmanganato\.com\/[\w\/-]+", res.url): - manga = self.get_manga_info_chapmanganato(soup, manga) + manga = self.get_manga_info_manganato(soup, manga) return manga def get_manga_info_mangakakalot(self, soup: BeautifulSoup, manga: Manga): + """Dedicated manga info retrieval for Mangakakalot""" + description = soup.find("div", {"id": "noidungm"}) description = description.contents[2].strip() @@ -104,7 +113,9 @@ def get_manga_info_mangakakalot(self, soup: BeautifulSoup, manga: Manga): return manga - def get_manga_info_chapmanganato(self, soup: BeautifulSoup, manga: Manga): + def get_manga_info_manganato(self, soup: BeautifulSoup, manga: Manga): + """Dedicated manga info retrieval for Manganato""" + description = soup.find("div", {"id": "panel-story-info-description"}) description = description.contents[2].strip() @@ -258,7 +269,8 @@ def generate_headers(chapter: Chapter) -> dict: "sec-fetch-mode": "no-cors", "sec-fetch-site": "cross-site", "sec-gpc": "1", - "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36", + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \ + Chrome/110.0.0.0 Safari/537.36", } return cf_headers diff --git a/extensions/nhentai/ext.py b/extensions/nhentai/ext.py index e0435d2..a81aca4 100644 --- a/extensions/nhentai/ext.py +++ b/extensions/nhentai/ext.py @@ -1,27 +1,33 @@ import re from typing import List +from datetime import datetime +import cloudscraper import requests from bs4 import BeautifulSoup -from datetime import datetime # local files -from models import Chapter, Extension, Manga, Tag, SearchResult, ParseResult -import extensions.nhentai.account as account -import extensions.nhentai.gallery as gallery +from models import Chapter, Extension, Manga, Tag, SearchResult +from extensions.nhentai import account +from extensions.nhentai import gallery import core NAME = "nhentai" +cf_scraper = cloudscraper.CloudScraper() class NHentai(Extension): + """Extension for https://nhentai.net""" + # initialises pickled variables stored_session = core.read_pickle("nhentai", "session") session = stored_session if stored_session else requests.Session() - def parse_url(self, query: str) -> dict: - return super().parse_url(query) + def parse_url(self, url: str) -> dict: + print("nhentai parse_url") def parse_gallery(self, url): + """Handles home page URL""" + res = self.session.get(url) res.close() soup = BeautifulSoup(res.text, "html.parser") @@ -38,13 +44,14 @@ def search(self, query: str, page: int, cover: bool = False): f"https://nhentai.net/search/?q={query}+language%3Aenglish&page={page}" ) - res = self.session.get(search_url) + # res = self.session.get(search_url) + res = cf_scraper.get(search_url) res.close() soup = BeautifulSoup(res.text, "html.parser") last_btn = soup.find("a", "last") - if last_btn == None: + if last_btn is None: last_page = True else: regex = r"page=([0-9]+)" @@ -53,15 +60,15 @@ def search(self, query: str, page: int, cover: bool = False): manga_list = [] - gallery = soup.find_all("div", "gallery") + gallery_divs = soup.find_all("div", "gallery") - for item in gallery: + for item in gallery_divs: # gets id from "/g/{id}" - id = re.search(r"\/g\/([0-9]+)\/", item.find("a")["href"]).group(1) + manga_id = re.search(r"\/g\/([0-9]+)\/", item.find("a")["href"]).group(1) title = item.find("div", "caption").string manga = Manga() - manga.id = id + manga.id = manga_id manga.title = title if cover: diff --git a/extensions/nhentai/gallery.py b/extensions/nhentai/gallery.py index 3437501..0490528 100644 --- a/extensions/nhentai/gallery.py +++ b/extensions/nhentai/gallery.py @@ -2,7 +2,6 @@ import re import requests from bs4 import BeautifulSoup -from extensions.mangadex.ext import API_URL import core @@ -21,9 +20,6 @@ def favourite(session: requests.Session, manga_id: str): session (requests.Session): Session object containing login session manga_id (str): Manga ID to be (un)favourited """ - - global gallery_url, res, soup, gallery_headers - if not manga_id in gallery_url: set_globals(session, manga_id) @@ -45,23 +41,23 @@ def favourite(session: requests.Session, manga_id: str): return - res = session.post( + fav_res = session.post( f"https://nhentai.net/api/gallery/{manga_id}/favorite", headers=gallery_headers ) - res.close() - check_fav_res(res, title) + fav_res.close() + check_fav_res(fav_res, title) -def check_fav_res(res: requests.Response, title: str): +def check_fav_res(fav_res: requests.Response, title: str): """Checks favourite status from HTTP Response Args: - res (requests.Response): HTTP Response after trying to (un)favourite a manga + res_fav (requests.Response): HTTP Response after trying to (un)favourite a manga title (str): Title of manga to be (un)favourited """ - if res.status_code == 200: - data = json.loads(res.text) + if fav_res.status_code == 200: + data = json.loads(fav_res.text) print(title, end=" has been ") print("favourited" if data["favorited"] else "unfavourited") else: @@ -77,7 +73,7 @@ def comment(session: requests.Session, manga_id: str): manga_id (str): Manga ID of manga to be commented """ - global gallery_url, res, soup, gallery_headers + global res if not manga_id in gallery_url: set_globals(session, manga_id) @@ -106,19 +102,18 @@ def comment(session: requests.Session, manga_id: str): def undo_comment(session: requests.Session): + """Removes last comment made""" last_comment = core.read_pickle("nhentai", "last_comment") - if last_comment == None: + if last_comment is None: return - global gallery_url, res, soup, gallery_headers - if not last_comment["manga_id"] in gallery_url: set_globals(session, last_comment["manga_id"]) delete_url = f"https://nhentai.net/api/comments/{last_comment['comment_id']}/delete" - res = session.post(delete_url, headers=gallery_headers) - res.close() - data = json.loads(res.text) + _res = session.post(delete_url, headers=gallery_headers) + _res.close() + data = json.loads(_res.text) if data["success"]: print("Last comment deleted") diff --git a/main.py b/main.py index 28c394c..0fff65e 100644 --- a/main.py +++ b/main.py @@ -12,7 +12,6 @@ import extensions.nhentai.ext as nhentaiExt import extensions.hitomi.ext as hitomiExt - # to match the object via string ext_dict = { mangadexExt.NAME: mangadexExt.Mangadex(), @@ -22,11 +21,7 @@ } -# keeps the active extension -ext_active: Extension = None - - -def main_parse_url(url: str): +def main_parse_url(ext: Extension, url: str): """Parse URL string given and proceeds accordingly either as a whole manga or single chapter Args: @@ -37,67 +32,57 @@ def main_parse_url(url: str): print("A URL was not provided") return - global ext_active - - parse_result = ext_active.parse_url(url) + parse_result = ext.parse_url(url) # if type is manga, output should be a dict with keys "title" and "manga_id" - if parse_result.type == ParseResult._MANGA: - main_get_manga_info(parse_result.item) + if parse_result.type == ParseResult.MANGA: + main_get_manga_info(ext, parse_result.item) # if type is chapter, output should be a regular chapter object - elif parse_result.type == ParseResult._CHAPTER: - core.download_chapters(ext_active, [parse_result.item]) + elif parse_result.type == ParseResult.CHAPTER: + core.download_chapters(ext, [parse_result.item]) -def main_search(query: str): +def main_search(ext: Extension, query: str): """Searches for a manga by the given query string Args: query (str): Query string to search for manga """ - global ext_active - - manga = core.search(ext_active, query) + manga = core.search(ext, query) if manga: - main_get_manga_info(manga) + main_get_manga_info(ext, manga) -def main_random(): +def main_random(ext: Extension): """Retrieves a random manga from the extension's website, not compatible with all extensions""" - global ext_active - - manga = core.random_manga(ext_active) + manga = core.random_manga(ext) if manga: - main_get_manga_info(manga) + main_get_manga_info(ext, manga) -def main_get_manga_info(manga: Manga): +def main_get_manga_info(ext: Extension, manga: Manga): """Gets chapters available for download and proceeds to downloading them Args: manga (Manga): models.Manga object with only id and title attributes populated """ - global ext_active - - valid_chapters = core.get_manga_info(ext_active, manga) + valid_chapters = core.get_manga_info(ext, manga) if valid_chapters: - core.download_chapters(ext_active, valid_chapters) + core.download_chapters(ext, valid_chapters) -def check_extension(): +def check_extension(ext: Extension): """Checks whether an active extension has been set""" - global ext_active - - if ext_active == None: - print("You have no initialised the program with an extension (i.e. mangadex)") + if ext is None: + print("You have no initialised the program with an extension (i.e. --mangadex)") sys.exit(0) @@ -173,8 +158,7 @@ def generate_arg_parser() -> argparse.ArgumentParser: def parse_arguments(): """Parses arguments from argparse""" - - global ext_active + ext = None parser = generate_arg_parser() args, unknown_args = parser.parse_known_args() @@ -183,31 +167,31 @@ def parse_arguments(): for arg in args: val = args[arg] - if not val == None: + if val is not None: # sets active extension - if arg in ext_dict.keys() and val == True: + if arg in ext_dict and val is True: # ensures pickle has key for active extension core.check_pickle(arg) - ext_active = ext_dict[arg] - # ext_active.arg_handler(unknown_args) + ext = ext_dict[arg] + ext.arg_handler(unknown_args) # catch and runs standardised functions elif arg == "search": - check_extension() - main_search(val) + check_extension(ext) + main_search(ext, val) - elif arg == "random" and val == True: - check_extension() - main_random() + elif arg == "random" and val is True: + check_extension(ext) + main_random(ext) elif arg == "parse_url": - check_extension() - main_parse_url(val.strip()) + check_extension(ext) + main_parse_url(ext, val.strip()) # if it's the remaining arguments, treat them as custom extension arguments elif arg == "ext_args": - check_extension() - ext_active.arg_handler(val) + check_extension(ext) + ext.arg_handler(val) if __name__ == "__main__": diff --git a/models/extension.py b/models/extension.py index 0cf2a98..8c09ac2 100644 --- a/models/extension.py +++ b/models/extension.py @@ -12,11 +12,11 @@ class Extension(ABC): """ @abstractmethod - def parse_url(self, query: str) -> ParseResult: + def parse_url(self, url: str) -> ParseResult: """Checks URL string whether it is a manga or chapter page Args: - query (str): URL string to parse + url (str): URL string to parse Raises: NotImplementedError: When method isn't implemented by the subclass @@ -35,7 +35,9 @@ def search(self, query: str, page: int, cover: bool = False) -> SearchResult: Args: query (str): Query string to search for manga page (int): Page of search results - cover (bool, optional): Boolean flag to indicate whether cover page should be retrieved to models.Manga object. Defaults to False. + cover (bool, optional): Boolean flag to indicate whether cover page + should be retrieved to models.Manga object. + Defaults to False. Raises: NotImplementedError: When method isn't implemented by the subclass diff --git a/models/manga.py b/models/manga.py index 905257b..baaaff4 100644 --- a/models/manga.py +++ b/models/manga.py @@ -19,6 +19,12 @@ def __init__(self): self.chapters: List[Chapter] = [] def add_attribute(self, name: str, value: str): + """Adds attribute tag to Manga object + + Args: + name (str): Attribute name + value (str): Attribute value + """ setattr(self, name, value) @@ -32,7 +38,8 @@ def __init__(self, pre_download: bool): """Constructor for Chapter class Args: - pre_download (bool): Boolean flag indicate segment of code should be run before downloading + pre_download (bool): Boolean flag indicate segment of code + should be run before downloading """ self.pre_download = pre_download @@ -41,6 +48,7 @@ def __init__(self, pre_download: bool): self.title: str = "" self.scanlator: str = "" self.date: str = "" + self.manga_id: str = "" self.manga_title: str = "" self.foldername: str = "" self.page_urls: List[str] = [] @@ -50,7 +58,8 @@ def __init__(self, pre_download: bool): self.cloudflare = False def add_attribute(self, name: str, value: str): - """Allow for additional attributes to be added to models.Chapter object upon extension's needs + """Allow for additional attributes to be added to models. + Chapter object upon extension's needs Args: name (str): Name of new attribute @@ -62,7 +71,7 @@ def add_attribute(self, name: str, value: str): class Tag: """This is the basic Tag class for manga tags, could be used for a GUI in the future""" - def __init__(self, name: str, id: str): + def __init__(self, name: str, tag_id: str): """Constructor for Tag class Args: @@ -71,4 +80,4 @@ def __init__(self, name: str, id: str): """ self.name = name - self.id = id + self.id = tag_id diff --git a/models/results.py b/models/results.py index 34bd7f7..4cc7929 100644 --- a/models/results.py +++ b/models/results.py @@ -6,24 +6,24 @@ class ParseResult: """Class containing results of parsing either manga or chapter""" - _MANGA = 0 - _CHAPTER = 1 + MANGA = 0 + CHAPTER = 1 - def __init__(self, type: int, item: Union[Manga, Chapter]): + def __init__(self, res_type: int, item: Union[Manga, Chapter]): """Constructor for ParseResult class Args: type (int): Either the _MANGA or _CHAPTER constants depending on parsing results - item (Union[Manga, Chapter]): The Manga or Chapter object retrieved from parsing the webpage + item (Union[Manga, Chapter]): Manga/Chapter object retrieved from parsing the webpage """ super().__init__() - - self.type = type + self.type = res_type self.item = item class SearchResult: + """Class containing search results from extension""" def __init__(self, manga_list: List[Manga], last_page: bool): diff --git a/tests/test_mangadex.py b/tests/test_mangadex.py index fe8848e..49199ee 100644 --- a/tests/test_mangadex.py +++ b/tests/test_mangadex.py @@ -1,7 +1,8 @@ import json import os -import requests import unittest +import re +import requests import core import extensions.mangadex.account as account @@ -15,28 +16,28 @@ class TestExtension(unittest.TestCase): def setUp(self) -> None: self.mangadex = mangadexExt.Mangadex() self.session = core.read_pickle("mangadex", "session") - self.API_URL = "https://api.mangadex.org" + self.api_url = "https://api.mangadex.org" self.manga_id = "fe5b40a2-061e-4f09-8f04-86e26aae5649" self.chapter_id = "1f9b078c-27b2-4abf-8ddd-7e08f835d202" - if self.session == None: + if self.session is None: self.session = requests.Session() return super().setUp() def test_parse_url(self): - MANGA_URL = f"https://mangadex.org/title/{self.manga_id}" - CHAPT_URL = f"https://mangadex.org/chapter/{self.chapter_id}" + manga_url = f"https://mangadex.org/title/{self.manga_id}" + chapt_url = f"https://mangadex.org/chapter/{self.chapter_id}" # tests manga url parsing - res = self.mangadex.parse_url(MANGA_URL) - check_type = res.type == ParseResult._MANGA + res = self.mangadex.parse_url(manga_url) + check_type = res.type == ParseResult.MANGA check_item = isinstance(res.item, Manga) self.assertTrue(check_type and check_item) # tests chapter url parsing - res = self.mangadex.parse_url(CHAPT_URL) - check_type = res.type == ParseResult._CHAPTER + res = self.mangadex.parse_url(chapt_url) + check_type = res.type == ParseResult.CHAPTER check_item = isinstance(res.item, Chapter) self.assertTrue(check_type and check_item) @@ -65,15 +66,15 @@ def test_get_manga_info(self): manga = self.mangadex.get_manga_info(manga) # checks all items in 'chapters' key is a models.Chapter object - allChapters = all(isinstance(chapter, Chapter) for chapter in manga.chapters) + all_chapters = all(isinstance(chapter, Chapter) for chapter in manga.chapters) # checks all items in 'tags' key is a Tag object - allTags = all(isinstance(tag, Tag) for tag in manga.tags) + all_tags = all(isinstance(tag, Tag) for tag in manga.tags) - self.assertTrue(allChapters and allTags) + self.assertTrue(all_chapters and all_tags) def test_get_chapter(self): - chapter_list_url = f"{self.API_URL}/chapter/?manga={self.manga_id}&limit=100&translatedLanguage[]=en" + chapter_list_url = f"{self.api_url}/chapter/?manga={self.manga_id}&limit=100&translatedLanguage[]=en" res = self.session.get(chapter_list_url) res.close() @@ -88,7 +89,7 @@ def test_get_chapter(self): self.assertEqual(chapter.title, "The Witches' Tanabata isn't Sweet") def test_get_scanlator(self): - chapter_list_url = f"{self.API_URL}/chapter/{self.chapter_id}" + chapter_list_url = f"{self.api_url}/chapter/{self.chapter_id}" res = self.session.get(chapter_list_url) res.close() @@ -103,26 +104,26 @@ def test_pre_download(self): chapter = self.get_chapter() # ensures all page_urls are valid - check = all(core.is_url(url) for url in chapter.page_urls) + check = all(core.is_url(re.sub(r"\s", "", url)) for url in chapter.page_urls) self.assertTrue(check) def test_download(self): - DOWNLOAD_PATH = "./downloads/unittest" + download_path = "./downloads/unittest" # get chapter with all information needed to download chapter = self.get_chapter() # downloads only 1 page to sample page = chapter.page_urls[0] - core.download_page(page, DOWNLOAD_PATH, 1) + core.download_page(page, download_path, 1) # gets filesize of page - size = os.path.getsize(f"{DOWNLOAD_PATH}/1.{page.split('.')[-1]}") + size = os.path.getsize(f"{download_path}/1.{page.split('.')[-1]}") self.assertEqual(size, 290716) - os.remove(f"{DOWNLOAD_PATH}/1.{page.split('.')[-1]}") - os.rmdir(DOWNLOAD_PATH) + os.remove(f"{download_path}/1.{page.split('.')[-1]}") + os.rmdir(download_path) def test_get_random(self): manga = self.mangadex.get_random() @@ -131,10 +132,10 @@ def test_get_random(self): self.assertIsNotNone(manga.id) def test_get_formatted_date(self): - DATETIME = "2018-04-11T20:23:32+00:00" - date = mangadexExt.format_date(DATETIME) + datetime = "2018-04-11T20:23:32+00:00" + date = mangadexExt.format_date(datetime) - self.assertEquals(date, "11/04/2018") + self.assertEqual(date, "11/04/2018") def get_chapter(self) -> Chapter: """Helper method for retrieving an attribute-populated models.Chapter object @@ -143,7 +144,7 @@ def get_chapter(self) -> Chapter: Chapter: Populated models.Chapter object """ - chapter_list_url = f"{self.API_URL}/chapter/?manga={self.manga_id}&limit=100&translatedLanguage[]=en" + chapter_list_url = f"{self.api_url}/chapter/?manga={self.manga_id}&limit=100&translatedLanguage[]=en" res = self.session.get(chapter_list_url) res.close() @@ -161,41 +162,42 @@ def setUp(self) -> None: # reads login session for every test in this class self.session = core.read_pickle("mangadex", "session") - if self.session == None: + if self.session is None: self.session = requests.Session() return super().setUp() def test_login(self): - USERNAME = "unittestusername" - PASSWORD = "password" + username = "unittestusername" + password = "password" # self.session's cookies should update after calling login assert self.session is not None - account.login(self.session, USERNAME, PASSWORD, "") - self.assertTrue("Login" in self.session.cookies._cookies[""]["/"]) + account.login(self.session, username, password, "True") + cookies = self.session.cookies.get_dict() + self.assertTrue("Login" in cookies and cookies["Login"] == "true") def test_mark_chapter(self): # mark chapter read and unread # Umineko Tsubasa Ch. 1 - MANGA_ID = "fe5b40a2-061e-4f09-8f04-86e26aae5649" - CHAPTER_ID = "1f9b078c-27b2-4abf-8ddd-7e08f835d202" - marked = account.mark_chapter_read(self.session, MANGA_ID, CHAPTER_ID) - unmarked = account.mark_chapter_unread(self.session, MANGA_ID, CHAPTER_ID) + manga_id = "fe5b40a2-061e-4f09-8f04-86e26aae5649" + chapter_id = "1f9b078c-27b2-4abf-8ddd-7e08f835d202" + marked = account.mark_chapter_read(self.session, manga_id, chapter_id) + unmarked = account.mark_chapter_unread(self.session, manga_id, chapter_id) self.assertTrue(marked and unmarked) def test_update_reading_status(self): # Umineko Tsubasa - MANGA_ID = "fe5b40a2-061e-4f09-8f04-86e26aae5649" + manga_id = "fe5b40a2-061e-4f09-8f04-86e26aae5649" results = [] # cycle through indexes 0-5 for index in range(6): - res = account.update_reading_status(self.session, MANGA_ID, index) + res = account.update_reading_status(self.session, manga_id, index) results.append(res) - self.assertTrue(all(res == True for res in results)) + self.assertTrue(all(res is True for res in results)) if __name__ == "__main__": diff --git a/tests/test_mangakakalot.py b/tests/test_mangakakalot.py index 5f7afbd..555cb22 100644 --- a/tests/test_mangakakalot.py +++ b/tests/test_mangakakalot.py @@ -3,7 +3,7 @@ import core import extensions.mangakakalot.ext as mangakakalotExt -from models import Chapter, Manga, Tag, SearchResult, ParseResult +from models import Chapter, Manga, Tag, SearchResult # testing methods defined in Mangakakalot class @@ -33,17 +33,17 @@ def test_search(self): def test_get_manga_info(self): manga = Manga() manga.title = "Umineko no Naku Koro ni Tsubasa" - manga.id = "https://mangakakalot.com/manga/umineko_no_naku_koro_ni_tsubasa" + manga.id = "https://mangakakalot.com/read-qmcrz2535k1685034552" manga = self.mangakakalot.get_manga_info(manga) # checks all items in 'chapters' key is a models.Chapter object - allChapters = all(isinstance(item, Chapter) for item in manga.chapters) + all_chapters = all(isinstance(item, Chapter) for item in manga.chapters) # checks all items in 'tags' key is a Tag object - allTags = all(isinstance(tag, Tag) for tag in manga.tags) + all_tags = all(isinstance(tag, Tag) for tag in manga.tags) - self.assertTrue(allChapters and allTags) + self.assertTrue(all_chapters and all_tags) def test_pre_download(self): # only populating id since it's all we need for predownload @@ -62,10 +62,10 @@ def test_pre_download(self): self.assertTrue(page_check) def test_download(self): - DOWNLOAD_PATH = "./downloads/unittest" + download_path = "./downloads/unittest" manga = Manga() - manga.id = "https://mangakakalot.com/manga/umineko_no_naku_koro_ni_tsubasa" + manga.id = "https://mangakakalot.com/read-qmcrz2535k1685034552" manga = self.mangakakalot.get_manga_info(manga) # downloads only chapter 1 @@ -76,15 +76,15 @@ def test_download(self): page = chapter.page_urls[0] cf = chapter.cloudflare headers = chapter.headers - core.download_page(page, DOWNLOAD_PATH, 1, cloudflare=cf, headers=headers) + core.download_page(page, download_path, 1, cloudflare=cf, headers=headers) # gets filesize of page - size = os.path.getsize(f"{DOWNLOAD_PATH}/1.{page.split('.')[-1]}") + size = os.path.getsize(f"{download_path}/1.{page.split('.')[-1]}") self.assertEqual(size, 130200) - os.remove(f"{DOWNLOAD_PATH}/1.{page.split('.')[-1]}") - os.rmdir(DOWNLOAD_PATH) + os.remove(f"{download_path}/1.{page.split('.')[-1]}") + os.rmdir(download_path) if __name__ == "__main__": diff --git a/tests/test_nhentai.py b/tests/test_nhentai.py index e654a75..7cd4459 100644 --- a/tests/test_nhentai.py +++ b/tests/test_nhentai.py @@ -3,7 +3,7 @@ import core import extensions.nhentai.ext as nhentai -from models import Chapter, Manga, Tag, SearchResult, ParseResult +from models import Chapter, Manga, Tag, SearchResult class TestExtension(unittest.TestCase): @@ -32,15 +32,15 @@ def test_get_manga_info(self): manga = self.get_manga() # checks all items in 'chapters' key is a models.Chapter object - allChapters = all(isinstance(chapter, Chapter) for chapter in manga.chapters) + all_chapters = all(isinstance(chapter, Chapter) for chapter in manga.chapters) # checks all items in 'tags' key is a Tag object - allTags = all(isinstance(tag, Tag) for tag in manga.tags) + all_tags = all(isinstance(tag, Tag) for tag in manga.tags) - self.assertTrue(allChapters and allTags) + self.assertTrue(all_chapters and all_tags) def test_download(self): - DOWNLOAD_PATH = "./downloads/unittest" + download_path = "./downloads/unittest" # retrieves chapter with all information needed to download manga = self.get_manga() @@ -48,15 +48,15 @@ def test_download(self): # downloads only 1 page to sample page = chapter.page_urls[0] - core.download_page(page, DOWNLOAD_PATH, 1) + core.download_page(page, download_path, 1) # gets filesize of page - size = os.path.getsize(f"{DOWNLOAD_PATH}/1.{page.split('.')[-1]}") + size = os.path.getsize(f"{download_path}/1.{page.split('.')[-1]}") self.assertEqual(size, 394241) - os.remove(f"{DOWNLOAD_PATH}/1.{page.split('.')[-1]}") - os.rmdir(DOWNLOAD_PATH) + os.remove(f"{download_path}/1.{page.split('.')[-1]}") + os.rmdir(download_path) def test_get_random(self): manga = self.nhentai.get_random()