-
-
Notifications
You must be signed in to change notification settings - Fork 291
/
engnovel.py
124 lines (105 loc) · 4.43 KB
/
engnovel.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
# -*- coding: utf-8 -*-
import logging
from bs4 import Tag
from lncrawl.core.crawler import Crawler
import urllib.parse
logger = logging.getLogger(__name__)
class EngNovel(Crawler):
base_url = "https://engnovel.com/"
def search_novel(self, query):
soup = self.get_soup(
f"https://engnovel.com/?s={urllib.parse.quote_plus(query)}"
)
results = []
for novel in soup.select("div.caption"):
latest = novel.select_one("small.btn-xs.label-primary")
span_full = latest.select_one("span")
if span_full:
span_full.extract()
results.append(
{
"title": novel.select_one("a")["title"],
"url": self.absolute_url(novel.select_one("a")["href"]),
"info": latest.text,
}
)
return results
def read_novel_info(self):
logger.debug("Visiting %s", self.novel_url)
soup = self.get_soup(self.novel_url)
possible_title = soup.select_one("h3.title")
assert possible_title, "No novel title"
self.novel_title = possible_title.text.strip()
logger.info("Novel title: %s", self.novel_title)
# <div class="wallpaper" style=
# "background:url(https://engnovel.com/wp-content/uploads/2019/07/lord-of-the-mysteries.jpeg) center no-repeat;background-size:cover"
# >
possible_image = soup.select_one("div.wallpaper")
if isinstance(possible_image, Tag):
self.novel_cover = self.absolute_url(
possible_image["style"].split("url(")[-1].split(")")[0]
)
logger.info("Novel cover: %s", self.novel_cover)
self.novel_author = ", ".join(
[
a.text.strip()
for a in soup.select('div.info-chitiet a[href*="novel-author"]')
]
)
logger.info("%s", self.novel_author)
# <div class="desc-text" itemprop="description">
# <div class="short-content"><span class="glyphicon glyphicon-info-sign"></span> SUMMARY</div>
# <p>In the waves of steam and machinery...</p>
# <p>Firearms, cannons, battleships, airships...</p>
# <br>
# You can also read this novel here:
# <a target="_blank" href="https://enovelon.com">Read free novels online</a>
# </div>
synopsis = soup.select_one("div.desc-text")
for e in synopsis.find_all():
if e.name != "p":
e.extract()
for string in synopsis.strings:
if string.strip() == "You can also read this novel here:":
string.replace_with("")
break
self.novel_synopsis = self.cleaner.extract_contents(synopsis)
logger.info("%s", self.novel_synopsis)
# <a href="https://engnovel.com/action-novels" title="Action Novels" itemprop="genre">Action</a>
self.novel_tags = [
a.text.strip() for a in soup.select('div.info-chitiet a[itemprop="genre"]')
]
logger.info("Tags: %s", self.novel_tags)
novel_id = soup.select_one("#id_post")["value"]
# <input name="total-page" type="hidden" value="36">
for page in range(
1, int(soup.select_one("input[name='total-page']")["value"]) + 1
):
data = {
"action": "tw_ajax",
"type": "pagination",
"id": novel_id,
"page": page,
}
chapter_list = self.make_soup(
self.post_response(
"https://engnovel.com/wp-admin/admin-ajax.php", data
).json()["list_chap"]
)
for a in chapter_list.select("ul.list-chapter li a"):
chap_id = len(self.chapters) + 1
vol_id = len(self.chapters) // 100 + 1
if len(self.chapters) % 100 == 0:
self.volumes.append({"id": vol_id})
self.chapters.append(
{
"id": chap_id,
"volume": vol_id,
"title": a["title"],
"url": self.absolute_url(a["href"]),
}
)
def download_chapter_body(self, chapter):
soup = self.get_soup(chapter["url"])
contents = soup.select_one("div.chapter-content")
return self.cleaner.extract_contents(contents)