-
Notifications
You must be signed in to change notification settings - Fork 1
/
dblp_xml_processing.py
558 lines (495 loc) · 26.5 KB
/
dblp_xml_processing.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
import logging
import dateutil.parser
import config as cfg
from pyhelpers import tools
import plac
import json
import sys
from lxml import etree
import gzip
import datetime
# modules to extract acm papers
from bs4 import BeautifulSoup
from urllib.request import Request, urlopen
# import time to set a sleeping mode to avoid HTTP error: 503/403/303
import time
import random
import json
import re
tools.setup_logging(file_name="xml_processor.log")
# add the number of access in acm to set sleep mode
num_of_access_in_acm = 0
# add the number of access in springer to set sleep mode
num_of_access_in_springer = 0
# add the number of access in aaai to set sleep mode
num_of_access_in_aaai = 0
# add the number of access in icwsm site to set sleep mode
num_of_access_in_icwsm = 0
# add the number of access in ieee site to set sleep mode
num_of_access_in_ieee = 0
num_of_access = 0
numOfPDFobtained = 0
numOfPDFobtainedInThisSession = 0
class XmlProcessing:
def __init__(self, book_titles, journals):
"""
Initialize the xml processing thingy
"""
# set to true if you want to persist to a local mongo DB (default connection)
self.storeToMongo = cfg.storeToMongo
if book_titles is None:
# GET THE VENUES WE LIKE from config.py
self.book_titles = cfg.booktitles
print('Conference of Interest: {}'.format(cfg.booktitles))
else:
self.book_titles = book_titles
print('Conference of Interest: {}'.format(self.book_titles))
if journals is None:
# GET THE VENUES WE LIKE from config.py
self.journals = cfg.journals
print('Journals of Interest: {}'.format(cfg.journals))
else:
self.journals = journals
print('Journals of Interest: {}'.format(self.journals))
# create all the folders
tools.create_all_folders()
# initialize connection to local mongoDB, database is named pub
if self.storeToMongo:
self.db = tools.connect_to_mongo()
else:
self.db = None
# set to true if you want to skip downloading EE entries (pdf URLs) which have been accessed before
# (either successfully or unsuccessfully)
# this only works if storeToMongo is set to True because the MongoDB must be accessed for that.
# (if you set storeToMongo to false, I will
# just assume that MongoDB is simply not active / there
self.skipPreviouslyAccessedURLs = cfg.skipPreviouslyAccessedURLs
# if they are new additions of papers this variable will be True
# if it is true then proceed to the text_extraction phase
self.newPapersIn = False
# the categories you are interested in
self.CATEGORIES = cfg.CATEGORIES
# the categories you are NOT interested in
self.SKIP_CATEGORIES = cfg.SKIP_CATEGORIES
# the fields which should be in your each data item / mongo entry
self.DATA_ITEMS = cfg.DATA_ITEMS
# prints out a progress every X attempted downloads (including skips which had been downloaded before)
self.statusEveryXdownloads = cfg.statusEveryXdownloads
self.statusEveryXxmlLoops = cfg.statusEveryXxmlLoops
self.filters = {}
# enabledScrapers = ["pdf", "acm", "springer", "ieee", "aaai", 'icwsm']
self.enabledScrapers = {"pdf"}
# tools.create_all_folders()
# just a counter
global numOfPDFobtained
global numOfPDFobtainedInThisSession
# numOfPDFobtained = 0
# numOfPDFobtainedInThisSession = 0
# In each update attempt the xml files are deleted
# and downloaded again overwrite= True
# get xml files
tools.downloadFileWithProgress('http://dblp.uni-trier.de/xml/dblp.xml.gz', incrementKB=10 * 1024,
folder=cfg.folder_dblp_xml, overwrite=cfg.overwriteDBLP_XML)
tools.downloadFileWithProgress('http://dblp.uni-trier.de/xml/dblp.dtd', incrementKB=10 * 1024,
folder=cfg.folder_dblp_xml, overwrite=cfg.overwriteDBLP_XML)
# open xml and iterate over xml tree to extract relevant stuff
with gzip.open(cfg.folder_dblp_xml + "dblp.xml.gz", 'rb') as file:
context = etree.iterparse(file, dtd_validation=True, events=("start", "end"))
self.fast_iter2(context, self.db)
def clear_element(self, element):
"""
helper just used for parsing the XML
:param element:
:return:
"""
element.clear()
while element.getprevious() is not None:
del element.getparent()[0]
def extract_paper_elements(self, context):
"""
helper just used for parsing the XML
:param context:
:return:
"""
for event, element in context:
if element.tag in self.CATEGORIES:
yield element
self.clear_element(element)
def fast_iter2(self, context, db):
"""
main XML parser: iterates over the xml and extracts relevant info
:param context:
:param db:
:return:
"""
global paperCounter
try:
for paperCounter, element in enumerate(self.extract_paper_elements(context)):
# extract basic metadata from the dblp XML
paper = {
'type': element.tag
# 'mdate': element.get("mdate"),
}
if 'key' in element.attrib:
paper['dblpkey'] = element.get('key', default=None)
else:
paper['dblpkey'] = None
if paper['dblpkey'] is not None:
paper['dblpkey'] = tools.normalizeDBLPkey(paper['dblpkey'])
paper['authors'] = [author.text for author in element.findall("author")]
for data_item in self.DATA_ITEMS:
data = element.find(data_item)
if data is not None:
paper[data_item] = data.text
if paper['type'] not in self.SKIP_CATEGORIES:
# try to download and store the thing if it is not in one of the skipped categories
self.download_and_store(paper, db)
# print(paperCounter, paperCounter % statusEveryXxmlLoops)
if (paperCounter % self.statusEveryXxmlLoops) == 0:
print('.', end="")
sys.stdout.flush()
except etree.XMLSyntaxError:
print("End of XML file")
def download_and_store(self, paper, db):
"""
stores stuff in mongo db, and downloads the PDF
:param paper:
:param db:
:return:
"""
# global filters
global skip
# the ee XML tag indicates that this paper has some kind of source attached (this will usually be an URL)
if 'ee' in paper:
# Do we want to skip this file? There are lots of reasons, see below...
# Skipping means we will not try to download it
skip = False
# filters have been set
"""
if len(filters) > 0:
for k, v in filters.items():
if k == 'scraper':
self.enabledScrapers.add(v)
continue
if not (k in paper and paper[k] == v):
skip = True
if not skip:
if "dblpkey" in paper:
print("Filter matched: " + str(paper["dblpkey"]))
"""
# do NOT skip if paper has a key, an ee entry
if (not skip and type(paper['dblpkey']) is str and type(paper['ee']) is str) and \
(('book_title' in paper and paper['book_title'] in self.book_titles) or
('journal' in paper and paper['journal'] in self.journals)):
# check if it one of our supported types. IMPORTANT: ADD NEW TYPES HERE IF WE HAVE THEM!
# also here we are checking if the resolved doi belonges in to one of our crawlers
# if yes then we proceed with the download otherwise we store the url in a file
# with the not-supported repositories
# check if the paper was already successfully downloaded
# downloadinfo is the dictionary which is later stored in the Mongo "downloads" collection to memorize
# which URLs have been accessed, and if that was successful or not
downloadinfo = {
'_id': paper['ee'],
'url': paper['ee'],
'dblpkey': paper['dblpkey'],
'lastaccessed': datetime.datetime.now(),
'success': True
}
print("Publication matched: " + str(paper["dblpkey"]))
skip = False
filename = paper['dblpkey'] + ".pdf"
req = ""
actual_url = ""
url_open = ""
# down_info = db.downloads.find_one({'dblpkey': paper['dblpkey']})
if self.skipPreviouslyAccessedURLs and self.storeToMongo:
result = db.downloads.find_one({'_id': downloadinfo['_id']})
if result is None:
skip = False
# if it wasn't successful try once more
elif result['success'] is False:
last_access = result['lastaccessed']
current_Date = downloadinfo['lastaccessed']
days_previous_check = (current_Date - last_access).days
skip = True
####### change it later!!!!!!
if days_previous_check >= 30:
skip = False
print()
print("Paper: {}, Last Check: {} days ago!".format(paper['dblpkey'], days_previous_check))
print()
# check if the download date was greater than 30 days
# elif number_of_Days < 2:
# skip = False
else:
skip = True
if result['success']:
skip = True
global numOfPDFobtained
global paperCounter
global numOfPDFobtainedInThisSession
numOfPDFobtained += 1
if numOfPDFobtained % self.statusEveryXdownloads is 0:
logging.info(
'DBLP XML PROGRESS: XML Paper Entries {} PDFs {} PDFs in this Session {} '
.format(paperCounter, numOfPDFobtained, numOfPDFobtainedInThisSession))
else:
skip = False # url not in download collection of mongo db
if skip is False:
pub_info = db.publications.find_one({'dblpkey': paper['dblpkey']})
if pub_info is None:
skip = False
try:
req = Request(paper['ee'], headers={'User-Agent': 'Mozilla/5.0'})
url_open = urlopen(req)
# if url_open.status != 200:
# skip = True
# raise BaseException("HTTPError {}".format(url_open.status))
# else:
# downloadinfo = {}
actual_url = url_open.geturl()
global num_of_access
# Here we need to add a time delay because we access the
# sleep for a random duration of time between 60 and 360 seconds
rndm_time = int(random.uniform(60, 360))
print(
"Crawler sleeps for {} min - Times Access Repositories: {}".format(
float(rndm_time / int(60)),
num_of_access))
num_of_access += 1
time.sleep(rndm_time)
if (paper['ee'].lower().endswith("pdf") and "pdf" in self.enabledScrapers) or (
"ieee" in str(actual_url)) or ("springer" in actual_url) or (
"acm" in actual_url) or paper['ee'].startswith("http://www.aaai.org") \
or paper['ee'].startswith("http://www.icwsm.org"):
filename = paper['dblpkey'] + ".pdf"
skip = False
# decide if we want to skip this entry
# (e.g., it has been accessed before and we are in the mood for skipping)
else:
skip = True # this ee entry is not interesting to us
print("{}, Repository not supported: {}".format(paper['dblpkey'], actual_url))
downloadinfo['success'] = False
downloadinfo['error'] = "{}, Repository not supported: {}".format(paper['dblpkey'],
actual_url)
db.downloads.replace_one({'_id': downloadinfo['_id']}, downloadinfo, upsert=True)
with open(cfg.folder_log + "not_supported_repos.txt", 'a', encoding='UTF-8') as f:
f.write(actual_url)
f.write("\n")
except BaseException:
logging.exception(
'Cannot download or store ' + paper['ee'] + " with dblpkey: " + paper['dblpkey'],
exc_info=True)
skip = True # error with the url_open so skip the download
print("first try catch!!! skip: {}".format(skip))
if self.storeToMongo:
downloadinfo['success'] = False
ex = sys.exc_info()
downloadinfo['error'] = repr(ex)
db.downloads.replace_one({'_id': downloadinfo['_id']}, downloadinfo, upsert=True)
else:
db.downloads.replace_one({'_id': downloadinfo['_id']}, downloadinfo)
skip = True
else:
print("{} already in DB".format(paper['dblpkey']))
skip = True # already exist in the db
# Do the Download and store to MongoDB
# print("Proceed with: {} : the download and store: Skip: {}".format(paper['dblpkey'],skip))
if not skip:
try:
# download based on type. IMPORTANT: Add supported types here, and also a few lines above!
if paper['ee'].lower().endswith("pdf") and "pdf" in self.enabledScrapers:
# Normal PDF download
self.newPapersIn = True # There are new additions
skipped = not tools.downloadFile(downloadinfo['url'], overwrite=False,
folder=cfg.folder_pdf,
localfilename=filename)
elif "springer" in actual_url:
# go to springer crawller
self.newPapersIn = True # There are new additions
global num_of_access_in_springer
num_of_access_in_springer += 1
print("{}, publisher: Springer, #Access: {}".format(paper['dblpkey'],
num_of_access_in_springer))
skipped = not self.extract_paper_from_SPRINGER(url_open, filename)
elif "acm" in actual_url:
# go to acm crawler
self.newPapersIn = True # There are new additions
global num_of_access_in_acm
num_of_access_in_acm += 1
print("{}, publisher: ACM, #Access: {}".format(paper['dblpkey'], num_of_access_in_acm))
skipped = not self.extract_paper_from_ACM(url_open, filename)
elif "ieee" in actual_url:
# go to ieee crawler
self.newPapersIn = True # There are new additions
global num_of_access_in_ieee
num_of_access_in_ieee += 1
print("{}, publisher: IEEE, #Access: {}".format(paper['dblpkey'], num_of_access_in_ieee))
skipped = not self.extract_paper_from_IEEE(url_open, filename)
elif paper['ee'].startswith("http://www.aaai.org"):
# go to aaai crawler
self.newPapersIn = True # There are new additions
global num_of_access_in_aaai
num_of_access_in_aaai += 1
print("{}, publisher: AAAI, #Access: {}".format(paper['dblpkey'], num_of_access_in_aaai))
skipped = not self.extract_paper_from_AAAI(actual_url, filename)
elif paper['ee'].startswith("http://www.icwsm.org"):
# got to icwsm crawler
self.newPapersIn = True # There are new additions
global num_of_access_in_icwsm
num_of_access_in_icwsm += 1
print("{}, publisher: ICWSM, #Access: {}".format(paper['dblpkey'], num_of_access_in_icwsm))
skipped = not self.extract_paper_from_ICWSM(paper['ee'], filename)
else:
skipped = True
if skipped:
logging.info(' Used local PDF copy for ' + paper['dblpkey'])
else:
logging.info(' Downloaded ' + paper['dblpkey'])
# global numOfPDFobtainedInThisSession
numOfPDFobtainedInThisSession += 1
# store
if self.storeToMongo:
# set additional data
paper['_id'] = paper['dblpkey']
# store to mongo
db.publications.replace_one({'_id': paper['_id']}, paper, upsert=True)
db.downloads.replace_one({'_id': downloadinfo['_id']}, downloadinfo, upsert=True)
except BaseException:
logging.exception(
'Cannot download or store ' + paper['ee'] + " with dblpkey: " + paper['dblpkey'],
exc_info=True)
print("second try catch")
if self.storeToMongo:
downloadinfo['success'] = False
ex = sys.exc_info()
downloadinfo['error'] = repr(ex)
db.downloads.replace_one({'_id': downloadinfo['_id']}, downloadinfo, upsert=True)
def extract_paper_from_ICWSM(self, req, filename):
"""
this function will access a given url and will find the link of the pdf.
Attention: WORKS ONLY IN THE TU DELFT NETWORK or VPN
:param paper_url: e.g. "http://www.icwsm.org/papers/paper54.html"
:return:
"""
# reguest to the url, add headers to avoid HTTP Error: 403 Forbidden
# the site will strike you out because you are a robot!
# req = Request(paper_url ,headers={'User-Agent': 'Mozilla/5.0'})
webpage = req.read()
# parse the html code
soup = BeautifulSoup(webpage, 'html.parser')
# select only the link tags
for link in soup.find_all('a', href=True, text='PDF'):
# the name of in the link tag is "FullTextPDF"
prefix = "http://www.icwsm.org/papers/"
suffix = link.get("href")
pdf_link = prefix + suffix
print("Access in " + pdf_link)
return tools.downloadFile(url=pdf_link, folder=cfg.folder_pdf, overwrite=False,
localfilename=filename, printOutput=False)
raise BaseException(req.geturl() + ' does not contain a valid ICWSM download link.')
def extract_paper_from_AAAI(self, req, filename):
"""
this function will access a given url and will find the link of the pdf.
Attention: WORKS ONLY IN THE TU DELFT NETWORK or VPN
:param paper_url: e.g. "http://www.aaai.org/ocs/index.php/ICWSM/ICWSM16/paper/view/13130"
:return:
"""
# reguest to the url, add headers to avoid HTTP Error: 403 Forbidden
# the site will strike you out because you are a robot!
paper_url = req
if "viewPaper" not in paper_url:
paper_url = paper_url.replace("view", "viewPaper")
req = Request(paper_url, headers={'User-Agent': 'Mozilla/5.0'})
webpage = urlopen(req).read()
# parse the html code
soup = BeautifulSoup(webpage, 'html.parser')
# select only the link tags
for link in soup.find_all('a', href=True, text='PDF'):
# the name of in the link tag is "FullTextPDF"
pdf_link = link.get("href").replace("view", "download")
print("Access in " + pdf_link)
return tools.downloadFile(url=pdf_link, folder=cfg.folder_pdf, overwrite=False,
localfilename=filename, printOutput=False)
raise BaseException(paper_url + ' does not contain a valid AAAI download link.')
# python3 dblp_xml_processing.py -filter="{'booktitle' : 'ICSE', 'scraper' : 'acm'}"
def extract_paper_from_IEEE(self, req, filename):
"""
this function will access a given url and will find the link of the pdf.
Attention: WORKS ONLY IN THE TU DELFT NETWORK or VPN
:param paper_url: e.g. "http://dx.doi.org/10.1007/BF00264597"
:return:
"""
# reguest to the url, add headers to avoid HTTP Error: 403 Forbidden
# the site will strike you out because you are a robot!
webpage = req.read()
# parse the html code
soup = BeautifulSoup(webpage, 'html.parser')
menus = json.loads(re.search(r"global.document.metadata\s*=\s*(.*);", soup.getText()).group(1))
pdfpath = str(menus['pdfPath']).replace("iel", "ielx")
pdf_link = "http://ieeexplore.ieee.org" + pdfpath
print("Access in " + pdf_link)
return tools.downloadFile(url=pdf_link, folder=cfg.folder_pdf, overwrite=False,
localfilename=filename, printOutput=False)
# raise BaseException(req.geturl() + ' does not contain a valid IEEE download link.')
def extract_paper_from_SPRINGER(self, req, filename):
"""
this function will access a given url and will find the link of the pdf.
Attention: WORKS ONLY IN THE TU DELFT NETWORK or VPN
:param paper_url: e.g. "http://dx.doi.org/10.1007/BF00264597"
:return:
"""
# reguest to the url, add headers to avoid HTTP Error: 403 Forbidden
# the site will strike you out because you are a robot!
# req = Request(paper_url, headers={'User-Agent': 'Mozilla/5.0'})
webpage = req.read()
# parse the html code
soup = BeautifulSoup(webpage, 'html.parser')
# select only the link tags
for link in soup.find_all('a'):
# the name of in the link tag is "FullTextPDF"
if str(link.get('href')).endswith('.pdf'):
# for instance : "/content/pdf/10.1007%2FBF00264597.pdf"
href_link = link.get('href')
prefix = "http://link.springer.com"
pdf_link = prefix + href_link
print("Access in " + pdf_link)
return tools.downloadFile(url=pdf_link, folder=cfg.folder_pdf, overwrite=False,
localfilename=filename, printOutput=False)
raise BaseException(req.geturl() + ' does not contain a valid SPRINGER download link.')
def extract_paper_from_ACM(self, req, filename):
"""
this function will access a given url and will find the link of the pdf.
Attention: WORKS ONLY IN THE TU DELFT NETWORK or VPN
:param paper_url: DOI link of and ACM Page
:return:
"""
# reguest to the url, add headers to avoid HTTP Error: 403 Forbidden
# the site will strike you out because you are a robot!
# req = Request(paper_url ,headers={'User-Agent': 'Mozilla/5.0'})
webpage = req.read()
# parse the html code
soup = BeautifulSoup(webpage, 'html.parser')
# select only the link tags
for link in soup.find_all('a'):
# the name of in the link tag is "FullTextPDF"
if str(link.get('name')).endswith('PDF') or str(link.get('name')).endswith('Pdf') \
or str(link.get('name')).endswith('pdf'):
href_link = link.get('href')
prefix = "http://dl.acm.org/"
pdf_link = prefix + href_link
# To avoid any conflicts I am taking the id of the link and the
# ftid and I concatinate them together. I put +3 and +6 to
# exclude the "id=" and "ftid="
pdf_id = href_link[href_link.find("id=") + 3: href_link.find("&f")]
file_id = href_link[href_link.find("ftid=") + 6: href_link.find("&d")]
# localfilename = pdf_id + '_' + file_id+'.pdf'
# folder = "C:/Users/User/Documents/acm_pdfs/"
print("donwload file " + pdf_link)
return tools.downloadFile(url=pdf_link, folder=cfg.folder_pdf, overwrite=False,
localfilename=filename, printOutput=False)
raise BaseException(req.geturl() + ' does not contain a valid ACM download link.')
# global filters
# if filter:
# self.filters = json.loads(filter.replace("'", '"'))
# print("Using filters " + str(filters))