Merge pull request #320 from GSA/iis-date-parser

add more date parser for IIS server
ckan · Nov 10, 2023 · 7350dbd · 7350dbd
2 parents fe13b30 + 0beefa2
commit 7350dbd
Showing 1 changed file with 22 additions and 11 deletions.
diff --git a/ckanext/spatial/harvesters/waf.py b/ckanext/spatial/harvesters/waf.py
@@ -244,17 +244,26 @@ def fetch_stage(self, harvest_object):
         ,adjacent=False, joinString=' ').setResultsName('date')
         )
 
-iis =      parse.SkipTo("<br>").suppress() \
-         + parse.OneOrMore("<br>").suppress() \
-         + parse.Optional(parse.Combine(
-           parse.Word(parse.alphanums+'/') +
-           parse.Word(parse.alphanums+':') +
-           parse.Word(parse.alphas)
-         , adjacent=False, joinString=' ').setResultsName('date')
-         ) \
-         + parse.Word(parse.nums).suppress() \
-         + parse.Literal('<A HREF=').suppress() \
-         + parse.quotedString.setParseAction(parse.removeQuotes).setResultsName('url')
+iis     = parse.SkipTo("<br>").suppress() \
+        + parse.OneOrMore("<br>").suppress() \
+        + parse.Optional(parse.Combine(
+            parse.Word(parse.alphanums+'/') +
+            parse.Word(parse.alphanums+':') +
+            parse.Word(parse.alphas)
+        , adjacent=False, joinString=' ').setResultsName('date')
+        ) \
+        + parse.Optional(parse.Combine(
+            parse.Word(parse.alphas+',') +
+            parse.Word(parse.alphas) +
+            parse.Word(parse.nums+',') +
+            parse.Word(parse.nums) +
+            parse.Word(parse.nums+':') +
+            parse.Word(parse.alphas)
+        , adjacent=False, joinString=' ').setResultsName('date')
+        ) \
+        + parse.Word(parse.nums).suppress() \
+        + parse.Literal('<A HREF=').suppress() \
+        + parse.quotedString.setParseAction(parse.removeQuotes).setResultsName('url')
 
 other = parse.SkipTo(parse.CaselessLiteral("<a href="), include=True).suppress() \
         + parse.quotedString.setParseAction(parse.removeQuotes).setResultsName('url')
@@ -328,6 +337,8 @@ def _extract_waf(content, base_url, scraper, results = None, depth=0):
             except Exception as e:
                 raise
                 date = None
+        if not date:
+            log.debug('failed to get date for %s', url)
         results.append((urljoin(base_url, record.url), date))
 
     return results