Skip to content

Commit

Permalink
[joyreactor] improve error handling for faulty JSON (#148)
Browse files Browse the repository at this point in the history
- remove all ASCII escape codes, not just \n and \r
- ignore faulty posts instead of letting the exception propagate
  • Loading branch information
mikf committed Jan 3, 2019
1 parent a36f52a commit 8753627
Showing 1 changed file with 8 additions and 6 deletions.
14 changes: 8 additions & 6 deletions gallery_dl/extractor/joyreactor.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,19 +62,21 @@ def _pagination(self, url):
return
url = self.root + path

@staticmethod
def _parse_post(post):
def _parse_post(self, post):
post, _, script = post.partition('<script type="application/ld+json">')
images = text.extract_iter(post, '<div class="image">', '</div>')
script = script[:script.index("</")].strip()

try:
data = json.loads(script)
except ValueError:
data = json.loads(script
.replace("\\", "\\\\")
.replace("\n", "")
.replace("\r", ""))
try:
mapping = dict.fromkeys(range(32))
script = script.translate(mapping).replace("\\", "\\\\")
data = json.loads(script)
except ValueError as exc:
self.log.warning("Unable to parse post: %s", exc)
return

num = 0
date = data["datePublished"]
Expand Down

0 comments on commit 8753627

Please sign in to comment.