Skip to content

Commit

Permalink
Fix encoding detection and exception on empty files
Browse files Browse the repository at this point in the history
The encoding detection code was trying to catch encoding-related
exceptions when the file is opened. This doesn't make sense, because
at this point no data has been read, therefore no encoding errors can be
detected. Instead, catch encoding-related exceptions when the file
contents are read.

Also avoid bailing out with `Exception('Unknown encoding')` on empty
files.
  • Loading branch information
DimitriPapadopoulos committed Dec 11, 2021
1 parent d8948ad commit c8196ea
Showing 1 changed file with 19 additions and 23 deletions.
42 changes: 19 additions & 23 deletions codespell_lib/_codespell.py
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ def open_with_chardet(self, filename):
self.encdetector.reset()
with codecs.open(filename, 'rb') as f:
for line in f:
self.encdetector.feed(line)
self.encdet
if self.encdetector.done:
break
self.encdetector.close()
Expand All @@ -200,30 +200,26 @@ def open_with_chardet(self, filename):
return lines, encoding

def open_with_internal(self, filename):
curr = 0
while True:
try:
f = codecs.open(filename, 'r', encoding=encodings[curr])
except UnicodeDecodeError:
if not self.quiet_level & QuietLevels.ENCODING:
print("WARNING: Decoding file using encoding=%s failed: %s"
% (encodings[curr], filename,), file=sys.stderr)
try:
print("WARNING: Trying next encoding %s"
% encodings[curr + 1], file=sys.stderr)
except IndexError:
pass

curr += 1
else:
lines = f.readlines()
f.close()
break
if not lines:
encoding = None
first_try = True
for encoding in encodings:
if first_try:
first_try = False
elif not self.quiet_level & QuietLevels.ENCODING:
print("WARNING: Trying next encoding %s"
% encoding, file=sys.stderr)
with codecs.open(filename, 'r', encoding=encoding) as f:
try:
lines = f.readlines()
except UnicodeDecodeError:
if not self.quiet_level & QuietLevels.ENCODING:
print("WARNING: Decoding file using encoding=%s failed: %s"
% (encoding, filename,), file=sys.stderr)
else:
break
else:
raise Exception('Unknown encoding')

encoding = encodings[curr]

return lines, encoding

# -.-:-.-:-.-:-.:-.-:-.-:-.-:-.-:-.:-.-:-.-:-.-:-.-:-.:-.-:-
Expand Down

0 comments on commit c8196ea

Please sign in to comment.