Skip to content

Commit

Permalink
Improve _HTMLWordTruncator. Fix #2863
Browse files Browse the repository at this point in the history
Use more than one unicode block in _word_regex, making word count
function behave properly with CJK, cyrillic and more latin characters
when generating summary.
  • Loading branch information
ImBearChild committed Sep 28, 2021
1 parent 332be6e commit 2417f85
Show file tree
Hide file tree
Showing 3 changed files with 31 additions and 3 deletions.
3 changes: 3 additions & 0 deletions RELEASE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Release type: minor

Improve word count behavior when generating summary.
10 changes: 10 additions & 0 deletions pelican/tests/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,16 @@ def test_truncate_html_words(self):
utils.truncate_html_words('word ' * 100, 20),
'word ' * 20 + '…')

# Plain text with Unicode content.
self.assertEqual(
utils.truncate_html_words('我愿意这样,朋友——我独自远行,\
不但没有你,并且再没有别的影在黑暗里。', 12),
'我愿意这样,朋友——我独自远行' + ' …')
self.assertEqual(
utils.truncate_html_words(
'Ты мелькнула, ты предстала, Снова сердце задрожало,', 3),
'Ты мелькнула, ты' + ' …')

# Words enclosed or intervaled by HTML tags.
self.assertEqual(
utils.truncate_html_words('<p>' + 'word ' * 100 + '</p>', 20),
Expand Down
21 changes: 18 additions & 3 deletions pelican/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ class memoized:
(not reevaluated).
"""

def __init__(self, func):
self.func = func
self.cache = {}
Expand Down Expand Up @@ -408,7 +409,21 @@ def posixize_path(rel_path):

class _HTMLWordTruncator(HTMLParser):

_word_regex = re.compile(r"\w[\w'-]*", re.U)
_word_regex = re.compile(r"(({SBC})({SBC}|-|')*)|{DBC}".format(
# SBC means Latin-like characters. A word contains a few characters.
# ASCII |Extended Latin | Cyrillic
SBC="[0-9a-zA-Z]|[\u00C0-\u024f]|[\u0400-\u04FF]",
# DBC means CJK-like characters. An character can stand for a word.
DBC=("([\u4E00-\u9FFF])|" # CJK Unified Ideographs
"([\u3400-\u4DBF])|" # CJK Unified Ideographs Extension A
"([\uF900\uFAFF])|" # CJK Compatibility Ideographs
"([\U00020000-\U0002A6DF])|" # CJK Unified Ideographs Extension B
"([\U0002F800-\U0002FA1F])|" # CJK Compatibility Ideographs Supplement
"([\u3040-\u30FF])|" # Hiragana and Katakana
"([\u1100-\u11FF])|" # Hangul Jamo
"([\uAC00-\uD7FF])|" # Hangul Compatibility Jamo
"([\u3130-\u318F])" # Hangul Syllables
)), re.UNICODE)
_word_prefix_regex = re.compile(r'\w', re.U)
_singlets = ('br', 'col', 'link', 'base', 'img', 'param', 'area',
'hr', 'input')
Expand Down Expand Up @@ -818,8 +833,8 @@ def check(self):
}
)
logger.warning(
'No valid files found in content for the active readers:\n'
+ '\n'.join(reader_descs))
'No valid files found in content for the active readers:\n'
+ '\n'.join(reader_descs))

if result.get('theme') is None:
logger.warning('Empty theme folder. Using `basic` theme.')
Expand Down

0 comments on commit 2417f85

Please sign in to comment.