find-mismatched-diacritics

#!/usr/bin/env python3

import argparse
import os
import fnmatch
import unicodedata
import regex

def main():
	parser = argparse.ArgumentParser(description="Find words with mismatched diacritics in XHTML files.  For example, \"cafe\" in one file and \"café\" in another.")
	parser.add_argument("-v", "--verbose", action="store_true", help="increase output verbosity")
	parser.add_argument("targets", metavar="TARGET", nargs="+", help="an XHTML file, or a directory containing XHTML files")
	args = parser.parse_args()

	for target in args.targets:
		target = os.path.abspath(target)

		if args.verbose:
			print("Processing {} ...".format(target), end="", flush=True)

		target_filenames = set()
		if os.path.isdir(target):
			for root, _, filenames in os.walk(target):
				for filename in fnmatch.filter(filenames, "*.xhtml"):
					target_filenames.add(os.path.join(root, filename))
		else:
			target_filenames.add(target)

		accented_words = set()

		for filename in target_filenames:
			with open(filename, "r+", encoding="utf-8") as file:
				xhtml = file.read()

				decomposed_xhtml = unicodedata.normalize("NFKD", xhtml)

				pattern = regex.compile(r"\b\w*\p{M}\w*\b")
				for decomposed_word in pattern.findall(decomposed_xhtml):
					word = unicodedata.normalize("NFKC", decomposed_word)

					if len(word) > 2:
						accented_words.add(word.lower())

		mismatches = {}

		#Now iterate over the list and search files for unaccented versions of the words
		if len(accented_words) > 0:
			for filename in target_filenames:
				with open(filename, "r", encoding="utf-8") as file:
					text = file.read()

					for accented_word in accented_words:
						plain_word = regex.sub(r"\p{M}", "", unicodedata.normalize("NFKD", accented_word))

						pattern = regex.compile(r"\b" + plain_word + r"\b", regex.IGNORECASE)
						if pattern.search(text) != None:
							mismatches[accented_word] = plain_word

		if len(mismatches) > 0:
			if args.verbose:
				print("")
			else:
				print(target)

			for accented_word, plain_word in sorted(mismatches.items()):
				print("\tFound {} and {}".format(accented_word, plain_word))
		else:
			if args.verbose:
				print(" OK")

if __name__ == "__main__":
	main()