Skip to content

Commit

Permalink
tests fix: remove dummy pickled data (#161)
Browse files Browse the repository at this point in the history
  • Loading branch information
adbar authored Nov 6, 2024
1 parent 91130df commit fae9d47
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 7 deletions.
4 changes: 4 additions & 0 deletions tests/test_dictionary_pickler.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,3 +39,7 @@ def test_logic() -> None:
os_handle, temp_outputfile = tempfile.mkstemp(suffix=".pkl", text=True)
dictionary_pickler._pickle_dict("zz", listpath, temp_outputfile)
dictionary_pickler._pickle_dict("zz", listpath, in_place=True)

# remove pickle file
filepath = dictionary_pickler._determine_pickle_path("zz")
os.remove(filepath)
18 changes: 11 additions & 7 deletions training/dictionary_pickler.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,16 @@ def _load_dict(
return _read_dict(filepath, langcode, silent)


def _determine_pickle_path(langcode: str = "en", in_place: bool = False) -> str:
filename = f"strategies/dictionaries/data/{langcode}.plzma"
directory = (
Path(simplemma.__file__).parent
if in_place
else Path(__file__).parent.parent / "simplemma"
)
return str(directory / filename)


def _pickle_dict(
langcode: str = "en",
listpath: str = "lists",
Expand All @@ -144,13 +154,7 @@ def _pickle_dict(
if langcode not in ("lt", "sw"):
mydict = dict(sorted(mydict.items(), key=itemgetter(1)))
if filepath is None:
filename = f"strategies/dictionaries/data/{langcode}.plzma"
directory = (
Path(simplemma.__file__).parent
if in_place
else Path(__file__).parent.parent / "simplemma"
)
filepath = str(directory / filename)
filepath = _determine_pickle_path(langcode, in_place)
with lzma.open(filepath, "wb") as filehandle: # , filters=my_filters, preset=9
pickle.dump(mydict, filehandle, protocol=4)
LOGGER.debug("%s %s", langcode, len(mydict))
Expand Down

0 comments on commit fae9d47

Please sign in to comment.