Skip to content

Commit

Permalink
added utf-8 encoding to dataset_repository (#402)
Browse files Browse the repository at this point in the history
  • Loading branch information
JohannesWesch authored Jan 22, 2024
1 parent 538399f commit 2683396
Showing 1 changed file with 3 additions and 3 deletions.
6 changes: 3 additions & 3 deletions src/intelligence_layer/core/evaluation/dataset_repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ def example(
if not self._fs.exists(example_path):
return None

with self._fs.open(example_path, "r") as examples_file:
with self._fs.open(example_path, "r", encoding="utf-8") as examples_file:
# Mypy does not accept dynamic types
for example in examples_file:
validated_example = Example[input_type, expected_output_type].model_validate_json(json_data=example) # type: ignore
Expand All @@ -48,7 +48,7 @@ def create_dataset(self, examples: Iterable[Example[Input, ExpectedOutput]]) ->
if self._fs.exists(dataset_path):
raise ValueError(f"Dataset name {dataset_id} already taken")

with self._fs.open(dataset_path, "w") as examples_file:
with self._fs.open(dataset_path, "w", encoding="utf-8") as examples_file:
for example in examples:
serialized_result = JsonSerializer(root=example)
text = serialized_result.model_dump_json() + "\n"
Expand All @@ -65,7 +65,7 @@ def examples_by_id(
if not self._fs.exists(example_path):
return None

with self._fs.open(example_path, "r") as examples_file:
with self._fs.open(example_path, "r", encoding="utf-8") as examples_file:
# Mypy does not accept dynamic types
examples = [Example[input_type, expected_output_type].model_validate_json(json_data=example) for example in examples_file] # type: ignore

Expand Down

0 comments on commit 2683396

Please sign in to comment.