diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f1411663..982a184a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,12 +36,16 @@ - This is not supported for the `OpenTelemetryTracer` because of technical incompatibilities. - All exported spans now contain the status of the span. - We now support python 3.12 + - Add `description` parameter to `Evaluator.evaluate_runs` and `Runner.run_dataset` to allow individual descriptions without the need to create a new `Evaluator` or `Runner`. ### Fixes - The document index client now correctly URL-encodes document names in its queries. - The `ArgillaEvaluator` not properly supports `dataset_name` - Update outdated `how_to_human_evaluation_via_argilla.ipynb` - Fix bug in `FileSystemBasedRepository` causing spurious mkdir failure if the file actually exists + - Update broken README links to Read The Docs + - The `evaluation` tutorial contained a broken multi-label classify example. This was fixed. + ### Deprecations ... diff --git a/README.md b/README.md index 79c35b412..e7067d01a 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ The key features of the Intelligence Layer are: - [How to use the Intelligence Layer in Docker](#how-to-use-the-intelligence-layer-in-docker) - [Via the GitHub repository](#via-the-github-repository) - [Getting started](#getting-started) - - [Setup LLM access](#setup-llm-access) + - [Setup LLM access](#setup-llm-access) - [Tutorial Notebooks](#tutorial-notebooks) - [How-Tos](#how-tos) - [Models](#models) @@ -176,20 +176,20 @@ To give you a starting point for using the Intelligence Layer, we provide some p | Type | Task | Description | |-----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Classify | [EmbeddingBasedClassify](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.use_cases.html#intelligence_layer.use_cases.EmbeddingBasedClassify) | Classify a short text by computing its similarity with example texts for each class. | -| Classify | [PromptBasedClassify](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.use_cases.html#intelligence_layer.use_cases.PromptBasedClassify) | Classify a short text by assessing each class' probability using zero-shot prompting. | -| Classify | [PromptBasedClassifyWithDefinitions](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.use_cases.html#intelligence_layer.use_cases.PromptBasedClassifyWithDefinitions) | Classify a short text by assessing each class' probability using zero-shot prompting. Each class is defined by a natural language description. | -| Classify | [KeywordExtract](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.use_cases.html#intelligence_layer.use_cases.KeywordExtract) | Generate matching labels for a short text. | -| QA | [MultipleChunkRetrieverQa](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.use_cases.html#intelligence_layer.use_cases.MultipleChunkRetrieverQa) | Answer a question based on an entire knowledge base. Recommended for most RAG-QA use-cases. | -| QA | [LongContextQa](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.use_cases.html#intelligence_layer.use_cases.LongContextQa) | Answer a question based on one document of any length. | -| QA | [MultipleChunkQa](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.use_cases.html#intelligence_layer.use_cases.MultipleChunkQa) | Answer a question based on a list of short texts. | -| QA | [SingleChunkQa](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.use_cases.html#intelligence_layer.use_cases.SingleChunkQa) | Answer a question based on a short text. | -| QA | [RetrieverBasedQa (deprecated)](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.use_cases.html#intelligence_layer.use_cases.RetrieverBasedQa) | Answer a question based on a document base using a [BaseRetriever](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.connectors.html#intelligence_layer.connectors.BaseRetriever) implementation. | -| Search | [Search](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.use_cases.html#intelligence_layer.use_cases.Search) | Search for texts in a document base using a [BaseRetriever](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.connectors.html#intelligence_layer.connectors.BaseRetriever) implementation. | -| Search | [ExpandChunks](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.use_cases.html#intelligence_layer.use_cases.ExpandChunks) | Expand chunks retrieved with a [BaseRetriever](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.connectors.html#intelligence_layer.connectors.BaseRetriever) implementation. | -| Summarize | [SteerableLongContextSummarize](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.use_cases.html#intelligence_layer.use_cases.SteerableLongContextSummarize) | Condense a long text into a summary with a natural language instruction. | -| Summarize | [SteerableSingleChunkSummarize](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.use_cases.html#intelligence_layer.use_cases.SteerableSingleChunkSummarize) | Condense a short text into a summary with a natural language instruction. | -| Summarize | [RecursiveSummarize](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.use_cases.html#intelligence_layer.use_cases.RecursiveSummarize) | Recursively condense a text into a summary. | +| Classify | [EmbeddingBasedClassify](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.examples.html#intelligence_layer.examples.EmbeddingBasedClassify) | Classify a short text by computing its similarity with example texts for each class. | +| Classify | [PromptBasedClassify](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.examples.html#intelligence_layer.examples.PromptBasedClassify) | Classify a short text by assessing each class' probability using zero-shot prompting. | +| Classify | [PromptBasedClassifyWithDefinitions](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.examples.html#intelligence_layer.examples.PromptBasedClassifyWithDefinitions) | Classify a short text by assessing each class' probability using zero-shot prompting. Each class is defined by a natural language description. | +| Classify | [KeywordExtract](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.examples.html#intelligence_layer.examples.KeywordExtract) | Generate matching labels for a short text. | +| QA | [MultipleChunkRetrieverQa](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.examples.html#intelligence_layer.examples.MultipleChunkRetrieverQa) | Answer a question based on an entire knowledge base. Recommended for most RAG-QA use-cases. | +| QA | [LongContextQa](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.examples.html#intelligence_layer.examples.LongContextQa) | Answer a question based on one document of any length. | +| QA | [MultipleChunkQa](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.examples.html#intelligence_layer.examples.MultipleChunkQa) | Answer a question based on a list of short texts. | +| QA | [SingleChunkQa](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.examples.html#intelligence_layer.examples.SingleChunkQa) | Answer a question based on a short text. | +| QA | [RetrieverBasedQa (deprecated)](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.examples.html#intelligence_layer.examples.RetrieverBasedQa) | Answer a question based on a document base using a [BaseRetriever](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.connectors.html#intelligence_layer.connectors.BaseRetriever) implementation. | +| Search | [Search](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.examples.html#intelligence_layer.examples.Search) | Search for texts in a document base using a [BaseRetriever](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.connectors.html#intelligence_layer.connectors.BaseRetriever) implementation. | +| Search | [ExpandChunks](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.examples.html#intelligence_layer.examples.ExpandChunks) | Expand chunks retrieved with a [BaseRetriever](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.connectors.html#intelligence_layer.connectors.BaseRetriever) implementation. | +| Summarize | [SteerableLongContextSummarize](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.examples.html#intelligence_layer.examples.SteerableLongContextSummarize) | Condense a long text into a summary with a natural language instruction. | +| Summarize | [SteerableSingleChunkSummarize](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.examples.html#intelligence_layer.examples.SteerableSingleChunkSummarize) | Condense a short text into a summary with a natural language instruction. | +| Summarize | [RecursiveSummarize](https://aleph-alpha-intelligence-layer.readthedocs-hosted.com/en/latest/intelligence_layer.examples.html#intelligence_layer.examples.RecursiveSummarize) | Recursively condense a text into a summary. | Note that we do not expect the above use cases to solve all of your issues. Instead, we encourage you to think of our pre-configured use cases as a foundation to fast-track your development process. diff --git a/docs/index.rst b/docs/index.rst index 68eb2fb3f..38d20788e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,11 +6,20 @@ Welcome to Intelligence Layer's documentation! ============================================== +This documentation provides API-level code documentation for the Intelligence Layer, covering modules such as connectors, core, evaluation, and examples. + +A comprehensive overview of the Intelligence Layer can be found on GitHub, including `tutorials `_ and `how-tos `_. +This code documentation is intended to serve as a reference companion to the main documentation, providing detailed information on the Intelligence Layer's API and implementation. + .. toctree:: :maxdepth: 1 :caption: Contents: - intelligence_layer + intelligence_layer.connectors + intelligence_layer.core + intelligence_layer.evaluation + intelligence_layer.examples + Indices and tables ================== diff --git a/docs/intelligence_layer.connectors.rst b/docs/intelligence_layer.connectors.rst index 0e9570b37..f38f9fe72 100644 --- a/docs/intelligence_layer.connectors.rst +++ b/docs/intelligence_layer.connectors.rst @@ -1,4 +1,4 @@ -intelligence\_layer.connectors package +intelligence\_layer.connectors ====================================== Module contents diff --git a/docs/intelligence_layer.core.rst b/docs/intelligence_layer.core.rst index cce640699..e1dbd4dfe 100644 --- a/docs/intelligence_layer.core.rst +++ b/docs/intelligence_layer.core.rst @@ -1,4 +1,4 @@ -intelligence\_layer.core package +intelligence\_layer.core ================================ Module contents diff --git a/docs/intelligence_layer.evaluation.rst b/docs/intelligence_layer.evaluation.rst index a871a800a..47a5cae5e 100644 --- a/docs/intelligence_layer.evaluation.rst +++ b/docs/intelligence_layer.evaluation.rst @@ -1,4 +1,4 @@ -intelligence\_layer.evaluation package +intelligence\_layer.evaluation ====================================== diff --git a/docs/intelligence_layer.use_cases.rst b/docs/intelligence_layer.examples.rst similarity index 75% rename from docs/intelligence_layer.use_cases.rst rename to docs/intelligence_layer.examples.rst index b620902f5..dacae35b8 100644 --- a/docs/intelligence_layer.use_cases.rst +++ b/docs/intelligence_layer.examples.rst @@ -1,4 +1,4 @@ -intelligence\_layer.use\_cases package +intelligence\_layer.examples ====================================== diff --git a/docs/intelligence_layer.rst b/docs/intelligence_layer.rst deleted file mode 100644 index 0e4ba8eef..000000000 --- a/docs/intelligence_layer.rst +++ /dev/null @@ -1,18 +0,0 @@ -intelligence\_layer package -=========================== - -Subpackages ------------ - -.. toctree:: - :maxdepth: 3 - - intelligence_layer.connectors - intelligence_layer.core - intelligence_layer.evaluation - intelligence_layer.use_cases - -Module contents ---------------- - -.. automodule:: intelligence_layer diff --git a/poetry.lock b/poetry.lock index f5713dfe3..480519ceb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1556,13 +1556,13 @@ typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "t [[package]] name = "hypercorn" -version = "0.17.2" +version = "0.17.3" description = "A ASGI Server based on Hyper libraries and inspired by Gunicorn" optional = false python-versions = ">=3.8" files = [ - {file = "hypercorn-0.17.2-py3-none-any.whl", hash = "sha256:eaa97b13c1fd8857cf31af34640704382ce3ed53cff6b3192869ea5f8244c6b1"}, - {file = "hypercorn-0.17.2.tar.gz", hash = "sha256:3827860f7e0cf0320a1288f0762c10483617de19dae08cfc3fbeb2b3323a6ffb"}, + {file = "hypercorn-0.17.3-py3-none-any.whl", hash = "sha256:059215dec34537f9d40a69258d323f56344805efb462959e727152b0aa504547"}, + {file = "hypercorn-0.17.3.tar.gz", hash = "sha256:1b37802ee3ac52d2d85270700d565787ab16cf19e1462ccfa9f089ca17574165"}, ] [package.dependencies] @@ -1579,7 +1579,7 @@ wsproto = ">=0.14.0" docs = ["pydata_sphinx_theme", "sphinxcontrib_mermaid"] h3 = ["aioquic (>=0.9.0,<1.0)"] trio = ["trio (>=0.22.0)"] -uvloop = ["uvloop"] +uvloop = ["uvloop (>=0.18)"] [[package]] name = "hyperframe" @@ -3757,18 +3757,18 @@ files = [ [[package]] name = "pydantic" -version = "2.7.1" +version = "2.7.2" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic-2.7.1-py3-none-any.whl", hash = "sha256:e029badca45266732a9a79898a15ae2e8b14840b1eabbb25844be28f0b33f3d5"}, - {file = "pydantic-2.7.1.tar.gz", hash = "sha256:e9dbb5eada8abe4d9ae5f46b9939aead650cd2b68f249bb3a8139dbe125803cc"}, + {file = "pydantic-2.7.2-py3-none-any.whl", hash = "sha256:834ab954175f94e6e68258537dc49402c4a5e9d0409b9f1b86b7e934a8372de7"}, + {file = "pydantic-2.7.2.tar.gz", hash = "sha256:71b2945998f9c9b7919a45bde9a50397b289937d215ae141c1d0903ba7149fd7"}, ] [package.dependencies] annotated-types = ">=0.4.0" -pydantic-core = "2.18.2" +pydantic-core = "2.18.3" typing-extensions = ">=4.6.1" [package.extras] @@ -3776,90 +3776,90 @@ email = ["email-validator (>=2.0.0)"] [[package]] name = "pydantic-core" -version = "2.18.2" +version = "2.18.3" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_core-2.18.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:9e08e867b306f525802df7cd16c44ff5ebbe747ff0ca6cf3fde7f36c05a59a81"}, - {file = "pydantic_core-2.18.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f0a21cbaa69900cbe1a2e7cad2aa74ac3cf21b10c3efb0fa0b80305274c0e8a2"}, - {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0680b1f1f11fda801397de52c36ce38ef1c1dc841a0927a94f226dea29c3ae3d"}, - {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:95b9d5e72481d3780ba3442eac863eae92ae43a5f3adb5b4d0a1de89d42bb250"}, - {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fcf5cd9c4b655ad666ca332b9a081112cd7a58a8b5a6ca7a3104bc950f2038"}, - {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b5155ff768083cb1d62f3e143b49a8a3432e6789a3abee8acd005c3c7af1c74"}, - {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:553ef617b6836fc7e4df130bb851e32fe357ce36336d897fd6646d6058d980af"}, - {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b89ed9eb7d616ef5714e5590e6cf7f23b02d0d539767d33561e3675d6f9e3857"}, - {file = "pydantic_core-2.18.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:75f7e9488238e920ab6204399ded280dc4c307d034f3924cd7f90a38b1829563"}, - {file = "pydantic_core-2.18.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ef26c9e94a8c04a1b2924149a9cb081836913818e55681722d7f29af88fe7b38"}, - {file = "pydantic_core-2.18.2-cp310-none-win32.whl", hash = "sha256:182245ff6b0039e82b6bb585ed55a64d7c81c560715d1bad0cbad6dfa07b4027"}, - {file = "pydantic_core-2.18.2-cp310-none-win_amd64.whl", hash = "sha256:e23ec367a948b6d812301afc1b13f8094ab7b2c280af66ef450efc357d2ae543"}, - {file = "pydantic_core-2.18.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:219da3f096d50a157f33645a1cf31c0ad1fe829a92181dd1311022f986e5fbe3"}, - {file = "pydantic_core-2.18.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cc1cfd88a64e012b74e94cd00bbe0f9c6df57049c97f02bb07d39e9c852e19a4"}, - {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05b7133a6e6aeb8df37d6f413f7705a37ab4031597f64ab56384c94d98fa0e90"}, - {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:224c421235f6102e8737032483f43c1a8cfb1d2f45740c44166219599358c2cd"}, - {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b14d82cdb934e99dda6d9d60dc84a24379820176cc4a0d123f88df319ae9c150"}, - {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2728b01246a3bba6de144f9e3115b532ee44bd6cf39795194fb75491824a1413"}, - {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:470b94480bb5ee929f5acba6995251ada5e059a5ef3e0dfc63cca287283ebfa6"}, - {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:997abc4df705d1295a42f95b4eec4950a37ad8ae46d913caeee117b6b198811c"}, - {file = "pydantic_core-2.18.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:75250dbc5290e3f1a0f4618db35e51a165186f9034eff158f3d490b3fed9f8a0"}, - {file = "pydantic_core-2.18.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4456f2dca97c425231d7315737d45239b2b51a50dc2b6f0c2bb181fce6207664"}, - {file = "pydantic_core-2.18.2-cp311-none-win32.whl", hash = "sha256:269322dcc3d8bdb69f054681edff86276b2ff972447863cf34c8b860f5188e2e"}, - {file = "pydantic_core-2.18.2-cp311-none-win_amd64.whl", hash = "sha256:800d60565aec896f25bc3cfa56d2277d52d5182af08162f7954f938c06dc4ee3"}, - {file = "pydantic_core-2.18.2-cp311-none-win_arm64.whl", hash = "sha256:1404c69d6a676245199767ba4f633cce5f4ad4181f9d0ccb0577e1f66cf4c46d"}, - {file = "pydantic_core-2.18.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:fb2bd7be70c0fe4dfd32c951bc813d9fe6ebcbfdd15a07527796c8204bd36242"}, - {file = "pydantic_core-2.18.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6132dd3bd52838acddca05a72aafb6eab6536aa145e923bb50f45e78b7251043"}, - {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d904828195733c183d20a54230c0df0eb46ec746ea1a666730787353e87182"}, - {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c9bd70772c720142be1020eac55f8143a34ec9f82d75a8e7a07852023e46617f"}, - {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2b8ed04b3582771764538f7ee7001b02e1170223cf9b75dff0bc698fadb00cf3"}, - {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e6dac87ddb34aaec85f873d737e9d06a3555a1cc1a8e0c44b7f8d5daeb89d86f"}, - {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ca4ae5a27ad7a4ee5170aebce1574b375de390bc01284f87b18d43a3984df72"}, - {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:886eec03591b7cf058467a70a87733b35f44707bd86cf64a615584fd72488b7c"}, - {file = "pydantic_core-2.18.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ca7b0c1f1c983e064caa85f3792dd2fe3526b3505378874afa84baf662e12241"}, - {file = "pydantic_core-2.18.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4b4356d3538c3649337df4074e81b85f0616b79731fe22dd11b99499b2ebbdf3"}, - {file = "pydantic_core-2.18.2-cp312-none-win32.whl", hash = "sha256:8b172601454f2d7701121bbec3425dd71efcb787a027edf49724c9cefc14c038"}, - {file = "pydantic_core-2.18.2-cp312-none-win_amd64.whl", hash = "sha256:b1bd7e47b1558ea872bd16c8502c414f9e90dcf12f1395129d7bb42a09a95438"}, - {file = "pydantic_core-2.18.2-cp312-none-win_arm64.whl", hash = "sha256:98758d627ff397e752bc339272c14c98199c613f922d4a384ddc07526c86a2ec"}, - {file = "pydantic_core-2.18.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:9fdad8e35f278b2c3eb77cbdc5c0a49dada440657bf738d6905ce106dc1de439"}, - {file = "pydantic_core-2.18.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1d90c3265ae107f91a4f279f4d6f6f1d4907ac76c6868b27dc7fb33688cfb347"}, - {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:390193c770399861d8df9670fb0d1874f330c79caaca4642332df7c682bf6b91"}, - {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:82d5d4d78e4448683cb467897fe24e2b74bb7b973a541ea1dcfec1d3cbce39fb"}, - {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4774f3184d2ef3e14e8693194f661dea5a4d6ca4e3dc8e39786d33a94865cefd"}, - {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4d938ec0adf5167cb335acb25a4ee69a8107e4984f8fbd2e897021d9e4ca21b"}, - {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0e8b1be28239fc64a88a8189d1df7fad8be8c1ae47fcc33e43d4be15f99cc70"}, - {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:868649da93e5a3d5eacc2b5b3b9235c98ccdbfd443832f31e075f54419e1b96b"}, - {file = "pydantic_core-2.18.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:78363590ef93d5d226ba21a90a03ea89a20738ee5b7da83d771d283fd8a56761"}, - {file = "pydantic_core-2.18.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:852e966fbd035a6468fc0a3496589b45e2208ec7ca95c26470a54daed82a0788"}, - {file = "pydantic_core-2.18.2-cp38-none-win32.whl", hash = "sha256:6a46e22a707e7ad4484ac9ee9f290f9d501df45954184e23fc29408dfad61350"}, - {file = "pydantic_core-2.18.2-cp38-none-win_amd64.whl", hash = "sha256:d91cb5ea8b11607cc757675051f61b3d93f15eca3cefb3e6c704a5d6e8440f4e"}, - {file = "pydantic_core-2.18.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:ae0a8a797a5e56c053610fa7be147993fe50960fa43609ff2a9552b0e07013e8"}, - {file = "pydantic_core-2.18.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:042473b6280246b1dbf530559246f6842b56119c2926d1e52b631bdc46075f2a"}, - {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a388a77e629b9ec814c1b1e6b3b595fe521d2cdc625fcca26fbc2d44c816804"}, - {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e25add29b8f3b233ae90ccef2d902d0ae0432eb0d45370fe315d1a5cf231004b"}, - {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f459a5ce8434614dfd39bbebf1041952ae01da6bed9855008cb33b875cb024c0"}, - {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eff2de745698eb46eeb51193a9f41d67d834d50e424aef27df2fcdee1b153845"}, - {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8309f67285bdfe65c372ea3722b7a5642680f3dba538566340a9d36e920b5f0"}, - {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f93a8a2e3938ff656a7c1bc57193b1319960ac015b6e87d76c76bf14fe0244b4"}, - {file = "pydantic_core-2.18.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:22057013c8c1e272eb8d0eebc796701167d8377441ec894a8fed1af64a0bf399"}, - {file = "pydantic_core-2.18.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cfeecd1ac6cc1fb2692c3d5110781c965aabd4ec5d32799773ca7b1456ac636b"}, - {file = "pydantic_core-2.18.2-cp39-none-win32.whl", hash = "sha256:0d69b4c2f6bb3e130dba60d34c0845ba31b69babdd3f78f7c0c8fae5021a253e"}, - {file = "pydantic_core-2.18.2-cp39-none-win_amd64.whl", hash = "sha256:d9319e499827271b09b4e411905b24a426b8fb69464dfa1696258f53a3334641"}, - {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a1874c6dd4113308bd0eb568418e6114b252afe44319ead2b4081e9b9521fe75"}, - {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:ccdd111c03bfd3666bd2472b674c6899550e09e9f298954cfc896ab92b5b0e6d"}, - {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e18609ceaa6eed63753037fc06ebb16041d17d28199ae5aba0052c51449650a9"}, - {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e5c584d357c4e2baf0ff7baf44f4994be121e16a2c88918a5817331fc7599d7"}, - {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:43f0f463cf89ace478de71a318b1b4f05ebc456a9b9300d027b4b57c1a2064fb"}, - {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:e1b395e58b10b73b07b7cf740d728dd4ff9365ac46c18751bf8b3d8cca8f625a"}, - {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:0098300eebb1c837271d3d1a2cd2911e7c11b396eac9661655ee524a7f10587b"}, - {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:36789b70d613fbac0a25bb07ab3d9dba4d2e38af609c020cf4d888d165ee0bf3"}, - {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3f9a801e7c8f1ef8718da265bba008fa121243dfe37c1cea17840b0944dfd72c"}, - {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:3a6515ebc6e69d85502b4951d89131ca4e036078ea35533bb76327f8424531ce"}, - {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20aca1e2298c56ececfd8ed159ae4dde2df0781988c97ef77d5c16ff4bd5b400"}, - {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:223ee893d77a310a0391dca6df00f70bbc2f36a71a895cecd9a0e762dc37b349"}, - {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2334ce8c673ee93a1d6a65bd90327588387ba073c17e61bf19b4fd97d688d63c"}, - {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:cbca948f2d14b09d20268cda7b0367723d79063f26c4ffc523af9042cad95592"}, - {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b3ef08e20ec49e02d5c6717a91bb5af9b20f1805583cb0adfe9ba2c6b505b5ae"}, - {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:c6fdc8627910eed0c01aed6a390a252fe3ea6d472ee70fdde56273f198938374"}, - {file = "pydantic_core-2.18.2.tar.gz", hash = "sha256:2e29d20810dfc3043ee13ac7d9e25105799817683348823f305ab3f349b9386e"}, + {file = "pydantic_core-2.18.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:744697428fcdec6be5670460b578161d1ffe34743a5c15656be7ea82b008197c"}, + {file = "pydantic_core-2.18.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:37b40c05ced1ba4218b14986fe6f283d22e1ae2ff4c8e28881a70fb81fbfcda7"}, + {file = "pydantic_core-2.18.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:544a9a75622357076efb6b311983ff190fbfb3c12fc3a853122b34d3d358126c"}, + {file = "pydantic_core-2.18.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e2e253af04ceaebde8eb201eb3f3e3e7e390f2d275a88300d6a1959d710539e2"}, + {file = "pydantic_core-2.18.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:855ec66589c68aa367d989da5c4755bb74ee92ccad4fdb6af942c3612c067e34"}, + {file = "pydantic_core-2.18.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d3e42bb54e7e9d72c13ce112e02eb1b3b55681ee948d748842171201a03a98a"}, + {file = "pydantic_core-2.18.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6ac9ffccc9d2e69d9fba841441d4259cb668ac180e51b30d3632cd7abca2b9b"}, + {file = "pydantic_core-2.18.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c56eca1686539fa0c9bda992e7bd6a37583f20083c37590413381acfc5f192d6"}, + {file = "pydantic_core-2.18.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:17954d784bf8abfc0ec2a633108207ebc4fa2df1a0e4c0c3ccbaa9bb01d2c426"}, + {file = "pydantic_core-2.18.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:98ed737567d8f2ecd54f7c8d4f8572ca7c7921ede93a2e52939416170d357812"}, + {file = "pydantic_core-2.18.3-cp310-none-win32.whl", hash = "sha256:9f9e04afebd3ed8c15d67a564ed0a34b54e52136c6d40d14c5547b238390e779"}, + {file = "pydantic_core-2.18.3-cp310-none-win_amd64.whl", hash = "sha256:45e4ffbae34f7ae30d0047697e724e534a7ec0a82ef9994b7913a412c21462a0"}, + {file = "pydantic_core-2.18.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:b9ebe8231726c49518b16b237b9fe0d7d361dd221302af511a83d4ada01183ab"}, + {file = "pydantic_core-2.18.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b8e20e15d18bf7dbb453be78a2d858f946f5cdf06c5072453dace00ab652e2b2"}, + {file = "pydantic_core-2.18.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c0d9ff283cd3459fa0bf9b0256a2b6f01ac1ff9ffb034e24457b9035f75587cb"}, + {file = "pydantic_core-2.18.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2f7ef5f0ebb77ba24c9970da18b771711edc5feaf00c10b18461e0f5f5949231"}, + {file = "pydantic_core-2.18.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73038d66614d2e5cde30435b5afdced2b473b4c77d4ca3a8624dd3e41a9c19be"}, + {file = "pydantic_core-2.18.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6afd5c867a74c4d314c557b5ea9520183fadfbd1df4c2d6e09fd0d990ce412cd"}, + {file = "pydantic_core-2.18.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd7df92f28d351bb9f12470f4c533cf03d1b52ec5a6e5c58c65b183055a60106"}, + {file = "pydantic_core-2.18.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:80aea0ffeb1049336043d07799eace1c9602519fb3192916ff525b0287b2b1e4"}, + {file = "pydantic_core-2.18.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaee40f25bba38132e655ffa3d1998a6d576ba7cf81deff8bfa189fb43fd2bbe"}, + {file = "pydantic_core-2.18.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9128089da8f4fe73f7a91973895ebf2502539d627891a14034e45fb9e707e26d"}, + {file = "pydantic_core-2.18.3-cp311-none-win32.whl", hash = "sha256:fec02527e1e03257aa25b1a4dcbe697b40a22f1229f5d026503e8b7ff6d2eda7"}, + {file = "pydantic_core-2.18.3-cp311-none-win_amd64.whl", hash = "sha256:58ff8631dbab6c7c982e6425da8347108449321f61fe427c52ddfadd66642af7"}, + {file = "pydantic_core-2.18.3-cp311-none-win_arm64.whl", hash = "sha256:3fc1c7f67f34c6c2ef9c213e0f2a351797cda98249d9ca56a70ce4ebcaba45f4"}, + {file = "pydantic_core-2.18.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f0928cde2ae416a2d1ebe6dee324709c6f73e93494d8c7aea92df99aab1fc40f"}, + {file = "pydantic_core-2.18.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0bee9bb305a562f8b9271855afb6ce00223f545de3d68560b3c1649c7c5295e9"}, + {file = "pydantic_core-2.18.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e862823be114387257dacbfa7d78547165a85d7add33b446ca4f4fae92c7ff5c"}, + {file = "pydantic_core-2.18.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6a36f78674cbddc165abab0df961b5f96b14461d05feec5e1f78da58808b97e7"}, + {file = "pydantic_core-2.18.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba905d184f62e7ddbb7a5a751d8a5c805463511c7b08d1aca4a3e8c11f2e5048"}, + {file = "pydantic_core-2.18.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7fdd362f6a586e681ff86550b2379e532fee63c52def1c666887956748eaa326"}, + {file = "pydantic_core-2.18.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:24b214b7ee3bd3b865e963dbed0f8bc5375f49449d70e8d407b567af3222aae4"}, + {file = "pydantic_core-2.18.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:691018785779766127f531674fa82bb368df5b36b461622b12e176c18e119022"}, + {file = "pydantic_core-2.18.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:60e4c625e6f7155d7d0dcac151edf5858102bc61bf959d04469ca6ee4e8381bd"}, + {file = "pydantic_core-2.18.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a4e651e47d981c1b701dcc74ab8fec5a60a5b004650416b4abbef13db23bc7be"}, + {file = "pydantic_core-2.18.3-cp312-none-win32.whl", hash = "sha256:ffecbb5edb7f5ffae13599aec33b735e9e4c7676ca1633c60f2c606beb17efc5"}, + {file = "pydantic_core-2.18.3-cp312-none-win_amd64.whl", hash = "sha256:2c8333f6e934733483c7eddffdb094c143b9463d2af7e6bd85ebcb2d4a1b82c6"}, + {file = "pydantic_core-2.18.3-cp312-none-win_arm64.whl", hash = "sha256:7a20dded653e516a4655f4c98e97ccafb13753987434fe7cf044aa25f5b7d417"}, + {file = "pydantic_core-2.18.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:eecf63195be644b0396f972c82598cd15693550f0ff236dcf7ab92e2eb6d3522"}, + {file = "pydantic_core-2.18.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2c44efdd3b6125419c28821590d7ec891c9cb0dff33a7a78d9d5c8b6f66b9702"}, + {file = "pydantic_core-2.18.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e59fca51ffbdd1638b3856779342ed69bcecb8484c1d4b8bdb237d0eb5a45e2"}, + {file = "pydantic_core-2.18.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:70cf099197d6b98953468461d753563b28e73cf1eade2ffe069675d2657ed1d5"}, + {file = "pydantic_core-2.18.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:63081a49dddc6124754b32a3774331467bfc3d2bd5ff8f10df36a95602560361"}, + {file = "pydantic_core-2.18.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:370059b7883485c9edb9655355ff46d912f4b03b009d929220d9294c7fd9fd60"}, + {file = "pydantic_core-2.18.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a64faeedfd8254f05f5cf6fc755023a7e1606af3959cfc1a9285744cc711044"}, + {file = "pydantic_core-2.18.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:19d2e725de0f90d8671f89e420d36c3dd97639b98145e42fcc0e1f6d492a46dc"}, + {file = "pydantic_core-2.18.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:67bc078025d70ec5aefe6200ef094576c9d86bd36982df1301c758a9fff7d7f4"}, + {file = "pydantic_core-2.18.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:adf952c3f4100e203cbaf8e0c907c835d3e28f9041474e52b651761dc248a3c0"}, + {file = "pydantic_core-2.18.3-cp38-none-win32.whl", hash = "sha256:9a46795b1f3beb167eaee91736d5d17ac3a994bf2215a996aed825a45f897558"}, + {file = "pydantic_core-2.18.3-cp38-none-win_amd64.whl", hash = "sha256:200ad4e3133cb99ed82342a101a5abf3d924722e71cd581cc113fe828f727fbc"}, + {file = "pydantic_core-2.18.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:304378b7bf92206036c8ddd83a2ba7b7d1a5b425acafff637172a3aa72ad7083"}, + {file = "pydantic_core-2.18.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c826870b277143e701c9ccf34ebc33ddb4d072612683a044e7cce2d52f6c3fef"}, + {file = "pydantic_core-2.18.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e201935d282707394f3668380e41ccf25b5794d1b131cdd96b07f615a33ca4b1"}, + {file = "pydantic_core-2.18.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5560dda746c44b48bf82b3d191d74fe8efc5686a9ef18e69bdabccbbb9ad9442"}, + {file = "pydantic_core-2.18.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6b32c2a1f8032570842257e4c19288eba9a2bba4712af542327de9a1204faff8"}, + {file = "pydantic_core-2.18.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:929c24e9dea3990bc8bcd27c5f2d3916c0c86f5511d2caa69e0d5290115344a9"}, + {file = "pydantic_core-2.18.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1a8376fef60790152564b0eab376b3e23dd6e54f29d84aad46f7b264ecca943"}, + {file = "pydantic_core-2.18.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dccf3ef1400390ddd1fb55bf0632209d39140552d068ee5ac45553b556780e06"}, + {file = "pydantic_core-2.18.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:41dbdcb0c7252b58fa931fec47937edb422c9cb22528f41cb8963665c372caf6"}, + {file = "pydantic_core-2.18.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:666e45cf071669fde468886654742fa10b0e74cd0fa0430a46ba6056b24fb0af"}, + {file = "pydantic_core-2.18.3-cp39-none-win32.whl", hash = "sha256:f9c08cabff68704a1b4667d33f534d544b8a07b8e5d039c37067fceb18789e78"}, + {file = "pydantic_core-2.18.3-cp39-none-win_amd64.whl", hash = "sha256:4afa5f5973e8572b5c0dcb4e2d4fda7890e7cd63329bd5cc3263a25c92ef0026"}, + {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:77319771a026f7c7d29c6ebc623de889e9563b7087911b46fd06c044a12aa5e9"}, + {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:df11fa992e9f576473038510d66dd305bcd51d7dd508c163a8c8fe148454e059"}, + {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d531076bdfb65af593326ffd567e6ab3da145020dafb9187a1d131064a55f97c"}, + {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d33ce258e4e6e6038f2b9e8b8a631d17d017567db43483314993b3ca345dcbbb"}, + {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1f9cd7f5635b719939019be9bda47ecb56e165e51dd26c9a217a433e3d0d59a9"}, + {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:cd4a032bb65cc132cae1fe3e52877daecc2097965cd3914e44fbd12b00dae7c5"}, + {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:82f2718430098bcdf60402136c845e4126a189959d103900ebabb6774a5d9fdb"}, + {file = "pydantic_core-2.18.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c0037a92cf0c580ed14e10953cdd26528e8796307bb8bb312dc65f71547df04d"}, + {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b95a0972fac2b1ff3c94629fc9081b16371dad870959f1408cc33b2f78ad347a"}, + {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:a62e437d687cc148381bdd5f51e3e81f5b20a735c55f690c5be94e05da2b0d5c"}, + {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b367a73a414bbb08507da102dc2cde0fa7afe57d09b3240ce82a16d608a7679c"}, + {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ecce4b2360aa3f008da3327d652e74a0e743908eac306198b47e1c58b03dd2b"}, + {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bd4435b8d83f0c9561a2a9585b1de78f1abb17cb0cef5f39bf6a4b47d19bafe3"}, + {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:616221a6d473c5b9aa83fa8982745441f6a4a62a66436be9445c65f241b86c94"}, + {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:7e6382ce89a92bc1d0c0c5edd51e931432202b9080dc921d8d003e616402efd1"}, + {file = "pydantic_core-2.18.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:ff58f379345603d940e461eae474b6bbb6dab66ed9a851ecd3cb3709bf4dcf6a"}, + {file = "pydantic_core-2.18.3.tar.gz", hash = "sha256:432e999088d85c8f36b9a3f769a8e2b57aabd817bbb729a90d1fe7f18f6f1f39"}, ] [package.dependencies] @@ -4389,13 +4389,13 @@ files = [ [[package]] name = "requests" -version = "2.32.2" +version = "2.32.3" description = "Python HTTP for Humans." optional = false python-versions = ">=3.8" files = [ - {file = "requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c"}, - {file = "requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289"}, + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, ] [package.dependencies] @@ -4577,28 +4577,28 @@ files = [ [[package]] name = "ruff" -version = "0.4.5" +version = "0.4.6" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.4.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:8f58e615dec58b1a6b291769b559e12fdffb53cc4187160a2fc83250eaf54e96"}, - {file = "ruff-0.4.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:84dd157474e16e3a82745d2afa1016c17d27cb5d52b12e3d45d418bcc6d49264"}, - {file = "ruff-0.4.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25f483ad9d50b00e7fd577f6d0305aa18494c6af139bce7319c68a17180087f4"}, - {file = "ruff-0.4.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:63fde3bf6f3ad4e990357af1d30e8ba2730860a954ea9282c95fc0846f5f64af"}, - {file = "ruff-0.4.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78e3ba4620dee27f76bbcad97067766026c918ba0f2d035c2fc25cbdd04d9c97"}, - {file = "ruff-0.4.5-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:441dab55c568e38d02bbda68a926a3d0b54f5510095c9de7f95e47a39e0168aa"}, - {file = "ruff-0.4.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1169e47e9c4136c997f08f9857ae889d614c5035d87d38fda9b44b4338909cdf"}, - {file = "ruff-0.4.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:755ac9ac2598a941512fc36a9070a13c88d72ff874a9781493eb237ab02d75df"}, - {file = "ruff-0.4.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f4b02a65985be2b34b170025a8b92449088ce61e33e69956ce4d316c0fe7cce0"}, - {file = "ruff-0.4.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:75a426506a183d9201e7e5664de3f6b414ad3850d7625764106f7b6d0486f0a1"}, - {file = "ruff-0.4.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:6e1b139b45e2911419044237d90b60e472f57285950e1492c757dfc88259bb06"}, - {file = "ruff-0.4.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a6f29a8221d2e3d85ff0c7b4371c0e37b39c87732c969b4d90f3dad2e721c5b1"}, - {file = "ruff-0.4.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:d6ef817124d72b54cc923f3444828ba24fa45c3164bc9e8f1813db2f3d3a8a11"}, - {file = "ruff-0.4.5-py3-none-win32.whl", hash = "sha256:aed8166c18b1a169a5d3ec28a49b43340949e400665555b51ee06f22813ef062"}, - {file = "ruff-0.4.5-py3-none-win_amd64.whl", hash = "sha256:b0b03c619d2b4350b4a27e34fd2ac64d0dabe1afbf43de57d0f9d8a05ecffa45"}, - {file = "ruff-0.4.5-py3-none-win_arm64.whl", hash = "sha256:9d15de3425f53161b3f5a5658d4522e4eee5ea002bf2ac7aa380743dd9ad5fba"}, - {file = "ruff-0.4.5.tar.gz", hash = "sha256:286eabd47e7d4d521d199cab84deca135557e6d1e0f0d01c29e757c3cb151b54"}, + {file = "ruff-0.4.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ef995583a038cd4a7edf1422c9e19118e2511b8ba0b015861b4abd26ec5367c5"}, + {file = "ruff-0.4.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:602ebd7ad909eab6e7da65d3c091547781bb06f5f826974a53dbe563d357e53c"}, + {file = "ruff-0.4.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f9ced5cbb7510fd7525448eeb204e0a22cabb6e99a3cb160272262817d49786"}, + {file = "ruff-0.4.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:04a80acfc862e0e1630c8b738e70dcca03f350bad9e106968a8108379e12b31f"}, + {file = "ruff-0.4.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:be47700ecb004dfa3fd4dcdddf7322d4e632de3c06cd05329d69c45c0280e618"}, + {file = "ruff-0.4.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:1ff930d6e05f444090a0139e4e13e1e2e1f02bd51bb4547734823c760c621e79"}, + {file = "ruff-0.4.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f13410aabd3b5776f9c5699f42b37a3a348d65498c4310589bc6e5c548dc8a2f"}, + {file = "ruff-0.4.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0cf5cc02d3ae52dfb0c8a946eb7a1d6ffe4d91846ffc8ce388baa8f627e3bd50"}, + {file = "ruff-0.4.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ea3424793c29906407e3cf417f28fc33f689dacbbadfb52b7e9a809dd535dcef"}, + {file = "ruff-0.4.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:1fa8561489fadf483ffbb091ea94b9c39a00ed63efacd426aae2f197a45e67fc"}, + {file = "ruff-0.4.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:4d5b914818d8047270308fe3e85d9d7f4a31ec86c6475c9f418fbd1624d198e0"}, + {file = "ruff-0.4.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:4f02284335c766678778475e7698b7ab83abaf2f9ff0554a07b6f28df3b5c259"}, + {file = "ruff-0.4.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:3a6a0a4f4b5f54fff7c860010ab3dd81425445e37d35701a965c0248819dde7a"}, + {file = "ruff-0.4.6-py3-none-win32.whl", hash = "sha256:9018bf59b3aa8ad4fba2b1dc0299a6e4e60a4c3bc62bbeaea222679865453062"}, + {file = "ruff-0.4.6-py3-none-win_amd64.whl", hash = "sha256:a769ae07ac74ff1a019d6bd529426427c3e30d75bdf1e08bb3d46ac8f417326a"}, + {file = "ruff-0.4.6-py3-none-win_arm64.whl", hash = "sha256:735a16407a1a8f58e4c5b913ad6102722e80b562dd17acb88887685ff6f20cf6"}, + {file = "ruff-0.4.6.tar.gz", hash = "sha256:a797a87da50603f71e6d0765282098245aca6e3b94b7c17473115167d8dfb0b7"}, ] [[package]] @@ -5446,13 +5446,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "uvicorn" -version = "0.29.0" +version = "0.30.0" description = "The lightning-fast ASGI server." optional = false python-versions = ">=3.8" files = [ - {file = "uvicorn-0.29.0-py3-none-any.whl", hash = "sha256:2c2aac7ff4f4365c206fd773a39bf4ebd1047c238f8b8268ad996829323473de"}, - {file = "uvicorn-0.29.0.tar.gz", hash = "sha256:6a69214c0b6a087462412670b3ef21224fa48cae0e452b5883e8e8bdfdd11dd0"}, + {file = "uvicorn-0.30.0-py3-none-any.whl", hash = "sha256:78fa0b5f56abb8562024a59041caeb555c86e48d0efdd23c3fe7de7a4075bdab"}, + {file = "uvicorn-0.30.0.tar.gz", hash = "sha256:f678dec4fa3a39706bbf49b9ec5fc40049d42418716cea52b53f07828a60aa37"}, ] [package.dependencies] @@ -6097,4 +6097,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "575503a7da9f18ee9a17aaa5c981f839696a00a4a99959bfef51dc2d31fda6bd" +content-hash = "16005dca5050be678ee578d80536d946881557f4e5e20f6f33c3177a04e97407" diff --git a/pyproject.toml b/pyproject.toml index 88c5c77eb..a1b2895df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ packages = [{ include = "intelligence_layer", from = "src" }] python = ">=3.10,<3.13" pydantic = "2.7.*" fastapi = "^0.111.0" -uvicorn = "^0.29.0" +uvicorn = "^0.30.0" aleph-alpha-client = ">=7.1.0" python-dotenv = "*" semantic-text-splitter = "^0.13.1" @@ -24,7 +24,7 @@ rich = "^13.7.1" nbconvert = "^7.16.4" datasets = "^2.19.1" jupyter = "^1.0.0" -requests = "^2.32.2" +requests = "^2.32.3" langdetect = "^1.0.9" pycountry = "23.12.11" opentelemetry-api = "^1.22.0" @@ -39,13 +39,13 @@ sacrebleu = "^2.4.2" # lint & format mypy = "^1.10.0" nbqa = "^1.8.5" -ruff = "^0.4.5" +ruff = "^0.4.6" pre-commit = "^3.7.1" # tests pytest = "^8.2" pytest-xdist = "^3.6.1" -hypercorn = "^0.17.2" +hypercorn = "^0.17.3" # typing httpx = "*" diff --git a/src/documentation/evaluation.ipynb b/src/documentation/evaluation.ipynb index 75eda7620..4289d6335 100644 --- a/src/documentation/evaluation.ipynb +++ b/src/documentation/evaluation.ipynb @@ -210,7 +210,7 @@ " )\n", " for item in data\n", " ],\n", - " dataset_name=\"tweet_topic_multi\",\n", + " dataset_name=\"tweet_topic_single\",\n", ")" ] }, @@ -279,10 +279,39 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "## Multilabel classification\n", + "\n", "As an alternative to the `PromptBasedClassify` we now gonne use the `EmbeddingBasedClassify` for multi label classifications.\n", - "In this case, we have to provide some example for each class.\n", + "In this case, we have to provide some example for each class and our examples need to contain a list of classes instead of a single class\n", "\n", - "We can even reuse our data repositories:" + "First, we will create a new dataset with more expected classes and a slightly different format:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dataset = dataset_repository.create_dataset(\n", + " examples=[\n", + " Example(\n", + " input=ClassifyInput(chunk=TextChunk(item[\"text\"]), labels=all_labels),\n", + " expected_output=item[\n", + " \"label_name\"\n", + " ], # <- difference here, we take all labels instead of a single one\n", + " )\n", + " for item in data\n", + " ],\n", + " dataset_name=\"tweet_topic_multi\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can then create examples for our labels and put them into our `EmbeddingBasedClassify`" ] }, { @@ -308,8 +337,16 @@ " LabelWithExamples(name=name, examples=examples)\n", " for name, examples in build_labels_and_examples(all_data[25:]).items()\n", " ],\n", - ")\n", - "eval_logic = MultiLabelClassifyEvaluationLogic(threshold=0.60)\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "eval_logic = MultiLabelClassifyEvaluationLogic(threshold=0.7)\n", "aggregation_logic = MultiLabelClassifyAggregationLogic()\n", "\n", "embedding_based_classify_runner = Runner(\n", @@ -352,15 +389,7 @@ " embedding_based_classify_evaluation_result.id\n", " )\n", ")\n", - "embedding_based_classify_aggregation_result.raise_on_evaluation_failure()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ + "embedding_based_classify_aggregation_result.raise_on_evaluation_failure()\n", "embedding_based_classify_aggregation_result.statistics.macro_avg" ] }, @@ -368,7 +397,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Apparently, our method has a great recall value, i.e. all relevant labels are retrieved. However, the low precision value indicates that we tend to falsely predict labels at times.\n", + "Apparently, our method has a great recall value, i.e. all relevant labels are retrieved. However, the low precision value indicates that we tend to falsely predict labels at times. This gives us an indicator that the threshold for our evaluation logic is probably too low with `0.6`\n", "\n", "Note, that the evaluation criteria for the multiple label approach are a lot harsher; we evaluate whether we correctly predict all labels & not just one of the correct ones!" ] @@ -402,7 +431,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.7" + "version": "3.12.2" } }, "nbformat": 4, diff --git a/src/intelligence_layer/evaluation/evaluation/evaluator/evaluator.py b/src/intelligence_layer/evaluation/evaluation/evaluator/evaluator.py index b539b1bef..c0f07852d 100644 --- a/src/intelligence_layer/evaluation/evaluation/evaluator/evaluator.py +++ b/src/intelligence_layer/evaluation/evaluation/evaluator/evaluator.py @@ -98,6 +98,7 @@ def evaluate_runs( num_examples: Optional[int] = None, abort_on_error: bool = False, skip_example_on_any_failure: bool = True, + description: Optional[str] = None, ) -> EvaluationOverview: """Evaluates all generated outputs in the run. @@ -116,6 +117,7 @@ def evaluate_runs( Always the first n runs stored in the evaluation repository. Defaults to None. abort_on_error: Flag to abort all evaluations when an error occurs. Defaults to False. skip_example_on_any_failure: Flag to skip evaluation on any example for which at least one run fails. Defaults to True. + description: Optional description of the evaluation. Defaults to None. Returns: EvaluationOverview: An overview of the evaluation. Individual :class:`Evaluation`s will not be @@ -150,6 +152,9 @@ def evaluate_runs( ) successful_evaluation_count = len(example_evaluations) - failed_evaluation_count + full_description = ( + self.description + " : " + description if description else self.description + ) overview = EvaluationOverview( run_overviews=frozenset(run_overviews), id=eval_id, @@ -157,7 +162,7 @@ def evaluate_runs( end_date=utc_now(), successful_evaluation_count=successful_evaluation_count, failed_evaluation_count=failed_evaluation_count, - description=self.description, + description=full_description, ) self._evaluation_repository.store_evaluation_overview(overview) diff --git a/src/intelligence_layer/evaluation/evaluation/evaluator/incremental_evaluator.py b/src/intelligence_layer/evaluation/evaluation/evaluator/incremental_evaluator.py index e58eebbee..18f62afd9 100644 --- a/src/intelligence_layer/evaluation/evaluation/evaluator/incremental_evaluator.py +++ b/src/intelligence_layer/evaluation/evaluation/evaluator/incremental_evaluator.py @@ -170,6 +170,7 @@ def evaluate_runs( num_examples: Optional[int] = None, abort_on_error: bool = False, skip_example_on_any_failure: bool = True, + description: Optional[str] = None, ) -> EvaluationOverview: self._evaluation_logic.set_previous_run_output_ids([]) return super().evaluate_runs( @@ -177,6 +178,7 @@ def evaluate_runs( num_examples=num_examples, skip_example_on_any_failure=skip_example_on_any_failure, abort_on_error=abort_on_error, + description=description, ) diff --git a/src/intelligence_layer/evaluation/run/runner.py b/src/intelligence_layer/evaluation/run/runner.py index a45b670fd..6f45d82e2 100644 --- a/src/intelligence_layer/evaluation/run/runner.py +++ b/src/intelligence_layer/evaluation/run/runner.py @@ -77,6 +77,7 @@ def run_dataset( num_examples: Optional[int] = None, abort_on_error: bool = False, max_workers: int = 10, + description: Optional[str] = None, ) -> RunOverview: """Generates all outputs for the provided dataset. @@ -90,6 +91,7 @@ def run_dataset( Always the first n examples will be taken. abort_on_error: Flag to abort all run when an error occurs. Defaults to False. max_workers: Number of examples that can be evaluated concurrently. Defaults to 10. + description: An optional description of the run. Defaults to None. Returns: An overview of the run. Outputs will not be returned but instead stored in the @@ -140,6 +142,9 @@ def run( run_id=run_id, example_id=example_id, output=output ), ) + full_description = ( + self.description + " : " + description if description else self.description + ) run_overview = RunOverview( dataset_id=dataset_id, id=run_id, @@ -147,7 +152,7 @@ def run( end=utc_now(), failed_example_count=failed_count, successful_example_count=successful_count, - description=self.description, + description=full_description, ) self._run_repository.store_run_overview(run_overview) return run_overview diff --git a/tests/evaluation/test_evaluator.py b/tests/evaluation/test_evaluator.py index fad8ba1cf..3b18f36c7 100644 --- a/tests/evaluation/test_evaluator.py +++ b/tests/evaluation/test_evaluator.py @@ -4,15 +4,7 @@ from pydantic import BaseModel from pytest import fixture -from intelligence_layer.core import ( - InMemoryTaskSpan, - InMemoryTracer, - Input, - NoOpTracer, - Output, - Task, - Tracer, -) +from intelligence_layer.core import Input, Output, Task, Tracer from intelligence_layer.evaluation import ( AggregatedEvaluation, AggregationLogic, @@ -241,6 +233,14 @@ def dataset_id( ).id +@fixture +def run_id( + dataset_id: str, + dummy_runner: Runner[str, str], +) -> str: + return dummy_runner.run_dataset(dataset_id).id + + @fixture def good_dataset_id( sequence_good_examples: Iterable[Example[str, None]], @@ -315,6 +315,37 @@ def test_eval_runs_returns_generic_statistics( assert evaluation_overview.failed_evaluation_count == 1 +def test_eval_runs_works_without_description( + run_id: str, + in_memory_dataset_repository: InMemoryDatasetRepository, + in_memory_run_repository: InMemoryRunRepository, + in_memory_evaluation_repository: InMemoryEvaluationRepository, + dummy_eval_logic: DummyEvaluationLogic, +) -> None: + evaluator = Evaluator( + in_memory_dataset_repository, + in_memory_run_repository, + in_memory_evaluation_repository, + "", + dummy_eval_logic, + ) + evaluation_overview = evaluator.evaluate_runs(run_id) + + assert evaluation_overview.description == evaluator.description + + +def test_eval_runs_uses_correct_description( + dummy_evaluator: Evaluator[str, str, None, DummyEvaluation], run_id: str +) -> None: + eval_description = "My evaluation description" + evaluation_overview = dummy_evaluator.evaluate_runs( + run_id, description=eval_description + ) + + assert dummy_evaluator.description in evaluation_overview.description + assert eval_description in evaluation_overview.description + + def test_eval_runs_keeps_example_for_eval_if_skip_flag_is_false( dummy_pairwise_evaluator: Evaluator[str, str, None, DummyEvaluation], dummy_runner: Runner[str, str], @@ -345,50 +376,25 @@ def test_eval_runs_keeps_example_for_eval_if_skip_flag_is_false( def test_evaluator_aborts_on_error( dummy_evaluator: Evaluator[str, str, None, DummyEvaluation], - dummy_aggregator: Aggregator[ - DummyEvaluation, DummyAggregatedEvaluationWithResultList - ], - dummy_runner: Runner[str, str], - dataset_id: str, + run_id: str, ) -> None: - run_overview = dummy_runner.run_dataset(dataset_id) - with pytest.raises(RuntimeError): - dummy_evaluator.evaluate_runs(run_overview.id, abort_on_error=True) - - -def test_eval_and_aggregate_runs_uses_passed_tracer( - dummy_evaluator: Evaluator[str, str, None, DummyEvaluation], - dummy_aggregator: Aggregator[ - DummyEvaluation, DummyAggregatedEvaluationWithResultList - ], - dataset_id: str, - dummy_runner: Runner[str, str], -) -> None: - in_memory_tracer = InMemoryTracer() - run_overview = dummy_runner.run_dataset(dataset_id, in_memory_tracer) - evaluation_overview = dummy_evaluator.evaluate_runs(run_overview.id) - dummy_aggregator.aggregate_evaluation(evaluation_overview.id) - - entries = in_memory_tracer.entries - assert len(entries) == 3 - assert all([isinstance(e, InMemoryTaskSpan) for e in entries]) + dummy_evaluator.evaluate_runs(run_id, abort_on_error=True) def test_eval_and_aggregate_runs_stores_example_evaluations( - dummy_runner: Runner[str, str], dummy_evaluator: Evaluator[str, str, None, DummyEvaluation], dummy_aggregator: Aggregator[ DummyEvaluation, DummyAggregatedEvaluationWithResultList ], dataset_id: str, + run_id: str, ) -> None: evaluation_repository = dummy_evaluator._evaluation_repository dataset_repository = dummy_evaluator._dataset_repository examples = list(dataset_repository.examples(dataset_id, str, type(None))) - run_overview = dummy_runner.run_dataset(dataset_id, NoOpTracer()) - evaluation_overview = dummy_evaluator.evaluate_runs(run_overview.id) + evaluation_overview = dummy_evaluator.evaluate_runs(run_id) aggregation_overview = dummy_aggregator.aggregate_evaluation(evaluation_overview.id) assert next(iter(aggregation_overview.evaluation_overviews)) == evaluation_overview @@ -416,13 +422,12 @@ def test_eval_and_aggregate_runs_stores_example_evaluations( def test_failed_evaluations_returns_only_failed_evaluations( - dummy_runner: Runner[str, str], dummy_evaluator: Evaluator[str, str, None, DummyEvaluation], dataset_id: str, + run_id: str, sequence_examples: Iterable[Example[str, None]], ) -> None: - run_overview = dummy_runner.run_dataset(dataset_id, NoOpTracer()) - evaluation_overview = dummy_evaluator.evaluate_runs(run_overview.id) + evaluation_overview = dummy_evaluator.evaluate_runs(run_id) failed_evaluations = list( dummy_evaluator.failed_evaluations(evaluation_overview.id) ) @@ -475,13 +480,12 @@ def test_eval_and_aggregate_runs_stores_aggregated_results( dummy_aggregator: Aggregator[ DummyEvaluation, DummyAggregatedEvaluationWithResultList ], - dummy_runner: Runner[str, str], dataset_id: str, + run_id: str, ) -> None: aggregation_repository = dummy_aggregator._aggregation_repository - run_overview = dummy_runner.run_dataset(dataset_id) - evaluation_overview = dummy_evaluator.evaluate_runs(run_overview.id) + evaluation_overview = dummy_evaluator.evaluate_runs(run_id) aggregation_overview = dummy_aggregator.aggregate_evaluation(evaluation_overview.id) loaded_evaluation_run_overview = aggregation_repository.aggregation_overview( aggregation_overview.id, DummyAggregatedEvaluationWithResultList diff --git a/tests/evaluation/test_runner.py b/tests/evaluation/test_runner.py index d07021b0b..dee8260ae 100644 --- a/tests/evaluation/test_runner.py +++ b/tests/evaluation/test_runner.py @@ -2,7 +2,7 @@ import pytest -from intelligence_layer.core import InMemoryTracer +from intelligence_layer.core import InMemoryTaskSpan, InMemoryTracer from intelligence_layer.evaluation import ( Example, InMemoryDatasetRepository, @@ -42,6 +42,41 @@ def test_runner_runs_dataset( assert failed_runs[0].example.id == examples[1].id +def test_runner_works_without_description( + in_memory_dataset_repository: InMemoryDatasetRepository, + in_memory_run_repository: InMemoryRunRepository, + sequence_examples: Iterable[Example[str, None]], +) -> None: + examples = list(sequence_examples) + task = DummyTask() + runner = Runner(task, in_memory_dataset_repository, in_memory_run_repository, "") + + dataset_id = in_memory_dataset_repository.create_dataset( + examples=examples, dataset_name="" + ).id + overview = runner.run_dataset(dataset_id) + assert overview.description is runner.description + + +def test_runner_has_correct_description( + in_memory_dataset_repository: InMemoryDatasetRepository, + in_memory_run_repository: InMemoryRunRepository, + sequence_examples: Iterable[Example[str, None]], +) -> None: + examples = list(sequence_examples) + task = DummyTask() + runner = Runner(task, in_memory_dataset_repository, in_memory_run_repository, "foo") + + dataset_id = in_memory_dataset_repository.create_dataset( + examples=examples, dataset_name="" + ).id + run_description = "bar" + overview = runner.run_dataset(dataset_id, description=run_description) + + assert runner.description in overview.description + assert run_description in overview.description + + def test_runner_aborts_on_error( in_memory_dataset_repository: InMemoryDatasetRepository, in_memory_run_repository: InMemoryRunRepository, @@ -83,3 +118,7 @@ def test_runner_runs_n_examples( assert overview.successful_example_count == 1 assert overview_with_tracer.successful_example_count == 1 assert overview_with_tracer.failed_example_count == 0 + + entries = tracer.entries + assert len(entries) == 1 + assert all([isinstance(e, InMemoryTaskSpan) for e in entries]) diff --git a/tests/use_cases/classify/test_classify.py b/tests/use_cases/classify/test_classify.py index 80fddae2a..882c61c04 100644 --- a/tests/use_cases/classify/test_classify.py +++ b/tests/use_cases/classify/test_classify.py @@ -19,6 +19,7 @@ InMemoryEvaluationRepository, InMemoryRunRepository, Runner, + SuccessfulExampleOutput, ) from intelligence_layer.examples import ( AggregatedLabelInfo, @@ -31,6 +32,7 @@ MultiLabelClassifyEvaluation, MultiLabelClassifyEvaluationLogic, MultiLabelClassifyOutput, + Probability, ) @@ -216,6 +218,69 @@ def classify_runner( ) +def test_multi_label_eval_logic_works_correctly() -> None: + threshold = 0.5 + eval_logic = MultiLabelClassifyEvaluationLogic(threshold=threshold) + tp = "aaaa" + tn = "bbbb" + fp = "cccc" + fn = "dddd" + expected_output: Sequence[str] = [tp, fn] + input_example = Example( + input=ClassifyInput(chunk=TextChunk("..."), labels=frozenset([tp, tn, fp, fn])), + expected_output=expected_output, + ) + input_output = SuccessfulExampleOutput( + run_id="", + example_id="", + output=MultiLabelClassifyOutput( + scores={ + tp: Probability(threshold + 0.1), + tn: Probability(threshold - 0.1), + fp: Probability(threshold + 0.1), + fn: Probability(threshold - 0.1), + } + ), + ) + res = eval_logic.do_evaluate(input_example, input_output) + assert tp in res.tp + assert tn in res.tn + assert fp in res.fp + assert fn in res.fn + + +def test_multi_label_eval_logic_works_if_everything_is_over_threshold() -> None: + threshold = 0.5 + eval_logic = MultiLabelClassifyEvaluationLogic(threshold=threshold) + tp = "aaaa" + tn = "bbbb" + fp = "cccc" + fn = "dddd" + expected_output: Sequence[str] = [tp, fn] + + input_example = Example( + input=ClassifyInput(chunk=TextChunk("..."), labels=frozenset([tp, tn, fp, fn])), + expected_output=expected_output, + ) + input_output = SuccessfulExampleOutput( + run_id="", + example_id="", + output=MultiLabelClassifyOutput( + scores={ + tp: Probability(threshold + 0.1), + tn: Probability(threshold + 0.1), + fp: Probability(threshold + 0.1), + fn: Probability(threshold + 0.1), + } + ), + ) + res = eval_logic.do_evaluate(input_example, input_output) + assert tp in res.tp + assert tn in res.fp + assert fp in res.fp + assert fn in res.tp + + def test_multi_label_classify_evaluator_single_example( single_entry_dataset_name: str, classify_evaluator: Evaluator[