From 75a999431e6a707b338fa892fd82eeac1f97f776 Mon Sep 17 00:00:00 2001 From: Krishnan <68746496+gemsteam@users.noreply.github.com> Date: Fri, 18 Oct 2024 02:52:49 +0530 Subject: [PATCH 01/18] source-castor-edc contribution from gemsteam (#46759) --- .../connectors/source-castor-edc/README.md | 36 + .../acceptance-test-config.yml | 17 + .../connectors/source-castor-edc/icon.svg | 7 + .../source-castor-edc/manifest.yaml | 2516 +++++++++++++++++ .../source-castor-edc/metadata.yaml | 35 + docs/integrations/sources/castor-edc.md | 48 + 6 files changed, 2659 insertions(+) create mode 100644 airbyte-integrations/connectors/source-castor-edc/README.md create mode 100644 airbyte-integrations/connectors/source-castor-edc/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-castor-edc/icon.svg create mode 100644 airbyte-integrations/connectors/source-castor-edc/manifest.yaml create mode 100644 airbyte-integrations/connectors/source-castor-edc/metadata.yaml create mode 100644 docs/integrations/sources/castor-edc.md diff --git a/airbyte-integrations/connectors/source-castor-edc/README.md b/airbyte-integrations/connectors/source-castor-edc/README.md new file mode 100644 index 000000000000..ca9376b23241 --- /dev/null +++ b/airbyte-integrations/connectors/source-castor-edc/README.md @@ -0,0 +1,36 @@ +# Castor EDC +This directory contains the manifest-only connector for [`source-castor-edc`](https://uk.castoredc.com/api#/). + +Documentation: https://uk.castoredc.com/api#/ + +## Authentication +Visit `https://YOUR_REGION.castoredc.com/account/settings` for getting your client id and secret + +## Usage +There are multiple ways to use this connector: +- You can use this connector as any other connector in Airbyte Marketplace. +- You can load this connector in `pyairbyte` using `get_source`! +- You can open this connector in Connector Builder, edit it, and publish to your workspaces. + +Please refer to the manifest-only connector documentation for more details. + +## Local Development +We recommend you use the Connector Builder to edit this connector. + +But, if you want to develop this connector locally, you can use the following steps. + +### Environment Setup +You will need `airbyte-ci` installed. You can find the documentation [here](airbyte-ci). + +### Build +This will create a dev image (`source-castor-edc:dev`) that you can use to test the connector locally. +```bash +airbyte-ci connectors --name=source-castor-edc build +``` + +### Test +This will run the acceptance tests for the connector. +```bash +airbyte-ci connectors --name=source-castor-edc test +``` + diff --git a/airbyte-integrations/connectors/source-castor-edc/acceptance-test-config.yml b/airbyte-integrations/connectors/source-castor-edc/acceptance-test-config.yml new file mode 100644 index 000000000000..ca0702484d70 --- /dev/null +++ b/airbyte-integrations/connectors/source-castor-edc/acceptance-test-config.yml @@ -0,0 +1,17 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-castor-edc:dev +acceptance_tests: + spec: + tests: + - spec_path: "manifest.yaml" + connection: + bypass_reason: "This is a builder contribution, and we do not have secrets at this time" + discovery: + bypass_reason: "This is a builder contribution, and we do not have secrets at this time" + basic_read: + bypass_reason: "This is a builder contribution, and we do not have secrets at this time" + incremental: + bypass_reason: "This is a builder contribution, and we do not have secrets at this time" + full_refresh: + bypass_reason: "This is a builder contribution, and we do not have secrets at this time" diff --git a/airbyte-integrations/connectors/source-castor-edc/icon.svg b/airbyte-integrations/connectors/source-castor-edc/icon.svg new file mode 100644 index 000000000000..3ab343648610 --- /dev/null +++ b/airbyte-integrations/connectors/source-castor-edc/icon.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/airbyte-integrations/connectors/source-castor-edc/manifest.yaml b/airbyte-integrations/connectors/source-castor-edc/manifest.yaml new file mode 100644 index 000000000000..cb637767b055 --- /dev/null +++ b/airbyte-integrations/connectors/source-castor-edc/manifest.yaml @@ -0,0 +1,2516 @@ +version: 5.12.0 + +type: DeclarativeSource + +description: "Documentation: https://uk.castoredc.com/api#/" + +check: + type: CheckStream + stream_names: + - user + +definitions: + streams: + user: + type: DeclarativeStream + name: user + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: user + http_method: GET + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + max_retries: 5 + response_filters: + - type: HttpResponseFilter + action: RATE_LIMITED + http_codes: + - 429 + error_message: Rate limits has been hit + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 5 + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - _embedded + - user + incremental_sync: + type: DatetimeBasedCursor + cursor_field: last_login + cursor_datetime_formats: + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%d %H:%M:%S" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config[\"start_date\"] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/user" + study: + type: DeclarativeStream + name: study + primary_key: + - crf_id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: study + http_method: GET + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + max_retries: 5 + response_filters: + - type: HttpResponseFilter + action: RATE_LIMITED + http_codes: + - 429 + error_message: Rate limits has been hit + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 5 + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - _embedded + - study + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 50 + start_from_page: 1 + inject_on_first_request: true + incremental_sync: + type: DatetimeBasedCursor + cursor_field: created_on + cursor_datetime_formats: + - "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%d %H:%M:%S" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config[\"start_date\"] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/study" + audit_trial: + type: DeclarativeStream + name: audit_trial + primary_key: + - uuid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: study/{{ stream_partition['study_id'] }}/audit-trail + http_method: GET + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + max_retries: 5 + response_filters: + - type: HttpResponseFilter + action: RATE_LIMITED + http_codes: + - 429 + error_message: Rate limits has been hit + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 5 + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - items + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 1 + inject_on_first_request: true + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: study_id + partition_field: study_id + stream: + $ref: "#/definitions/streams/study" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: date + cursor_datetime_formats: + - "%Y-%m-%d %H:%M:%S" + - "%Y-%m-%d %H:%M:%S.%f+00:00" + datetime_format: "%Y-%m-%d" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config[\"start_date\"] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: date_from + inject_into: request_parameter + end_time_option: + type: RequestOption + field_name: date_to + inject_into: request_parameter + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + transformations: + - type: AddFields + fields: + - path: + - date + value: "{{ record[\"datetime\"][\"date\"].split(\".\")[0] }}" + - type: AddFields + fields: + - path: + - uuid + value: "{{ now_utc() }}" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/audit_trial" + country: + type: DeclarativeStream + name: country + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: country + http_method: GET + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + max_retries: 5 + response_filters: + - type: HttpResponseFilter + action: RATE_LIMITED + http_codes: + - 429 + error_message: Rate limits has been hit + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 5 + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - results + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/country" + study_survey: + type: DeclarativeStream + name: study_survey + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: study/{{ stream_partition['study_id'] }}/survey + http_method: GET + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + max_retries: 5 + response_filters: + - type: HttpResponseFilter + action: RATE_LIMITED + http_codes: + - 429 + error_message: Rate limits has been hit + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 5 + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - _embedded + - surveys + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 1 + inject_on_first_request: true + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: study_id + partition_field: study_id + stream: + $ref: "#/definitions/streams/study" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/study_survey" + study_survey_package: + type: DeclarativeStream + name: study_survey_package + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: study/{{ stream_partition['study_id'] }}/survey-package + http_method: GET + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + max_retries: 5 + response_filters: + - type: HttpResponseFilter + action: RATE_LIMITED + http_codes: + - 429 + error_message: Rate limits has been hit + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 5 + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - _embedded + - survey_packages + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 1 + inject_on_first_request: true + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: study_id + partition_field: study_id + stream: + $ref: "#/definitions/streams/study" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/study_survey_package" + study_site: + type: DeclarativeStream + name: study_site + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: study/{{ stream_partition['study_id'] }}/site + http_method: GET + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + max_retries: 5 + response_filters: + - type: HttpResponseFilter + action: RATE_LIMITED + http_codes: + - 429 + error_message: Rate limits has been hit + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 5 + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - _embedded + - sites + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 1 + inject_on_first_request: true + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: study_id + partition_field: study_id + stream: + $ref: "#/definitions/streams/study" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/study_site" + study_fields: + type: DeclarativeStream + name: study_fields + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: study/{{ stream_partition['study_id'] }}/field + http_method: GET + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + max_retries: 5 + response_filters: + - type: HttpResponseFilter + action: RATE_LIMITED + http_codes: + - 429 + error_message: Rate limits has been hit + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 5 + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - _embedded + - fields + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 1 + inject_on_first_request: true + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: study_id + partition_field: study_id + stream: + $ref: "#/definitions/streams/study" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/study_fields" + study_field_dependency: + type: DeclarativeStream + name: study_field_dependency + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: study/{{ stream_partition['study_id'] }}/field-dependency + http_method: GET + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + max_retries: 5 + response_filters: + - type: HttpResponseFilter + action: RATE_LIMITED + http_codes: + - 429 + error_message: Rate limits has been hit + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 5 + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - _embedded + - fieldDependencies + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 1 + inject_on_first_request: true + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: study_id + partition_field: study_id + stream: + $ref: "#/definitions/streams/study" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/study_field_dependency" + study_field_validation: + type: DeclarativeStream + name: study_field_validation + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: study/{{ stream_partition['study_id'] }}/field-validation + http_method: GET + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + max_retries: 5 + response_filters: + - type: HttpResponseFilter + action: RATE_LIMITED + http_codes: + - 429 + error_message: Rate limits has been hit + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 5 + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - _embedded + - fieldValidations + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 1 + inject_on_first_request: true + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: study_id + partition_field: study_id + stream: + $ref: "#/definitions/streams/study" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/study_field_validation" + study_form: + type: DeclarativeStream + name: study_form + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: study/{{ stream_partition['study_id'] }}/form + http_method: GET + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + max_retries: 5 + response_filters: + - type: HttpResponseFilter + action: RATE_LIMITED + http_codes: + - 429 + error_message: Rate limits has been hit + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 5 + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - _embedded + - forms + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 1 + inject_on_first_request: true + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: study_id + partition_field: study_id + stream: + $ref: "#/definitions/streams/study" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/study_form" + study_role: + type: DeclarativeStream + name: study_role + primary_key: + - uuid + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: study/{{ stream_partition['study_id'] }}/role + http_method: GET + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + max_retries: 5 + response_filters: + - type: HttpResponseFilter + action: RATE_LIMITED + http_codes: + - 429 + error_message: Rate limits has been hit + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 5 + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - _embedded + - roles + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 1 + inject_on_first_request: true + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: study_id + partition_field: study_id + stream: + $ref: "#/definitions/streams/study" + transformations: + - type: AddFields + fields: + - path: + - uuid + value: "{{ now_utc() }}" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/study_role" + study_fieldoption_groups: + type: DeclarativeStream + name: study_fieldoption_groups + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: study/{{ stream_partition['study_id'] }}/field-optiongroup + http_method: GET + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + max_retries: 5 + response_filters: + - type: HttpResponseFilter + action: RATE_LIMITED + http_codes: + - 429 + error_message: Rate limits has been hit + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 5 + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - _embedded + - fieldOptionGroups + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 1 + inject_on_first_request: true + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: study_id + partition_field: study_id + stream: + $ref: "#/definitions/streams/study" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/study_fieldoption_groups" + study_statistics: + type: DeclarativeStream + name: study_statistics + primary_key: + - study_id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: study/{{ stream_partition['study_id'] }}/statistics + http_method: GET + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + max_retries: 5 + response_filters: + - type: HttpResponseFilter + action: RATE_LIMITED + http_codes: + - 429 + error_message: Rate limits has been hit + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 5 + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 1 + inject_on_first_request: true + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: study_id + partition_field: study_id + stream: + $ref: "#/definitions/streams/study" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/study_statistics" + study_user: + type: DeclarativeStream + name: study_user + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: study/{{ stream_partition['study_id'] }}/user + http_method: GET + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + max_retries: 5 + response_filters: + - type: HttpResponseFilter + action: RATE_LIMITED + http_codes: + - 429 + error_message: Rate limits has been hit + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 5 + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - _embedded + - studyUsers + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 1 + inject_on_first_request: true + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: study_id + partition_field: study_id + stream: + $ref: "#/definitions/streams/study" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: last_login + cursor_datetime_formats: + - "%Y-%m-%d %H:%M:%S" + - "%Y-%m-%d %H:%M:%S.%f+00:00" + datetime_format: "%Y-%m-%d" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config[\"start_date\"] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: date_from + inject_into: request_parameter + end_time_option: + type: RequestOption + field_name: date_to + inject_into: request_parameter + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/study_user" + study_visit: + type: DeclarativeStream + name: study_visit + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: study/{{ stream_partition['study_id'] }}/visit + http_method: GET + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + max_retries: 5 + response_filters: + - type: HttpResponseFilter + action: RATE_LIMITED + http_codes: + - 429 + error_message: Rate limits has been hit + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 5 + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - _embedded + - visits + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 1000 + start_from_page: 1 + inject_on_first_request: true + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: study_id + partition_field: study_id + stream: + $ref: "#/definitions/streams/study" + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/study_visit" + base_requester: + type: HttpRequester + url_base: https://{{ config['url_region'] }}.castoredc.com/api/ + authenticator: + type: OAuthAuthenticator + client_id: "{{ config[\"client_id\"] }}" + grant_type: client_credentials + client_secret: "{{ config[\"client_secret\"] }}" + refresh_request_body: {} + token_refresh_endpoint: https://{{ config['url_region'] }}.castoredc.com/oauth/token + +streams: + - $ref: "#/definitions/streams/user" + - $ref: "#/definitions/streams/study" + - $ref: "#/definitions/streams/audit_trial" + - $ref: "#/definitions/streams/country" + - $ref: "#/definitions/streams/study_survey" + - $ref: "#/definitions/streams/study_survey_package" + - $ref: "#/definitions/streams/study_site" + - $ref: "#/definitions/streams/study_fields" + - $ref: "#/definitions/streams/study_field_dependency" + - $ref: "#/definitions/streams/study_field_validation" + - $ref: "#/definitions/streams/study_form" + - $ref: "#/definitions/streams/study_role" + - $ref: "#/definitions/streams/study_fieldoption_groups" + - $ref: "#/definitions/streams/study_statistics" + - $ref: "#/definitions/streams/study_user" + - $ref: "#/definitions/streams/study_visit" + +spec: + type: Spec + connection_specification: + type: object + $schema: http://json-schema.org/draft-07/schema# + required: + - url_region + - client_id + - client_secret + - start_date + properties: + url_region: + type: string + description: The url region given at time of registration + enum: + - uk + - nl + - us + order: 0 + title: URL Region + default: uk + client_id: + type: string + description: Visit `https://YOUR_REGION.castoredc.com/account/settings` + order: 1 + title: Client ID + airbyte_secret: true + client_secret: + type: string + description: Visit `https://YOUR_REGION.castoredc.com/account/settings` + order: 2 + title: Client secret + airbyte_secret: true + start_date: + type: string + order: 3 + title: Start date + format: date-time + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ + additionalProperties: true + +metadata: + autoImportSchema: + user: true + study: true + audit_trial: true + country: true + study_survey: true + study_survey_package: true + study_site: true + study_fields: true + study_field_dependency: true + study_field_validation: true + study_form: true + study_role: true + study_fieldoption_groups: true + study_statistics: true + study_user: true + study_visit: true + testedStreams: + user: + hasRecords: true + streamHash: e4be83a47840afcbe60015a9fd79b8d6c24616cf + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + study: + hasRecords: true + streamHash: d1dfaa856b56acc506600310b760abe604b1e0da + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + audit_trial: + hasRecords: true + streamHash: 8e795e643c2cbd94f9086cc23768c9089c5cd0db + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + country: + hasRecords: true + streamHash: 4edeb56b6fc5345cf63c7edc8677aec159b9a7fd + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + study_survey: + hasRecords: true + streamHash: 1a2bcfb2eabebc65881f09bee49d55ed0439d689 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + study_survey_package: + hasRecords: true + streamHash: 5660ff42aa3c7ee93c08808f3f4c6c6b3151c694 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + study_site: + hasRecords: true + streamHash: a3884a1f449dc14e5aa53b35e96d3c6ea0d2c9c5 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + study_fields: + hasRecords: true + streamHash: 9bac3ca57ed2c96098ab751a2bb7e77c9e5fe2b9 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + study_field_dependency: + hasRecords: true + streamHash: dce6c36f9d7725f4d5436b6fe086a108238e016e + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + study_field_validation: + hasRecords: true + streamHash: 901fab3a36c14137abad6d19a1bca6475ca062f5 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + study_form: + hasRecords: true + streamHash: a32849d2a3df0337b7c55179d71ba83f2982e462 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + study_role: + hasRecords: true + streamHash: 083cbdd8c959c981e563384f95d5d50e0a47415b + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + study_fieldoption_groups: + hasRecords: true + streamHash: 05422c08007170811db015be8069f5b4f9306496 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + study_statistics: + hasRecords: true + streamHash: 48f4f4de6097cd64c524d3075733cc8ef83f4b93 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + study_user: + hasRecords: true + streamHash: 3628b2c81078905be23d8f2213c825ff65811587 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + study_visit: + hasRecords: true + streamHash: 2eef3043f6f22b55f69e870eaa1c887c29c61eb6 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + assist: {} + +schemas: + user: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + _links: + type: + - object + - "null" + properties: + self: + type: + - object + - "null" + properties: + href: + type: + - string + - "null" + email_address: + type: + - string + - "null" + entity_id: + type: + - string + - "null" + full_name: + type: + - string + - "null" + id: + type: string + last_login: + type: string + name_first: + type: + - string + - "null" + name_last: + type: + - string + - "null" + user_id: + type: + - string + - "null" + required: + - id + - last_login + study: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + version: + type: + - string + - "null" + _links: + type: + - object + - "null" + properties: + self: + type: + - object + - "null" + properties: + href: + type: + - string + - "null" + created_by: + type: + - string + - "null" + created_on: + type: string + crf_id: + type: string + domain: + type: + - string + - "null" + expected_centers: + type: + - number + - "null" + expected_records: + type: + - number + - "null" + gcp_enabled: + type: + - boolean + - "null" + live: + type: + - boolean + - "null" + main_contact: + type: + - string + - "null" + name: + type: + - string + - "null" + premium_support_enabled: + type: + - boolean + - "null" + randomization_enabled: + type: + - boolean + - "null" + slug: + type: + - string + - "null" + study_id: + type: + - string + - "null" + surveys_enabled: + type: + - boolean + - "null" + trial_registry_id: + type: + - string + - "null" + required: + - crf_id + - created_on + audit_trial: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + date: + type: string + datetime: + type: + - object + - "null" + properties: + date: + type: + - string + - "null" + timezone: + type: + - string + - "null" + timezone_type: + type: + - number + - "null" + event_details: + anyOf: + - type: array + - type: object + properties: + Additional Config: + type: string + Allow Step Navigation: + type: string + Auto Lock On Finish: + type: string + Country Id: + type: number + Createdby: + type: string + Createdon: + type: string + DependencyChilds: + type: string + DependencyParents: + type: string + Duration: + type: "null" + Encryption Enabled: + type: boolean + Exclude On Data Export: + type: number + Field Alignment: + type: string + Field Enforce Decimals: + type: "null" + Field Hidden: + type: number + Field Id: + type: string + Field Image: + type: + - "null" + - string + Field Info: + type: string + Field Label: + type: string + Field Length: + type: + - "null" + - number + Field Max: + type: + - "null" + - number + Field Max Label: + type: string + Field Min: + type: + - "null" + - number + Field Min Label: + type: string + Field Number: + type: number + Field Number Display: + type: "null" + Field Option Group: + type: string + Field Required: + type: string + Field Required Description: + type: "null" + Field Slider Step: + type: + - "null" + - number + Field Summary Template: + type: string + Field Type: + type: string + Field Type Description: + type: string + Field Units: + type: string + Field Validation Id: + type: "null" + Field Validation Operator: + type: string + Field Validation Text: + type: string + Field Validation Type: + type: string + Field Validation Value: + type: string + Field Value: + type: "null" + Field Variable Name: + type: + - "null" + - string + Field Visibility: + type: "null" + Footer Text: + type: "null" + Force Step Completion: + type: string + Form Numbering: + type: string + Form Sync Id: + type: string + Institute Abbreviation: + type: string + Institute Code: + type: + - "null" + - string + Institute Id: + type: string + Institute Name: + type: string + Label Bold If Required: + type: boolean + Name: + type: string + Option Groups: + type: "null" + Parent Id: + type: string + Randomize Option Order: + type: boolean + Report Allowed Name Edit: + type: number + Report Description: + type: string + Report Id: + type: string + Report Name: + type: string + Report Naming Strategy: + type: string + Report Step Description: + type: string + Report Step Id: + type: string + Report Step Name: + type: string + Report Step Number: + type: number + Report Type: + type: string + Searchability Enabled: + type: boolean + Show Step Navigator: + type: string + Step Desc: + type: string + Step Id: + type: string + Step Name: + type: string + Step Number: + type: number + Step Status: + type: string + Study Phase Desc: + type: "null" + Study Phase Id: + type: string + Study Phase Name: + type: string + Study Phase Number: + type: "null" + Study Phase Order: + type: number + Survey Auto Send: + type: number + Survey Id: + type: string + Survey Intro Text: + type: string + Survey Name: + type: string + Survey Package Id: + type: "null" + Survey Reminder: + type: number + Survey Required: + type: number + Survey Send After: + type: string + Survey Sender Address: + type: string + Survey Sender Name: + type: string + Survey Step Description: + type: string + Survey Step Footer Text: + type: "null" + Survey Step Id: + type: string + Survey Step Name: + type: string + Survey Step Number: + type: number + Updated By: + type: string + Updated On: + type: string + Updatedby: + type: string + Updatedon: + type: string + Validation: + type: string + event_type: + type: + - string + - "null" + user_email: + type: + - string + - "null" + user_id: + type: + - string + - "null" + user_name: + type: + - string + - "null" + uuid: + type: string + required: + - uuid + - date + country: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + country_cca2: + type: + - string + - "null" + country_cca3: + type: + - string + - "null" + country_id: + type: + - string + - "null" + country_name: + type: + - string + - "null" + country_tld: + type: + - string + - "null" + id: + type: string + required: + - id + study_survey: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + description: + type: + - string + - "null" + _links: + type: + - object + - "null" + properties: + self: + type: + - object + - "null" + properties: + href: + type: + - string + - "null" + id: + type: string + intro_text: + type: + - string + - "null" + name: + type: + - string + - "null" + outro_text: + type: + - string + - "null" + survey_forms: + type: + - array + - "null" + survey_id: + type: + - string + - "null" + required: + - id + study_survey_package: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + description: + type: + - string + - "null" + _embedded: + type: + - object + - "null" + properties: + surveys: + type: + - array + - "null" + items: + type: + - object + - "null" + properties: + description: + type: + - string + - "null" + _links: + type: + - object + - "null" + properties: + self: + type: + - object + - "null" + properties: + href: + type: + - string + - "null" + id: + type: + - string + - "null" + intro_text: + type: + - string + - "null" + name: + type: + - string + - "null" + outro_text: + type: + - string + - "null" + survey_forms: + type: + - array + - "null" + survey_id: + type: + - string + - "null" + _links: + type: + - object + - "null" + properties: + self: + type: + - object + - "null" + properties: + href: + type: + - string + - "null" + allow_open_survey_link: + type: + - boolean + - "null" + allow_step_navigation: + type: + - boolean + - "null" + as_needed: + type: + - boolean + - "null" + auto_lock_on_finish: + type: + - boolean + - "null" + auto_send: + type: + - boolean + - "null" + default_invitation: + type: + - string + - "null" + default_invitation_subject: + type: + - string + - "null" + field_pagination: + type: + - string + - "null" + finish_url: + type: + - string + - "null" + id: + type: string + intro_text: + type: + - string + - "null" + is_mobile: + type: + - boolean + - "null" + is_repeatable: + type: + - boolean + - "null" + is_resumable: + type: + - boolean + - "null" + is_training: + type: + - boolean + - "null" + name: + type: + - string + - "null" + outro_text: + type: + - string + - "null" + sender_email: + type: + - string + - "null" + sender_name: + type: + - string + - "null" + show_step_navigator: + type: + - boolean + - "null" + survey_package_id: + type: + - string + - "null" + required: + - id + study_site: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + _links: + type: + - object + - "null" + properties: + self: + type: + - object + - "null" + properties: + href: + type: + - string + - "null" + abbreviation: + type: + - string + - "null" + code: + type: + - string + - "null" + country_id: + type: + - number + - "null" + date_format: + type: + - string + - "null" + deleted: + type: + - boolean + - "null" + id: + type: string + name: + type: + - string + - "null" + order: + type: + - number + - "null" + site_id: + type: + - string + - "null" + required: + - id + study_fields: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + _links: + type: + - object + - "null" + properties: + self: + type: + - object + - "null" + properties: + href: + type: + - string + - "null" + additional_config: + type: + - string + - "null" + dependency_children: + type: + - array + - "null" + dependency_parents: + type: + - array + - "null" + exclude_on_data_export: + type: + - boolean + - "null" + field_hidden: + type: + - number + - "null" + field_id: + type: + - string + - "null" + field_image: + type: + - string + - "null" + field_info: + type: + - string + - "null" + field_label: + type: + - string + - "null" + field_length: + type: + - number + - "null" + field_max: + type: + - number + - "null" + field_max_label: + type: + - string + - "null" + field_min: + type: + - number + - "null" + field_min_label: + type: + - string + - "null" + field_number: + type: + - number + - "null" + field_required: + type: + - number + - "null" + field_slider_step: + type: + - string + - "null" + field_slider_step_value: + type: + - number + - "null" + field_summary_template: + type: + - string + - "null" + field_type: + type: + - string + - "null" + field_units: + type: + - string + - "null" + field_variable_name: + type: + - string + - "null" + id: + type: string + metadata_points: + type: + - array + - "null" + parent_id: + type: + - string + - "null" + randomize_option_order: + type: + - boolean + - "null" + report_id: + type: + - string + - "null" + validations: + type: + - array + - "null" + required: + - id + study_field_dependency: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + _links: + type: + - object + - "null" + properties: + self: + type: + - object + - "null" + properties: + href: + type: + - string + - "null" + child_id: + type: + - string + - "null" + id: + type: number + operator: + type: + - string + - "null" + parent_id: + type: + - string + - "null" + value: + type: + - string + - "null" + required: + - id + study_field_validation: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + _links: + type: + - object + - "null" + properties: + self: + type: + - object + - "null" + properties: + href: + type: + - string + - "null" + field_id: + type: + - string + - "null" + id: + type: number + operator: + type: + - string + - "null" + text: + type: + - string + - "null" + value: + type: + - string + - "null" + required: + - id + study_form: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + _embedded: + type: + - object + - "null" + properties: + visit: + type: + - object + - "null" + properties: + _links: + type: + - object + - "null" + properties: + self: + type: + - object + - "null" + properties: + href: + type: + - string + - "null" + id: + type: + - string + - "null" + visit_id: + type: + - string + - "null" + visit_name: + type: + - string + - "null" + visit_order: + type: + - number + - "null" + _links: + type: + - object + - "null" + properties: + self: + type: + - object + - "null" + properties: + href: + type: + - string + - "null" + form_description: + type: + - string + - "null" + form_id: + type: + - string + - "null" + form_name: + type: + - string + - "null" + form_order: + type: + - number + - "null" + id: + type: string + required: + - id + study_role: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + description: + type: + - string + - "null" + _links: + type: + - object + - "null" + properties: + self: + type: + - object + - "null" + properties: + href: + type: + - string + - "null" + name: + type: + - string + - "null" + permissions: + type: + - object + - "null" + properties: + add: + type: + - boolean + - "null" + decrypt: + type: + - boolean + - "null" + delete: + type: + - boolean + - "null" + edit: + type: + - boolean + - "null" + email_addresses: + type: + - boolean + - "null" + encrypt: + type: + - boolean + - "null" + export: + type: + - boolean + - "null" + lock: + type: + - boolean + - "null" + query: + type: + - boolean + - "null" + randomization_read: + type: + - boolean + - "null" + randomization_write: + type: + - boolean + - "null" + sdv: + type: + - boolean + - "null" + sign: + type: + - boolean + - "null" + survey_send: + type: + - boolean + - "null" + survey_view: + type: + - boolean + - "null" + view: + type: + - boolean + - "null" + uuid: + type: string + required: + - uuid + study_fieldoption_groups: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + description: + type: + - string + - "null" + _links: + type: + - object + - "null" + properties: + self: + type: + - object + - "null" + properties: + href: + type: + - string + - "null" + id: + type: string + layout: + type: + - boolean + - "null" + name: + type: + - string + - "null" + options: + type: + - array + - "null" + items: + type: + - object + - "null" + properties: + groupOrder: + type: + - number + - "null" + id: + type: + - string + - "null" + name: + type: + - string + - "null" + value: + type: + - string + - "null" + required: + - id + study_statistics: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + _links: + type: + - object + - "null" + properties: + self: + type: + - object + - "null" + properties: + href: + type: + - string + - "null" + records: + type: + - object + - "null" + properties: + institutes: + type: + - array + - "null" + items: + type: + - object + - "null" + properties: + institute_id: + type: + - string + - "null" + institute_name: + type: + - string + - "null" + record_count: + type: + - number + - "null" + total_count: + type: + - number + - "null" + study_id: + type: string + required: + - study_id + study_user: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + _links: + type: + - object + - "null" + properties: + self: + type: + - object + - "null" + properties: + href: + type: + - string + - "null" + email_address: + type: + - string + - "null" + entity_id: + type: + - string + - "null" + full_name: + type: + - string + - "null" + id: + type: string + last_login: + type: string + manage_permissions: + type: + - object + - "null" + properties: + manage_encryption: + type: + - boolean + - "null" + manage_form: + type: + - boolean + - "null" + manage_records: + type: + - boolean + - "null" + manage_settings: + type: + - boolean + - "null" + manage_users: + type: + - boolean + - "null" + name_first: + type: + - string + - "null" + name_last: + type: + - string + - "null" + role_assignments: + type: + - array + - "null" + items: + type: + - object + - "null" + properties: + role_id: + type: + - number + - "null" + role_name: + type: + - string + - "null" + site_id: + type: + - string + - "null" + user_id: + type: + - string + - "null" + required: + - id + - last_login + study_visit: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + _links: + type: + - object + - "null" + properties: + self: + type: + - object + - "null" + properties: + href: + type: + - string + - "null" + id: + type: string + visit_id: + type: + - string + - "null" + visit_name: + type: + - string + - "null" + visit_order: + type: + - number + - "null" + required: + - id diff --git a/airbyte-integrations/connectors/source-castor-edc/metadata.yaml b/airbyte-integrations/connectors/source-castor-edc/metadata.yaml new file mode 100644 index 000000000000..3d199cce6027 --- /dev/null +++ b/airbyte-integrations/connectors/source-castor-edc/metadata.yaml @@ -0,0 +1,35 @@ +metadataSpecVersion: "1.0" +data: + allowedHosts: + hosts: + - "https:" + registryOverrides: + oss: + enabled: true + cloud: + enabled: true + remoteRegistries: + pypi: + enabled: false + packageName: airbyte-source-castor-edc + connectorBuildOptions: + baseImage: docker.io/airbyte/source-declarative-manifest:5.13.0@sha256:ffc5977f59e1f38bf3f5dd70b6fa0520c2450ebf85153c5a8df315b8c918d5c3 + connectorSubtype: api + connectorType: source + definitionId: 2cb45514-7c10-439c-a198-aeb1ddab02cb + dockerImageTag: 0.0.1 + dockerRepository: airbyte/source-castor-edc + githubIssueLabel: source-castor-edc + icon: icon.svg + license: MIT + name: Castor EDC + releaseDate: 2024-10-12 + releaseStage: alpha + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/sources/castor-edc + tags: + - language:manifest-only + - cdk:low-code + ab_internal: + ql: 100 + sl: 100 diff --git a/docs/integrations/sources/castor-edc.md b/docs/integrations/sources/castor-edc.md new file mode 100644 index 000000000000..f9980616b421 --- /dev/null +++ b/docs/integrations/sources/castor-edc.md @@ -0,0 +1,48 @@ +# Castor EDC +This document provides the specifications of the [`source-castor-edc`](https://uk.castoredc.com/api#/), describing the currently available +API endpoints and methods. + +Documentation: https://uk.castoredc.com/api#/ + +## Authentication +Visit `https://YOUR_REGION.castoredc.com/account/settings` for getting your client id and secret + +## Configuration + +| Input | Type | Description | Default Value | +|-------|------|-------------|---------------| +| `url_region` | `string` | URL Region. The url region given at time of registration | uk | +| `client_id` | `string` | Client ID. Visit `https://YOUR_REGION.castoredc.com/account/settings` | | +| `client_secret` | `string` | Client secret. Visit `https://YOUR_REGION.castoredc.com/account/settings` | | +| `start_date` | `string` | Start date. | | + +## Streams +| Stream Name | Primary Key | Pagination | Supports Full Sync | Supports Incremental | +|-------------|-------------|------------|---------------------|----------------------| +| user | id | No pagination | ✅ | ✅ | +| study | crf_id | DefaultPaginator | ✅ | ✅ | +| audit_trial | uuid | DefaultPaginator | ✅ | ✅ | +| country | id | No pagination | ✅ | ❌ | +| study_survey | id | DefaultPaginator | ✅ | ❌ | +| study_survey_package | id | DefaultPaginator | ✅ | ❌ | +| study_site | id | DefaultPaginator | ✅ | ❌ | +| study_fields | id | DefaultPaginator | ✅ | ❌ | +| study_field_dependency | id | DefaultPaginator | ✅ | ❌ | +| study_field_validation | id | DefaultPaginator | ✅ | ❌ | +| study_form | id | DefaultPaginator | ✅ | ❌ | +| study_role | uuid | DefaultPaginator | ✅ | ❌ | +| study_fieldoption_groups | id | DefaultPaginator | ✅ | ❌ | +| study_statistics | study_id | DefaultPaginator | ✅ | ❌ | +| study_user | id | DefaultPaginator | ✅ | ✅ | +| study_visit | id | DefaultPaginator | ✅ | ❌ | + +## Changelog + +
+ Expand to review + +| Version | Date | Pull Request | Subject | +|------------------|-------------------|--------------|----------------| +| 0.0.1 | 2024-10-12 | [46759](https://github.com/airbytehq/airbyte/pull/46759) | Initial release by [@gemsteam](https://github.com/gemsteam) via Connector Builder | + +
From 5552db14f367952989430e6e7bfc154fc2721bd0 Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Thu, 17 Oct 2024 14:28:56 -0700 Subject: [PATCH 02/18] Bulk load CDK: move control flow from tasks into launcher (#46948) --- .../task/DestinationTaskExceptionHandler.kt | 11 ++++- .../cdk/load/task/DestinationTaskLauncher.kt | 10 ++++- .../cdk/load/task/implementor/FailSyncTask.kt | 24 ++++------- .../cdk/load/task/implementor/TeardownTask.kt | 42 ++++++------------- 4 files changed, 38 insertions(+), 49 deletions(-) diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/DestinationTaskExceptionHandler.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/DestinationTaskExceptionHandler.kt index 5a894bae4112..4d8d94a71cf2 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/DestinationTaskExceptionHandler.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/DestinationTaskExceptionHandler.kt @@ -12,9 +12,11 @@ import io.airbyte.cdk.load.state.SyncManager import io.airbyte.cdk.load.state.SyncSuccess import io.airbyte.cdk.load.task.implementor.FailStreamTaskFactory import io.airbyte.cdk.load.task.implementor.FailSyncTaskFactory +import io.airbyte.cdk.load.util.setOnce import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton +import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicReference import kotlinx.coroutines.CancellationException @@ -66,6 +68,7 @@ T : ScopedTask { val log = KotlinLogging.logger {} val onException = AtomicReference(suspend {}) + private val failSyncTaskEnqueued = AtomicBoolean(false) inner class SyncTaskWrapper( private val syncManager: SyncManager, @@ -163,8 +166,12 @@ T : ScopedTask { val task = failStreamTaskFactory.make(this, e, it, kill = true) taskScopeProvider.launch(NoHandlingWrapper(task)) } - val failSyncTask = failSyncTaskFactory.make(this, e) - taskScopeProvider.launch(NoHandlingWrapper(failSyncTask)) + if (failSyncTaskEnqueued.setOnce()) { + val failSyncTask = failSyncTaskFactory.make(this, e) + taskScopeProvider.launch(NoHandlingWrapper(failSyncTask)) + } else { + log.info { "Sync fail task already launched, not triggering a second one" } + } } override suspend fun handleStreamFailure(stream: DestinationStream, e: Exception) { diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/DestinationTaskLauncher.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/DestinationTaskLauncher.kt index 6c219ae19dfa..f2e01d62c24e 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/DestinationTaskLauncher.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/DestinationTaskLauncher.kt @@ -22,9 +22,11 @@ import io.airbyte.cdk.load.task.internal.InputConsumerTask import io.airbyte.cdk.load.task.internal.SpillToDiskTaskFactory import io.airbyte.cdk.load.task.internal.TimedForcedCheckpointFlushTask import io.airbyte.cdk.load.task.internal.UpdateCheckpointsTask +import io.airbyte.cdk.load.util.setOnce import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton +import java.util.concurrent.atomic.AtomicBoolean import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock @@ -108,6 +110,8 @@ class DefaultDestinationTaskLauncher( private val batchUpdateLock = Mutex() private val succeeded = Channel(Channel.UNLIMITED) + private val teardownIsEnqueued = AtomicBoolean(false) + private suspend fun enqueue(task: LeveledTask) { taskScopeProvider.launch(exceptionHandler.withExceptionHandling(task)) } @@ -214,7 +218,11 @@ class DefaultDestinationTaskLauncher( /** Called when a stream is closed. */ override suspend fun handleStreamClosed(stream: DestinationStream) { - enqueue(teardownTaskFactory.make(this)) + if (teardownIsEnqueued.setOnce()) { + enqueue(teardownTaskFactory.make(this)) + } else { + log.info { "Teardown task already enqueued, not enqueuing another one" } + } } /** Called exactly once when all streams are closed. */ diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/FailSyncTask.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/FailSyncTask.kt index cfcb20e51037..dd079f5f40f8 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/FailSyncTask.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/FailSyncTask.kt @@ -8,12 +8,10 @@ import io.airbyte.cdk.load.state.CheckpointManager import io.airbyte.cdk.load.state.SyncManager import io.airbyte.cdk.load.task.DestinationTaskExceptionHandler import io.airbyte.cdk.load.task.ImplementorScope -import io.airbyte.cdk.load.util.setOnce import io.airbyte.cdk.load.write.DestinationWriter import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton -import java.util.concurrent.atomic.AtomicBoolean interface FailSyncTask : ImplementorScope @@ -27,22 +25,17 @@ class DefaultFailSyncTask( private val exception: Exception, private val syncManager: SyncManager, private val checkpointManager: CheckpointManager<*, *>, - private val syncFailedHasRun: AtomicBoolean, ) : FailSyncTask { private val log = KotlinLogging.logger {} override suspend fun execute() { - if (syncFailedHasRun.setOnce()) { - // Ensure any remaining ready state gets captured: don't waste work! - checkpointManager.flushReadyCheckpointMessages() - val result = syncManager.markFailed(exception) // awaits stream completion - log.info { "Calling teardown with failure result $result" } - exceptionHandler.handleSyncFailed() - // Do this cleanup last, after all the tasks have had a decent chance to finish. - destinationWriter.teardown(result) - } else { - log.info { "Fail sync task already initiated, doing nothing." } - } + // Ensure any remaining ready state gets captured: don't waste work! + checkpointManager.flushReadyCheckpointMessages() + val result = syncManager.markFailed(exception) // awaits stream completion + log.info { "Calling teardown with failure result $result" } + exceptionHandler.handleSyncFailed() + // Do this cleanup last, after all the tasks have had a decent chance to finish. + destinationWriter.teardown(result) } } @@ -60,8 +53,6 @@ class DefaultFailSyncTaskFactory( private val checkpointManager: CheckpointManager<*, *>, private val destinationWriter: DestinationWriter ) : FailSyncTaskFactory { - private val syncFailedHasRun = AtomicBoolean(false) - override fun make( exceptionHandler: DestinationTaskExceptionHandler<*, *>, exception: Exception @@ -72,7 +63,6 @@ class DefaultFailSyncTaskFactory( exception, syncManager, checkpointManager, - syncFailedHasRun, ) } } diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/TeardownTask.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/TeardownTask.kt index 7bf839e8948d..04b9c5b2182d 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/TeardownTask.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/TeardownTask.kt @@ -9,12 +9,10 @@ import io.airbyte.cdk.load.state.SyncManager import io.airbyte.cdk.load.task.DestinationTaskLauncher import io.airbyte.cdk.load.task.ImplementorScope import io.airbyte.cdk.load.task.SyncLevel -import io.airbyte.cdk.load.util.setOnce import io.airbyte.cdk.load.write.DestinationWriter import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton -import java.util.concurrent.atomic.AtomicBoolean interface TeardownTask : SyncLevel, ImplementorScope @@ -28,30 +26,24 @@ class DefaultTeardownTask( private val syncManager: SyncManager, private val destination: DestinationWriter, private val taskLauncher: DestinationTaskLauncher, - private val teardownHasRun: AtomicBoolean, ) : TeardownTask { val log = KotlinLogging.logger {} override suspend fun execute() { - // Run the task exactly once, and only after all streams have closed. - if (teardownHasRun.setOnce()) { - syncManager.awaitInputProcessingComplete() - checkpointManager.awaitAllCheckpointsFlushed() + syncManager.awaitInputProcessingComplete() + checkpointManager.awaitAllCheckpointsFlushed() - log.info { "Teardown task awaiting stream completion" } - if (!syncManager.awaitAllStreamsCompletedSuccessfully()) { - log.info { "Streams failed to complete successfully, doing nothing." } - return - } - - log.info { "Starting teardown task" } - destination.teardown() - log.info { "Teardown task complete, marking sync succeeded." } - syncManager.markSucceeded() - taskLauncher.handleTeardownComplete() - } else { - log.info { "Teardown already initiated, doing nothing." } + log.info { "Teardown task awaiting stream completion" } + if (!syncManager.awaitAllStreamsCompletedSuccessfully()) { + log.info { "Streams failed to complete successfully, doing nothing." } + return } + + log.info { "Starting teardown task" } + destination.teardown() + log.info { "Teardown task complete, marking sync succeeded." } + syncManager.markSucceeded() + taskLauncher.handleTeardownComplete() } } @@ -66,15 +58,7 @@ class DefaultTeardownTaskFactory( private val syncManager: SyncManager, private val destination: DestinationWriter, ) : TeardownTaskFactory { - private val teardownHasRun = AtomicBoolean(false) - override fun make(taskLauncher: DestinationTaskLauncher): TeardownTask { - return DefaultTeardownTask( - checkpointManager, - syncManager, - destination, - taskLauncher, - teardownHasRun - ) + return DefaultTeardownTask(checkpointManager, syncManager, destination, taskLauncher) } } From f5413520bb0befc7bb14165f23efd988ff90ec06 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Thu, 17 Oct 2024 14:53:49 -0700 Subject: [PATCH 03/18] Chore(Source-S3): Release v4.9.0 RC (#46973) --- airbyte-integrations/connectors/source-s3/metadata.yaml | 4 ++-- airbyte-integrations/connectors/source-s3/pyproject.toml | 2 +- docs/integrations/sources/s3.md | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-s3/metadata.yaml b/airbyte-integrations/connectors/source-s3/metadata.yaml index 2fea1187d06f..4adbdbbf3546 100644 --- a/airbyte-integrations/connectors/source-s3/metadata.yaml +++ b/airbyte-integrations/connectors/source-s3/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: file connectorType: source definitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2 - dockerImageTag: 4.9.0-rc.1 + dockerImageTag: 4.9.0 dockerRepository: airbyte/source-s3 documentationUrl: https://docs.airbyte.com/integrations/sources/s3 githubIssueLabel: source-s3 @@ -30,7 +30,7 @@ data: releaseStage: generally_available releases: rolloutConfiguration: - enableProgressiveRollout: true + enableProgressiveRollout: false breakingChanges: 4.0.0: message: UX improvement, multi-stream support and deprecation of some parsing features diff --git a/airbyte-integrations/connectors/source-s3/pyproject.toml b/airbyte-integrations/connectors/source-s3/pyproject.toml index 123dd2652670..ca55b03047e6 100644 --- a/airbyte-integrations/connectors/source-s3/pyproject.toml +++ b/airbyte-integrations/connectors/source-s3/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "4.9.0-rc.1" +version = "4.9.0" name = "source-s3" description = "Source implementation for S3." authors = [ "Airbyte ",] diff --git a/docs/integrations/sources/s3.md b/docs/integrations/sources/s3.md index 2ff548532d81..19abf6dfff6c 100644 --- a/docs/integrations/sources/s3.md +++ b/docs/integrations/sources/s3.md @@ -339,6 +339,7 @@ This connector utilizes the open source [Unstructured](https://unstructured-io.g | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------| +| 4.9.0 | 2024-10-17 | [46973](https://github.com/airbytehq/airbyte/pull/46973) | Promote releae candidate. | | 4.9.0-rc.1 | 2024-10-14 | [46298](https://github.com/airbytehq/airbyte/pull/46298) | Migrate to CDK v5 | | 4.8.5 | 2024-10-12 | [46511](https://github.com/airbytehq/airbyte/pull/46511) | Update dependencies | | 4.8.4 | 2024-09-28 | [46131](https://github.com/airbytehq/airbyte/pull/46131) | Update dependencies | From 55a7522af78a87c6e7943df963151d40448c2af7 Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Thu, 17 Oct 2024 14:55:11 -0700 Subject: [PATCH 04/18] Bulk CDK: ConnectorRunner accepts explicit isTest flag (#46920) --- .../io/airbyte/cdk/AirbyteConnectorRunner.kt | 17 ++++++++++++++++- .../kotlin/io/airbyte/cdk/command/CliRunner.kt | 2 +- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/AirbyteConnectorRunner.kt b/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/AirbyteConnectorRunner.kt index bd61711e3745..41b8bf1f1bdf 100644 --- a/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/AirbyteConnectorRunner.kt +++ b/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/AirbyteConnectorRunner.kt @@ -58,7 +58,18 @@ sealed class AirbyteConnectorRunner( val testBeanDefinitions: Array>, val testProperties: Map = emptyMap(), ) { - val envs: Array = arrayOf(Environment.CLI, connectorType) + // Micronaut's TEST env detection relies on inspecting the stacktrace and checking for + // any junit calls. This doesn't work if we launch the connector from a different thread, e.g. + // `Dispatchers.IO`. Force the test env if needed. (Some tests launch the connector from the IO + // context to avoid blocking themselves.) + private val isTest = testBeanDefinitions.isNotEmpty() + val envs: Array = + arrayOf(Environment.CLI, connectorType) + + if (isTest) { + arrayOf(Environment.TEST) + } else { + emptyArray() + } inline fun run() { val picocliCommandLineFactory = PicocliCommandLineFactory(this) @@ -82,6 +93,10 @@ sealed class AirbyteConnectorRunner( ) .beanDefinitions(*testBeanDefinitions) .start() + // We can't rely on the isTest value from our constructor, + // because that won't autodetect junit in our stacktrace. + // So instead we ask micronaut (which will include if we explicitly added + // the TEST env). val isTest: Boolean = ctx.environment.activeNames.contains(Environment.TEST) val picocliFactory: CommandLine.IFactory = MicronautFactory(ctx) val picocliCommandLine: CommandLine = diff --git a/airbyte-cdk/bulk/core/base/src/testFixtures/kotlin/io/airbyte/cdk/command/CliRunner.kt b/airbyte-cdk/bulk/core/base/src/testFixtures/kotlin/io/airbyte/cdk/command/CliRunner.kt index 20a0b68c8c61..44950c69fe0f 100644 --- a/airbyte-cdk/bulk/core/base/src/testFixtures/kotlin/io/airbyte/cdk/command/CliRunner.kt +++ b/airbyte-cdk/bulk/core/base/src/testFixtures/kotlin/io/airbyte/cdk/command/CliRunner.kt @@ -65,7 +65,7 @@ data object CliRunner { args, testProperties, inputBeanDefinition, - out.beanDefinition + out.beanDefinition, ) } return CliRunnable(runnable, out.results) From 871db55443d9265a06ffdde5ad5a33ff9c6c1f3e Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Thu, 17 Oct 2024 15:04:17 -0700 Subject: [PATCH 05/18] Bulk load CDK: nondocker runner throws exception if destination fails (#46921) --- .../io/airbyte/cdk/AirbyteConnectorRunner.kt | 3 ++ .../cdk/ConnectorUncleanExitException.kt | 8 +++++ .../DestinationUncleanExitException.kt | 33 +++++++++++++++++++ .../DockerizedDestination.kt | 24 +------------- .../NonDockerizedDestination.kt | 7 +++- 5 files changed, 51 insertions(+), 24 deletions(-) create mode 100644 airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/ConnectorUncleanExitException.kt create mode 100644 airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DestinationUncleanExitException.kt diff --git a/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/AirbyteConnectorRunner.kt b/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/AirbyteConnectorRunner.kt index 41b8bf1f1bdf..214d0821e2d4 100644 --- a/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/AirbyteConnectorRunner.kt +++ b/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/AirbyteConnectorRunner.kt @@ -105,6 +105,9 @@ sealed class AirbyteConnectorRunner( if (!isTest) { // Required by the platform, otherwise syncs may hang. exitProcess(exitCode) + } else if (exitCode != 0) { + // Otherwise, propagate failure to test callers. + throw ConnectorUncleanExitException(exitCode) } } } diff --git a/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/ConnectorUncleanExitException.kt b/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/ConnectorUncleanExitException.kt new file mode 100644 index 000000000000..19c108b8966d --- /dev/null +++ b/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/ConnectorUncleanExitException.kt @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk + +class ConnectorUncleanExitException(val exitCode: Int) : + Exception("Destination process exited uncleanly: $exitCode") diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DestinationUncleanExitException.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DestinationUncleanExitException.kt new file mode 100644 index 000000000000..f07c7cba70b6 --- /dev/null +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DestinationUncleanExitException.kt @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.load.test.util.destination_process + +import io.airbyte.protocol.models.v0.AirbyteErrorTraceMessage +import io.airbyte.protocol.models.v0.AirbyteTraceMessage + +class DestinationUncleanExitException( + exitCode: Int, + traceMessages: List +) : + Exception( + """ + Destination process exited uncleanly: $exitCode + Trace messages: + """.trimIndent() + // explicit concat because otherwise trimIndent behaves badly + + traceMessages + ) { + companion object { + // generic type erasure strikes again >.> + // this can't just be a second constructor, because both constructors + // would have signature `traceMessages: List`. + // so we have to pull this into a companion object function. + fun of(exitCode: Int, traceMessages: List) = + DestinationUncleanExitException( + exitCode, + traceMessages.filter { it.type == AirbyteTraceMessage.Type.ERROR }.map { it.error } + ) + } +} diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DockerizedDestination.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DockerizedDestination.kt index 576e4ec8ed5c..dd310cccafc8 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DockerizedDestination.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DockerizedDestination.kt @@ -8,10 +8,8 @@ import com.google.common.collect.Lists import io.airbyte.cdk.command.ConfigurationSpecification import io.airbyte.cdk.output.BufferingOutputConsumer import io.airbyte.cdk.util.Jsons -import io.airbyte.protocol.models.v0.AirbyteErrorTraceMessage import io.airbyte.protocol.models.v0.AirbyteLogMessage import io.airbyte.protocol.models.v0.AirbyteMessage -import io.airbyte.protocol.models.v0.AirbyteTraceMessage import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Requires @@ -207,32 +205,12 @@ class DockerizedDestination( stderrDrained.join() val exitCode = process.exitValue() if (exitCode != 0) { - // Hey look, it's possible to extract the error from a failed destination process! - // because "destination exit code 1" is the least-helpful error message. - val filteredTraces = - destinationOutput - .traces() - .filter { it.type == AirbyteTraceMessage.Type.ERROR } - .map { it.error } - throw DestinationUncleanExitException(exitCode, filteredTraces) + throw DestinationUncleanExitException.of(exitCode, destinationOutput.traces()) } } } } -class DestinationUncleanExitException( - exitCode: Int, - traceMessages: List -) : - Exception( - """ - Destination process exited uncleanly: $exitCode - Trace messages: - """.trimIndent() - // explicit concat because otherwise trimIndent behaves badly - + traceMessages - ) - @Singleton @Requires(env = [DOCKERIZED_TEST_ENV]) class DockerizedDestinationFactory( diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/NonDockerizedDestination.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/NonDockerizedDestination.kt index bd669e863df8..60ebaaa03270 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/NonDockerizedDestination.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/NonDockerizedDestination.kt @@ -4,6 +4,7 @@ package io.airbyte.cdk.load.test.util.destination_process +import io.airbyte.cdk.ConnectorUncleanExitException import io.airbyte.cdk.command.CliRunnable import io.airbyte.cdk.command.CliRunner import io.airbyte.cdk.command.ConfigurationSpecification @@ -53,7 +54,11 @@ class NonDockerizedDestination( } override suspend fun run() { - destination.run() + try { + destination.run() + } catch (e: ConnectorUncleanExitException) { + throw DestinationUncleanExitException.of(e.exitCode, destination.results.traces()) + } } override fun sendMessage(message: AirbyteMessage) { From 4a2e7c80121a0ff57abf3591c99ef6bbfbe7690c Mon Sep 17 00:00:00 2001 From: Johnny Schmidt Date: Thu, 17 Oct 2024 17:32:21 -0700 Subject: [PATCH 06/18] Bulk Load CDK: File format Spec & Streaming Upload, S3V2 Usage (#46957) --- .../MockDestinationBackend.kt | 9 +- .../cdk/load/command/DestinationStream.kt | 65 +++++++ .../cdk/load/data/AirbyteValueToJson.kt | 4 + ...DestinationRecordToAirbyteValueWithMeta.kt | 45 +++++ .../cdk/load/data/JsonToAirbyteValue.kt | 4 + .../io/airbyte/cdk/load/message/Batch.kt | 6 - .../cdk/load/message/DestinationMessage.kt | 8 + .../io/airbyte/cdk/load/util/JsonUtils.kt | 16 ++ .../DestinationTaskExceptionHandlerTest.kt | 1 - .../AirbyteValueWithMetaToOutputRecord.kt | 61 +++++++ .../load/test/util/DestinationDataDumper.kt | 13 +- .../cdk/load/test/util/IntegrationTest.kt | 6 +- .../BasicFunctionalityIntegrationTest.kt | 22 +-- .../ObjectStorageFormatSpecification.kt | 79 +++++++++ .../ObjectStorageUploadConfiguration.kt | 11 ++ .../object_storage/ObjectStorageClient.kt | 16 +- .../ObjectStoragePathFactory.kt | 140 +++++++++++++-- .../load/file/object_storage/RemoteObject.kt | 10 ++ .../io/airbyte/cdk/load/file/s3/S3Client.kt | 164 ++++++++++++++++-- .../dev_null/DevNullDestinationDataDumper.kt | 7 +- .../destination-s3-v2/metadata.yaml | 2 +- .../src/main/kotlin/S3V2Checker.kt | 26 ++- .../src/main/kotlin/S3V2Configuration.kt | 20 ++- .../src/main/kotlin/S3V2Specification.kt | 7 +- .../src/main/kotlin/S3V2Writer.kt | 54 +++++- .../destination/s3_v2/S3V2CheckTest.kt | 7 +- .../destination/s3_v2/S3V2TestUtils.kt | 18 ++ .../destination/s3_v2/S3V2WriteTest.kt | 55 +++++- .../resources/expected-spec-cloud.json | 56 +++++- .../resources/expected-spec-oss.json | 56 +++++- 30 files changed, 894 insertions(+), 94 deletions(-) create mode 100644 airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/DestinationRecordToAirbyteValueWithMeta.kt create mode 100644 airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/util/JsonUtils.kt create mode 100644 airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/AirbyteValueWithMetaToOutputRecord.kt create mode 100644 airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/command/object_storage/ObjectStorageFormatSpecification.kt create mode 100644 airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/command/object_storage/ObjectStorageUploadConfiguration.kt create mode 100644 airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/file/object_storage/RemoteObject.kt create mode 100644 airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2TestUtils.kt diff --git a/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockDestinationBackend.kt b/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockDestinationBackend.kt index fb7720bac2fc..c85a94ca1e39 100644 --- a/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockDestinationBackend.kt +++ b/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockDestinationBackend.kt @@ -4,6 +4,8 @@ package io.airbyte.cdk.load.mock_integration_test +import io.airbyte.cdk.command.ConfigurationSpecification +import io.airbyte.cdk.load.command.DestinationStream import io.airbyte.cdk.load.test.util.DestinationDataDumper import io.airbyte.cdk.load.test.util.OutputRecord import java.util.concurrent.ConcurrentHashMap @@ -25,9 +27,12 @@ object MockDestinationBackend { } object MockDestinationDataDumper : DestinationDataDumper { - override fun dumpRecords(streamName: String, streamNamespace: String?): List { + override fun dumpRecords( + spec: ConfigurationSpecification, + stream: DestinationStream + ): List { return MockDestinationBackend.readFile( - MockStreamLoader.getFilename(streamNamespace, streamName) + MockStreamLoader.getFilename(stream.descriptor.namespace, stream.descriptor.name) ) } } diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/command/DestinationStream.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/command/DestinationStream.kt index ec45955b9d58..71587934bbc4 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/command/DestinationStream.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/command/DestinationStream.kt @@ -6,7 +6,13 @@ package io.airbyte.cdk.load.command import io.airbyte.cdk.load.data.AirbyteType import io.airbyte.cdk.load.data.AirbyteTypeToJsonSchema +import io.airbyte.cdk.load.data.ArrayType +import io.airbyte.cdk.load.data.FieldType +import io.airbyte.cdk.load.data.IntegerType import io.airbyte.cdk.load.data.JsonSchemaToAirbyteType +import io.airbyte.cdk.load.data.ObjectType +import io.airbyte.cdk.load.data.StringType +import io.airbyte.cdk.load.message.DestinationRecord import io.airbyte.protocol.models.v0.AirbyteStream import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream import io.airbyte.protocol.models.v0.DestinationSyncMode @@ -40,6 +46,65 @@ data class DestinationStream( fun toPrettyString() = "$namespace.$name" } + /** + * This is the schema of what we currently write to destinations, but this might not reflect + * what actually exists, as many destinations have legacy data from before this schema was + * adopted. + */ + val schemaWithMeta: AirbyteType + get() = + ObjectType( + linkedMapOf( + DestinationRecord.Meta.COLUMN_NAME_AB_RAW_ID to + FieldType(StringType, nullable = false), + DestinationRecord.Meta.COLUMN_NAME_AB_EXTRACTED_AT to + FieldType(IntegerType, nullable = false), + DestinationRecord.Meta.COLUMN_NAME_AB_META to + FieldType( + nullable = false, + type = + ObjectType( + linkedMapOf( + "sync_id" to FieldType(IntegerType, nullable = false), + "changes" to + FieldType( + nullable = false, + type = + ArrayType( + FieldType( + nullable = false, + type = + ObjectType( + linkedMapOf( + "field" to + FieldType( + StringType, + nullable = false + ), + "change" to + FieldType( + StringType, + nullable = false + ), + "reason" to + FieldType( + StringType, + nullable = false + ), + ) + ) + ) + ) + ) + ) + ) + ), + DestinationRecord.Meta.COLUMN_NAME_AB_GENERATION_ID to + FieldType(IntegerType, nullable = false), + DestinationRecord.Meta.COLUMN_NAME_DATA to FieldType(schema, nullable = false), + ) + ) + /** * This is not fully round-trippable. Destinations don't care about most of the stuff in an * AirbyteStream (e.g. we don't care about defaultCursorField, we only care about the _actual_ diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/AirbyteValueToJson.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/AirbyteValueToJson.kt index d4e688dbda60..75b128cf1340 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/AirbyteValueToJson.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/AirbyteValueToJson.kt @@ -29,3 +29,7 @@ class AirbyteValueToJson { } } } + +fun AirbyteValue.toJson(): JsonNode { + return AirbyteValueToJson().convert(this) +} diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/DestinationRecordToAirbyteValueWithMeta.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/DestinationRecordToAirbyteValueWithMeta.kt new file mode 100644 index 000000000000..0c80d9b6f1e9 --- /dev/null +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/DestinationRecordToAirbyteValueWithMeta.kt @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.load.data + +import io.airbyte.cdk.load.command.DestinationCatalog +import io.airbyte.cdk.load.message.DestinationRecord +import io.airbyte.cdk.load.message.DestinationRecord.Meta +import jakarta.inject.Singleton +import java.util.* + +@Singleton +class DestinationRecordToAirbyteValueWithMeta(private val catalog: DestinationCatalog) { + fun decorate(record: DestinationRecord): ObjectValue { + val streamActual = catalog.getStream(record.stream.name, record.stream.namespace) + return ObjectValue( + linkedMapOf( + Meta.COLUMN_NAME_AB_RAW_ID to StringValue(UUID.randomUUID().toString()), + Meta.COLUMN_NAME_AB_EXTRACTED_AT to IntegerValue(record.emittedAtMs), + Meta.COLUMN_NAME_AB_META to + ObjectValue( + linkedMapOf( + "sync_id" to IntegerValue(streamActual.syncId), + "changes" to + ArrayValue( + record.meta?.changes?.map { + ObjectValue( + linkedMapOf( + "field" to StringValue(it.field), + "change" to StringValue(it.change.name), + "reason" to StringValue(it.reason.name) + ) + ) + } + ?: emptyList() + ) + ) + ), + Meta.COLUMN_NAME_AB_GENERATION_ID to IntegerValue(streamActual.generationId), + Meta.COLUMN_NAME_DATA to record.data + ) + ) + } +} diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/JsonToAirbyteValue.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/JsonToAirbyteValue.kt index a0872b4e493c..d7389dc1ee69 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/JsonToAirbyteValue.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/JsonToAirbyteValue.kt @@ -187,3 +187,7 @@ class JsonToAirbyteValue { } } } + +fun JsonNode.toAirbyteValue(schema: AirbyteType): AirbyteValue { + return JsonToAirbyteValue().convert(this, schema) +} diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/Batch.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/Batch.kt index 41e4f38ce737..794f81324141 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/Batch.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/Batch.kt @@ -73,12 +73,6 @@ abstract class StagedLocalFile() : Batch { override val state: Batch.State = Batch.State.LOCAL } -/** Represents a remote object containing persisted records. */ -abstract class RemoteObject() : Batch { - override val state: Batch.State = Batch.State.PERSISTED - abstract val key: String -} - /** * Represents a file of raw records staged to disk for pre-processing. Used internally by the * framework diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/DestinationMessage.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/DestinationMessage.kt index 2fd484bdb0ab..747d0e3b07ef 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/DestinationMessage.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/DestinationMessage.kt @@ -63,6 +63,14 @@ data class DestinationRecord( ) data class Meta(val changes: List?) { + companion object { + const val COLUMN_NAME_AB_RAW_ID: String = "_airbyte_raw_id" + const val COLUMN_NAME_AB_EXTRACTED_AT: String = "_airbyte_extracted_at" + const val COLUMN_NAME_AB_META: String = "_airbyte_meta" + const val COLUMN_NAME_AB_GENERATION_ID: String = "_airbyte_generation_id" + const val COLUMN_NAME_DATA: String = "_airbyte_data" + } + fun asProtocolObject(): AirbyteRecordMessageMeta = AirbyteRecordMessageMeta().also { if (changes != null) { diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/util/JsonUtils.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/util/JsonUtils.kt new file mode 100644 index 000000000000..682354cde481 --- /dev/null +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/util/JsonUtils.kt @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.load.util + +import com.fasterxml.jackson.databind.JsonNode +import io.airbyte.cdk.util.Jsons + +fun JsonNode.serializeToString(): String { + return Jsons.writeValueAsString(this) +} + +fun String.deserializeToNode(): JsonNode { + return Jsons.readTree(this) +} diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/DestinationTaskExceptionHandlerTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/DestinationTaskExceptionHandlerTest.kt index 1b3083dacb7e..61b0733920f3 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/DestinationTaskExceptionHandlerTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/DestinationTaskExceptionHandlerTest.kt @@ -125,7 +125,6 @@ class DestinationTaskExceptionHandlerTest where T : LeveledTask, T : ScopedTa (stream, exception, kill) -> stream to Pair(exception, kill) } - println(streamResults) catalog.streams.forEach { stream -> Assertions.assertTrue(streamResults[stream]!!.first is RuntimeException) Assertions.assertTrue(streamResults[stream]!!.second) diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/AirbyteValueWithMetaToOutputRecord.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/AirbyteValueWithMetaToOutputRecord.kt new file mode 100644 index 000000000000..8248daa09b0b --- /dev/null +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/AirbyteValueWithMetaToOutputRecord.kt @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.load.test.util + +import io.airbyte.cdk.load.data.AirbyteValue +import io.airbyte.cdk.load.data.ArrayValue +import io.airbyte.cdk.load.data.IntegerValue +import io.airbyte.cdk.load.data.ObjectValue +import io.airbyte.cdk.load.data.StringValue +import io.airbyte.cdk.load.message.DestinationRecord +import io.airbyte.protocol.models.v0.AirbyteRecordMessageMetaChange +import java.time.Instant +import java.util.* + +class AirbyteValueWithMetaToOutputRecord { + fun convert(value: ObjectValue): OutputRecord { + val meta = value.values[DestinationRecord.Meta.COLUMN_NAME_AB_META] as ObjectValue + return OutputRecord( + rawId = + UUID.fromString( + (value.values[DestinationRecord.Meta.COLUMN_NAME_AB_RAW_ID] as StringValue) + .value + ), + extractedAt = + Instant.ofEpochMilli( + (value.values[DestinationRecord.Meta.COLUMN_NAME_AB_EXTRACTED_AT] + as IntegerValue) + .value + ), + loadedAt = null, + data = value.values[DestinationRecord.Meta.COLUMN_NAME_DATA] as ObjectValue, + generationId = + (value.values[DestinationRecord.Meta.COLUMN_NAME_AB_GENERATION_ID] as IntegerValue) + .value, + airbyteMeta = + OutputRecord.Meta( + syncId = (meta.values["sync_id"] as IntegerValue).value, + changes = + (meta.values["changes"] as ArrayValue).values.map { + DestinationRecord.Change( + field = ((it as ObjectValue).values["field"] as StringValue).value, + change = + AirbyteRecordMessageMetaChange.Change.fromValue( + (it.values["change"] as StringValue).value + ), + reason = + AirbyteRecordMessageMetaChange.Reason.fromValue( + (it.values["reason"] as StringValue).value + ) + ) + } + ) + ) + } +} + +fun AirbyteValue.toOutputRecord(): OutputRecord { + return AirbyteValueWithMetaToOutputRecord().convert(this as ObjectValue) +} diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/DestinationDataDumper.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/DestinationDataDumper.kt index 93d68879df95..f02454d92b2d 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/DestinationDataDumper.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/DestinationDataDumper.kt @@ -4,11 +4,11 @@ package io.airbyte.cdk.load.test.util +import io.airbyte.cdk.command.ConfigurationSpecification +import io.airbyte.cdk.load.command.DestinationStream + fun interface DestinationDataDumper { - fun dumpRecords( - streamName: String, - streamNamespace: String?, - ): List + fun dumpRecords(spec: ConfigurationSpecification, stream: DestinationStream): List } /** @@ -16,7 +16,10 @@ fun interface DestinationDataDumper { * implementation to satisfy the compiler. */ object FakeDataDumper : DestinationDataDumper { - override fun dumpRecords(streamName: String, streamNamespace: String?): List { + override fun dumpRecords( + spec: ConfigurationSpecification, + stream: DestinationStream + ): List { throw NotImplementedError() } } diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/IntegrationTest.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/IntegrationTest.kt index 7ba77b868d0b..7c15388a4e8d 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/IntegrationTest.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/IntegrationTest.kt @@ -104,13 +104,13 @@ abstract class IntegrationTest( } fun dumpAndDiffRecords( + config: ConfigurationSpecification, canonicalExpectedRecords: List, - streamName: String, - streamNamespace: String?, + stream: DestinationStream, primaryKey: List>, cursor: List?, ) { - val actualRecords: List = dataDumper.dumpRecords(streamName, streamNamespace) + val actualRecords: List = dataDumper.dumpRecords(config, stream) val expectedRecords: List = canonicalExpectedRecords.map { recordMangler.mapRecord(it) } diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/write/BasicFunctionalityIntegrationTest.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/write/BasicFunctionalityIntegrationTest.kt index 2672a956b3fd..c881602a9df8 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/write/BasicFunctionalityIntegrationTest.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/write/BasicFunctionalityIntegrationTest.kt @@ -38,17 +38,19 @@ abstract class BasicFunctionalityIntegrationTest( ) : IntegrationTest(dataDumper, destinationCleaner, recordMangler, nameMapper) { @Test open fun testBasicWrite() { + val stream = + DestinationStream( + DestinationStream.Descriptor(randomizedNamespace, "test_stream"), + Append, + ObjectTypeWithoutSchema, + generationId = 0, + minimumGenerationId = 0, + syncId = 42, + ) val messages = runSync( config, - DestinationStream( - DestinationStream.Descriptor(randomizedNamespace, "test_stream"), - Append, - ObjectTypeWithoutSchema, - generationId = 0, - minimumGenerationId = 0, - syncId = 42, - ), + stream, listOf( DestinationRecord( namespace = randomizedNamespace, @@ -98,6 +100,7 @@ abstract class BasicFunctionalityIntegrationTest( { if (verifyDataWriting) { dumpAndDiffRecords( + config, listOf( OutputRecord( extractedAt = 1234, @@ -121,8 +124,7 @@ abstract class BasicFunctionalityIntegrationTest( ) ) ), - "test_stream", - randomizedNamespace, + stream, primaryKey = listOf(listOf("id")), cursor = null, ) diff --git a/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/command/object_storage/ObjectStorageFormatSpecification.kt b/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/command/object_storage/ObjectStorageFormatSpecification.kt new file mode 100644 index 000000000000..9f51099e8d23 --- /dev/null +++ b/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/command/object_storage/ObjectStorageFormatSpecification.kt @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.load.command.object_storage + +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonPropertyDescription +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonTypeInfo +import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle + +interface ObjectStorageFormatSpecificationProvider { + @get:JsonSchemaTitle("Output Format") + @get:JsonPropertyDescription( + "Format of the data output. See here for more details", + ) + val format: ObjectStorageFormatSpecification + + fun toObjectStorageFormatConfiguration(): ObjectStorageFormatConfiguration { + return when (format) { + is JsonFormatSpecification -> JsonFormatConfiguration() + is CSVFormatSpecification -> CSVFormatConfiguration() + is AvroFormatSpecification -> AvroFormatConfiguration() + is ParquetFormatSpecification -> ParquetFormatConfiguration() + } + } +} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.EXISTING_PROPERTY, + property = "format_type" +) +@JsonSubTypes( + JsonSubTypes.Type(value = JsonFormatSpecification::class, name = "JSONL"), + JsonSubTypes.Type(value = CSVFormatSpecification::class, name = "CSV"), + JsonSubTypes.Type(value = AvroFormatSpecification::class, name = "AVRO"), + JsonSubTypes.Type(value = ParquetFormatSpecification::class, name = "PARQUET") +) +sealed class ObjectStorageFormatSpecification { + @JsonSchemaTitle("Format Type") @JsonProperty("format_type") val formatType: String = "JSONL" +} + +@JsonSchemaTitle("JSON Lines: Newline-delimited JSON") +class JsonFormatSpecification : ObjectStorageFormatSpecification() + +@JsonSchemaTitle("CSV: Comma-Separated Values") +class CSVFormatSpecification : ObjectStorageFormatSpecification() + +@JsonSchemaTitle("Avro: Apache Avro") +class AvroFormatSpecification : ObjectStorageFormatSpecification() + +@JsonSchemaTitle("Parquet: Columnar Storage") +class ParquetFormatSpecification : ObjectStorageFormatSpecification() + +interface OutputFormatConfigurationProvider { + val outputFormat: ObjectStorageFormatConfiguration +} + +sealed interface ObjectStorageFormatConfiguration { + val extension: String +} + +data class JsonFormatConfiguration(override val extension: String = "jsonl") : + ObjectStorageFormatConfiguration + +data class CSVFormatConfiguration(override val extension: String = "csv") : + ObjectStorageFormatConfiguration + +data class AvroFormatConfiguration(override val extension: String = "avro") : + ObjectStorageFormatConfiguration + +data class ParquetFormatConfiguration(override val extension: String = "parquet") : + ObjectStorageFormatConfiguration + +interface ObjectStorageFormatConfigurationProvider { + val objectStorageFormatConfiguration: ObjectStorageFormatConfiguration +} diff --git a/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/command/object_storage/ObjectStorageUploadConfiguration.kt b/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/command/object_storage/ObjectStorageUploadConfiguration.kt new file mode 100644 index 000000000000..d868ca1c9e9c --- /dev/null +++ b/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/command/object_storage/ObjectStorageUploadConfiguration.kt @@ -0,0 +1,11 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.load.command.object_storage + +data class ObjectStorageUploadConfiguration(val streamingUploadPartSize: Long) + +interface ObjectStorageUploadConfigurationProvider { + val objectStorageUploadConfiguration: ObjectStorageUploadConfiguration +} diff --git a/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/file/object_storage/ObjectStorageClient.kt b/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/file/object_storage/ObjectStorageClient.kt index 3d46dc26601d..7ddaa5e2b583 100644 --- a/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/file/object_storage/ObjectStorageClient.kt +++ b/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/file/object_storage/ObjectStorageClient.kt @@ -4,11 +4,19 @@ package io.airbyte.cdk.load.file.object_storage -import io.airbyte.cdk.load.message.RemoteObject +import java.io.InputStream import kotlinx.coroutines.flow.Flow -interface ObjectStorageClient { +interface ObjectStorageClient, U : ObjectStorageStreamingUploadWriter> { suspend fun list(prefix: String): Flow - suspend fun put(key: String, bytes: ByteArray) - suspend fun delete(key: String) + suspend fun move(remoteObject: T, toKey: String): T + suspend fun get(key: String, block: (InputStream) -> U): U + suspend fun put(key: String, bytes: ByteArray): T + suspend fun delete(remoteObject: T) + suspend fun streamingUpload(key: String, collector: suspend U.() -> Unit): T +} + +interface ObjectStorageStreamingUploadWriter { + suspend fun write(bytes: ByteArray) + suspend fun write(string: String) = write(string.toByteArray(Charsets.UTF_8)) } diff --git a/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/file/object_storage/ObjectStoragePathFactory.kt b/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/file/object_storage/ObjectStoragePathFactory.kt index b2284920a4e5..1d5d93cca17e 100644 --- a/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/file/object_storage/ObjectStoragePathFactory.kt +++ b/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/file/object_storage/ObjectStoragePathFactory.kt @@ -5,6 +5,7 @@ package io.airbyte.cdk.load.file.object_storage import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.command.object_storage.ObjectStorageFormatConfigurationProvider import io.airbyte.cdk.load.command.object_storage.ObjectStoragePathConfigurationProvider import io.airbyte.cdk.load.file.TimeProvider import io.micronaut.context.annotation.Secondary @@ -14,51 +15,137 @@ import java.nio.file.Paths import java.time.Instant import java.time.ZoneId import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter import java.util.* @Singleton @Secondary class ObjectStoragePathFactory( pathConfigProvider: ObjectStoragePathConfigurationProvider, - timeProvider: TimeProvider + formatConfigProvider: ObjectStorageFormatConfigurationProvider? = null, + timeProvider: TimeProvider? = null, ) { - private val loadedAt = Instant.ofEpochMilli(timeProvider.currentTimeMillis()) + private val loadedAt = timeProvider?.let { Instant.ofEpochMilli(it.currentTimeMillis()) } private val pathConfig = pathConfigProvider.objectStoragePathConfiguration + private val defaultExtension = formatConfigProvider?.objectStorageFormatConfiguration?.extension inner class VariableContext( val stream: DestinationStream, - val time: Instant = loadedAt, + val time: Instant? = loadedAt, + val extension: String? = null, + val partNumber: Long? = null ) - data class PathVariable(val variable: String, val provider: (VariableContext) -> String) { - fun toMacro(): String = "\${$variable}" + interface Variable { + val provider: (VariableContext) -> String + fun toMacro(): String + fun maybeApply(source: String, context: VariableContext): String { + val macro = toMacro() + if (source.contains(macro)) { + return source.replace(macro, provider(context)) + } + return source + } + } + + data class PathVariable( + val variable: String, + override val provider: (VariableContext) -> String + ) : Variable { + override fun toMacro(): String = "\${$variable}" + } + + data class FileVariable( + val variable: String, + override val provider: (VariableContext) -> String + ) : Variable { + override fun toMacro(): String = "{$variable}" } companion object { + private val DATE_FORMATTER: DateTimeFormatter = + DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneId.systemDefault()) + private val TIMESTAMP_FORMATTER: DateTimeFormatter = + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSX").withZone(ZoneId.of("UTC")) + const val DEFAULT_STAGING_PREFIX_SUFFIX = "__airbyte_tmp" const val DEFAULT_PATH_FORMAT = "\${NAMESPACE}/\${STREAM_NAME}/\${YEAR}_\${MONTH}_\${DAY}_\${EPOCH}_" + const val DEFAULT_FILE_FORMAT = "{part_number}{format_extension}" val PATH_VARIABLES = // TODO: Vet that these match the past format exactly (eg day = 5 versus 05, etc) listOf( PathVariable("NAMESPACE") { it.stream.descriptor.namespace ?: "" }, PathVariable("STREAM_NAME") { it.stream.descriptor.name }, PathVariable("YEAR") { - ZonedDateTime.ofInstant(it.time, ZoneId.of("UTC")).year.toString() + it.time?.let { t -> + ZonedDateTime.ofInstant(t, ZoneId.of("UTC")).year.toString() + } + ?: throw IllegalArgumentException("time is required when {YEAR} is present") }, PathVariable("MONTH") { - ZonedDateTime.ofInstant(it.time, ZoneId.of("UTC")).monthValue.toString() + it.time?.let { t -> + ZonedDateTime.ofInstant(t, ZoneId.of("UTC")).monthValue.toString() + } + ?: throw IllegalArgumentException( + "time is required when {MONTH} is present" + ) }, PathVariable("DAY") { - ZonedDateTime.ofInstant(it.time, ZoneId.of("UTC")).dayOfMonth.toString() + it.time?.let { t -> + ZonedDateTime.ofInstant(t, ZoneId.of("UTC")).dayOfMonth.toString() + } + ?: throw IllegalArgumentException("time is required when {DAY} is present") + }, + PathVariable("HOUR") { + it.time?.toEpochMilli()?.let { t -> t / 1000 / 60 / 60 }?.toString() + ?: throw IllegalArgumentException("time is required when {HOUR} is present") + }, + PathVariable("MINUTE") { + (it.time?.toEpochMilli()?.let { t -> t / 1000 / 60 }?.toString() + ?: throw IllegalArgumentException( + "time is required when {MINUTE} is present" + )) + }, + PathVariable("SECOND") { + (it.time?.toEpochMilli()?.div(1000)?.toString() + ?: throw IllegalArgumentException( + "time is required when {SECOND} is present" + )) + }, + PathVariable("MILLISECOND") { + it.time?.toEpochMilli()?.toString() + ?: throw IllegalArgumentException( + "time is required when {MILLISECOND} is present" + ) + }, + PathVariable("EPOCH") { + it.time?.toEpochMilli()?.toString() + ?: throw IllegalArgumentException( + "time is required when {EPOCH} is present" + ) }, - PathVariable("HOUR") { (it.time.toEpochMilli() / 1000 / 60 / 60).toString() }, - PathVariable("MINUTE") { (it.time.toEpochMilli() / 1000 / 60).toString() }, - PathVariable("SECOND") { (it.time.toEpochMilli() / 1000).toString() }, - PathVariable("MILLISECOND") { it.time.toEpochMilli().toString() }, - PathVariable("EPOCH") { it.time.toEpochMilli().toString() }, PathVariable("UUID") { UUID.randomUUID().toString() } ) + val FILENAME_VARIABLES = + listOf( + FileVariable("date") { DATE_FORMATTER.format(it.time) }, + FileVariable("timestamp") { TIMESTAMP_FORMATTER.format(it.time) }, + FileVariable("part_number") { + it.partNumber?.toString() + ?: throw IllegalArgumentException( + "part_number is required when {part_number} is present" + ) + }, + FileVariable("sync_id") { it.stream.syncId.toString() }, + FileVariable("format_extension") { it.extension?.let { ext -> ".$ext" } ?: "" } + ) + + fun from(config: T, timeProvider: TimeProvider? = null): ObjectStoragePathFactory where + T : ObjectStoragePathConfigurationProvider, + T : ObjectStorageFormatConfigurationProvider { + return ObjectStoragePathFactory(config, config, timeProvider) + } } fun getStagingDirectory(stream: DestinationStream): Path { @@ -74,10 +161,33 @@ class ObjectStoragePathFactory( return Paths.get(pathConfig.prefix, path) } + fun getPathToFile( + stream: DestinationStream, + partNumber: Long?, + isStaging: Boolean = false, + extension: String? = defaultExtension + ): Path { + val path = + if (isStaging) { + getStagingDirectory(stream) + } else { + getFinalDirectory(stream) + } + val context = VariableContext(stream, extension = extension, partNumber = partNumber) + val fileName = getFormattedFileName(context) + return path.resolve(fileName) + } + private fun getFormattedPath(stream: DestinationStream): String { val pattern = pathConfig.pathSuffixPattern ?: DEFAULT_PATH_FORMAT - return PATH_VARIABLES.fold(pattern) { acc, variable -> - acc.replace(variable.toMacro(), variable.provider(VariableContext(stream))) + val context = VariableContext(stream) + return PATH_VARIABLES.fold(pattern) { acc, variable -> variable.maybeApply(acc, context) } + } + + private fun getFormattedFileName(context: VariableContext): String { + val pattern = pathConfig.fileNamePattern ?: DEFAULT_FILE_FORMAT + return FILENAME_VARIABLES.fold(pattern) { acc, variable -> + variable.maybeApply(acc, context) } } } diff --git a/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/file/object_storage/RemoteObject.kt b/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/file/object_storage/RemoteObject.kt new file mode 100644 index 000000000000..c20429c7aab1 --- /dev/null +++ b/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/file/object_storage/RemoteObject.kt @@ -0,0 +1,10 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.load.file.object_storage + +interface RemoteObject { + val key: String + val storageConfig: C +} diff --git a/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt b/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt index 3d2241a673c1..41390bedd5fa 100644 --- a/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt +++ b/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt @@ -5,29 +5,50 @@ package io.airbyte.cdk.load.file.s3 import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider +import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.CompletedMultipartUpload +import aws.sdk.kotlin.services.s3.model.CompletedPart +import aws.sdk.kotlin.services.s3.model.CopyObjectRequest +import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadResponse +import aws.sdk.kotlin.services.s3.model.DeleteObjectRequest +import aws.sdk.kotlin.services.s3.model.GetObjectRequest import aws.sdk.kotlin.services.s3.model.ListObjectsRequest import aws.sdk.kotlin.services.s3.model.PutObjectRequest +import aws.sdk.kotlin.services.s3.model.UploadPartRequest import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.content.toInputStream import edu.umd.cs.findbugs.annotations.SuppressFBWarnings import io.airbyte.cdk.load.command.aws.AWSAccessKeyConfigurationProvider +import io.airbyte.cdk.load.command.object_storage.ObjectStorageUploadConfiguration +import io.airbyte.cdk.load.command.object_storage.ObjectStorageUploadConfigurationProvider import io.airbyte.cdk.load.command.s3.S3BucketConfiguration import io.airbyte.cdk.load.command.s3.S3BucketConfigurationProvider import io.airbyte.cdk.load.file.object_storage.ObjectStorageClient -import io.airbyte.cdk.load.message.RemoteObject +import io.airbyte.cdk.load.file.object_storage.ObjectStorageStreamingUploadWriter +import io.airbyte.cdk.load.file.object_storage.RemoteObject +import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Factory import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton +import java.io.ByteArrayOutputStream +import java.io.InputStream import kotlinx.coroutines.flow.flow -data class S3Object( - override val key: String, -) : RemoteObject() +data class S3Object(override val key: String, override val storageConfig: S3BucketConfiguration) : + RemoteObject { + val keyWithBucketName + get() = "${storageConfig.s3BucketName}/$key" +} @SuppressFBWarnings("NP_NONNULL_PARAM_VIOLATION", justification = "Kotlin async continuation") class S3Client( private val client: aws.sdk.kotlin.services.s3.S3Client, - private val bucketConfig: S3BucketConfiguration -) : ObjectStorageClient { + val bucketConfig: S3BucketConfiguration, + private val uploadConfig: ObjectStorageUploadConfiguration?, +) : ObjectStorageClient { + private val log = KotlinLogging.logger {} + override suspend fun list(prefix: String) = flow { var request = ListObjectsRequest { bucket = bucketConfig.s3BucketName @@ -38,7 +59,7 @@ class S3Client( val response = client.listObjects(request) response.contents?.forEach { obj -> lastKey = obj.key - emit(S3Object(obj.key!!)) + emit(S3Object(obj.key!!, bucketConfig)) } // null contents => empty list, not error if (client.listObjects(request).isTruncated == false) { break @@ -47,30 +68,137 @@ class S3Client( } } - override suspend fun put(key: String, bytes: ByteArray) { + override suspend fun move(remoteObject: S3Object, toKey: String): S3Object { + val request = CopyObjectRequest { + bucket = bucketConfig.s3BucketName + this.key = toKey + copySource = remoteObject.keyWithBucketName + } + client.copyObject(request) + delete(remoteObject) + return S3Object(toKey, bucketConfig) + } + + override suspend fun get(key: String, block: (InputStream) -> R): R { + val request = GetObjectRequest { + bucket = bucketConfig.s3BucketName + this.key = key + } + return client.getObject(request) { + val inputStream = + it.body?.toInputStream() + ?: throw IllegalStateException( + "S3 object body is null (this indicates a failure, not an empty object)" + ) + block(inputStream) + } + } + + override suspend fun put(key: String, bytes: ByteArray): S3Object { val request = PutObjectRequest { bucket = bucketConfig.s3BucketName this.key = key body = ByteStream.fromBytes(bytes) } client.putObject(request) + return S3Object(key, bucketConfig) } - override suspend fun delete(key: String) { - val request = - aws.sdk.kotlin.services.s3.model.DeleteObjectRequest { - bucket = bucketConfig.s3BucketName - this.key = key - } + override suspend fun delete(remoteObject: S3Object) { + val request = DeleteObjectRequest { + bucket = remoteObject.storageConfig.s3BucketName + this.key = remoteObject.key + } client.deleteObject(request) } + + override suspend fun streamingUpload( + key: String, + collector: suspend S3MultipartUpload.Writer.() -> Unit + ): S3Object { + val request = CreateMultipartUploadRequest { + this.bucket = bucketConfig.s3BucketName + this.key = key + } + val response = client.createMultipartUpload(request) + S3MultipartUpload(response).upload(collector) + return S3Object(key, bucketConfig) + } + + inner class S3MultipartUpload(private val response: CreateMultipartUploadResponse) { + private val uploadedParts = mutableListOf() + private var inputBuffer = ByteArrayOutputStream() + private val partSize = + uploadConfig?.streamingUploadPartSize + ?: throw IllegalStateException("Streaming upload part size is not configured") + + suspend fun upload(collector: suspend S3MultipartUpload.Writer.() -> Unit) { + log.info { + "Starting multipart upload to ${response.bucket}/${response.key} (${response.uploadId}" + } + collector.invoke(Writer()) + complete() + } + + inner class Writer : ObjectStorageStreamingUploadWriter { + override suspend fun write(bytes: ByteArray) { + inputBuffer.write(bytes) + if (inputBuffer.size() >= partSize) { + uploadPart() + } + } + } + + private suspend fun uploadPart() { + val partNumber = uploadedParts.size + 1 + val request = UploadPartRequest { + uploadId = response.uploadId + bucket = response.bucket + key = response.key + body = ByteStream.fromBytes(inputBuffer.toByteArray()) + this.partNumber = partNumber + } + val uploadResponse = client.uploadPart(request) + uploadedParts.add( + CompletedPart { + this.partNumber = partNumber + this.eTag = uploadResponse.eTag + } + ) + inputBuffer.reset() + } + + private suspend fun complete() { + if (inputBuffer.size() > 0) { + uploadPart() + } + log.info { + "Completing multipart upload to ${response.bucket}/${response.key} (${response.uploadId}" + } + val request = CompleteMultipartUploadRequest { + uploadId = response.uploadId + bucket = response.bucket + key = response.key + this.multipartUpload = CompletedMultipartUpload { parts = uploadedParts } + } + client.completeMultipartUpload(request) + } + } } @Factory class S3ClientFactory( private val keyConfig: AWSAccessKeyConfigurationProvider, - private val bucketConfig: S3BucketConfigurationProvider + private val bucketConfig: S3BucketConfigurationProvider, + private val uploadConifg: ObjectStorageUploadConfigurationProvider? = null, ) { + companion object { + fun make(config: T) where + T : S3BucketConfigurationProvider, + T : AWSAccessKeyConfigurationProvider, + T : ObjectStorageUploadConfigurationProvider = + S3ClientFactory(config, config, config).make() + } @Singleton @Secondary @@ -86,6 +214,10 @@ class S3ClientFactory( credentialsProvider = credentials } - return S3Client(client, bucketConfig.s3BucketConfiguration) + return S3Client( + client, + bucketConfig.s3BucketConfiguration, + uploadConifg?.objectStorageUploadConfiguration + ) } } diff --git a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestinationDataDumper.kt b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestinationDataDumper.kt index c322c5e26e57..d7f44e1f61e4 100644 --- a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestinationDataDumper.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullDestinationDataDumper.kt @@ -4,11 +4,16 @@ package io.airbyte.integrations.destination.dev_null +import io.airbyte.cdk.command.ConfigurationSpecification +import io.airbyte.cdk.load.command.DestinationStream import io.airbyte.cdk.load.test.util.DestinationDataDumper import io.airbyte.cdk.load.test.util.OutputRecord object DevNullDestinationDataDumper : DestinationDataDumper { - override fun dumpRecords(streamName: String, streamNamespace: String?): List { + override fun dumpRecords( + spec: ConfigurationSpecification, + stream: DestinationStream + ): List { // E2e destination doesn't actually write records, so we shouldn't even // have tests that try to read back the records throw NotImplementedError() diff --git a/airbyte-integrations/connectors/destination-s3-v2/metadata.yaml b/airbyte-integrations/connectors/destination-s3-v2/metadata.yaml index 550514a74bef..744f472028ae 100644 --- a/airbyte-integrations/connectors/destination-s3-v2/metadata.yaml +++ b/airbyte-integrations/connectors/destination-s3-v2/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: file connectorType: destination definitionId: d6116991-e809-4c7c-ae09-c64712df5b66 - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 dockerRepository: airbyte/destination-s3-v2 githubIssueLabel: destination-s3-v2 icon: s3.svg diff --git a/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Checker.kt b/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Checker.kt index e866e3e04c68..773184d10216 100644 --- a/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Checker.kt +++ b/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Checker.kt @@ -5,35 +5,43 @@ package io.airbyte.integrations.destination.s3_v2 import io.airbyte.cdk.load.check.DestinationChecker +import io.airbyte.cdk.load.command.object_storage.JsonFormatConfiguration +import io.airbyte.cdk.load.file.TimeProvider import io.airbyte.cdk.load.file.object_storage.ObjectStoragePathFactory -import io.airbyte.cdk.load.file.s3.S3Client +import io.airbyte.cdk.load.file.s3.S3ClientFactory +import io.airbyte.cdk.load.file.s3.S3Object import io.github.oshai.kotlinlogging.KotlinLogging +import io.micronaut.context.exceptions.ConfigurationException import jakarta.inject.Singleton import kotlinx.coroutines.flow.toList import kotlinx.coroutines.runBlocking @Singleton -class S3V2Checker( - private val s3Client: S3Client, - private val pathFactory: ObjectStoragePathFactory -) : DestinationChecker { +class S3V2Checker(private val timeProvider: TimeProvider) : DestinationChecker { private val log = KotlinLogging.logger {} override fun check(config: S3V2Configuration) { runBlocking { + if (config.objectStorageFormatConfiguration !is JsonFormatConfiguration) { + throw ConfigurationException("Currently only JSON format is supported") + } + val s3Client = S3ClientFactory.make(config) + val pathFactory = ObjectStoragePathFactory.from(config, timeProvider) val path = pathFactory.getStagingDirectory(mockStream()) val key = path.resolve("_EMPTY").toString() log.info { "Checking if destination can write to $path" } + var s3Object: S3Object? = null try { - s3Client.put(key, byteArrayOf()) + s3Object = s3Client.put(key, byteArrayOf()) val results = s3Client.list(path.toString()).toList() if (results.isEmpty() || results.find { it.key == key } == null) { throw IllegalStateException("Failed to write to S3 bucket") } - log.info { "Successfully wrote: $results" } + log.info { "Successfully wrote test file: $results" } } finally { - s3Client.delete(key) - s3Client.list(path.toString()).toList() + s3Object?.also { s3Client.delete(it) } + val results = s3Client.list(path.toString()).toList() + log.info { "Successfully removed test tile: $results" } } } } diff --git a/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Configuration.kt b/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Configuration.kt index 4a642ec8cc56..28937de8ae72 100644 --- a/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Configuration.kt +++ b/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Configuration.kt @@ -8,22 +8,35 @@ import io.airbyte.cdk.load.command.DestinationConfiguration import io.airbyte.cdk.load.command.DestinationConfigurationFactory import io.airbyte.cdk.load.command.aws.AWSAccessKeyConfiguration import io.airbyte.cdk.load.command.aws.AWSAccessKeyConfigurationProvider +import io.airbyte.cdk.load.command.object_storage.ObjectStorageFormatConfiguration +import io.airbyte.cdk.load.command.object_storage.ObjectStorageFormatConfigurationProvider import io.airbyte.cdk.load.command.object_storage.ObjectStoragePathConfiguration import io.airbyte.cdk.load.command.object_storage.ObjectStoragePathConfigurationProvider +import io.airbyte.cdk.load.command.object_storage.ObjectStorageUploadConfiguration +import io.airbyte.cdk.load.command.object_storage.ObjectStorageUploadConfigurationProvider import io.airbyte.cdk.load.command.s3.S3BucketConfiguration import io.airbyte.cdk.load.command.s3.S3BucketConfigurationProvider import io.micronaut.context.annotation.Factory import jakarta.inject.Singleton data class S3V2Configuration( + // Client-facing configuration override val awsAccessKeyConfiguration: AWSAccessKeyConfiguration, override val s3BucketConfiguration: S3BucketConfiguration, - override val objectStoragePathConfiguration: ObjectStoragePathConfiguration + override val objectStoragePathConfiguration: ObjectStoragePathConfiguration, + override val objectStorageFormatConfiguration: ObjectStorageFormatConfiguration, + + // Internal configuration + override val objectStorageUploadConfiguration: ObjectStorageUploadConfiguration = + ObjectStorageUploadConfiguration(5L * 1024 * 1024), + override val recordBatchSizeBytes: Long = 200L * 1024 * 1024 ) : DestinationConfiguration(), AWSAccessKeyConfigurationProvider, S3BucketConfigurationProvider, - ObjectStoragePathConfigurationProvider + ObjectStoragePathConfigurationProvider, + ObjectStorageFormatConfigurationProvider, + ObjectStorageUploadConfigurationProvider @Singleton class S3V2ConfigurationFactory : @@ -32,7 +45,8 @@ class S3V2ConfigurationFactory : return S3V2Configuration( awsAccessKeyConfiguration = pojo.toAWSAccessKeyConfiguration(), s3BucketConfiguration = pojo.toS3BucketConfiguration(), - objectStoragePathConfiguration = pojo.toObjectStoragePathConfiguration() + objectStoragePathConfiguration = pojo.toObjectStoragePathConfiguration(), + objectStorageFormatConfiguration = pojo.toObjectStorageFormatConfiguration() ) } } diff --git a/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Specification.kt b/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Specification.kt index 8045fff4e4ad..b8cb45d78154 100644 --- a/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Specification.kt +++ b/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Specification.kt @@ -7,6 +7,9 @@ package io.airbyte.integrations.destination.s3_v2 import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import io.airbyte.cdk.command.ConfigurationSpecification import io.airbyte.cdk.load.command.aws.AWSAccessKeySpecification +import io.airbyte.cdk.load.command.object_storage.JsonFormatSpecification +import io.airbyte.cdk.load.command.object_storage.ObjectStorageFormatSpecification +import io.airbyte.cdk.load.command.object_storage.ObjectStorageFormatSpecificationProvider import io.airbyte.cdk.load.command.s3.S3BucketRegion import io.airbyte.cdk.load.command.s3.S3BucketSpecification import io.airbyte.cdk.load.command.s3.S3PathSpecification @@ -20,12 +23,14 @@ class S3V2Specification : ConfigurationSpecification(), AWSAccessKeySpecification, S3BucketSpecification, - S3PathSpecification { + S3PathSpecification, + ObjectStorageFormatSpecificationProvider { override val accessKeyId: String = "" override val secretAccessKey: String = "" override val s3BucketName: String = "" override val s3BucketPath: String = "" override val s3BucketRegion: S3BucketRegion = S3BucketRegion.`us-west-1` + override val format: ObjectStorageFormatSpecification = JsonFormatSpecification() override val s3Endpoint: String? = null override val s3PathFormat: String? = null override val fileNamePattern: String? = null diff --git a/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Writer.kt b/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Writer.kt index fb34ea5500e2..2c8df1b0af31 100644 --- a/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Writer.kt +++ b/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Writer.kt @@ -4,24 +4,72 @@ package io.airbyte.integrations.destination.s3_v2 +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.data.DestinationRecordToAirbyteValueWithMeta +import io.airbyte.cdk.load.data.toJson +import io.airbyte.cdk.load.file.object_storage.ObjectStoragePathFactory +import io.airbyte.cdk.load.file.s3.S3Client +import io.airbyte.cdk.load.file.s3.S3Object import io.airbyte.cdk.load.message.Batch import io.airbyte.cdk.load.message.DestinationRecord -import io.airbyte.cdk.load.message.SimpleBatch +import io.airbyte.cdk.load.util.serializeToString import io.airbyte.cdk.load.write.DestinationWriter import io.airbyte.cdk.load.write.StreamLoader import jakarta.inject.Singleton +import java.util.concurrent.atomic.AtomicLong @Singleton -class S3V2Writer : DestinationWriter { +class S3V2Writer( + private val s3Client: S3Client, + private val pathFactory: ObjectStoragePathFactory, + private val recordDecorator: DestinationRecordToAirbyteValueWithMeta +) : DestinationWriter { + sealed interface S3V2Batch : Batch + data class StagedObject( + override val state: Batch.State = Batch.State.PERSISTED, + val s3Object: S3Object, + val partNumber: Long + ) : S3V2Batch + data class FinalizedObject( + override val state: Batch.State = Batch.State.COMPLETE, + val s3Object: S3Object, + ) : S3V2Batch + override fun createStreamLoader(stream: DestinationStream): StreamLoader { return S3V2StreamLoader(stream) } + @SuppressFBWarnings("NP_NONNULL_PARAM_VIOLATION", justification = "Kotlin async continuation") inner class S3V2StreamLoader(override val stream: DestinationStream) : StreamLoader { + private val partNumber = AtomicLong(0L) // TODO: Get from destination state + override suspend fun processRecords( records: Iterator, totalSizeBytes: Long - ): Batch = SimpleBatch(state = Batch.State.COMPLETE) + ): Batch { + val partNumber = partNumber.getAndIncrement() + val key = pathFactory.getPathToFile(stream, partNumber, isStaging = true).toString() + val s3Object = + s3Client.streamingUpload(key) { + records.forEach { + val serialized = recordDecorator.decorate(it).toJson().serializeToString() + write(serialized) + write("\n") + } + } + return StagedObject(s3Object = s3Object, partNumber = partNumber) + } + + override suspend fun processBatch(batch: Batch): Batch { + val stagedObject = batch as StagedObject + val finalKey = + pathFactory + .getPathToFile(stream, stagedObject.partNumber, isStaging = false) + .toString() + val newObject = s3Client.move(stagedObject.s3Object, finalKey) + val finalizedObject = FinalizedObject(s3Object = newObject) + return finalizedObject + } } } diff --git a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2CheckTest.kt b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2CheckTest.kt index 04d722e82cda..528088495c00 100644 --- a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2CheckTest.kt +++ b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2CheckTest.kt @@ -13,12 +13,7 @@ class S3V2CheckTest : CheckIntegrationTest( S3V2Specification::class.java, successConfigFilenames = - listOf( - CheckTestConfig( - "secrets/s3_dest_v2_minimal_required_config.json", - TestDeploymentMode.CLOUD - ) - ), + listOf(CheckTestConfig(S3V2TestUtils.MINIMAL_CONFIG_PATH, TestDeploymentMode.CLOUD)), failConfigFilenamesAndFailureReasons = emptyMap() ) { @Test diff --git a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2TestUtils.kt b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2TestUtils.kt new file mode 100644 index 000000000000..2d72b272c73f --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2TestUtils.kt @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.s3_v2 + +import io.airbyte.cdk.command.ValidatedJsonUtils +import java.nio.file.Files +import java.nio.file.Path + +object S3V2TestUtils { + const val MINIMAL_CONFIG_PATH = "secrets/s3_dest_v2_minimal_required_config.json" + val minimalConfig: S3V2Specification = + ValidatedJsonUtils.parseOne( + S3V2Specification::class.java, + Files.readString(Path.of(MINIMAL_CONFIG_PATH)), + ) +} diff --git a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2WriteTest.kt b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2WriteTest.kt index 0068da59f22f..666d5a12782a 100644 --- a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2WriteTest.kt +++ b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2WriteTest.kt @@ -4,20 +4,33 @@ package io.airbyte.integrations.destination.s3_v2 +import io.airbyte.cdk.command.ConfigurationSpecification +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.data.toAirbyteValue +import io.airbyte.cdk.load.file.object_storage.ObjectStoragePathFactory +import io.airbyte.cdk.load.file.s3.S3ClientFactory +import io.airbyte.cdk.load.file.s3.S3Object import io.airbyte.cdk.load.test.util.DestinationDataDumper import io.airbyte.cdk.load.test.util.NoopDestinationCleaner import io.airbyte.cdk.load.test.util.NoopExpectedRecordMapper import io.airbyte.cdk.load.test.util.OutputRecord +import io.airbyte.cdk.load.test.util.toOutputRecord +import io.airbyte.cdk.load.util.deserializeToNode import io.airbyte.cdk.load.write.BasicFunctionalityIntegrationTest +import java.io.InputStream +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext import org.junit.jupiter.api.Test class S3V2WriteTest : BasicFunctionalityIntegrationTest( - S3V2Specification(), + S3V2TestUtils.minimalConfig, S3V2DataDumper, NoopDestinationCleaner, NoopExpectedRecordMapper, - verifyDataWriting = false ) { @Test override fun testBasicWrite() { @@ -26,9 +39,39 @@ class S3V2WriteTest : } object S3V2DataDumper : DestinationDataDumper { - override fun dumpRecords(streamName: String, streamNamespace: String?): List { - // E2e destination doesn't actually write records, so we shouldn't even - // have tests that try to read back the records - throw NotImplementedError() + override fun dumpRecords( + spec: ConfigurationSpecification, + stream: DestinationStream + ): List { + val config = + S3V2ConfigurationFactory().makeWithoutExceptionHandling(spec as S3V2Specification) + val s3Client = S3ClientFactory.make(config) + // Note: because we cannot mock wall time in docker, this + // path code cannot contain time-based macros. + // TODO: add pattern matching to the path factory. + val pathFactory = ObjectStoragePathFactory.from(config) + val prefix = pathFactory.getFinalDirectory(stream).toString() + return runBlocking { + withContext(Dispatchers.IO) { + s3Client + .list(prefix) + .map { listedObject: S3Object -> + s3Client.get(listedObject.key) { objectData: InputStream -> + objectData + .bufferedReader() + .lineSequence() + .map { line -> + line + .deserializeToNode() + .toAirbyteValue(stream.schemaWithMeta) + .toOutputRecord() + } + .toList() + } + } + .toList() + .flatten() + } + } } } diff --git a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/resources/expected-spec-cloud.json b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/resources/expected-spec-cloud.json index a00c7993db95..2b93394a99a8 100644 --- a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/resources/expected-spec-cloud.json +++ b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/resources/expected-spec-cloud.json @@ -6,6 +6,60 @@ "type" : "object", "additionalProperties" : true, "properties" : { + "format" : { + "oneOf" : [ { + "title" : "JSON Lines: Newline-delimited JSON", + "type" : "object", + "additionalProperties" : true, + "properties" : { + "format_type" : { + "type" : "string", + "enum" : [ "JSONL" ], + "default" : "JSONL" + } + }, + "required" : [ "format_type" ] + }, { + "title" : "CSV: Comma-Separated Values", + "type" : "object", + "additionalProperties" : true, + "properties" : { + "format_type" : { + "type" : "string", + "enum" : [ "CSV" ], + "default" : "CSV" + } + }, + "required" : [ "format_type" ] + }, { + "title" : "Avro: Apache Avro", + "type" : "object", + "additionalProperties" : true, + "properties" : { + "format_type" : { + "type" : "string", + "enum" : [ "AVRO" ], + "default" : "AVRO" + } + }, + "required" : [ "format_type" ] + }, { + "title" : "Parquet: Columnar Storage", + "type" : "object", + "additionalProperties" : true, + "properties" : { + "format_type" : { + "type" : "string", + "enum" : [ "PARQUET" ], + "default" : "PARQUET" + } + }, + "required" : [ "format_type" ] + } ], + "description" : "Format of the data output. See here for more details", + "title" : "Output Format", + "type" : "object" + }, "access_key_id" : { "type" : "string", "description" : "The access key ID to access the S3 bucket. Airbyte requires Read and Write permissions to the given bucket. Read more here.", @@ -63,7 +117,7 @@ "examples" : [ "__staging/data_sync/test" ] } }, - "required" : [ "access_key_id", "secret_access_key", "s3_bucket_name", "s3_bucket_path", "s3_bucket_region" ] + "required" : [ "format", "access_key_id", "secret_access_key", "s3_bucket_name", "s3_bucket_path", "s3_bucket_region" ] }, "supportsIncremental" : true, "supportsNormalization" : false, diff --git a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/resources/expected-spec-oss.json b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/resources/expected-spec-oss.json index a00c7993db95..2b93394a99a8 100644 --- a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/resources/expected-spec-oss.json +++ b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/resources/expected-spec-oss.json @@ -6,6 +6,60 @@ "type" : "object", "additionalProperties" : true, "properties" : { + "format" : { + "oneOf" : [ { + "title" : "JSON Lines: Newline-delimited JSON", + "type" : "object", + "additionalProperties" : true, + "properties" : { + "format_type" : { + "type" : "string", + "enum" : [ "JSONL" ], + "default" : "JSONL" + } + }, + "required" : [ "format_type" ] + }, { + "title" : "CSV: Comma-Separated Values", + "type" : "object", + "additionalProperties" : true, + "properties" : { + "format_type" : { + "type" : "string", + "enum" : [ "CSV" ], + "default" : "CSV" + } + }, + "required" : [ "format_type" ] + }, { + "title" : "Avro: Apache Avro", + "type" : "object", + "additionalProperties" : true, + "properties" : { + "format_type" : { + "type" : "string", + "enum" : [ "AVRO" ], + "default" : "AVRO" + } + }, + "required" : [ "format_type" ] + }, { + "title" : "Parquet: Columnar Storage", + "type" : "object", + "additionalProperties" : true, + "properties" : { + "format_type" : { + "type" : "string", + "enum" : [ "PARQUET" ], + "default" : "PARQUET" + } + }, + "required" : [ "format_type" ] + } ], + "description" : "Format of the data output. See here for more details", + "title" : "Output Format", + "type" : "object" + }, "access_key_id" : { "type" : "string", "description" : "The access key ID to access the S3 bucket. Airbyte requires Read and Write permissions to the given bucket. Read more here.", @@ -63,7 +117,7 @@ "examples" : [ "__staging/data_sync/test" ] } }, - "required" : [ "access_key_id", "secret_access_key", "s3_bucket_name", "s3_bucket_path", "s3_bucket_region" ] + "required" : [ "format", "access_key_id", "secret_access_key", "s3_bucket_name", "s3_bucket_path", "s3_bucket_region" ] }, "supportsIncremental" : true, "supportsNormalization" : false, From 7b12647a4291eaf2b8df60e296d06ba9fc0867ae Mon Sep 17 00:00:00 2001 From: Johnny Schmidt Date: Thu, 17 Oct 2024 17:41:30 -0700 Subject: [PATCH 07/18] Bulk Load CDK: Cleanup: Files/Objects no longer Batches (#46960) --- .../io/airbyte/cdk/load/message/Batch.kt | 19 ------------ .../cdk/load/task/DestinationTaskLauncher.kt | 17 ++++------- .../task/implementor/ProcessRecordsTask.kt | 30 ++++++++----------- .../cdk/load/task/internal/SpillToDiskTask.kt | 15 ++++++---- .../load/task/DestinationTaskLauncherTest.kt | 23 +++++++------- .../airbyte/cdk/load/task/MockTaskLauncher.kt | 9 +++--- .../implementor/ProcessRecordsTaskTest.kt | 6 ++-- .../load/task/internal/SpillToDiskTaskTest.kt | 16 +++++----- 8 files changed, 56 insertions(+), 79 deletions(-) diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/Batch.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/Batch.kt index 794f81324141..9b4ac55bef1e 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/Batch.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/message/Batch.kt @@ -7,7 +7,6 @@ package io.airbyte.cdk.load.message import com.google.common.collect.Range import com.google.common.collect.RangeSet import com.google.common.collect.TreeRangeSet -import io.airbyte.cdk.load.file.LocalFile /** * Represents an accumulated batch of records in some stage of processing. @@ -47,7 +46,6 @@ import io.airbyte.cdk.load.file.LocalFile */ interface Batch { enum class State { - SPILLED, LOCAL, PERSISTED, COMPLETE @@ -66,23 +64,6 @@ interface Batch { /** Simple batch: use if you need no other metadata for processing. */ data class SimpleBatch(override val state: Batch.State) : Batch -/** Represents a file of records locally staged. */ -abstract class StagedLocalFile() : Batch { - abstract val localFile: LocalFile - abstract val totalSizeBytes: Long - override val state: Batch.State = Batch.State.LOCAL -} - -/** - * Represents a file of raw records staged to disk for pre-processing. Used internally by the - * framework - */ -data class SpilledRawMessagesLocalFile( - override val localFile: LocalFile, - override val totalSizeBytes: Long, - override val state: Batch.State = Batch.State.SPILLED -) : StagedLocalFile() - /** * Internally-used wrapper for tracking the association between a batch and the range of records it * contains. diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/DestinationTaskLauncher.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/DestinationTaskLauncher.kt index f2e01d62c24e..f82fc6368914 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/DestinationTaskLauncher.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/DestinationTaskLauncher.kt @@ -9,7 +9,6 @@ import io.airbyte.cdk.load.command.DestinationCatalog import io.airbyte.cdk.load.command.DestinationStream import io.airbyte.cdk.load.message.Batch import io.airbyte.cdk.load.message.BatchEnvelope -import io.airbyte.cdk.load.message.SpilledRawMessagesLocalFile import io.airbyte.cdk.load.state.SyncManager import io.airbyte.cdk.load.task.implementor.CloseStreamTaskFactory import io.airbyte.cdk.load.task.implementor.OpenStreamTaskFactory @@ -20,6 +19,7 @@ import io.airbyte.cdk.load.task.implementor.TeardownTaskFactory import io.airbyte.cdk.load.task.internal.FlushCheckpointsTaskFactory import io.airbyte.cdk.load.task.internal.InputConsumerTask import io.airbyte.cdk.load.task.internal.SpillToDiskTaskFactory +import io.airbyte.cdk.load.task.internal.SpilledRawMessagesLocalFile import io.airbyte.cdk.load.task.internal.TimedForcedCheckpointFlushTask import io.airbyte.cdk.load.task.internal.UpdateCheckpointsTask import io.airbyte.cdk.load.util.setOnce @@ -34,11 +34,7 @@ import kotlinx.coroutines.sync.withLock interface DestinationTaskLauncher : TaskLauncher { suspend fun handleSetupComplete() suspend fun handleStreamStarted(stream: DestinationStream) - suspend fun handleNewSpilledFile( - stream: DestinationStream, - wrapped: BatchEnvelope, - endOfStream: Boolean - ) + suspend fun handleNewSpilledFile(stream: DestinationStream, file: SpilledRawMessagesLocalFile) suspend fun handleNewBatch(stream: DestinationStream, wrapped: BatchEnvelope<*>) suspend fun handleStreamClosed(stream: DestinationStream) suspend fun handleTeardownComplete() @@ -168,13 +164,12 @@ class DefaultDestinationTaskLauncher( /** Called for each new spilled file. */ override suspend fun handleNewSpilledFile( stream: DestinationStream, - wrapped: BatchEnvelope, - endOfStream: Boolean + file: SpilledRawMessagesLocalFile ) { - log.info { "Starting process records task for ${stream.descriptor}, file ${wrapped.batch}" } - val task = processRecordsTaskFactory.make(this, stream, wrapped) + log.info { "Starting process records task for ${stream.descriptor}, file $file" } + val task = processRecordsTaskFactory.make(this, stream, file) enqueue(task) - if (!endOfStream) { + if (!file.endOfStream) { log.info { "End-of-stream not reached, restarting spill-to-disk task for $stream" } val spillTask = spillToDiskTaskFactory.make(this, stream) enqueue(spillTask) diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/ProcessRecordsTask.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/ProcessRecordsTask.kt index 05830995a764..d668ab698136 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/ProcessRecordsTask.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/implementor/ProcessRecordsTask.kt @@ -12,11 +12,11 @@ import io.airbyte.cdk.load.message.DestinationRecord import io.airbyte.cdk.load.message.DestinationStreamAffinedMessage import io.airbyte.cdk.load.message.DestinationStreamComplete import io.airbyte.cdk.load.message.DestinationStreamIncomplete -import io.airbyte.cdk.load.message.SpilledRawMessagesLocalFile import io.airbyte.cdk.load.state.SyncManager import io.airbyte.cdk.load.task.DestinationTaskLauncher import io.airbyte.cdk.load.task.ImplementorScope import io.airbyte.cdk.load.task.StreamLevel +import io.airbyte.cdk.load.task.internal.SpilledRawMessagesLocalFile import io.airbyte.cdk.load.write.StreamLoader import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Secondary @@ -35,7 +35,7 @@ interface ProcessRecordsTask : StreamLevel, ImplementorScope class DefaultProcessRecordsTask( override val stream: DestinationStream, private val taskLauncher: DestinationTaskLauncher, - private val fileEnvelope: BatchEnvelope, + private val file: SpilledRawMessagesLocalFile, private val deserializer: Deserializer, private val syncManager: SyncManager, ) : ProcessRecordsTask { @@ -45,10 +45,10 @@ class DefaultProcessRecordsTask( log.info { "Fetching stream loader for ${stream.descriptor}" } val streamLoader = syncManager.getOrAwaitStreamLoader(stream.descriptor) - log.info { "Processing records from ${fileEnvelope.batch.localFile}" } - val nextBatch = + log.info { "Processing records from $file" } + val batch = try { - fileEnvelope.batch.localFile.toFileReader().use { reader -> + file.localFile.toFileReader().use { reader -> val records = reader .lines() @@ -67,14 +67,14 @@ class DefaultProcessRecordsTask( } .map { it as DestinationRecord } .iterator() - streamLoader.processRecords(records, fileEnvelope.batch.totalSizeBytes) + streamLoader.processRecords(records, file.totalSizeBytes) } } finally { - log.info { "Processing completed, deleting ${fileEnvelope.batch.localFile}" } - fileEnvelope.batch.localFile.delete() + log.info { "Processing completed, deleting $file" } + file.localFile.delete() } - val wrapped = fileEnvelope.withBatch(nextBatch) + val wrapped = BatchEnvelope(batch, file.indexRange) taskLauncher.handleNewBatch(stream, wrapped) } } @@ -83,7 +83,7 @@ interface ProcessRecordsTaskFactory { fun make( taskLauncher: DestinationTaskLauncher, stream: DestinationStream, - fileEnvelope: BatchEnvelope, + file: SpilledRawMessagesLocalFile, ): ProcessRecordsTask } @@ -96,14 +96,8 @@ class DefaultProcessRecordsTaskFactory( override fun make( taskLauncher: DestinationTaskLauncher, stream: DestinationStream, - fileEnvelope: BatchEnvelope, + file: SpilledRawMessagesLocalFile, ): ProcessRecordsTask { - return DefaultProcessRecordsTask( - stream, - taskLauncher, - fileEnvelope, - deserializer, - syncManager - ) + return DefaultProcessRecordsTask(stream, taskLauncher, file, deserializer, syncManager) } } diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/internal/SpillToDiskTask.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/internal/SpillToDiskTask.kt index 4e1950e5466c..34b09ff501ce 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/internal/SpillToDiskTask.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/task/internal/SpillToDiskTask.kt @@ -7,12 +7,11 @@ package io.airbyte.cdk.load.task.internal import com.google.common.collect.Range import io.airbyte.cdk.load.command.DestinationConfiguration import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.file.LocalFile import io.airbyte.cdk.load.file.TempFileProvider -import io.airbyte.cdk.load.message.BatchEnvelope import io.airbyte.cdk.load.message.DestinationRecordWrapped import io.airbyte.cdk.load.message.MessageQueueSupplier import io.airbyte.cdk.load.message.QueueReader -import io.airbyte.cdk.load.message.SpilledRawMessagesLocalFile import io.airbyte.cdk.load.message.StreamCompleteWrapped import io.airbyte.cdk.load.message.StreamRecordWrapped import io.airbyte.cdk.load.state.FlushStrategy @@ -98,9 +97,8 @@ class DefaultSpillToDiskTask( return } - val batch = SpilledRawMessagesLocalFile(tmpFile, sizeBytes) - val wrapped = BatchEnvelope(batch, range) - launcher.handleNewSpilledFile(stream, wrapped, endOfStream) + val file = SpilledRawMessagesLocalFile(tmpFile, sizeBytes, range, endOfStream) + launcher.handleNewSpilledFile(stream, file) } } @@ -130,3 +128,10 @@ class DefaultSpillToDiskTaskFactory( ) } } + +data class SpilledRawMessagesLocalFile( + val localFile: LocalFile, + val totalSizeBytes: Long, + val indexRange: Range, + val endOfStream: Boolean = false +) diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/DestinationTaskLauncherTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/DestinationTaskLauncherTest.kt index 6a08140cf79e..a4768c675da0 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/DestinationTaskLauncherTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/DestinationTaskLauncherTest.kt @@ -12,7 +12,6 @@ import io.airbyte.cdk.load.command.MockDestinationCatalogFactory import io.airbyte.cdk.load.file.DefaultLocalFile import io.airbyte.cdk.load.message.Batch import io.airbyte.cdk.load.message.BatchEnvelope -import io.airbyte.cdk.load.message.SpilledRawMessagesLocalFile import io.airbyte.cdk.load.state.SyncManager import io.airbyte.cdk.load.task.implementor.CloseStreamTask import io.airbyte.cdk.load.task.implementor.CloseStreamTaskFactory @@ -38,6 +37,7 @@ import io.airbyte.cdk.load.task.internal.FlushCheckpointsTaskFactory import io.airbyte.cdk.load.task.internal.InputConsumerTask import io.airbyte.cdk.load.task.internal.SpillToDiskTask import io.airbyte.cdk.load.task.internal.SpillToDiskTaskFactory +import io.airbyte.cdk.load.task.internal.SpilledRawMessagesLocalFile import io.airbyte.cdk.load.task.internal.TimedForcedCheckpointFlushTask import io.airbyte.cdk.load.task.internal.UpdateCheckpointsTask import io.micronaut.context.annotation.Primary @@ -167,7 +167,7 @@ class DestinationTaskLauncherTest where T : LeveledTask, T : ScopedTask { override fun make( taskLauncher: DestinationTaskLauncher, stream: DestinationStream, - fileEnvelope: BatchEnvelope + file: SpilledRawMessagesLocalFile ): ProcessRecordsTask { return object : ProcessRecordsTask { override val stream: DestinationStream = stream @@ -354,10 +354,11 @@ class DestinationTaskLauncherTest where T : LeveledTask, T : ScopedTask { fun testHandleSpilledFileCompleteNotEndOfStream() = runTest { taskLauncher.handleNewSpilledFile( MockDestinationCatalogFactory.stream1, - BatchEnvelope( - SpilledRawMessagesLocalFile(DefaultLocalFile(Path("not/a/real/file")), 100L) - ), - false + SpilledRawMessagesLocalFile( + DefaultLocalFile(Path("not/a/real/file")), + 100L, + Range.singleton(0) + ) ) processRecordsTaskFactory.hasRun.receive() @@ -371,10 +372,12 @@ class DestinationTaskLauncherTest where T : LeveledTask, T : ScopedTask { launch { taskLauncher.handleNewSpilledFile( MockDestinationCatalogFactory.stream1, - BatchEnvelope( - SpilledRawMessagesLocalFile(DefaultLocalFile(Path("not/a/real/file")), 100L) - ), - true + SpilledRawMessagesLocalFile( + DefaultLocalFile(Path("not/a/real/file")), + 100L, + Range.singleton(0), + true + ) ) } diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/MockTaskLauncher.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/MockTaskLauncher.kt index 86edab735223..7561e31d3dc6 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/MockTaskLauncher.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/MockTaskLauncher.kt @@ -6,7 +6,7 @@ package io.airbyte.cdk.load.task import io.airbyte.cdk.load.command.DestinationStream import io.airbyte.cdk.load.message.BatchEnvelope -import io.airbyte.cdk.load.message.SpilledRawMessagesLocalFile +import io.airbyte.cdk.load.task.internal.SpilledRawMessagesLocalFile import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Requires import jakarta.inject.Singleton @@ -15,7 +15,7 @@ import jakarta.inject.Singleton @Primary @Requires(env = ["MockTaskLauncher"]) class MockTaskLauncher : DestinationTaskLauncher { - val spilledFiles = mutableListOf>() + val spilledFiles = mutableListOf() val batchEnvelopes = mutableListOf>() override suspend fun handleSetupComplete() { @@ -28,10 +28,9 @@ class MockTaskLauncher : DestinationTaskLauncher { override suspend fun handleNewSpilledFile( stream: DestinationStream, - wrapped: BatchEnvelope, - endOfStream: Boolean + file: SpilledRawMessagesLocalFile ) { - spilledFiles.add(wrapped) + spilledFiles.add(file) } override suspend fun handleNewBatch(stream: DestinationStream, wrapped: BatchEnvelope<*>) { diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/implementor/ProcessRecordsTaskTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/implementor/ProcessRecordsTaskTest.kt index ab7029c9944a..2ef466f6addd 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/implementor/ProcessRecordsTaskTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/implementor/ProcessRecordsTaskTest.kt @@ -10,13 +10,12 @@ import io.airbyte.cdk.load.command.MockDestinationCatalogFactory import io.airbyte.cdk.load.data.IntegerValue import io.airbyte.cdk.load.file.MockTempFileProvider import io.airbyte.cdk.load.message.Batch -import io.airbyte.cdk.load.message.BatchEnvelope import io.airbyte.cdk.load.message.Deserializer import io.airbyte.cdk.load.message.DestinationMessage import io.airbyte.cdk.load.message.DestinationRecord -import io.airbyte.cdk.load.message.SpilledRawMessagesLocalFile import io.airbyte.cdk.load.state.SyncManager import io.airbyte.cdk.load.task.MockTaskLauncher +import io.airbyte.cdk.load.task.internal.SpilledRawMessagesLocalFile import io.airbyte.cdk.load.write.StreamLoader import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Requires @@ -100,12 +99,13 @@ class ProcessRecordsTaskTest { SpilledRawMessagesLocalFile( localFile = mockFile, totalSizeBytes = byteSize, + indexRange = Range.closed(0, recordCount) ) val task = processRecordsTaskFactory.make( taskLauncher = launcher, stream = MockDestinationCatalogFactory.stream1, - fileEnvelope = BatchEnvelope(file, Range.closed(0, 1024)) + file = file ) mockFile.linesToRead = (0 until recordCount).map { "$it" }.toMutableList() diff --git a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/internal/SpillToDiskTaskTest.kt b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/internal/SpillToDiskTaskTest.kt index 383e345f1ad0..995ccc572876 100644 --- a/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/internal/SpillToDiskTaskTest.kt +++ b/airbyte-cdk/bulk/core/load/src/test/kotlin/io/airbyte/cdk/load/task/internal/SpillToDiskTaskTest.kt @@ -104,16 +104,16 @@ class SpillToDiskTaskTest { .execute() Assertions.assertEquals(2, mockTaskLauncher.spilledFiles.size) - Assertions.assertEquals(1024, mockTaskLauncher.spilledFiles[0].batch.totalSizeBytes) - Assertions.assertEquals(512, mockTaskLauncher.spilledFiles[1].batch.totalSizeBytes) + Assertions.assertEquals(1024, mockTaskLauncher.spilledFiles[0].totalSizeBytes) + Assertions.assertEquals(512, mockTaskLauncher.spilledFiles[1].totalSizeBytes) - val env1 = mockTaskLauncher.spilledFiles[0] - val env2 = mockTaskLauncher.spilledFiles[1] - Assertions.assertEquals(1024, env1.batch.totalSizeBytes) - Assertions.assertEquals(512, env2.batch.totalSizeBytes) + val spilled1 = mockTaskLauncher.spilledFiles[0] + val spilled2 = mockTaskLauncher.spilledFiles[1] + Assertions.assertEquals(1024, spilled1.totalSizeBytes) + Assertions.assertEquals(512, spilled2.totalSizeBytes) - val file1 = env1.batch.localFile as MockTempFileProvider.MockLocalFile - val file2 = env2.batch.localFile as MockTempFileProvider.MockLocalFile + val file1 = spilled1.localFile as MockTempFileProvider.MockLocalFile + val file2 = spilled2.localFile as MockTempFileProvider.MockLocalFile Assertions.assertTrue(file1.writersCreated[0].isClosed) Assertions.assertTrue(file2.writersCreated[0].isClosed) From a4847dfe49eae0cbb8c1b1cf1becff56c551f1cc Mon Sep 17 00:00:00 2001 From: Marius Posta Date: Thu, 17 Oct 2024 19:36:01 -0700 Subject: [PATCH 08/18] bulk-cdk: add feature flag environments (#46692) --- .../io/airbyte/cdk/AirbyteConnectorRunner.kt | 41 +++++------ .../cdk/ConnectorUncleanExitException.kt | 1 + .../io/airbyte/cdk/command/FeatureFlag.kt | 64 +++++++++++++++++ .../io/airbyte/cdk/command/CliRunner.kt | 13 ++-- .../cdk/load/check/CheckIntegrationTest.kt | 12 ++-- .../io/airbyte/cdk/load/spec/SpecTest.kt | 17 +++-- .../destination_process/DestinationProcess.kt | 8 +-- .../DockerizedDestination.kt | 69 ++++++++++--------- .../NonDockerizedDestination.kt | 16 ++--- .../destination-dev-null/metadata.yaml | 2 +- .../dev_null/DevNullSpecification.kt | 5 +- .../dev_null/DevNullCheckIntegrationTest.kt | 10 +-- .../destination/s3_v2/S3V2CheckTest.kt | 9 ++- .../connectors/source-mysql-v2/metadata.yaml | 2 +- .../source/mysql/MysqlSourceConfiguration.kt | 21 +++++- .../MysqlSourceConfigurationSpecification.kt | 6 +- .../mysql/MysqlSourceConfigurationTest.kt | 31 +++++++-- .../MysqlSourceTestConfigurationFactory.kt | 5 +- docs/integrations/destinations/dev-null.md | 1 + 19 files changed, 221 insertions(+), 112 deletions(-) create mode 100644 airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/command/FeatureFlag.kt diff --git a/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/AirbyteConnectorRunner.kt b/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/AirbyteConnectorRunner.kt index 214d0821e2d4..218ef56c1723 100644 --- a/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/AirbyteConnectorRunner.kt +++ b/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/AirbyteConnectorRunner.kt @@ -2,13 +2,13 @@ package io.airbyte.cdk import io.airbyte.cdk.command.ConnectorCommandLinePropertySource +import io.airbyte.cdk.command.FeatureFlag import io.airbyte.cdk.command.MetadataYamlPropertySource import io.micronaut.configuration.picocli.MicronautFactory import io.micronaut.context.ApplicationContext import io.micronaut.context.RuntimeBeanDefinition import io.micronaut.context.env.CommandLinePropertySource import io.micronaut.context.env.Environment -import io.micronaut.context.env.MapPropertySource import io.micronaut.core.cli.CommandLine as MicronautCommandLine import java.nio.file.Path import kotlin.system.exitProcess @@ -21,9 +21,11 @@ import picocli.CommandLine.Model.UsageMessageSpec class AirbyteSourceRunner( /** CLI args. */ args: Array, + /** Environment variables. */ + systemEnv: Map = System.getenv(), /** Micronaut bean definition overrides, used only for tests. */ vararg testBeanDefinitions: RuntimeBeanDefinition<*>, -) : AirbyteConnectorRunner("source", args, testBeanDefinitions) { +) : AirbyteConnectorRunner("source", args, systemEnv, testBeanDefinitions) { companion object { @JvmStatic fun run(vararg args: String) { @@ -36,10 +38,11 @@ class AirbyteSourceRunner( class AirbyteDestinationRunner( /** CLI args. */ args: Array, - testEnvironments: Map = emptyMap(), + /** Environment variables. */ + systemEnv: Map = System.getenv(), /** Micronaut bean definition overrides, used only for tests. */ vararg testBeanDefinitions: RuntimeBeanDefinition<*>, -) : AirbyteConnectorRunner("destination", args, testBeanDefinitions, testEnvironments) { +) : AirbyteConnectorRunner("destination", args, systemEnv, testBeanDefinitions) { companion object { @JvmStatic fun run(vararg args: String) { @@ -55,21 +58,18 @@ class AirbyteDestinationRunner( sealed class AirbyteConnectorRunner( val connectorType: String, val args: Array, + systemEnv: Map, val testBeanDefinitions: Array>, - val testProperties: Map = emptyMap(), ) { - // Micronaut's TEST env detection relies on inspecting the stacktrace and checking for - // any junit calls. This doesn't work if we launch the connector from a different thread, e.g. - // `Dispatchers.IO`. Force the test env if needed. (Some tests launch the connector from the IO - // context to avoid blocking themselves.) - private val isTest = testBeanDefinitions.isNotEmpty() val envs: Array = arrayOf(Environment.CLI, connectorType) + - if (isTest) { - arrayOf(Environment.TEST) - } else { - emptyArray() - } + // Set feature flag environments. + FeatureFlag.active(systemEnv).map { it.micronautEnvironmentName } + + // Micronaut's TEST env detection relies on inspecting the stacktrace and checking for + // any junit calls. This doesn't work if we launch the connector from a different + // thread, e.g. `Dispatchers.IO`. Force the test env if needed. Some tests launch the + // connector from the IO context to avoid blocking themselves. + listOfNotNull(Environment.TEST.takeIf { testBeanDefinitions.isNotEmpty() }) inline fun run() { val picocliCommandLineFactory = PicocliCommandLineFactory(this) @@ -84,7 +84,6 @@ sealed class AirbyteConnectorRunner( ApplicationContext.builder(R::class.java, *envs) .propertySources( *listOfNotNull( - MapPropertySource("additional_properties", testProperties), airbytePropertySource, commandLinePropertySource, MetadataYamlPropertySource(), @@ -93,10 +92,6 @@ sealed class AirbyteConnectorRunner( ) .beanDefinitions(*testBeanDefinitions) .start() - // We can't rely on the isTest value from our constructor, - // because that won't autodetect junit in our stacktrace. - // So instead we ask micronaut (which will include if we explicitly added - // the TEST env). val isTest: Boolean = ctx.environment.activeNames.contains(Environment.TEST) val picocliFactory: CommandLine.IFactory = MicronautFactory(ctx) val picocliCommandLine: CommandLine = @@ -105,8 +100,10 @@ sealed class AirbyteConnectorRunner( if (!isTest) { // Required by the platform, otherwise syncs may hang. exitProcess(exitCode) - } else if (exitCode != 0) { - // Otherwise, propagate failure to test callers. + } + // At this point, we're in a test. + if (exitCode != 0) { + // Propagate failure to test callers. throw ConnectorUncleanExitException(exitCode) } } diff --git a/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/ConnectorUncleanExitException.kt b/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/ConnectorUncleanExitException.kt index 19c108b8966d..626ee21cde29 100644 --- a/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/ConnectorUncleanExitException.kt +++ b/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/ConnectorUncleanExitException.kt @@ -4,5 +4,6 @@ package io.airbyte.cdk +/** This is used only in tests. */ class ConnectorUncleanExitException(val exitCode: Int) : Exception("Destination process exited uncleanly: $exitCode") diff --git a/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/command/FeatureFlag.kt b/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/command/FeatureFlag.kt new file mode 100644 index 000000000000..4d7fde405b13 --- /dev/null +++ b/airbyte-cdk/bulk/core/base/src/main/kotlin/io/airbyte/cdk/command/FeatureFlag.kt @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.command + +import io.micronaut.context.annotation.Factory +import io.micronaut.context.env.Environment +import jakarta.inject.Singleton +import java.util.EnumSet + +/** + * An enum of all feature flags, currently these are set via environment vars. + * + * Micronaut can inject a Set singleton of all active feature flags. + */ +enum class FeatureFlag( + val micronautEnvironmentName: String, + val envVar: EnvVar, + val requiredEnvVarValue: String, + private val transformActualValue: (String) -> String = { it } +) { + + /** [AIRBYTE_CLOUD_DEPLOYMENT] is active when the connector is running in Airbyte Cloud. */ + AIRBYTE_CLOUD_DEPLOYMENT( + micronautEnvironmentName = AIRBYTE_CLOUD_ENV, + envVar = EnvVar.DEPLOYMENT_MODE, + requiredEnvVarValue = "CLOUD", + transformActualValue = { it.trim().uppercase() }, + ); + + /** Environment variable binding shell declaration which activates the feature flag. */ + val envVarBindingDeclaration: String + get() = "${envVar.name}=$requiredEnvVarValue" + + enum class EnvVar(val defaultValue: String = "") { + DEPLOYMENT_MODE + } + + companion object { + internal fun active(systemEnv: Map): List = + entries.filter { featureFlag: FeatureFlag -> + val envVar: EnvVar = featureFlag.envVar + val envVarValue: String = systemEnv[envVar.name] ?: envVar.defaultValue + featureFlag.transformActualValue(envVarValue) == featureFlag.requiredEnvVarValue + } + } + + @Factory + private class MicronautFactory { + + @Singleton + fun active(environment: Environment): Set = + EnumSet.noneOf(FeatureFlag::class.java).apply { + addAll( + FeatureFlag.entries.filter { + environment.activeNames.contains(it.micronautEnvironmentName) + } + ) + } + } +} + +const val AIRBYTE_CLOUD_ENV = "airbyte-cloud" diff --git a/airbyte-cdk/bulk/core/base/src/testFixtures/kotlin/io/airbyte/cdk/command/CliRunner.kt b/airbyte-cdk/bulk/core/base/src/testFixtures/kotlin/io/airbyte/cdk/command/CliRunner.kt index 44950c69fe0f..4d72c69df974 100644 --- a/airbyte-cdk/bulk/core/base/src/testFixtures/kotlin/io/airbyte/cdk/command/CliRunner.kt +++ b/airbyte-cdk/bulk/core/base/src/testFixtures/kotlin/io/airbyte/cdk/command/CliRunner.kt @@ -36,11 +36,12 @@ data object CliRunner { config: ConfigurationSpecification? = null, catalog: ConfiguredAirbyteCatalog? = null, state: List? = null, + vararg featureFlags: FeatureFlag, ): CliRunnable { val out = CliRunnerOutputStream() val runnable: Runnable = makeRunnable(op, config, catalog, state) { args: Array -> - AirbyteSourceRunner(args, out.beanDefinition) + AirbyteSourceRunner(args, featureFlags.systemEnv, out.beanDefinition) } return CliRunnable(runnable, out.results) } @@ -52,7 +53,7 @@ data object CliRunner { catalog: ConfiguredAirbyteCatalog? = null, state: List? = null, inputStream: InputStream, - testProperties: Map = emptyMap(), + vararg featureFlags: FeatureFlag, ): CliRunnable { val inputBeanDefinition: RuntimeBeanDefinition = RuntimeBeanDefinition.builder(InputStream::class.java) { inputStream } @@ -63,7 +64,7 @@ data object CliRunner { makeRunnable(op, config, catalog, state) { args: Array -> AirbyteDestinationRunner( args, - testProperties, + featureFlags.systemEnv, inputBeanDefinition, out.beanDefinition, ) @@ -77,6 +78,7 @@ data object CliRunner { config: ConfigurationSpecification? = null, catalog: ConfiguredAirbyteCatalog? = null, state: List? = null, + featureFlags: Set = setOf(), vararg input: AirbyteMessage, ): CliRunnable { val inputJsonBytes: ByteArray = @@ -88,7 +90,7 @@ data object CliRunner { baos.toByteArray() } val inputStream: InputStream = ByteArrayInputStream(inputJsonBytes) - return destination(op, config, catalog, state, inputStream) + return destination(op, config, catalog, state, inputStream, *featureFlags.toTypedArray()) } private fun makeRunnable( @@ -120,6 +122,9 @@ data object CliRunner { } } + private val Array.systemEnv: Map + get() = toSet().map { it.envVar.name to it.requiredEnvVarValue }.toMap() + private fun inputFile(contents: Any?): Path? = contents?.let { Files.createTempFile(null, null).also { file -> diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/check/CheckIntegrationTest.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/check/CheckIntegrationTest.kt index f2e8ed655528..3d9b1d017177 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/check/CheckIntegrationTest.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/check/CheckIntegrationTest.kt @@ -5,12 +5,12 @@ package io.airbyte.cdk.load.check import io.airbyte.cdk.command.ConfigurationSpecification +import io.airbyte.cdk.command.FeatureFlag import io.airbyte.cdk.command.ValidatedJsonUtils import io.airbyte.cdk.load.test.util.FakeDataDumper import io.airbyte.cdk.load.test.util.IntegrationTest import io.airbyte.cdk.load.test.util.NoopDestinationCleaner import io.airbyte.cdk.load.test.util.NoopExpectedRecordMapper -import io.airbyte.cdk.load.test.util.destination_process.TestDeploymentMode import io.airbyte.protocol.models.v0.AirbyteConnectionStatus import io.airbyte.protocol.models.v0.AirbyteMessage import java.nio.charset.StandardCharsets @@ -23,7 +23,7 @@ import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertAll -data class CheckTestConfig(val configPath: String, val deploymentMode: TestDeploymentMode) +data class CheckTestConfig(val configPath: String, val featureFlags: Set = emptySet()) open class CheckIntegrationTest( val configurationClass: Class, @@ -37,14 +37,14 @@ open class CheckIntegrationTest( ) { @Test open fun testSuccessConfigs() { - for ((path, deploymentMode) in successConfigFilenames) { + for ((path, featureFlags) in successConfigFilenames) { val fileContents = Files.readString(Path.of(path), StandardCharsets.UTF_8) val config = ValidatedJsonUtils.parseOne(configurationClass, fileContents) val process = destinationProcessFactory.createDestinationProcess( "check", config = config, - deploymentMode = deploymentMode, + featureFlags = featureFlags.toTypedArray(), ) runBlocking { process.run() } val messages = process.readMessages() @@ -66,14 +66,14 @@ open class CheckIntegrationTest( @Test open fun testFailConfigs() { for ((checkTestConfig, failurePattern) in failConfigFilenamesAndFailureReasons) { - val (path, deploymentMode) = checkTestConfig + val (path, featureFlags) = checkTestConfig val fileContents = Files.readString(Path.of(path)) val config = ValidatedJsonUtils.parseOne(configurationClass, fileContents) val process = destinationProcessFactory.createDestinationProcess( "check", config = config, - deploymentMode = deploymentMode, + featureFlags = featureFlags.toTypedArray(), ) runBlocking { process.run() } val messages = process.readMessages() diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/spec/SpecTest.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/spec/SpecTest.kt index 06d678c18aa8..d2dd15ae9541 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/spec/SpecTest.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/spec/SpecTest.kt @@ -12,11 +12,11 @@ import com.deblock.jsondiff.matcher.LenientJsonObjectPartialMatcher import com.deblock.jsondiff.matcher.StrictJsonArrayPartialMatcher import com.deblock.jsondiff.matcher.StrictPrimitivePartialMatcher import com.deblock.jsondiff.viewer.OnlyErrorDiffViewer +import io.airbyte.cdk.command.FeatureFlag import io.airbyte.cdk.load.test.util.FakeDataDumper import io.airbyte.cdk.load.test.util.IntegrationTest import io.airbyte.cdk.load.test.util.NoopDestinationCleaner import io.airbyte.cdk.load.test.util.NoopExpectedRecordMapper -import io.airbyte.cdk.load.test.util.destination_process.TestDeploymentMode import io.airbyte.cdk.util.Jsons import io.airbyte.protocol.models.v0.AirbyteMessage import java.nio.file.Files @@ -42,16 +42,18 @@ abstract class SpecTest : ) { @Test fun testSpecOss() { - testSpec(TestDeploymentMode.OSS) + testSpec("expected-spec-oss.json") } @Test fun testSpecCloud() { - testSpec(TestDeploymentMode.CLOUD) + testSpec("expected-spec-cloud.json", FeatureFlag.AIRBYTE_CLOUD_DEPLOYMENT) } - private fun testSpec(deploymentMode: TestDeploymentMode) { - val expectedSpecFilename = "expected-spec-${deploymentMode.name.lowercase()}.json" + private fun testSpec( + expectedSpecFilename: String, + vararg featureFlags: FeatureFlag, + ) { val expectedSpecPath = Path.of("src/test-integration/resources", expectedSpecFilename) if (!Files.exists(expectedSpecPath)) { @@ -59,10 +61,7 @@ abstract class SpecTest : } val expectedSpec = Files.readString(expectedSpecPath) val process = - destinationProcessFactory.createDestinationProcess( - "spec", - deploymentMode = deploymentMode, - ) + destinationProcessFactory.createDestinationProcess("spec", featureFlags = featureFlags) runBlocking { process.run() } val messages = process.readMessages() val specMessages = messages.filter { it.type == AirbyteMessage.Type.SPEC } diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DestinationProcess.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DestinationProcess.kt index 2829b519e412..42ec5c0d3218 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DestinationProcess.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DestinationProcess.kt @@ -6,6 +6,7 @@ package io.airbyte.cdk.load.test.util.destination_process import edu.umd.cs.findbugs.annotations.SuppressFBWarnings import io.airbyte.cdk.command.ConfigurationSpecification +import io.airbyte.cdk.command.FeatureFlag import io.airbyte.cdk.load.test.util.IntegrationTest import io.airbyte.protocol.models.v0.AirbyteMessage import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog @@ -40,11 +41,6 @@ interface DestinationProcess { suspend fun shutdown() } -enum class TestDeploymentMode { - CLOUD, - OSS -} - @SuppressFBWarnings("NP_NONNULL_RETURN_VIOLATION", "good old lateinit") abstract class DestinationProcessFactory { /** @@ -58,6 +54,6 @@ abstract class DestinationProcessFactory { command: String, config: ConfigurationSpecification? = null, catalog: ConfiguredAirbyteCatalog? = null, - deploymentMode: TestDeploymentMode = TestDeploymentMode.OSS, + vararg featureFlags: FeatureFlag, ): DestinationProcess } diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DockerizedDestination.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DockerizedDestination.kt index dd310cccafc8..35eafc396068 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DockerizedDestination.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DockerizedDestination.kt @@ -4,8 +4,8 @@ package io.airbyte.cdk.load.test.util.destination_process -import com.google.common.collect.Lists import io.airbyte.cdk.command.ConfigurationSpecification +import io.airbyte.cdk.command.FeatureFlag import io.airbyte.cdk.output.BufferingOutputConsumer import io.airbyte.cdk.util.Jsons import io.airbyte.protocol.models.v0.AirbyteLogMessage @@ -36,8 +36,8 @@ class DockerizedDestination( command: String, config: ConfigurationSpecification?, catalog: ConfiguredAirbyteCatalog?, - testDeploymentMode: TestDeploymentMode, private val testName: String, + vararg featureFlags: FeatureFlag, ) : DestinationProcess { private val process: Process private val destinationOutput = BufferingOutputConsumer(Clock.systemDefaultZone()) @@ -85,35 +85,38 @@ class DockerizedDestination( logger.info { "Creating docker container $containerName" } val cmd: MutableList = - Lists.newArrayList( - "docker", - "run", - "--rm", - "--init", - "-i", - "-w", - "/data/job", - "--log-driver", - "none", - "--name", - containerName, - "--network", - "host", - "-v", - String.format("%s:%s", workspaceRoot, "/data"), - "-v", - String.format("%s:%s", localRoot, "/local"), - "-e", - "DEPLOYMENT_MODE=$testDeploymentMode", - // Yes, we hardcode the job ID to 0. - // Also yes, this is available in the configured catalog - // via the syncId property. - // Also also yes, we're relying on this env var >.> - "-e", - "WORKER_JOB_ID=0", - imageTag, - command, - ) + (listOf( + "docker", + "run", + "--rm", + "--init", + "-i", + "-w", + "/data/job", + "--log-driver", + "none", + "--name", + containerName, + "--network", + "host", + "-v", + String.format("%s:%s", workspaceRoot, "/data"), + "-v", + String.format("%s:%s", localRoot, "/local"), + ) + + featureFlags.flatMap { listOf("-e", it.envVarBindingDeclaration) } + + listOf( + + // Yes, we hardcode the job ID to 0. + // Also yes, this is available in the configured catalog + // via the syncId property. + // Also also yes, we're relying on this env var >.> + "-e", + "WORKER_JOB_ID=0", + imageTag, + command, + )) + .toMutableList() fun addInput(paramName: String, fileContents: Any) { Files.write( @@ -230,15 +233,15 @@ class DockerizedDestinationFactory( command: String, config: ConfigurationSpecification?, catalog: ConfiguredAirbyteCatalog?, - deploymentMode: TestDeploymentMode, + vararg featureFlags: FeatureFlag, ): DestinationProcess { return DockerizedDestination( "$imageName:$imageVersion", command, config, catalog, - deploymentMode, testName, + *featureFlags, ) } } diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/NonDockerizedDestination.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/NonDockerizedDestination.kt index 60ebaaa03270..7908e7268f29 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/NonDockerizedDestination.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/NonDockerizedDestination.kt @@ -8,6 +8,7 @@ import io.airbyte.cdk.ConnectorUncleanExitException import io.airbyte.cdk.command.CliRunnable import io.airbyte.cdk.command.CliRunner import io.airbyte.cdk.command.ConfigurationSpecification +import io.airbyte.cdk.command.FeatureFlag import io.airbyte.protocol.models.Jsons import io.airbyte.protocol.models.v0.AirbyteMessage import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog @@ -21,7 +22,7 @@ class NonDockerizedDestination( command: String, config: ConfigurationSpecification?, catalog: ConfiguredAirbyteCatalog?, - testDeploymentMode: TestDeploymentMode, + vararg featureFlags: FeatureFlag, ) : DestinationProcess { private val destinationStdinPipe: PrintWriter private val destination: CliRunnable @@ -36,20 +37,13 @@ class NonDockerizedDestination( // from PrintWriter(outputStream) ). // Thanks, spotbugs. PrintWriter(PipedOutputStream(destinationStdin), false, Charsets.UTF_8) - val testEnvironments = - when (testDeploymentMode) { - // the env var is DEPLOYMENT_MODE, which micronaut parses to - // a property called deployment.mode. - TestDeploymentMode.CLOUD -> mapOf("deployment.mode" to "CLOUD") - TestDeploymentMode.OSS -> mapOf("deployment.mode" to "OSS") - } destination = CliRunner.destination( command, config = config, catalog = catalog, - testProperties = testEnvironments, inputStream = destinationStdin, + featureFlags = featureFlags, ) } @@ -82,9 +76,9 @@ class NonDockerizedDestinationFactory : DestinationProcessFactory() { command: String, config: ConfigurationSpecification?, catalog: ConfiguredAirbyteCatalog?, - deploymentMode: TestDeploymentMode, + vararg featureFlags: FeatureFlag, ): DestinationProcess { // TODO pass test name into the destination process - return NonDockerizedDestination(command, config, catalog, deploymentMode) + return NonDockerizedDestination(command, config, catalog, *featureFlags) } } diff --git a/airbyte-integrations/connectors/destination-dev-null/metadata.yaml b/airbyte-integrations/connectors/destination-dev-null/metadata.yaml index 3d9d3085ae37..bff0c2b7f320 100644 --- a/airbyte-integrations/connectors/destination-dev-null/metadata.yaml +++ b/airbyte-integrations/connectors/destination-dev-null/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: file connectorType: destination definitionId: a7bcc9d8-13b3-4e49-b80d-d020b90045e3 - dockerImageTag: 0.7.6 + dockerImageTag: 0.7.7 dockerRepository: airbyte/destination-dev-null githubIssueLabel: destination-dev-null icon: airbyte.svg diff --git a/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullSpecification.kt b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullSpecification.kt index 0eb567f84171..bd3e1434338a 100644 --- a/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullSpecification.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/main/kotlin/io/airbyte/integrations/destination/dev_null/DevNullSpecification.kt @@ -12,6 +12,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo import com.fasterxml.jackson.annotation.JsonValue import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaInject import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle +import io.airbyte.cdk.command.AIRBYTE_CLOUD_ENV import io.airbyte.cdk.command.ConfigurationSpecification import io.airbyte.cdk.load.spec.DestinationSpecificationExtension import io.airbyte.protocol.models.v0.DestinationSyncMode @@ -45,7 +46,7 @@ sealed class DevNullSpecification : ConfigurationSpecification() { */ @JsonSchemaTitle("E2E Test Destination Spec") @Singleton -@Requires(property = "deployment.mode", pattern = "(?i)oss") +@Requires(notEnv = [AIRBYTE_CLOUD_ENV]) class DevNullSpecificationOss : DevNullSpecification() { @JsonProperty("test_destination") @JsonSchemaTitle("Test Destination") @@ -201,7 +202,7 @@ data class FailingDestination( /** The cloud variant is more restricted: it only allows for a single destination type. */ @JsonSchemaTitle("E2E Test Destination Spec") @Singleton -@Requires(property = "deployment.mode", pattern = "(?i)cloud") +@Requires(env = [AIRBYTE_CLOUD_ENV]) class DevNullSpecificationCloud : DevNullSpecification() { @JsonProperty("test_destination") @JsonSchemaTitle("Test Destination") diff --git a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullCheckIntegrationTest.kt b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullCheckIntegrationTest.kt index 5d3539ad251c..1e06c9ff8a86 100644 --- a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullCheckIntegrationTest.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullCheckIntegrationTest.kt @@ -4,9 +4,9 @@ package io.airbyte.integrations.destination.dev_null +import io.airbyte.cdk.command.FeatureFlag import io.airbyte.cdk.load.check.CheckIntegrationTest import io.airbyte.cdk.load.check.CheckTestConfig -import io.airbyte.cdk.load.test.util.destination_process.TestDeploymentMode import java.util.regex.Pattern import org.junit.jupiter.api.Test @@ -15,14 +15,16 @@ class DevNullCheckIntegrationTest : DevNullSpecificationOss::class.java, successConfigFilenames = listOf( - CheckTestConfig(DevNullTestUtils.LOGGING_CONFIG_PATH, TestDeploymentMode.OSS), + CheckTestConfig(DevNullTestUtils.LOGGING_CONFIG_PATH), ), failConfigFilenamesAndFailureReasons = mapOf( // cloud doesn't support logging mode, so this should fail // when trying to parse the config - CheckTestConfig(DevNullTestUtils.LOGGING_CONFIG_PATH, TestDeploymentMode.CLOUD) to - Pattern.compile("Value 'LOGGING' is not defined in the schema") + CheckTestConfig( + DevNullTestUtils.LOGGING_CONFIG_PATH, + setOf(FeatureFlag.AIRBYTE_CLOUD_DEPLOYMENT) + ) to Pattern.compile("Value 'LOGGING' is not defined in the schema") ), ) { diff --git a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2CheckTest.kt b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2CheckTest.kt index 528088495c00..3bac6fb726f9 100644 --- a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2CheckTest.kt +++ b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2CheckTest.kt @@ -4,16 +4,21 @@ package io.airbyte.integrations.destination.s3_v2 +import io.airbyte.cdk.command.FeatureFlag import io.airbyte.cdk.load.check.CheckIntegrationTest import io.airbyte.cdk.load.check.CheckTestConfig -import io.airbyte.cdk.load.test.util.destination_process.TestDeploymentMode import org.junit.jupiter.api.Test class S3V2CheckTest : CheckIntegrationTest( S3V2Specification::class.java, successConfigFilenames = - listOf(CheckTestConfig(S3V2TestUtils.MINIMAL_CONFIG_PATH, TestDeploymentMode.CLOUD)), + listOf( + CheckTestConfig( + S3V2TestUtils.MINIMAL_CONFIG_PATH, + setOf(FeatureFlag.AIRBYTE_CLOUD_DEPLOYMENT) + ) + ), failConfigFilenamesAndFailureReasons = emptyMap() ) { @Test diff --git a/airbyte-integrations/connectors/source-mysql-v2/metadata.yaml b/airbyte-integrations/connectors/source-mysql-v2/metadata.yaml index 1a73f0b8ae0c..571fe85d7588 100644 --- a/airbyte-integrations/connectors/source-mysql-v2/metadata.yaml +++ b/airbyte-integrations/connectors/source-mysql-v2/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: 561393ed-7e3a-4d0d-8b8b-90ded371754c - dockerImageTag: 0.0.25 + dockerImageTag: 0.0.26 dockerRepository: airbyte/source-mysql-v2 documentationUrl: https://docs.airbyte.com/integrations/sources/mysql githubIssueLabel: source-mysql-v2 diff --git a/airbyte-integrations/connectors/source-mysql-v2/src/main/kotlin/io/airbyte/integrations/source/mysql/MysqlSourceConfiguration.kt b/airbyte-integrations/connectors/source-mysql-v2/src/main/kotlin/io/airbyte/integrations/source/mysql/MysqlSourceConfiguration.kt index 0c20f18dadcf..e4ca3972b0a6 100644 --- a/airbyte-integrations/connectors/source-mysql-v2/src/main/kotlin/io/airbyte/integrations/source/mysql/MysqlSourceConfiguration.kt +++ b/airbyte-integrations/connectors/source-mysql-v2/src/main/kotlin/io/airbyte/integrations/source/mysql/MysqlSourceConfiguration.kt @@ -4,13 +4,16 @@ package io.airbyte.integrations.source.mysql import io.airbyte.cdk.ConfigErrorException import io.airbyte.cdk.command.CdcSourceConfiguration import io.airbyte.cdk.command.ConfigurationSpecificationSupplier +import io.airbyte.cdk.command.FeatureFlag import io.airbyte.cdk.command.JdbcSourceConfiguration import io.airbyte.cdk.command.SourceConfiguration import io.airbyte.cdk.command.SourceConfigurationFactory import io.airbyte.cdk.ssh.SshConnectionOptions +import io.airbyte.cdk.ssh.SshNoTunnelMethod import io.airbyte.cdk.ssh.SshTunnelMethodConfiguration import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Factory +import jakarta.inject.Inject import jakarta.inject.Singleton import java.net.URLDecoder import java.nio.charset.StandardCharsets @@ -68,8 +71,11 @@ enum class InvalidCdcCursorPositionBehavior { } @Singleton -class MysqlSourceConfigurationFactory : +class MysqlSourceConfigurationFactory @Inject constructor(val featureFlags: Set) : SourceConfigurationFactory { + + constructor() : this(emptySet()) + override fun makeWithoutExceptionHandling( pojo: MysqlSourceConfigurationSpecification, ): MysqlSourceConfiguration { @@ -99,7 +105,18 @@ class MysqlSourceConfigurationFactory : val encryption: Encryption = pojo.getEncryptionValue() val jdbcEncryption = when (encryption) { - is EncryptionPreferred -> MysqlJdbcEncryption(sslMode = SSLMode.PREFERRED) + is EncryptionPreferred -> { + if ( + featureFlags.contains(FeatureFlag.AIRBYTE_CLOUD_DEPLOYMENT) && + sshTunnel is SshNoTunnelMethod + ) { + throw ConfigErrorException( + "Connection from Airbyte Cloud requires " + + "SSL encryption or an SSH tunnel." + ) + } + MysqlJdbcEncryption(sslMode = SSLMode.PREFERRED) + } is EncryptionRequired -> MysqlJdbcEncryption(sslMode = SSLMode.REQUIRED) is SslVerifyCertificate -> MysqlJdbcEncryption( diff --git a/airbyte-integrations/connectors/source-mysql-v2/src/main/kotlin/io/airbyte/integrations/source/mysql/MysqlSourceConfigurationSpecification.kt b/airbyte-integrations/connectors/source-mysql-v2/src/main/kotlin/io/airbyte/integrations/source/mysql/MysqlSourceConfigurationSpecification.kt index 6ab78608d870..dd1a9a953f63 100644 --- a/airbyte-integrations/connectors/source-mysql-v2/src/main/kotlin/io/airbyte/integrations/source/mysql/MysqlSourceConfigurationSpecification.kt +++ b/airbyte-integrations/connectors/source-mysql-v2/src/main/kotlin/io/airbyte/integrations/source/mysql/MysqlSourceConfigurationSpecification.kt @@ -286,17 +286,17 @@ class SslVerifyIdentity : Encryption { @ConfigurationProperties("$CONNECTOR_CONFIG_PREFIX.ssl_mode") class MicronautPropertiesFriendlyEncryption { - var encryptionMethod: String = "preferred" + var mode: String = "preferred" var sslCertificate: String? = null @JsonValue fun asEncryption(): Encryption = - when (encryptionMethod) { + when (mode) { "preferred" -> EncryptionPreferred "required" -> EncryptionRequired "verify_ca" -> SslVerifyCertificate().also { it.sslCertificate = sslCertificate!! } "verify_identity" -> SslVerifyIdentity().also { it.sslCertificate = sslCertificate!! } - else -> throw ConfigErrorException("invalid value $encryptionMethod") + else -> throw ConfigErrorException("invalid value $mode") } } diff --git a/airbyte-integrations/connectors/source-mysql-v2/src/test/kotlin/io/airbyte/integrations/source/mysql/MysqlSourceConfigurationTest.kt b/airbyte-integrations/connectors/source-mysql-v2/src/test/kotlin/io/airbyte/integrations/source/mysql/MysqlSourceConfigurationTest.kt index f58dba793066..98fd548677fa 100644 --- a/airbyte-integrations/connectors/source-mysql-v2/src/test/kotlin/io/airbyte/integrations/source/mysql/MysqlSourceConfigurationTest.kt +++ b/airbyte-integrations/connectors/source-mysql-v2/src/test/kotlin/io/airbyte/integrations/source/mysql/MysqlSourceConfigurationTest.kt @@ -1,10 +1,11 @@ /* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ package io.airbyte.integrations.source.mysql +import io.airbyte.cdk.ConfigErrorException +import io.airbyte.cdk.command.AIRBYTE_CLOUD_ENV import io.airbyte.cdk.command.ConfigurationSpecificationSupplier import io.airbyte.cdk.command.SourceConfigurationFactory import io.airbyte.cdk.ssh.SshNoTunnelMethod -import io.airbyte.cdk.ssh.SshPasswordAuthTunnelMethod import io.micronaut.context.annotation.Property import io.micronaut.context.env.Environment import io.micronaut.test.extensions.junit5.annotation.MicronautTest @@ -13,7 +14,7 @@ import java.time.Duration import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test -@MicronautTest(environments = [Environment.TEST], rebuildContext = true) +@MicronautTest(environments = [Environment.TEST, AIRBYTE_CLOUD_ENV], rebuildContext = true) class MysqlSourceConfigurationTest { @Inject lateinit var pojoSupplier: @@ -24,7 +25,16 @@ class MysqlSourceConfigurationTest { SourceConfigurationFactory @Test - @Property(name = "airbyte.connector.config.json", value = CONFIG) + @Property(name = "airbyte.connector.config.host", value = "localhost") + @Property(name = "airbyte.connector.config.port", value = "12345") + @Property(name = "airbyte.connector.config.username", value = "FOO") + @Property(name = "airbyte.connector.config.password", value = "BAR") + @Property(name = "airbyte.connector.config.database", value = "SYSTEM") + @Property(name = "airbyte.connector.config.ssl_mode.mode", value = "required") + @Property( + name = "airbyte.connector.config.jdbc_url_params", + value = "theAnswerToLiveAndEverything=42&sessionVariables=max_execution_time=10000&foo=bar&" + ) fun testParseJdbcParameters() { val pojo: MysqlSourceConfigurationSpecification = pojoSupplier.get() @@ -33,7 +43,7 @@ class MysqlSourceConfigurationTest { Assertions.assertEquals(config.realHost, "localhost") Assertions.assertEquals(config.realPort, 12345) Assertions.assertEquals(config.namespaces, setOf("SYSTEM")) - Assertions.assertTrue(config.sshTunnel is SshPasswordAuthTunnelMethod) + Assertions.assertTrue(config.sshTunnel is SshNoTunnelMethod) Assertions.assertEquals(config.jdbcProperties["user"], "FOO") Assertions.assertEquals(config.jdbcProperties["password"], "BAR") @@ -46,6 +56,19 @@ class MysqlSourceConfigurationTest { Assertions.assertEquals(config.jdbcProperties["foo"], "bar") } + @Test + @Property(name = "airbyte.connector.config.host", value = "localhost") + @Property(name = "airbyte.connector.config.port", value = "12345") + @Property(name = "airbyte.connector.config.username", value = "FOO") + @Property(name = "airbyte.connector.config.password", value = "BAR") + @Property(name = "airbyte.connector.config.database", value = "SYSTEM") + fun testAirbyteCloudDeployment() { + val pojo: MysqlSourceConfigurationSpecification = pojoSupplier.get() + Assertions.assertThrows(ConfigErrorException::class.java) { + factory.makeWithoutExceptionHandling(pojo) + } + } + @Test @Property(name = "airbyte.connector.config.json", value = CONFIG_V1) fun testParseConfigFromV1() { diff --git a/airbyte-integrations/connectors/source-mysql-v2/src/test/kotlin/io/airbyte/integrations/source/mysql/MysqlSourceTestConfigurationFactory.kt b/airbyte-integrations/connectors/source-mysql-v2/src/test/kotlin/io/airbyte/integrations/source/mysql/MysqlSourceTestConfigurationFactory.kt index 7bac9f03e2cc..60930cb82823 100644 --- a/airbyte-integrations/connectors/source-mysql-v2/src/test/kotlin/io/airbyte/integrations/source/mysql/MysqlSourceTestConfigurationFactory.kt +++ b/airbyte-integrations/connectors/source-mysql-v2/src/test/kotlin/io/airbyte/integrations/source/mysql/MysqlSourceTestConfigurationFactory.kt @@ -1,6 +1,7 @@ /* Copyright (c) 2024 Airbyte, Inc., all rights reserved. */ package io.airbyte.integrations.source.mysql +import io.airbyte.cdk.command.FeatureFlag import io.airbyte.cdk.command.SourceConfigurationFactory import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Requires @@ -11,12 +12,12 @@ import java.time.Duration @Singleton @Requires(env = [Environment.TEST]) @Primary -class MysqlSourceTestConfigurationFactory : +class MysqlSourceTestConfigurationFactory(val featureFlags: Set) : SourceConfigurationFactory { override fun makeWithoutExceptionHandling( pojo: MysqlSourceConfigurationSpecification, ): MysqlSourceConfiguration = - MysqlSourceConfigurationFactory() + MysqlSourceConfigurationFactory(featureFlags) .makeWithoutExceptionHandling(pojo) .copy( maxConcurrency = 1, diff --git a/docs/integrations/destinations/dev-null.md b/docs/integrations/destinations/dev-null.md index 33bc05b56998..f9e98018a139 100644 --- a/docs/integrations/destinations/dev-null.md +++ b/docs/integrations/destinations/dev-null.md @@ -49,6 +49,7 @@ The OSS and Cloud variants have the same version number starting from version `0 | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------| +| 0.7.7 | 2024-10-17 | [46692](https://github.com/airbytehq/airbyte/pull/46692) | Internal code changes | | 0.7.6 | 2024-10-08 | [46683](https://github.com/airbytehq/airbyte/pull/46683) | Bugfix: pick up checkpoint safety check fix | | 0.7.5 | 2024-10-08 | [46683](https://github.com/airbytehq/airbyte/pull/46683) | Bugfix: checkpoints in order, all checkpoints processed before shutdown | | 0.7.4 | 2024-10-08 | [46650](https://github.com/airbytehq/airbyte/pull/46650) | Internal code changes | From f07571f15f1bdbba86ad5e324e829a89b7d07cd6 Mon Sep 17 00:00:00 2001 From: Artem Inzhyyants <36314070+artem1205@users.noreply.github.com> Date: Fri, 18 Oct 2024 11:10:44 +0200 Subject: [PATCH 09/18] ref(source-facebook-marketing): raise exception on missing stream (#46546) Signed-off-by: Artem Inzhyyants --- .../source-facebook-marketing/metadata.yaml | 2 +- .../source-facebook-marketing/poetry.lock | 1883 +++++++++++------ .../source-facebook-marketing/pyproject.toml | 8 +- .../config_migrations.py | 23 +- .../source_facebook_marketing/source.py | 2 +- .../streams/base_insight_streams.py | 8 +- .../streams/base_streams.py | 25 +- .../test_ads_insights_action_product_id.py | 8 +- .../integration/test_include_deleted.py | 8 +- .../unit_tests/integration/test_videos.py | 8 +- .../unit_tests/integration/utils.py | 2 +- .../unit_tests/test_base_insight_streams.py | 20 +- .../unit_tests/test_base_streams.py | 2 +- .../unit_tests/test_client.py | 3 +- .../unit_tests/test_config_migrations.py | 22 +- .../unit_tests/test_source.py | 5 +- .../unit_tests/utils.py | 2 +- .../sources/facebook-marketing.md | 77 +- 18 files changed, 1323 insertions(+), 785 deletions(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml index a73b772ec1fd..8115981aa4d7 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c - dockerImageTag: 3.3.15 + dockerImageTag: 3.3.16 dockerRepository: airbyte/source-facebook-marketing documentationUrl: https://docs.airbyte.com/integrations/sources/facebook-marketing githubIssueLabel: source-facebook-marketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/poetry.lock b/airbyte-integrations/connectors/source-facebook-marketing/poetry.lock index f9fe6a6362d8..6902c2c6c67f 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/poetry.lock +++ b/airbyte-integrations/connectors/source-facebook-marketing/poetry.lock @@ -1,100 +1,127 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. + +[[package]] +name = "aiohappyeyeballs" +version = "2.4.3" +description = "Happy Eyeballs for asyncio" +optional = false +python-versions = ">=3.8" +files = [ + {file = "aiohappyeyeballs-2.4.3-py3-none-any.whl", hash = "sha256:8a7a83727b2756f394ab2895ea0765a0a8c475e3c71e98d43d76f22b4b435572"}, + {file = "aiohappyeyeballs-2.4.3.tar.gz", hash = "sha256:75cf88a15106a5002a8eb1dab212525c00d1f4c0fa96e551c9fbe6f09a621586"}, +] [[package]] name = "aiohttp" -version = "3.9.5" +version = "3.10.9" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.8" files = [ - {file = "aiohttp-3.9.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fcde4c397f673fdec23e6b05ebf8d4751314fa7c24f93334bf1f1364c1c69ac7"}, - {file = "aiohttp-3.9.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d6b3f1fabe465e819aed2c421a6743d8debbde79b6a8600739300630a01bf2c"}, - {file = "aiohttp-3.9.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6ae79c1bc12c34082d92bf9422764f799aee4746fd7a392db46b7fd357d4a17a"}, - {file = "aiohttp-3.9.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d3ebb9e1316ec74277d19c5f482f98cc65a73ccd5430540d6d11682cd857430"}, - {file = "aiohttp-3.9.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84dabd95154f43a2ea80deffec9cb44d2e301e38a0c9d331cc4aa0166fe28ae3"}, - {file = "aiohttp-3.9.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c8a02fbeca6f63cb1f0475c799679057fc9268b77075ab7cf3f1c600e81dd46b"}, - {file = "aiohttp-3.9.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c26959ca7b75ff768e2776d8055bf9582a6267e24556bb7f7bd29e677932be72"}, - {file = "aiohttp-3.9.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:714d4e5231fed4ba2762ed489b4aec07b2b9953cf4ee31e9871caac895a839c0"}, - {file = "aiohttp-3.9.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e7a6a8354f1b62e15d48e04350f13e726fa08b62c3d7b8401c0a1314f02e3558"}, - {file = "aiohttp-3.9.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c413016880e03e69d166efb5a1a95d40f83d5a3a648d16486592c49ffb76d0db"}, - {file = "aiohttp-3.9.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ff84aeb864e0fac81f676be9f4685f0527b660f1efdc40dcede3c251ef1e867f"}, - {file = "aiohttp-3.9.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ad7f2919d7dac062f24d6f5fe95d401597fbb015a25771f85e692d043c9d7832"}, - {file = "aiohttp-3.9.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:702e2c7c187c1a498a4e2b03155d52658fdd6fda882d3d7fbb891a5cf108bb10"}, - {file = "aiohttp-3.9.5-cp310-cp310-win32.whl", hash = "sha256:67c3119f5ddc7261d47163ed86d760ddf0e625cd6246b4ed852e82159617b5fb"}, - {file = "aiohttp-3.9.5-cp310-cp310-win_amd64.whl", hash = "sha256:471f0ef53ccedec9995287f02caf0c068732f026455f07db3f01a46e49d76bbb"}, - {file = "aiohttp-3.9.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e0ae53e33ee7476dd3d1132f932eeb39bf6125083820049d06edcdca4381f342"}, - {file = "aiohttp-3.9.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c088c4d70d21f8ca5c0b8b5403fe84a7bc8e024161febdd4ef04575ef35d474d"}, - {file = "aiohttp-3.9.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:639d0042b7670222f33b0028de6b4e2fad6451462ce7df2af8aee37dcac55424"}, - {file = "aiohttp-3.9.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f26383adb94da5e7fb388d441bf09c61e5e35f455a3217bfd790c6b6bc64b2ee"}, - {file = "aiohttp-3.9.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:66331d00fb28dc90aa606d9a54304af76b335ae204d1836f65797d6fe27f1ca2"}, - {file = "aiohttp-3.9.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4ff550491f5492ab5ed3533e76b8567f4b37bd2995e780a1f46bca2024223233"}, - {file = "aiohttp-3.9.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f22eb3a6c1080d862befa0a89c380b4dafce29dc6cd56083f630073d102eb595"}, - {file = "aiohttp-3.9.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a81b1143d42b66ffc40a441379387076243ef7b51019204fd3ec36b9f69e77d6"}, - {file = "aiohttp-3.9.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f64fd07515dad67f24b6ea4a66ae2876c01031de91c93075b8093f07c0a2d93d"}, - {file = "aiohttp-3.9.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:93e22add827447d2e26d67c9ac0161756007f152fdc5210277d00a85f6c92323"}, - {file = "aiohttp-3.9.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:55b39c8684a46e56ef8c8d24faf02de4a2b2ac60d26cee93bc595651ff545de9"}, - {file = "aiohttp-3.9.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4715a9b778f4293b9f8ae7a0a7cef9829f02ff8d6277a39d7f40565c737d3771"}, - {file = "aiohttp-3.9.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:afc52b8d969eff14e069a710057d15ab9ac17cd4b6753042c407dcea0e40bf75"}, - {file = "aiohttp-3.9.5-cp311-cp311-win32.whl", hash = "sha256:b3df71da99c98534be076196791adca8819761f0bf6e08e07fd7da25127150d6"}, - {file = "aiohttp-3.9.5-cp311-cp311-win_amd64.whl", hash = "sha256:88e311d98cc0bf45b62fc46c66753a83445f5ab20038bcc1b8a1cc05666f428a"}, - {file = "aiohttp-3.9.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:c7a4b7a6cf5b6eb11e109a9755fd4fda7d57395f8c575e166d363b9fc3ec4678"}, - {file = "aiohttp-3.9.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:0a158704edf0abcac8ac371fbb54044f3270bdbc93e254a82b6c82be1ef08f3c"}, - {file = "aiohttp-3.9.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d153f652a687a8e95ad367a86a61e8d53d528b0530ef382ec5aaf533140ed00f"}, - {file = "aiohttp-3.9.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:82a6a97d9771cb48ae16979c3a3a9a18b600a8505b1115cfe354dfb2054468b4"}, - {file = "aiohttp-3.9.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60cdbd56f4cad9f69c35eaac0fbbdf1f77b0ff9456cebd4902f3dd1cf096464c"}, - {file = "aiohttp-3.9.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8676e8fd73141ded15ea586de0b7cda1542960a7b9ad89b2b06428e97125d4fa"}, - {file = "aiohttp-3.9.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da00da442a0e31f1c69d26d224e1efd3a1ca5bcbf210978a2ca7426dfcae9f58"}, - {file = "aiohttp-3.9.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18f634d540dd099c262e9f887c8bbacc959847cfe5da7a0e2e1cf3f14dbf2daf"}, - {file = "aiohttp-3.9.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:320e8618eda64e19d11bdb3bd04ccc0a816c17eaecb7e4945d01deee2a22f95f"}, - {file = "aiohttp-3.9.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:2faa61a904b83142747fc6a6d7ad8fccff898c849123030f8e75d5d967fd4a81"}, - {file = "aiohttp-3.9.5-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:8c64a6dc3fe5db7b1b4d2b5cb84c4f677768bdc340611eca673afb7cf416ef5a"}, - {file = "aiohttp-3.9.5-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:393c7aba2b55559ef7ab791c94b44f7482a07bf7640d17b341b79081f5e5cd1a"}, - {file = "aiohttp-3.9.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c671dc117c2c21a1ca10c116cfcd6e3e44da7fcde37bf83b2be485ab377b25da"}, - {file = "aiohttp-3.9.5-cp312-cp312-win32.whl", hash = "sha256:5a7ee16aab26e76add4afc45e8f8206c95d1d75540f1039b84a03c3b3800dd59"}, - {file = "aiohttp-3.9.5-cp312-cp312-win_amd64.whl", hash = "sha256:5ca51eadbd67045396bc92a4345d1790b7301c14d1848feaac1d6a6c9289e888"}, - {file = "aiohttp-3.9.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:694d828b5c41255e54bc2dddb51a9f5150b4eefa9886e38b52605a05d96566e8"}, - {file = "aiohttp-3.9.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0605cc2c0088fcaae79f01c913a38611ad09ba68ff482402d3410bf59039bfb8"}, - {file = "aiohttp-3.9.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4558e5012ee03d2638c681e156461d37b7a113fe13970d438d95d10173d25f78"}, - {file = "aiohttp-3.9.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dbc053ac75ccc63dc3a3cc547b98c7258ec35a215a92bd9f983e0aac95d3d5b"}, - {file = "aiohttp-3.9.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4109adee842b90671f1b689901b948f347325045c15f46b39797ae1bf17019de"}, - {file = "aiohttp-3.9.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6ea1a5b409a85477fd8e5ee6ad8f0e40bf2844c270955e09360418cfd09abac"}, - {file = "aiohttp-3.9.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3c2890ca8c59ee683fd09adf32321a40fe1cf164e3387799efb2acebf090c11"}, - {file = "aiohttp-3.9.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3916c8692dbd9d55c523374a3b8213e628424d19116ac4308e434dbf6d95bbdd"}, - {file = "aiohttp-3.9.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8d1964eb7617907c792ca00b341b5ec3e01ae8c280825deadbbd678447b127e1"}, - {file = "aiohttp-3.9.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d5ab8e1f6bee051a4bf6195e38a5c13e5e161cb7bad83d8854524798bd9fcd6e"}, - {file = "aiohttp-3.9.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:52c27110f3862a1afbcb2af4281fc9fdc40327fa286c4625dfee247c3ba90156"}, - {file = "aiohttp-3.9.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:7f64cbd44443e80094309875d4f9c71d0401e966d191c3d469cde4642bc2e031"}, - {file = "aiohttp-3.9.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8b4f72fbb66279624bfe83fd5eb6aea0022dad8eec62b71e7bf63ee1caadeafe"}, - {file = "aiohttp-3.9.5-cp38-cp38-win32.whl", hash = "sha256:6380c039ec52866c06d69b5c7aad5478b24ed11696f0e72f6b807cfb261453da"}, - {file = "aiohttp-3.9.5-cp38-cp38-win_amd64.whl", hash = "sha256:da22dab31d7180f8c3ac7c7635f3bcd53808f374f6aa333fe0b0b9e14b01f91a"}, - {file = "aiohttp-3.9.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:1732102949ff6087589408d76cd6dea656b93c896b011ecafff418c9661dc4ed"}, - {file = "aiohttp-3.9.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c6021d296318cb6f9414b48e6a439a7f5d1f665464da507e8ff640848ee2a58a"}, - {file = "aiohttp-3.9.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:239f975589a944eeb1bad26b8b140a59a3a320067fb3cd10b75c3092405a1372"}, - {file = "aiohttp-3.9.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b7b30258348082826d274504fbc7c849959f1989d86c29bc355107accec6cfb"}, - {file = "aiohttp-3.9.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd2adf5c87ff6d8b277814a28a535b59e20bfea40a101db6b3bdca7e9926bc24"}, - {file = "aiohttp-3.9.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e9a3d838441bebcf5cf442700e3963f58b5c33f015341f9ea86dcd7d503c07e2"}, - {file = "aiohttp-3.9.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e3a1ae66e3d0c17cf65c08968a5ee3180c5a95920ec2731f53343fac9bad106"}, - {file = "aiohttp-3.9.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9c69e77370cce2d6df5d12b4e12bdcca60c47ba13d1cbbc8645dd005a20b738b"}, - {file = "aiohttp-3.9.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0cbf56238f4bbf49dab8c2dc2e6b1b68502b1e88d335bea59b3f5b9f4c001475"}, - {file = "aiohttp-3.9.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d1469f228cd9ffddd396d9948b8c9cd8022b6d1bf1e40c6f25b0fb90b4f893ed"}, - {file = "aiohttp-3.9.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:45731330e754f5811c314901cebdf19dd776a44b31927fa4b4dbecab9e457b0c"}, - {file = "aiohttp-3.9.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:3fcb4046d2904378e3aeea1df51f697b0467f2aac55d232c87ba162709478c46"}, - {file = "aiohttp-3.9.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8cf142aa6c1a751fcb364158fd710b8a9be874b81889c2bd13aa8893197455e2"}, - {file = "aiohttp-3.9.5-cp39-cp39-win32.whl", hash = "sha256:7b179eea70833c8dee51ec42f3b4097bd6370892fa93f510f76762105568cf09"}, - {file = "aiohttp-3.9.5-cp39-cp39-win_amd64.whl", hash = "sha256:38d80498e2e169bc61418ff36170e0aad0cd268da8b38a17c4cf29d254a8b3f1"}, - {file = "aiohttp-3.9.5.tar.gz", hash = "sha256:edea7d15772ceeb29db4aff55e482d4bcfb6ae160ce144f2682de02f6d693551"}, + {file = "aiohttp-3.10.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8b3fb28a9ac8f2558760d8e637dbf27aef1e8b7f1d221e8669a1074d1a266bb2"}, + {file = "aiohttp-3.10.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:91aa966858593f64c8a65cdefa3d6dc8fe3c2768b159da84c1ddbbb2c01ab4ef"}, + {file = "aiohttp-3.10.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:63649309da83277f06a15bbdc2a54fbe75efb92caa2c25bb57ca37762789c746"}, + {file = "aiohttp-3.10.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3e7fabedb3fe06933f47f1538df7b3a8d78e13d7167195f51ca47ee12690373"}, + {file = "aiohttp-3.10.9-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5c070430fda1a550a1c3a4c2d7281d3b8cfc0c6715f616e40e3332201a253067"}, + {file = "aiohttp-3.10.9-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:51d0a4901b27272ae54e42067bc4b9a90e619a690b4dc43ea5950eb3070afc32"}, + {file = "aiohttp-3.10.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fec5fac7aea6c060f317f07494961236434928e6f4374e170ef50b3001e14581"}, + {file = "aiohttp-3.10.9-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:172ad884bb61ad31ed7beed8be776eb17e7fb423f1c1be836d5cb357a096bf12"}, + {file = "aiohttp-3.10.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d646fdd74c25bbdd4a055414f0fe32896c400f38ffbdfc78c68e62812a9e0257"}, + {file = "aiohttp-3.10.9-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e86260b76786c28acf0b5fe31c8dca4c2add95098c709b11e8c35b424ebd4f5b"}, + {file = "aiohttp-3.10.9-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:c7d7cafc11d70fdd8801abfc2ff276744ae4cb39d8060b6b542c7e44e5f2cfc2"}, + {file = "aiohttp-3.10.9-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:fc262c3df78c8ff6020c782d9ce02e4bcffe4900ad71c0ecdad59943cba54442"}, + {file = "aiohttp-3.10.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:482c85cf3d429844396d939b22bc2a03849cb9ad33344689ad1c85697bcba33a"}, + {file = "aiohttp-3.10.9-cp310-cp310-win32.whl", hash = "sha256:aeebd3061f6f1747c011e1d0b0b5f04f9f54ad1a2ca183e687e7277bef2e0da2"}, + {file = "aiohttp-3.10.9-cp310-cp310-win_amd64.whl", hash = "sha256:fa430b871220dc62572cef9c69b41e0d70fcb9d486a4a207a5de4c1f25d82593"}, + {file = "aiohttp-3.10.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:16e6a51d8bc96b77f04a6764b4ad03eeef43baa32014fce71e882bd71302c7e4"}, + {file = "aiohttp-3.10.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8bd9125dd0cc8ebd84bff2be64b10fdba7dc6fd7be431b5eaf67723557de3a31"}, + {file = "aiohttp-3.10.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dcf354661f54e6a49193d0b5653a1b011ba856e0b7a76bda2c33e4c6892f34ea"}, + {file = "aiohttp-3.10.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42775de0ca04f90c10c5c46291535ec08e9bcc4756f1b48f02a0657febe89b10"}, + {file = "aiohttp-3.10.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:87d1e4185c5d7187684d41ebb50c9aeaaaa06ca1875f4c57593071b0409d2444"}, + {file = "aiohttp-3.10.9-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c2695c61cf53a5d4345a43d689f37fc0f6d3a2dc520660aec27ec0f06288d1f9"}, + {file = "aiohttp-3.10.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a3f063b41cc06e8d0b3fcbbfc9c05b7420f41287e0cd4f75ce0a1f3d80729e6"}, + {file = "aiohttp-3.10.9-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d37f4718002863b82c6f391c8efd4d3a817da37030a29e2682a94d2716209de"}, + {file = "aiohttp-3.10.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2746d8994ebca1bdc55a1e998feff4e94222da709623bb18f6e5cfec8ec01baf"}, + {file = "aiohttp-3.10.9-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6f3c6648aa123bcd73d6f26607d59967b607b0da8ffcc27d418a4b59f4c98c7c"}, + {file = "aiohttp-3.10.9-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:558b3d223fd631ad134d89adea876e7fdb4c93c849ef195049c063ada82b7d08"}, + {file = "aiohttp-3.10.9-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:4e6cb75f8ddd9c2132d00bc03c9716add57f4beff1263463724f6398b813e7eb"}, + {file = "aiohttp-3.10.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:608cecd8d58d285bfd52dbca5b6251ca8d6ea567022c8a0eaae03c2589cd9af9"}, + {file = "aiohttp-3.10.9-cp311-cp311-win32.whl", hash = "sha256:36d4fba838be5f083f5490ddd281813b44d69685db910907636bc5dca6322316"}, + {file = "aiohttp-3.10.9-cp311-cp311-win_amd64.whl", hash = "sha256:8be1a65487bdfc285bd5e9baf3208c2132ca92a9b4020e9f27df1b16fab998a9"}, + {file = "aiohttp-3.10.9-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:4fd16b30567c5b8e167923be6e027eeae0f20cf2b8a26b98a25115f28ad48ee0"}, + {file = "aiohttp-3.10.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:40ff5b7660f903dc587ed36ef08a88d46840182d9d4b5694e7607877ced698a1"}, + {file = "aiohttp-3.10.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4edc3fd701e2b9a0d605a7b23d3de4ad23137d23fc0dbab726aa71d92f11aaaf"}, + {file = "aiohttp-3.10.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e525b69ee8a92c146ae5b4da9ecd15e518df4d40003b01b454ad694a27f498b5"}, + {file = "aiohttp-3.10.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5002a02c17fcfd796d20bac719981d2fca9c006aac0797eb8f430a58e9d12431"}, + {file = "aiohttp-3.10.9-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fd4ceeae2fb8cabdd1b71c82bfdd39662473d3433ec95b962200e9e752fb70d0"}, + {file = "aiohttp-3.10.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d6e395c3d1f773cf0651cd3559e25182eb0c03a2777b53b4575d8adc1149c6e9"}, + {file = "aiohttp-3.10.9-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bbdb8def5268f3f9cd753a265756f49228a20ed14a480d151df727808b4531dd"}, + {file = "aiohttp-3.10.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f82ace0ec57c94aaf5b0e118d4366cff5889097412c75aa14b4fd5fc0c44ee3e"}, + {file = "aiohttp-3.10.9-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:6ebdc3b3714afe1b134b3bbeb5f745eed3ecbcff92ab25d80e4ef299e83a5465"}, + {file = "aiohttp-3.10.9-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f9ca09414003c0e96a735daa1f071f7d7ed06962ef4fa29ceb6c80d06696d900"}, + {file = "aiohttp-3.10.9-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1298b854fd31d0567cbb916091be9d3278168064fca88e70b8468875ef9ff7e7"}, + {file = "aiohttp-3.10.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:60ad5b8a7452c0f5645c73d4dad7490afd6119d453d302cd5b72b678a85d6044"}, + {file = "aiohttp-3.10.9-cp312-cp312-win32.whl", hash = "sha256:1a0ee6c0d590c917f1b9629371fce5f3d3f22c317aa96fbdcce3260754d7ea21"}, + {file = "aiohttp-3.10.9-cp312-cp312-win_amd64.whl", hash = "sha256:c46131c6112b534b178d4e002abe450a0a29840b61413ac25243f1291613806a"}, + {file = "aiohttp-3.10.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2bd9f3eac515c16c4360a6a00c38119333901b8590fe93c3257a9b536026594d"}, + {file = "aiohttp-3.10.9-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8cc0d13b4e3b1362d424ce3f4e8c79e1f7247a00d792823ffd640878abf28e56"}, + {file = "aiohttp-3.10.9-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ba1a599255ad6a41022e261e31bc2f6f9355a419575b391f9655c4d9e5df5ff5"}, + {file = "aiohttp-3.10.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:776e9f3c9b377fcf097c4a04b241b15691e6662d850168642ff976780609303c"}, + {file = "aiohttp-3.10.9-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8debb45545ad95b58cc16c3c1cc19ad82cffcb106db12b437885dbee265f0ab5"}, + {file = "aiohttp-3.10.9-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c2555e4949c8d8782f18ef20e9d39730d2656e218a6f1a21a4c4c0b56546a02e"}, + {file = "aiohttp-3.10.9-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c54dc329cd44f7f7883a9f4baaefe686e8b9662e2c6c184ea15cceee587d8d69"}, + {file = "aiohttp-3.10.9-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e709d6ac598c5416f879bb1bae3fd751366120ac3fa235a01de763537385d036"}, + {file = "aiohttp-3.10.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:17c272cfe7b07a5bb0c6ad3f234e0c336fb53f3bf17840f66bd77b5815ab3d16"}, + {file = "aiohttp-3.10.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0c21c82df33b264216abffff9f8370f303dab65d8eee3767efbbd2734363f677"}, + {file = "aiohttp-3.10.9-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9331dd34145ff105177855017920dde140b447049cd62bb589de320fd6ddd582"}, + {file = "aiohttp-3.10.9-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ac3196952c673822ebed8871cf8802e17254fff2a2ed4835d9c045d9b88c5ec7"}, + {file = "aiohttp-3.10.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2c33fa6e10bb7ed262e3ff03cc69d52869514f16558db0626a7c5c61dde3c29f"}, + {file = "aiohttp-3.10.9-cp313-cp313-win32.whl", hash = "sha256:a14e4b672c257a6b94fe934ee62666bacbc8e45b7876f9dd9502d0f0fe69db16"}, + {file = "aiohttp-3.10.9-cp313-cp313-win_amd64.whl", hash = "sha256:a35ed3d03910785f7d9d6f5381f0c24002b2b888b298e6f941b2fc94c5055fcd"}, + {file = "aiohttp-3.10.9-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5f392ef50e22c31fa49b5a46af7f983fa3f118f3eccb8522063bee8bfa6755f8"}, + {file = "aiohttp-3.10.9-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d1f5c9169e26db6a61276008582d945405b8316aae2bb198220466e68114a0f5"}, + {file = "aiohttp-3.10.9-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8d9d10d10ec27c0d46ddaecc3c5598c4db9ce4e6398ca872cdde0525765caa2f"}, + {file = "aiohttp-3.10.9-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d97273a52d7f89a75b11ec386f786d3da7723d7efae3034b4dda79f6f093edc1"}, + {file = "aiohttp-3.10.9-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d271f770b52e32236d945911b2082f9318e90ff835d45224fa9e28374303f729"}, + {file = "aiohttp-3.10.9-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7003f33f5f7da1eb02f0446b0f8d2ccf57d253ca6c2e7a5732d25889da82b517"}, + {file = "aiohttp-3.10.9-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6e00c8a92e7663ed2be6fcc08a2997ff06ce73c8080cd0df10cc0321a3168d7"}, + {file = "aiohttp-3.10.9-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a61df62966ce6507aafab24e124e0c3a1cfbe23c59732987fc0fd0d71daa0b88"}, + {file = "aiohttp-3.10.9-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:60555211a006d26e1a389222e3fab8cd379f28e0fbf7472ee55b16c6c529e3a6"}, + {file = "aiohttp-3.10.9-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:d15a29424e96fad56dc2f3abed10a89c50c099f97d2416520c7a543e8fddf066"}, + {file = "aiohttp-3.10.9-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:a19caae0d670771ea7854ca30df76f676eb47e0fd9b2ee4392d44708f272122d"}, + {file = "aiohttp-3.10.9-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:99f9678bf0e2b1b695e8028fedac24ab6770937932eda695815d5a6618c37e04"}, + {file = "aiohttp-3.10.9-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:2914caa46054f3b5ff910468d686742ff8cff54b8a67319d75f5d5945fd0a13d"}, + {file = "aiohttp-3.10.9-cp38-cp38-win32.whl", hash = "sha256:0bc059ecbce835630e635879f5f480a742e130d9821fbe3d2f76610a6698ee25"}, + {file = "aiohttp-3.10.9-cp38-cp38-win_amd64.whl", hash = "sha256:e883b61b75ca6efc2541fcd52a5c8ccfe288b24d97e20ac08fdf343b8ac672ea"}, + {file = "aiohttp-3.10.9-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:fcd546782d03181b0b1d20b43d612429a90a68779659ba8045114b867971ab71"}, + {file = "aiohttp-3.10.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:85711eec2d875cd88c7eb40e734c4ca6d9ae477d6f26bd2b5bb4f7f60e41b156"}, + {file = "aiohttp-3.10.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:02d1d6610588bcd743fae827bd6f2e47e0d09b346f230824b4c6fb85c6065f9c"}, + {file = "aiohttp-3.10.9-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3668d0c2a4d23fb136a753eba42caa2c0abbd3d9c5c87ee150a716a16c6deec1"}, + {file = "aiohttp-3.10.9-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d7c071235a47d407b0e93aa6262b49422dbe48d7d8566e1158fecc91043dd948"}, + {file = "aiohttp-3.10.9-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ac74e794e3aee92ae8f571bfeaa103a141e409863a100ab63a253b1c53b707eb"}, + {file = "aiohttp-3.10.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bbf94d4a0447705b7775417ca8bb8086cc5482023a6e17cdc8f96d0b1b5aba6"}, + {file = "aiohttp-3.10.9-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb0b2d5d51f96b6cc19e6ab46a7b684be23240426ae951dcdac9639ab111b45e"}, + {file = "aiohttp-3.10.9-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:e83dfefb4f7d285c2d6a07a22268344a97d61579b3e0dce482a5be0251d672ab"}, + {file = "aiohttp-3.10.9-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:f0a44bb40b6aaa4fb9a5c1ee07880570ecda2065433a96ccff409c9c20c1624a"}, + {file = "aiohttp-3.10.9-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:c2b627d3c8982691b06d89d31093cee158c30629fdfebe705a91814d49b554f8"}, + {file = "aiohttp-3.10.9-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:03690541e4cc866eef79626cfa1ef4dd729c5c1408600c8cb9e12e1137eed6ab"}, + {file = "aiohttp-3.10.9-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:ad3675c126f2a95bde637d162f8231cff6bc0bc9fbe31bd78075f9ff7921e322"}, + {file = "aiohttp-3.10.9-cp39-cp39-win32.whl", hash = "sha256:1321658f12b6caffafdc35cfba6c882cb014af86bef4e78c125e7e794dfb927b"}, + {file = "aiohttp-3.10.9-cp39-cp39-win_amd64.whl", hash = "sha256:9fdf5c839bf95fc67be5794c780419edb0dbef776edcfc6c2e5e2ffd5ee755fa"}, + {file = "aiohttp-3.10.9.tar.gz", hash = "sha256:143b0026a9dab07a05ad2dd9e46aa859bffdd6348ddc5967b42161168c24f857"}, ] [package.dependencies] +aiohappyeyeballs = ">=2.3.0" aiosignal = ">=1.1.2" async-timeout = {version = ">=4.0,<5.0", markers = "python_version < \"3.11\""} attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" -yarl = ">=1.0,<2.0" +yarl = ">=1.12.0,<2.0" [package.extras] -speedups = ["Brotli", "aiodns", "brotlicffi"] +speedups = ["Brotli", "aiodns (>=3.2.0)", "brotlicffi"] [[package]] name = "aiosignal" @@ -112,17 +139,17 @@ frozenlist = ">=1.1.0" [[package]] name = "airbyte-cdk" -version = "3.5.0" +version = "5.11.1" description = "A framework for writing Airbyte Connectors." optional = false -python-versions = "<4.0,>=3.9" +python-versions = "<4.0,>=3.10" files = [ - {file = "airbyte_cdk-3.5.0-py3-none-any.whl", hash = "sha256:1dbf4074762488e91eb8e00891a4ff887bbec242c8f231b41ca8baa1ecb6e1e8"}, - {file = "airbyte_cdk-3.5.0.tar.gz", hash = "sha256:4077a72c277b76cb1c614d8fac158c7ceb60dfc89c377390d9b21d2ca41ad719"}, + {file = "airbyte_cdk-5.11.1-py3-none-any.whl", hash = "sha256:efddee85179128cb7d65b11a9a4aba353ea5b01daaa56fc3069d12ce156d2857"}, + {file = "airbyte_cdk-5.11.1.tar.gz", hash = "sha256:0cc1cdc1d50909bbb2791a9c389c0f3db32474502addf65eb745d87af7d36fd9"}, ] [package.dependencies] -airbyte-protocol-models-pdv2 = ">=0.12.2,<0.13.0" +airbyte-protocol-models-dataclasses = ">=0.13,<0.14" backoff = "*" cachetools = "*" cryptography = ">=42.0.5,<43.0.0" @@ -134,6 +161,9 @@ Jinja2 = ">=3.1.2,<3.2.0" jsonref = ">=0.2,<0.3" jsonschema = ">=3.2.0,<3.3.0" langchain_core = "0.1.42" +nltk = "3.8.1" +orjson = ">=3.10.7,<4.0.0" +pandas = "2.2.2" pendulum = "<3.0.0" pydantic = ">=2.7,<3.0" pyjwt = ">=2.8.0,<3.0.0" @@ -143,27 +173,25 @@ pytz = "2024.1" PyYAML = ">=6.0.1,<7.0.0" requests = "*" requests_cache = "*" +serpyco-rs = ">=1.10.2,<2.0.0" wcmatch = "8.4" [package.extras] -file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (>=15.0.0,<15.1.0)", "pytesseract (==0.3.10)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (>=15.0.0,<15.1.0)", "pytesseract (==0.3.10)", "python-calamine (==0.2.3)", "python-snappy (==0.7.3)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] sphinx-docs = ["Sphinx (>=4.2,<4.3)", "sphinx-rtd-theme (>=1.0,<1.1)"] vector-db-based = ["cohere (==4.21)", "langchain (==0.1.16)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] [[package]] -name = "airbyte-protocol-models-pdv2" -version = "0.12.2" -description = "Declares the Airbyte Protocol." +name = "airbyte-protocol-models-dataclasses" +version = "0.13.0" +description = "Declares the Airbyte Protocol using Python Dataclasses. Dataclasses in Python have less performance overhead compared to Pydantic models, making them a more efficient choice for scenarios where speed and memory usage are critical" optional = false python-versions = ">=3.8" files = [ - {file = "airbyte_protocol_models_pdv2-0.12.2-py3-none-any.whl", hash = "sha256:8b3f9d0388928547cdf2e9134c0d589e4bcaa6f63bf71a21299f6824bfb7ad0e"}, - {file = "airbyte_protocol_models_pdv2-0.12.2.tar.gz", hash = "sha256:130c9ab289f3f53749ce63ff1abbfb67a44b7e5bd2794865315a2976138b672b"}, + {file = "airbyte_protocol_models_dataclasses-0.13.0-py3-none-any.whl", hash = "sha256:0aedb99ffc4f9aab0ce91bba2c292fa17cd8fd4b42eeba196d6a16c20bbbd7a5"}, + {file = "airbyte_protocol_models_dataclasses-0.13.0.tar.gz", hash = "sha256:72e67850d661e2808406aec5839b3158ebb94d3553b798dbdae1b4a278548d2f"}, ] -[package.dependencies] -pydantic = ">=2.7.2,<3.0.0" - [[package]] name = "annotated-types" version = "0.7.0" @@ -175,6 +203,28 @@ files = [ {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, ] +[[package]] +name = "anyio" +version = "4.6.0" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = false +python-versions = ">=3.9" +files = [ + {file = "anyio-4.6.0-py3-none-any.whl", hash = "sha256:c7d2e9d63e31599eeb636c8c5c03a7e108d73b345f064f1c19fdc87b79036a9a"}, + {file = "anyio-4.6.0.tar.gz", hash = "sha256:137b4559cbb034c477165047febb6ff83f390fc3b20bf181c1fc0a728cb8beeb"}, +] + +[package.dependencies] +exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} +idna = ">=2.8" +sniffio = ">=1.1" +typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""} + +[package.extras] +doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.21.0b1)"] +trio = ["trio (>=0.26.1)"] + [[package]] name = "async-timeout" version = "4.0.3" @@ -187,33 +237,34 @@ files = [ ] [[package]] -name = "atomicwrites" -version = "1.4.1" -description = "Atomic file writes." +name = "attributes-doc" +version = "0.4.0" +description = "PEP 224 implementation" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.8" files = [ - {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, + {file = "attributes-doc-0.4.0.tar.gz", hash = "sha256:b1576c94a714e9fc2c65c47cf10d0c8e1a5f7c4f5ae7f69006be108d95cbfbfb"}, + {file = "attributes_doc-0.4.0-py2.py3-none-any.whl", hash = "sha256:4c3007d9e58f3a6cb4b9c614c4d4ce2d92161581f28e594ddd8241cc3a113bdd"}, ] [[package]] name = "attrs" -version = "23.2.0" +version = "24.2.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.7" files = [ - {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, - {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, + {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, + {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, ] [package.extras] -cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] -dev = ["attrs[tests]", "pre-commit"] -docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] -tests = ["attrs[tests-no-zope]", "zope-interface"] -tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] -tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] +benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] [[package]] name = "backoff" @@ -228,13 +279,13 @@ files = [ [[package]] name = "bracex" -version = "2.4" +version = "2.5.post1" description = "Bash style brace expander." optional = false python-versions = ">=3.8" files = [ - {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, - {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, + {file = "bracex-2.5.post1-py3-none-any.whl", hash = "sha256:13e5732fec27828d6af308628285ad358047cec36801598368cb28bc631dbaf6"}, + {file = "bracex-2.5.post1.tar.gz", hash = "sha256:12c50952415bfa773d2d9ccb8e79651b8cdb1f31a42f6091b804f6ba2b4a66b6"}, ] [[package]] @@ -250,24 +301,24 @@ files = [ [[package]] name = "cachetools" -version = "5.3.3" +version = "5.5.0" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" files = [ - {file = "cachetools-5.3.3-py3-none-any.whl", hash = "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945"}, - {file = "cachetools-5.3.3.tar.gz", hash = "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105"}, + {file = "cachetools-5.5.0-py3-none-any.whl", hash = "sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292"}, + {file = "cachetools-5.5.0.tar.gz", hash = "sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a"}, ] [[package]] name = "cattrs" -version = "23.2.3" +version = "24.1.2" description = "Composable complex class support for attrs and dataclasses." optional = false python-versions = ">=3.8" files = [ - {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, - {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, + {file = "cattrs-24.1.2-py3-none-any.whl", hash = "sha256:67c7495b760168d931a10233f979b28dc04daf853b30752246f4f8471c6d68d0"}, + {file = "cattrs-24.1.2.tar.gz", hash = "sha256:8028cfe1ff5382df59dd36474a86e02d817b06eaf8af84555441bac915d2ef85"}, ] [package.dependencies] @@ -279,6 +330,7 @@ typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_ver bson = ["pymongo (>=4.4.0)"] cbor2 = ["cbor2 (>=5.4.6)"] msgpack = ["msgpack (>=1.0.5)"] +msgspec = ["msgspec (>=0.18.5)"] orjson = ["orjson (>=3.9.2)"] pyyaml = ["pyyaml (>=6.0)"] tomlkit = ["tomlkit (>=0.11.8)"] @@ -286,74 +338,89 @@ ujson = ["ujson (>=5.7.0)"] [[package]] name = "certifi" -version = "2024.7.4" +version = "2024.8.30" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, - {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, + {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, + {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, ] [[package]] name = "cffi" -version = "1.16.0" +version = "1.17.1" description = "Foreign Function Interface for Python calling C code." optional = false python-versions = ">=3.8" files = [ - {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, - {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, - {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, - {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, - {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, - {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, - {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, - {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, - {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, - {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, - {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, - {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, - {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, - {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, - {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, - {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, - {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, - {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, - {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, - {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, - {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, - {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, - {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, - {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, - {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, - {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, - {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, - {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, - {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, - {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, - {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, - {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, + {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, + {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17"}, + {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8"}, + {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e"}, + {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be"}, + {file = "cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c"}, + {file = "cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15"}, + {file = "cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401"}, + {file = "cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b"}, + {file = "cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655"}, + {file = "cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0"}, + {file = "cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4"}, + {file = "cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93"}, + {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3"}, + {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8"}, + {file = "cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65"}, + {file = "cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903"}, + {file = "cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e"}, + {file = "cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd"}, + {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed"}, + {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9"}, + {file = "cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d"}, + {file = "cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a"}, + {file = "cffi-1.17.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:636062ea65bd0195bc012fea9321aca499c0504409f413dc88af450b57ffd03b"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7eac2ef9b63c79431bc4b25f1cd649d7f061a28808cbc6c47b534bd789ef964"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e221cf152cff04059d011ee126477f0d9588303eb57e88923578ace7baad17f9"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31000ec67d4221a71bd3f67df918b1f88f676f1c3b535a7eb473255fdc0b83fc"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f17be4345073b0a7b8ea599688f692ac3ef23ce28e5df79c04de519dbc4912c"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2b1fac190ae3ebfe37b979cc1ce69c81f4e4fe5746bb401dca63a9062cdaf1"}, + {file = "cffi-1.17.1-cp38-cp38-win32.whl", hash = "sha256:7596d6620d3fa590f677e9ee430df2958d2d6d6de2feeae5b20e82c00b76fbf8"}, + {file = "cffi-1.17.1-cp38-cp38-win_amd64.whl", hash = "sha256:78122be759c3f8a014ce010908ae03364d00a1f81ab5c7f4a7a5120607ea56e1"}, + {file = "cffi-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b2ab587605f4ba0bf81dc0cb08a41bd1c0a5906bd59243d56bad7668a6fc6c16"}, + {file = "cffi-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:28b16024becceed8c6dfbc75629e27788d8a3f9030691a1dbf9821a128b22c36"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d599671f396c4723d016dbddb72fe8e0397082b0a77a4fab8028923bec050e8"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca74b8dbe6e8e8263c0ffd60277de77dcee6c837a3d0881d8c1ead7268c9e576"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98e3969bcff97cae1b2def8ba499ea3d6f31ddfdb7635374834cf89a1a08ecf0"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdf5ce3acdfd1661132f2a9c19cac174758dc2352bfe37d98aa7512c6b7178b3"}, + {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9755e4345d1ec879e3849e62222a18c7174d65a6a92d5b346b1863912168b595"}, + {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f1e22e8c4419538cb197e4dd60acc919d7696e5ef98ee4da4e01d3f8cfa4cc5a"}, + {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c03e868a0b3bc35839ba98e74211ed2b05d2119be4e8a0f224fba9384f1fe02e"}, + {file = "cffi-1.17.1-cp39-cp39-win32.whl", hash = "sha256:e31ae45bc2e29f6b2abd0de1cc3b9d5205aa847cafaecb8af1476a609a2f6eb7"}, + {file = "cffi-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662"}, + {file = "cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824"}, ] [package.dependencies] @@ -458,6 +525,20 @@ files = [ {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, ] +[[package]] +name = "click" +version = "8.1.7" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + [[package]] name = "colorama" version = "0.4.6" @@ -719,17 +800,77 @@ files = [ {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, ] +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + +[[package]] +name = "httpcore" +version = "1.0.6" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpcore-1.0.6-py3-none-any.whl", hash = "sha256:27b59625743b85577a8c0e10e55b50b5368a4f2cfe8cc7bcfa9cf00829c2682f"}, + {file = "httpcore-1.0.6.tar.gz", hash = "sha256:73f6dbd6eb8c21bbf7ef8efad555481853f5f6acdeaff1edb0694289269ee17f"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.13,<0.15" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<1.0)"] + +[[package]] +name = "httpx" +version = "0.27.2" +description = "The next generation HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpx-0.27.2-py3-none-any.whl", hash = "sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0"}, + {file = "httpx-0.27.2.tar.gz", hash = "sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2"}, +] + +[package.dependencies] +anyio = "*" +certifi = "*" +httpcore = "==1.*" +idna = "*" +sniffio = "*" + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +zstd = ["zstandard (>=0.18.0)"] + [[package]] name = "idna" -version = "3.7" +version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" files = [ - {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, - {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, + {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, + {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, ] +[package.extras] +all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] + [[package]] name = "iniconfig" version = "2.0.0" @@ -772,6 +913,17 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "joblib" +version = "1.4.2" +description = "Lightweight pipelining with Python functions" +optional = false +python-versions = ">=3.8" +files = [ + {file = "joblib-1.4.2-py3-none-any.whl", hash = "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6"}, + {file = "joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"}, +] + [[package]] name = "jsonpatch" version = "1.33" @@ -853,246 +1005,347 @@ extended-testing = ["jinja2 (>=3,<4)"] [[package]] name = "langsmith" -version = "0.1.85" +version = "0.1.132" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." optional = false python-versions = "<4.0,>=3.8.1" files = [ - {file = "langsmith-0.1.85-py3-none-any.whl", hash = "sha256:c1f94384f10cea96f7b4d33fd3db7ec180c03c7468877d50846f881d2017ff94"}, - {file = "langsmith-0.1.85.tar.gz", hash = "sha256:acff31f9e53efa48586cf8e32f65625a335c74d7c4fa306d1655ac18452296f6"}, + {file = "langsmith-0.1.132-py3-none-any.whl", hash = "sha256:2320894203675c1c292b818cbecf68b69e47a9f7814d4e950237d1faaafd5dee"}, + {file = "langsmith-0.1.132.tar.gz", hash = "sha256:007b8fac469138abdba89db931900a26c5d316640e27ff4660d28c92a766aae1"}, ] [package.dependencies] +httpx = ">=0.23.0,<1" orjson = ">=3.9.14,<4.0.0" pydantic = {version = ">=1,<3", markers = "python_full_version < \"3.12.4\""} requests = ">=2,<3" +requests-toolbelt = ">=1.0.0,<2.0.0" [[package]] name = "markupsafe" -version = "2.1.5" +version = "3.0.0" description = "Safely add untrusted strings to HTML/XML markup." optional = false -python-versions = ">=3.7" -files = [ - {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, - {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +python-versions = ">=3.9" +files = [ + {file = "MarkupSafe-3.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:380faf314c3c84c1682ca672e6280c6c59e92d0bc13dc71758ffa2de3cd4e252"}, + {file = "MarkupSafe-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1ee9790be6f62121c4c58bbced387b0965ab7bffeecb4e17cc42ef290784e363"}, + {file = "MarkupSafe-3.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ddf5cb8e9c00d9bf8b0c75949fb3ff9ea2096ba531693e2e87336d197fdb908"}, + {file = "MarkupSafe-3.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b36473a2d3e882d1873ea906ce54408b9588dc2c65989664e6e7f5a2de353d7"}, + {file = "MarkupSafe-3.0.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dba0f83119b9514bc37272ad012f0cc03f0805cc6a2bea7244e19250ac8ff29f"}, + {file = "MarkupSafe-3.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:409535e0521c4630d5b5a1bf284e9d3c76d2fc2f153ebb12cf3827797798cc99"}, + {file = "MarkupSafe-3.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:64a7c7856c3a409011139b17d137c2924df4318dab91ee0530800819617c4381"}, + {file = "MarkupSafe-3.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4deea1d9169578917d1f35cdb581bc7bab56a7e8c5be2633bd1b9549c3c22a01"}, + {file = "MarkupSafe-3.0.0-cp310-cp310-win32.whl", hash = "sha256:3cd0bba31d484fe9b9d77698ddb67c978704603dc10cdc905512af308cfcca6b"}, + {file = "MarkupSafe-3.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:4ca04c60006867610a06575b46941ae616b19da0adc85b9f8f3d9cbd7a3da385"}, + {file = "MarkupSafe-3.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e64b390a306f9e849ee809f92af6a52cda41741c914358e0e9f8499d03741526"}, + {file = "MarkupSafe-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c524203207f5b569df06c96dafdc337228921ee8c3cc5f6e891d024c6595352"}, + {file = "MarkupSafe-3.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c409691696bec2b5e5c9efd9593c99025bf2f317380bf0d993ee0213516d908a"}, + {file = "MarkupSafe-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64f7d04410be600aa5ec0626d73d43e68a51c86500ce12917e10fd013e258df5"}, + {file = "MarkupSafe-3.0.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:105ada43a61af22acb8774514c51900dc820c481cc5ba53f17c09d294d9c07ca"}, + {file = "MarkupSafe-3.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a5fd5500d4e4f7cc88d8c0f2e45126c4307ed31e08f8ec521474f2fd99d35ac3"}, + {file = "MarkupSafe-3.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:25396abd52b16900932e05b7104bcdc640a4d96c914f39c3b984e5a17b01fba0"}, + {file = "MarkupSafe-3.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3efde9a8c56c3b6e5f3fa4baea828f8184970c7c78480fedb620d804b1c31e5c"}, + {file = "MarkupSafe-3.0.0-cp311-cp311-win32.whl", hash = "sha256:12ddac720b8965332d36196f6f83477c6351ba6a25d4aff91e30708c729350d7"}, + {file = "MarkupSafe-3.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:658fdf6022740896c403d45148bf0c36978c6b48c9ef8b1f8d0c7a11b6cdea86"}, + {file = "MarkupSafe-3.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d261ec38b8a99a39b62e0119ed47fe3b62f7691c500bc1e815265adc016438c1"}, + {file = "MarkupSafe-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e363440c8534bf2f2ef1b8fdc02037eb5fff8fce2a558519b22d6a3a38b3ec5e"}, + {file = "MarkupSafe-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7835de4c56066e096407a1852e5561f6033786dd987fa90dc384e45b9bd21295"}, + {file = "MarkupSafe-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6cc46a27d904c9be5732029769acf4b0af69345172ed1ef6d4db0c023ff603b"}, + {file = "MarkupSafe-3.0.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0411641d31aa6f7f0cc13f0f18b63b8dc08da5f3a7505972a42ab059f479ba3"}, + {file = "MarkupSafe-3.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b2a7afd24d408b907672015555bc10be2382e6c5f62a488e2d452da670bbd389"}, + {file = "MarkupSafe-3.0.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c8ab7efeff1884c5da8e18f743b667215300e09043820d11723718de0b7db934"}, + {file = "MarkupSafe-3.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8219e2207f6c188d15614ea043636c2b36d2d79bf853639c124a179412325a13"}, + {file = "MarkupSafe-3.0.0-cp312-cp312-win32.whl", hash = "sha256:59420b5a9a5d3fee483a32adb56d7369ae0d630798da056001be1e9f674f3aa6"}, + {file = "MarkupSafe-3.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:7ed789d0f7f11fcf118cf0acb378743dfdd4215d7f7d18837c88171405c9a452"}, + {file = "MarkupSafe-3.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:27d6a73682b99568916c54a4bfced40e7d871ba685b580ea04bbd2e405dfd4c5"}, + {file = "MarkupSafe-3.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:494a64efc535e147fcc713dba58eecfce3a79f1e93ebe81995b387f5cd9bc2e1"}, + {file = "MarkupSafe-3.0.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5243044a927e8a6bb28517838662a019cd7f73d7f106bbb37ab5e7fa8451a92"}, + {file = "MarkupSafe-3.0.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63dae84964a9a3d2610808cee038f435d9a111620c37ccf872c2fcaeca6865b3"}, + {file = "MarkupSafe-3.0.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dcbee57fedc9b2182c54ffc1c5eed316c3da8bbfeda8009e1b5d7220199d15da"}, + {file = "MarkupSafe-3.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f846fd7c241e5bd4161e2a483663eb66e4d8e12130fcdc052f310f388f1d61c6"}, + {file = "MarkupSafe-3.0.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:678fbceb202382aae42c1f0cd9f56b776bc20a58ae5b553ee1fe6b802983a1d6"}, + {file = "MarkupSafe-3.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bd9b8e458e2bab52f9ad3ab5dc8b689a3c84b12b2a2f64cd9a0dfe209fb6b42f"}, + {file = "MarkupSafe-3.0.0-cp313-cp313-win32.whl", hash = "sha256:1fd02f47596e00a372f5b4af2b4c45f528bade65c66dfcbc6e1ea1bfda758e98"}, + {file = "MarkupSafe-3.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:b94bec9eda10111ec7102ef909eca4f3c2df979643924bfe58375f560713a7d1"}, + {file = "MarkupSafe-3.0.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:509c424069dd037d078925b6815fc56b7271f3aaec471e55e6fa513b0a80d2aa"}, + {file = "MarkupSafe-3.0.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:81be2c0084d8c69e97e3c5d73ce9e2a6e523556f2a19c4e195c09d499be2f808"}, + {file = "MarkupSafe-3.0.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b43ac1eb9f91e0c14aac1d2ef0f76bc7b9ceea51de47536f61268191adf52ad7"}, + {file = "MarkupSafe-3.0.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b231255770723f1e125d63c14269bcd8b8136ecfb620b9a18c0297e046d0736"}, + {file = "MarkupSafe-3.0.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c182d45600556917f811aa019d834a89fe4b6f6255da2fd0bdcf80e970f95918"}, + {file = "MarkupSafe-3.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9f91c90f8f3bf436f81c12eeb4d79f9ddd263c71125e6ad71341906832a34386"}, + {file = "MarkupSafe-3.0.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:a7171d2b869e9be238ea318c196baf58fbf272704e9c1cd4be8c380eea963342"}, + {file = "MarkupSafe-3.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cb244adf2499aa37d5dc43431990c7f0b632d841af66a51d22bd89c437b60264"}, + {file = "MarkupSafe-3.0.0-cp313-cp313t-win32.whl", hash = "sha256:96e3ed550600185d34429477f1176cedea8293fa40e47fe37a05751bcb64c997"}, + {file = "MarkupSafe-3.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:1d151b9cf3307e259b749125a5a08c030ba15a8f1d567ca5bfb0e92f35e761f5"}, + {file = "MarkupSafe-3.0.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:23efb2be7221105c8eb0e905433414d2439cb0a8c5d5ca081c1c72acef0f5613"}, + {file = "MarkupSafe-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:81ee9c967956b9ea39b3a5270b7cb1740928d205b0dc72629164ce621b4debf9"}, + {file = "MarkupSafe-3.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5509a8373fed30b978557890a226c3d30569746c565b9daba69df80c160365a5"}, + {file = "MarkupSafe-3.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1c13c6c908811f867a8e9e66efb2d6c03d1cdd83e92788fe97f693c457dc44f"}, + {file = "MarkupSafe-3.0.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d7e63d1977d3806ce0a1a3e0099b089f61abdede5238ca6a3f3bf8877b46d095"}, + {file = "MarkupSafe-3.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:d2c099be5274847d606574234e494f23a359e829ba337ea9037c3a72b0851942"}, + {file = "MarkupSafe-3.0.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:e042ccf8fe5bf8b6a4b38b3f7d618eb10ea20402b0c9f4add9293408de447974"}, + {file = "MarkupSafe-3.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:98fb3a2bf525ad66db96745707b93ba0f78928b7a1cb2f1cb4b143bc7e2ba3b3"}, + {file = "MarkupSafe-3.0.0-cp39-cp39-win32.whl", hash = "sha256:a80c6740e1bfbe50cea7cbf74f48823bb57bd59d914ee22ff8a81963b08e62d2"}, + {file = "MarkupSafe-3.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:5d207ff5cceef77796f8aacd44263266248cf1fbc601441524d7835613f8abec"}, + {file = "markupsafe-3.0.0.tar.gz", hash = "sha256:03ff62dea2fef3eadf2f1853bc6332bcb0458d9608b11dfb1cd5aeda1c178ea6"}, ] [[package]] name = "multidict" -version = "6.0.5" +version = "6.1.0" description = "multidict implementation" optional = false +python-versions = ">=3.8" +files = [ + {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3380252550e372e8511d49481bd836264c009adb826b23fefcc5dd3c69692f60"}, + {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:99f826cbf970077383d7de805c0681799491cb939c25450b9b5b3ced03ca99f1"}, + {file = "multidict-6.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a114d03b938376557927ab23f1e950827c3b893ccb94b62fd95d430fd0e5cf53"}, + {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1c416351ee6271b2f49b56ad7f308072f6f44b37118d69c2cad94f3fa8a40d5"}, + {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6b5d83030255983181005e6cfbac1617ce9746b219bc2aad52201ad121226581"}, + {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3e97b5e938051226dc025ec80980c285b053ffb1e25a3db2a3aa3bc046bf7f56"}, + {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d618649d4e70ac6efcbba75be98b26ef5078faad23592f9b51ca492953012429"}, + {file = "multidict-6.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10524ebd769727ac77ef2278390fb0068d83f3acb7773792a5080f2b0abf7748"}, + {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ff3827aef427c89a25cc96ded1759271a93603aba9fb977a6d264648ebf989db"}, + {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:06809f4f0f7ab7ea2cabf9caca7d79c22c0758b58a71f9d32943ae13c7ace056"}, + {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:f179dee3b863ab1c59580ff60f9d99f632f34ccb38bf67a33ec6b3ecadd0fd76"}, + {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:aaed8b0562be4a0876ee3b6946f6869b7bcdb571a5d1496683505944e268b160"}, + {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3c8b88a2ccf5493b6c8da9076fb151ba106960a2df90c2633f342f120751a9e7"}, + {file = "multidict-6.1.0-cp310-cp310-win32.whl", hash = "sha256:4a9cb68166a34117d6646c0023c7b759bf197bee5ad4272f420a0141d7eb03a0"}, + {file = "multidict-6.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:20b9b5fbe0b88d0bdef2012ef7dee867f874b72528cf1d08f1d59b0e3850129d"}, + {file = "multidict-6.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3efe2c2cb5763f2f1b275ad2bf7a287d3f7ebbef35648a9726e3b69284a4f3d6"}, + {file = "multidict-6.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c7053d3b0353a8b9de430a4f4b4268ac9a4fb3481af37dfe49825bf45ca24156"}, + {file = "multidict-6.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:27e5fc84ccef8dfaabb09d82b7d179c7cf1a3fbc8a966f8274fcb4ab2eb4cadb"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e2b90b43e696f25c62656389d32236e049568b39320e2735d51f08fd362761b"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d83a047959d38a7ff552ff94be767b7fd79b831ad1cd9920662db05fec24fe72"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d1a9dd711d0877a1ece3d2e4fea11a8e75741ca21954c919406b44e7cf971304"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec2abea24d98246b94913b76a125e855eb5c434f7c46546046372fe60f666351"}, + {file = "multidict-6.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4867cafcbc6585e4b678876c489b9273b13e9fff9f6d6d66add5e15d11d926cb"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5b48204e8d955c47c55b72779802b219a39acc3ee3d0116d5080c388970b76e3"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:d8fff389528cad1618fb4b26b95550327495462cd745d879a8c7c2115248e399"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a7a9541cd308eed5e30318430a9c74d2132e9a8cb46b901326272d780bf2d423"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:da1758c76f50c39a2efd5e9859ce7d776317eb1dd34317c8152ac9251fc574a3"}, + {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c943a53e9186688b45b323602298ab727d8865d8c9ee0b17f8d62d14b56f0753"}, + {file = "multidict-6.1.0-cp311-cp311-win32.whl", hash = "sha256:90f8717cb649eea3504091e640a1b8568faad18bd4b9fcd692853a04475a4b80"}, + {file = "multidict-6.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:82176036e65644a6cc5bd619f65f6f19781e8ec2e5330f51aa9ada7504cc1926"}, + {file = "multidict-6.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b04772ed465fa3cc947db808fa306d79b43e896beb677a56fb2347ca1a49c1fa"}, + {file = "multidict-6.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6180c0ae073bddeb5a97a38c03f30c233e0a4d39cd86166251617d1bbd0af436"}, + {file = "multidict-6.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:071120490b47aa997cca00666923a83f02c7fbb44f71cf7f136df753f7fa8761"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50b3a2710631848991d0bf7de077502e8994c804bb805aeb2925a981de58ec2e"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b58c621844d55e71c1b7f7c498ce5aa6985d743a1a59034c57a905b3f153c1ef"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55b6d90641869892caa9ca42ff913f7ff1c5ece06474fbd32fb2cf6834726c95"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b820514bfc0b98a30e3d85462084779900347e4d49267f747ff54060cc33925"}, + {file = "multidict-6.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10a9b09aba0c5b48c53761b7c720aaaf7cf236d5fe394cd399c7ba662d5f9966"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1e16bf3e5fc9f44632affb159d30a437bfe286ce9e02754759be5536b169b305"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:76f364861c3bfc98cbbcbd402d83454ed9e01a5224bb3a28bf70002a230f73e2"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:820c661588bd01a0aa62a1283f20d2be4281b086f80dad9e955e690c75fb54a2"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:0e5f362e895bc5b9e67fe6e4ded2492d8124bdf817827f33c5b46c2fe3ffaca6"}, + {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3ec660d19bbc671e3a6443325f07263be452c453ac9e512f5eb935e7d4ac28b3"}, + {file = "multidict-6.1.0-cp312-cp312-win32.whl", hash = "sha256:58130ecf8f7b8112cdb841486404f1282b9c86ccb30d3519faf301b2e5659133"}, + {file = "multidict-6.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:188215fc0aafb8e03341995e7c4797860181562380f81ed0a87ff455b70bf1f1"}, + {file = "multidict-6.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d569388c381b24671589335a3be6e1d45546c2988c2ebe30fdcada8457a31008"}, + {file = "multidict-6.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:052e10d2d37810b99cc170b785945421141bf7bb7d2f8799d431e7db229c385f"}, + {file = "multidict-6.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f90c822a402cb865e396a504f9fc8173ef34212a342d92e362ca498cad308e28"}, + {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b225d95519a5bf73860323e633a664b0d85ad3d5bede6d30d95b35d4dfe8805b"}, + {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:23bfd518810af7de1116313ebd9092cb9aa629beb12f6ed631ad53356ed6b86c"}, + {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c09fcfdccdd0b57867577b719c69e347a436b86cd83747f179dbf0cc0d4c1f3"}, + {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf6bea52ec97e95560af5ae576bdac3aa3aae0b6758c6efa115236d9e07dae44"}, + {file = "multidict-6.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57feec87371dbb3520da6192213c7d6fc892d5589a93db548331954de8248fd2"}, + {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0c3f390dc53279cbc8ba976e5f8035eab997829066756d811616b652b00a23a3"}, + {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:59bfeae4b25ec05b34f1956eaa1cb38032282cd4dfabc5056d0a1ec4d696d3aa"}, + {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b2f59caeaf7632cc633b5cf6fc449372b83bbdf0da4ae04d5be36118e46cc0aa"}, + {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:37bb93b2178e02b7b618893990941900fd25b6b9ac0fa49931a40aecdf083fe4"}, + {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4e9f48f58c2c523d5a06faea47866cd35b32655c46b443f163d08c6d0ddb17d6"}, + {file = "multidict-6.1.0-cp313-cp313-win32.whl", hash = "sha256:3a37ffb35399029b45c6cc33640a92bef403c9fd388acce75cdc88f58bd19a81"}, + {file = "multidict-6.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:e9aa71e15d9d9beaad2c6b9319edcdc0a49a43ef5c0a4c8265ca9ee7d6c67774"}, + {file = "multidict-6.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:db7457bac39421addd0c8449933ac32d8042aae84a14911a757ae6ca3eef1392"}, + {file = "multidict-6.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d094ddec350a2fb899fec68d8353c78233debde9b7d8b4beeafa70825f1c281a"}, + {file = "multidict-6.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5845c1fd4866bb5dd3125d89b90e57ed3138241540897de748cdf19de8a2fca2"}, + {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9079dfc6a70abe341f521f78405b8949f96db48da98aeb43f9907f342f627cdc"}, + {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3914f5aaa0f36d5d60e8ece6a308ee1c9784cd75ec8151062614657a114c4478"}, + {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c08be4f460903e5a9d0f76818db3250f12e9c344e79314d1d570fc69d7f4eae4"}, + {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d093be959277cb7dee84b801eb1af388b6ad3ca6a6b6bf1ed7585895789d027d"}, + {file = "multidict-6.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3702ea6872c5a2a4eeefa6ffd36b042e9773f05b1f37ae3ef7264b1163c2dcf6"}, + {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:2090f6a85cafc5b2db085124d752757c9d251548cedabe9bd31afe6363e0aff2"}, + {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:f67f217af4b1ff66c68a87318012de788dd95fcfeb24cc889011f4e1c7454dfd"}, + {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:189f652a87e876098bbc67b4da1049afb5f5dfbaa310dd67c594b01c10388db6"}, + {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:6bb5992037f7a9eff7991ebe4273ea7f51f1c1c511e6a2ce511d0e7bdb754492"}, + {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:ac10f4c2b9e770c4e393876e35a7046879d195cd123b4f116d299d442b335bcd"}, + {file = "multidict-6.1.0-cp38-cp38-win32.whl", hash = "sha256:e27bbb6d14416713a8bd7aaa1313c0fc8d44ee48d74497a0ff4c3a1b6ccb5167"}, + {file = "multidict-6.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:22f3105d4fb15c8f57ff3959a58fcab6ce36814486500cd7485651230ad4d4ef"}, + {file = "multidict-6.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:4e18b656c5e844539d506a0a06432274d7bd52a7487e6828c63a63d69185626c"}, + {file = "multidict-6.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a185f876e69897a6f3325c3f19f26a297fa058c5e456bfcff8015e9a27e83ae1"}, + {file = "multidict-6.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ab7c4ceb38d91570a650dba194e1ca87c2b543488fe9309b4212694174fd539c"}, + {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e617fb6b0b6953fffd762669610c1c4ffd05632c138d61ac7e14ad187870669c"}, + {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16e5f4bf4e603eb1fdd5d8180f1a25f30056f22e55ce51fb3d6ad4ab29f7d96f"}, + {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4c035da3f544b1882bac24115f3e2e8760f10a0107614fc9839fd232200b875"}, + {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:957cf8e4b6e123a9eea554fa7ebc85674674b713551de587eb318a2df3e00255"}, + {file = "multidict-6.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:483a6aea59cb89904e1ceabd2b47368b5600fb7de78a6e4a2c2987b2d256cf30"}, + {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:87701f25a2352e5bf7454caa64757642734da9f6b11384c1f9d1a8e699758057"}, + {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:682b987361e5fd7a139ed565e30d81fd81e9629acc7d925a205366877d8c8657"}, + {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ce2186a7df133a9c895dea3331ddc5ddad42cdd0d1ea2f0a51e5d161e4762f28"}, + {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:9f636b730f7e8cb19feb87094949ba54ee5357440b9658b2a32a5ce4bce53972"}, + {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:73eae06aa53af2ea5270cc066dcaf02cc60d2994bbb2c4ef5764949257d10f43"}, + {file = "multidict-6.1.0-cp39-cp39-win32.whl", hash = "sha256:1ca0083e80e791cffc6efce7660ad24af66c8d4079d2a750b29001b53ff59ada"}, + {file = "multidict-6.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:aa466da5b15ccea564bdab9c89175c762bc12825f4659c11227f515cee76fa4a"}, + {file = "multidict-6.1.0-py3-none-any.whl", hash = "sha256:48e171e52d1c4d33888e529b999e5900356b9ae588c2f09a52dcefb158b27506"}, + {file = "multidict-6.1.0.tar.gz", hash = "sha256:22ae2ebf9b0c69d206c003e2f6a914ea33f0a932d4aa16f236afc049d9958f4a"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} + +[[package]] +name = "nltk" +version = "3.8.1" +description = "Natural Language Toolkit" +optional = false python-versions = ">=3.7" files = [ - {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9"}, - {file = "multidict-6.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604"}, - {file = "multidict-6.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600"}, - {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c"}, - {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5"}, - {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f"}, - {file = "multidict-6.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae"}, - {file = "multidict-6.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182"}, - {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf"}, - {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442"}, - {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a"}, - {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef"}, - {file = "multidict-6.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc"}, - {file = "multidict-6.0.5-cp310-cp310-win32.whl", hash = "sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319"}, - {file = "multidict-6.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8"}, - {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba"}, - {file = "multidict-6.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e"}, - {file = "multidict-6.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd"}, - {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3"}, - {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf"}, - {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29"}, - {file = "multidict-6.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed"}, - {file = "multidict-6.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733"}, - {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f"}, - {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4"}, - {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1"}, - {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc"}, - {file = "multidict-6.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e"}, - {file = "multidict-6.0.5-cp311-cp311-win32.whl", hash = "sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c"}, - {file = "multidict-6.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea"}, - {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e"}, - {file = "multidict-6.0.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b"}, - {file = "multidict-6.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5"}, - {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450"}, - {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496"}, - {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a"}, - {file = "multidict-6.0.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226"}, - {file = "multidict-6.0.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271"}, - {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb"}, - {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef"}, - {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24"}, - {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6"}, - {file = "multidict-6.0.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda"}, - {file = "multidict-6.0.5-cp312-cp312-win32.whl", hash = "sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5"}, - {file = "multidict-6.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556"}, - {file = "multidict-6.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3"}, - {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5"}, - {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd"}, - {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e"}, - {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626"}, - {file = "multidict-6.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83"}, - {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a"}, - {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c"}, - {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5"}, - {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3"}, - {file = "multidict-6.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc"}, - {file = "multidict-6.0.5-cp37-cp37m-win32.whl", hash = "sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee"}, - {file = "multidict-6.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423"}, - {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54"}, - {file = "multidict-6.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d"}, - {file = "multidict-6.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7"}, - {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93"}, - {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8"}, - {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b"}, - {file = "multidict-6.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50"}, - {file = "multidict-6.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e"}, - {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89"}, - {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386"}, - {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453"}, - {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461"}, - {file = "multidict-6.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44"}, - {file = "multidict-6.0.5-cp38-cp38-win32.whl", hash = "sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241"}, - {file = "multidict-6.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c"}, - {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929"}, - {file = "multidict-6.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9"}, - {file = "multidict-6.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a"}, - {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1"}, - {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e"}, - {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046"}, - {file = "multidict-6.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c"}, - {file = "multidict-6.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40"}, - {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527"}, - {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9"}, - {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38"}, - {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479"}, - {file = "multidict-6.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c"}, - {file = "multidict-6.0.5-cp39-cp39-win32.whl", hash = "sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b"}, - {file = "multidict-6.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755"}, - {file = "multidict-6.0.5-py3-none-any.whl", hash = "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7"}, - {file = "multidict-6.0.5.tar.gz", hash = "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da"}, + {file = "nltk-3.8.1-py3-none-any.whl", hash = "sha256:fd5c9109f976fa86bcadba8f91e47f5e9293bd034474752e92a520f81c93dda5"}, + {file = "nltk-3.8.1.zip", hash = "sha256:1834da3d0682cba4f2cede2f9aad6b0fafb6461ba451db0efb6f9c39798d64d3"}, +] + +[package.dependencies] +click = "*" +joblib = "*" +regex = ">=2021.8.3" +tqdm = "*" + +[package.extras] +all = ["matplotlib", "numpy", "pyparsing", "python-crfsuite", "requests", "scikit-learn", "scipy", "twython"] +corenlp = ["requests"] +machine-learning = ["numpy", "python-crfsuite", "scikit-learn", "scipy"] +plot = ["matplotlib"] +tgrep = ["pyparsing"] +twitter = ["twython"] + +[[package]] +name = "numpy" +version = "2.1.2" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.10" +files = [ + {file = "numpy-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:30d53720b726ec36a7f88dc873f0eec8447fbc93d93a8f079dfac2629598d6ee"}, + {file = "numpy-2.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e8d3ca0a72dd8846eb6f7dfe8f19088060fcb76931ed592d29128e0219652884"}, + {file = "numpy-2.1.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:fc44e3c68ff00fd991b59092a54350e6e4911152682b4782f68070985aa9e648"}, + {file = "numpy-2.1.2-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:7c1c60328bd964b53f8b835df69ae8198659e2b9302ff9ebb7de4e5a5994db3d"}, + {file = "numpy-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6cdb606a7478f9ad91c6283e238544451e3a95f30fb5467fbf715964341a8a86"}, + {file = "numpy-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d666cb72687559689e9906197e3bec7b736764df6a2e58ee265e360663e9baf7"}, + {file = "numpy-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c6eef7a2dbd0abfb0d9eaf78b73017dbfd0b54051102ff4e6a7b2980d5ac1a03"}, + {file = "numpy-2.1.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:12edb90831ff481f7ef5f6bc6431a9d74dc0e5ff401559a71e5e4611d4f2d466"}, + {file = "numpy-2.1.2-cp310-cp310-win32.whl", hash = "sha256:a65acfdb9c6ebb8368490dbafe83c03c7e277b37e6857f0caeadbbc56e12f4fb"}, + {file = "numpy-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:860ec6e63e2c5c2ee5e9121808145c7bf86c96cca9ad396c0bd3e0f2798ccbe2"}, + {file = "numpy-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b42a1a511c81cc78cbc4539675713bbcf9d9c3913386243ceff0e9429ca892fe"}, + {file = "numpy-2.1.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:faa88bc527d0f097abdc2c663cddf37c05a1c2f113716601555249805cf573f1"}, + {file = "numpy-2.1.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:c82af4b2ddd2ee72d1fc0c6695048d457e00b3582ccde72d8a1c991b808bb20f"}, + {file = "numpy-2.1.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:13602b3174432a35b16c4cfb5de9a12d229727c3dd47a6ce35111f2ebdf66ff4"}, + {file = "numpy-2.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ebec5fd716c5a5b3d8dfcc439be82a8407b7b24b230d0ad28a81b61c2f4659a"}, + {file = "numpy-2.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2b49c3c0804e8ecb05d59af8386ec2f74877f7ca8fd9c1e00be2672e4d399b1"}, + {file = "numpy-2.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2cbba4b30bf31ddbe97f1c7205ef976909a93a66bb1583e983adbd155ba72ac2"}, + {file = "numpy-2.1.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8e00ea6fc82e8a804433d3e9cedaa1051a1422cb6e443011590c14d2dea59146"}, + {file = "numpy-2.1.2-cp311-cp311-win32.whl", hash = "sha256:5006b13a06e0b38d561fab5ccc37581f23c9511879be7693bd33c7cd15ca227c"}, + {file = "numpy-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:f1eb068ead09f4994dec71c24b2844f1e4e4e013b9629f812f292f04bd1510d9"}, + {file = "numpy-2.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d7bf0a4f9f15b32b5ba53147369e94296f5fffb783db5aacc1be15b4bf72f43b"}, + {file = "numpy-2.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b1d0fcae4f0949f215d4632be684a539859b295e2d0cb14f78ec231915d644db"}, + {file = "numpy-2.1.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:f751ed0a2f250541e19dfca9f1eafa31a392c71c832b6bb9e113b10d050cb0f1"}, + {file = "numpy-2.1.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:bd33f82e95ba7ad632bc57837ee99dba3d7e006536200c4e9124089e1bf42426"}, + {file = "numpy-2.1.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1b8cde4f11f0a975d1fd59373b32e2f5a562ade7cde4f85b7137f3de8fbb29a0"}, + {file = "numpy-2.1.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d95f286b8244b3649b477ac066c6906fbb2905f8ac19b170e2175d3d799f4df"}, + {file = "numpy-2.1.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ab4754d432e3ac42d33a269c8567413bdb541689b02d93788af4131018cbf366"}, + {file = "numpy-2.1.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e585c8ae871fd38ac50598f4763d73ec5497b0de9a0ab4ef5b69f01c6a046142"}, + {file = "numpy-2.1.2-cp312-cp312-win32.whl", hash = "sha256:9c6c754df29ce6a89ed23afb25550d1c2d5fdb9901d9c67a16e0b16eaf7e2550"}, + {file = "numpy-2.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:456e3b11cb79ac9946c822a56346ec80275eaf2950314b249b512896c0d2505e"}, + {file = "numpy-2.1.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a84498e0d0a1174f2b3ed769b67b656aa5460c92c9554039e11f20a05650f00d"}, + {file = "numpy-2.1.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4d6ec0d4222e8ffdab1744da2560f07856421b367928026fb540e1945f2eeeaf"}, + {file = "numpy-2.1.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:259ec80d54999cc34cd1eb8ded513cb053c3bf4829152a2e00de2371bd406f5e"}, + {file = "numpy-2.1.2-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:675c741d4739af2dc20cd6c6a5c4b7355c728167845e3c6b0e824e4e5d36a6c3"}, + {file = "numpy-2.1.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05b2d4e667895cc55e3ff2b56077e4c8a5604361fc21a042845ea3ad67465aa8"}, + {file = "numpy-2.1.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43cca367bf94a14aca50b89e9bc2061683116cfe864e56740e083392f533ce7a"}, + {file = "numpy-2.1.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:76322dcdb16fccf2ac56f99048af32259dcc488d9b7e25b51e5eca5147a3fb98"}, + {file = "numpy-2.1.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:32e16a03138cabe0cb28e1007ee82264296ac0983714094380b408097a418cfe"}, + {file = "numpy-2.1.2-cp313-cp313-win32.whl", hash = "sha256:242b39d00e4944431a3cd2db2f5377e15b5785920421993770cddb89992c3f3a"}, + {file = "numpy-2.1.2-cp313-cp313-win_amd64.whl", hash = "sha256:f2ded8d9b6f68cc26f8425eda5d3877b47343e68ca23d0d0846f4d312ecaa445"}, + {file = "numpy-2.1.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2ffef621c14ebb0188a8633348504a35c13680d6da93ab5cb86f4e54b7e922b5"}, + {file = "numpy-2.1.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:ad369ed238b1959dfbade9018a740fb9392c5ac4f9b5173f420bd4f37ba1f7a0"}, + {file = "numpy-2.1.2-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:d82075752f40c0ddf57e6e02673a17f6cb0f8eb3f587f63ca1eaab5594da5b17"}, + {file = "numpy-2.1.2-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:1600068c262af1ca9580a527d43dc9d959b0b1d8e56f8a05d830eea39b7c8af6"}, + {file = "numpy-2.1.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a26ae94658d3ba3781d5e103ac07a876b3e9b29db53f68ed7df432fd033358a8"}, + {file = "numpy-2.1.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13311c2db4c5f7609b462bc0f43d3c465424d25c626d95040f073e30f7570e35"}, + {file = "numpy-2.1.2-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:2abbf905a0b568706391ec6fa15161fad0fb5d8b68d73c461b3c1bab6064dd62"}, + {file = "numpy-2.1.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ef444c57d664d35cac4e18c298c47d7b504c66b17c2ea91312e979fcfbdfb08a"}, + {file = "numpy-2.1.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:bdd407c40483463898b84490770199d5714dcc9dd9b792f6c6caccc523c00952"}, + {file = "numpy-2.1.2-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:da65fb46d4cbb75cb417cddf6ba5e7582eb7bb0b47db4b99c9fe5787ce5d91f5"}, + {file = "numpy-2.1.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c193d0b0238638e6fc5f10f1b074a6993cb13b0b431f64079a509d63d3aa8b7"}, + {file = "numpy-2.1.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:a7d80b2e904faa63068ead63107189164ca443b42dd1930299e0d1cb041cec2e"}, + {file = "numpy-2.1.2.tar.gz", hash = "sha256:13532a088217fa624c99b843eeb54640de23b3414b14aa66d023805eb731066c"}, ] [[package]] name = "orjson" -version = "3.10.6" +version = "3.10.7" description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" optional = false python-versions = ">=3.8" files = [ - {file = "orjson-3.10.6-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:fb0ee33124db6eaa517d00890fc1a55c3bfe1cf78ba4a8899d71a06f2d6ff5c7"}, - {file = "orjson-3.10.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c1c4b53b24a4c06547ce43e5fee6ec4e0d8fe2d597f4647fc033fd205707365"}, - {file = "orjson-3.10.6-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eadc8fd310edb4bdbd333374f2c8fec6794bbbae99b592f448d8214a5e4050c0"}, - {file = "orjson-3.10.6-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:61272a5aec2b2661f4fa2b37c907ce9701e821b2c1285d5c3ab0207ebd358d38"}, - {file = "orjson-3.10.6-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:57985ee7e91d6214c837936dc1608f40f330a6b88bb13f5a57ce5257807da143"}, - {file = "orjson-3.10.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:633a3b31d9d7c9f02d49c4ab4d0a86065c4a6f6adc297d63d272e043472acab5"}, - {file = "orjson-3.10.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:1c680b269d33ec444afe2bdc647c9eb73166fa47a16d9a75ee56a374f4a45f43"}, - {file = "orjson-3.10.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f759503a97a6ace19e55461395ab0d618b5a117e8d0fbb20e70cfd68a47327f2"}, - {file = "orjson-3.10.6-cp310-none-win32.whl", hash = "sha256:95a0cce17f969fb5391762e5719575217bd10ac5a189d1979442ee54456393f3"}, - {file = "orjson-3.10.6-cp310-none-win_amd64.whl", hash = "sha256:df25d9271270ba2133cc88ee83c318372bdc0f2cd6f32e7a450809a111efc45c"}, - {file = "orjson-3.10.6-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b1ec490e10d2a77c345def52599311849fc063ae0e67cf4f84528073152bb2ba"}, - {file = "orjson-3.10.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55d43d3feb8f19d07e9f01e5b9be4f28801cf7c60d0fa0d279951b18fae1932b"}, - {file = "orjson-3.10.6-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac3045267e98fe749408eee1593a142e02357c5c99be0802185ef2170086a863"}, - {file = "orjson-3.10.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c27bc6a28ae95923350ab382c57113abd38f3928af3c80be6f2ba7eb8d8db0b0"}, - {file = "orjson-3.10.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d27456491ca79532d11e507cadca37fb8c9324a3976294f68fb1eff2dc6ced5a"}, - {file = "orjson-3.10.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05ac3d3916023745aa3b3b388e91b9166be1ca02b7c7e41045da6d12985685f0"}, - {file = "orjson-3.10.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1335d4ef59ab85cab66fe73fd7a4e881c298ee7f63ede918b7faa1b27cbe5212"}, - {file = "orjson-3.10.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4bbc6d0af24c1575edc79994c20e1b29e6fb3c6a570371306db0993ecf144dc5"}, - {file = "orjson-3.10.6-cp311-none-win32.whl", hash = "sha256:450e39ab1f7694465060a0550b3f6d328d20297bf2e06aa947b97c21e5241fbd"}, - {file = "orjson-3.10.6-cp311-none-win_amd64.whl", hash = "sha256:227df19441372610b20e05bdb906e1742ec2ad7a66ac8350dcfd29a63014a83b"}, - {file = "orjson-3.10.6-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:ea2977b21f8d5d9b758bb3f344a75e55ca78e3ff85595d248eee813ae23ecdfb"}, - {file = "orjson-3.10.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b6f3d167d13a16ed263b52dbfedff52c962bfd3d270b46b7518365bcc2121eed"}, - {file = "orjson-3.10.6-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f710f346e4c44a4e8bdf23daa974faede58f83334289df80bc9cd12fe82573c7"}, - {file = "orjson-3.10.6-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7275664f84e027dcb1ad5200b8b18373e9c669b2a9ec33d410c40f5ccf4b257e"}, - {file = "orjson-3.10.6-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0943e4c701196b23c240b3d10ed8ecd674f03089198cf503105b474a4f77f21f"}, - {file = "orjson-3.10.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:446dee5a491b5bc7d8f825d80d9637e7af43f86a331207b9c9610e2f93fee22a"}, - {file = "orjson-3.10.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:64c81456d2a050d380786413786b057983892db105516639cb5d3ee3c7fd5148"}, - {file = "orjson-3.10.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:960db0e31c4e52fa0fc3ecbaea5b2d3b58f379e32a95ae6b0ebeaa25b93dfd34"}, - {file = "orjson-3.10.6-cp312-none-win32.whl", hash = "sha256:a6ea7afb5b30b2317e0bee03c8d34c8181bc5a36f2afd4d0952f378972c4efd5"}, - {file = "orjson-3.10.6-cp312-none-win_amd64.whl", hash = "sha256:874ce88264b7e655dde4aeaacdc8fd772a7962faadfb41abe63e2a4861abc3dc"}, - {file = "orjson-3.10.6-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:66680eae4c4e7fc193d91cfc1353ad6d01b4801ae9b5314f17e11ba55e934183"}, - {file = "orjson-3.10.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:caff75b425db5ef8e8f23af93c80f072f97b4fb3afd4af44482905c9f588da28"}, - {file = "orjson-3.10.6-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3722fddb821b6036fd2a3c814f6bd9b57a89dc6337b9924ecd614ebce3271394"}, - {file = "orjson-3.10.6-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2c116072a8533f2fec435fde4d134610f806bdac20188c7bd2081f3e9e0133f"}, - {file = "orjson-3.10.6-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6eeb13218c8cf34c61912e9df2de2853f1d009de0e46ea09ccdf3d757896af0a"}, - {file = "orjson-3.10.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:965a916373382674e323c957d560b953d81d7a8603fbeee26f7b8248638bd48b"}, - {file = "orjson-3.10.6-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:03c95484d53ed8e479cade8628c9cea00fd9d67f5554764a1110e0d5aa2de96e"}, - {file = "orjson-3.10.6-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:e060748a04cccf1e0a6f2358dffea9c080b849a4a68c28b1b907f272b5127e9b"}, - {file = "orjson-3.10.6-cp38-none-win32.whl", hash = "sha256:738dbe3ef909c4b019d69afc19caf6b5ed0e2f1c786b5d6215fbb7539246e4c6"}, - {file = "orjson-3.10.6-cp38-none-win_amd64.whl", hash = "sha256:d40f839dddf6a7d77114fe6b8a70218556408c71d4d6e29413bb5f150a692ff7"}, - {file = "orjson-3.10.6-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:697a35a083c4f834807a6232b3e62c8b280f7a44ad0b759fd4dce748951e70db"}, - {file = "orjson-3.10.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd502f96bf5ea9a61cbc0b2b5900d0dd68aa0da197179042bdd2be67e51a1e4b"}, - {file = "orjson-3.10.6-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f215789fb1667cdc874c1b8af6a84dc939fd802bf293a8334fce185c79cd359b"}, - {file = "orjson-3.10.6-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2debd8ddce948a8c0938c8c93ade191d2f4ba4649a54302a7da905a81f00b56"}, - {file = "orjson-3.10.6-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5410111d7b6681d4b0d65e0f58a13be588d01b473822483f77f513c7f93bd3b2"}, - {file = "orjson-3.10.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb1f28a137337fdc18384079fa5726810681055b32b92253fa15ae5656e1dddb"}, - {file = "orjson-3.10.6-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:bf2fbbce5fe7cd1aa177ea3eab2b8e6a6bc6e8592e4279ed3db2d62e57c0e1b2"}, - {file = "orjson-3.10.6-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:79b9b9e33bd4c517445a62b90ca0cc279b0f1f3970655c3df9e608bc3f91741a"}, - {file = "orjson-3.10.6-cp39-none-win32.whl", hash = "sha256:30b0a09a2014e621b1adf66a4f705f0809358350a757508ee80209b2d8dae219"}, - {file = "orjson-3.10.6-cp39-none-win_amd64.whl", hash = "sha256:49e3bc615652617d463069f91b867a4458114c5b104e13b7ae6872e5f79d0844"}, - {file = "orjson-3.10.6.tar.gz", hash = "sha256:e54b63d0a7c6c54a5f5f726bc93a2078111ef060fec4ecbf34c5db800ca3b3a7"}, + {file = "orjson-3.10.7-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:74f4544f5a6405b90da8ea724d15ac9c36da4d72a738c64685003337401f5c12"}, + {file = "orjson-3.10.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34a566f22c28222b08875b18b0dfbf8a947e69df21a9ed5c51a6bf91cfb944ac"}, + {file = "orjson-3.10.7-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf6ba8ebc8ef5792e2337fb0419f8009729335bb400ece005606336b7fd7bab7"}, + {file = "orjson-3.10.7-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac7cf6222b29fbda9e3a472b41e6a5538b48f2c8f99261eecd60aafbdb60690c"}, + {file = "orjson-3.10.7-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de817e2f5fc75a9e7dd350c4b0f54617b280e26d1631811a43e7e968fa71e3e9"}, + {file = "orjson-3.10.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:348bdd16b32556cf8d7257b17cf2bdb7ab7976af4af41ebe79f9796c218f7e91"}, + {file = "orjson-3.10.7-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:479fd0844ddc3ca77e0fd99644c7fe2de8e8be1efcd57705b5c92e5186e8a250"}, + {file = "orjson-3.10.7-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fdf5197a21dd660cf19dfd2a3ce79574588f8f5e2dbf21bda9ee2d2b46924d84"}, + {file = "orjson-3.10.7-cp310-none-win32.whl", hash = "sha256:d374d36726746c81a49f3ff8daa2898dccab6596864ebe43d50733275c629175"}, + {file = "orjson-3.10.7-cp310-none-win_amd64.whl", hash = "sha256:cb61938aec8b0ffb6eef484d480188a1777e67b05d58e41b435c74b9d84e0b9c"}, + {file = "orjson-3.10.7-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:7db8539039698ddfb9a524b4dd19508256107568cdad24f3682d5773e60504a2"}, + {file = "orjson-3.10.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:480f455222cb7a1dea35c57a67578848537d2602b46c464472c995297117fa09"}, + {file = "orjson-3.10.7-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8a9c9b168b3a19e37fe2778c0003359f07822c90fdff8f98d9d2a91b3144d8e0"}, + {file = "orjson-3.10.7-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8de062de550f63185e4c1c54151bdddfc5625e37daf0aa1e75d2a1293e3b7d9a"}, + {file = "orjson-3.10.7-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6b0dd04483499d1de9c8f6203f8975caf17a6000b9c0c54630cef02e44ee624e"}, + {file = "orjson-3.10.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b58d3795dafa334fc8fd46f7c5dc013e6ad06fd5b9a4cc98cb1456e7d3558bd6"}, + {file = "orjson-3.10.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:33cfb96c24034a878d83d1a9415799a73dc77480e6c40417e5dda0710d559ee6"}, + {file = "orjson-3.10.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e724cebe1fadc2b23c6f7415bad5ee6239e00a69f30ee423f319c6af70e2a5c0"}, + {file = "orjson-3.10.7-cp311-none-win32.whl", hash = "sha256:82763b46053727a7168d29c772ed5c870fdae2f61aa8a25994c7984a19b1021f"}, + {file = "orjson-3.10.7-cp311-none-win_amd64.whl", hash = "sha256:eb8d384a24778abf29afb8e41d68fdd9a156cf6e5390c04cc07bbc24b89e98b5"}, + {file = "orjson-3.10.7-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:44a96f2d4c3af51bfac6bc4ef7b182aa33f2f054fd7f34cc0ee9a320d051d41f"}, + {file = "orjson-3.10.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76ac14cd57df0572453543f8f2575e2d01ae9e790c21f57627803f5e79b0d3c3"}, + {file = "orjson-3.10.7-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bdbb61dcc365dd9be94e8f7df91975edc9364d6a78c8f7adb69c1cdff318ec93"}, + {file = "orjson-3.10.7-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b48b3db6bb6e0a08fa8c83b47bc169623f801e5cc4f24442ab2b6617da3b5313"}, + {file = "orjson-3.10.7-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:23820a1563a1d386414fef15c249040042b8e5d07b40ab3fe3efbfbbcbcb8864"}, + {file = "orjson-3.10.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0c6a008e91d10a2564edbb6ee5069a9e66df3fbe11c9a005cb411f441fd2c09"}, + {file = "orjson-3.10.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d352ee8ac1926d6193f602cbe36b1643bbd1bbcb25e3c1a657a4390f3000c9a5"}, + {file = "orjson-3.10.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d2d9f990623f15c0ae7ac608103c33dfe1486d2ed974ac3f40b693bad1a22a7b"}, + {file = "orjson-3.10.7-cp312-none-win32.whl", hash = "sha256:7c4c17f8157bd520cdb7195f75ddbd31671997cbe10aee559c2d613592e7d7eb"}, + {file = "orjson-3.10.7-cp312-none-win_amd64.whl", hash = "sha256:1d9c0e733e02ada3ed6098a10a8ee0052dd55774de3d9110d29868d24b17faa1"}, + {file = "orjson-3.10.7-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:77d325ed866876c0fa6492598ec01fe30e803272a6e8b10e992288b009cbe149"}, + {file = "orjson-3.10.7-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ea2c232deedcb605e853ae1db2cc94f7390ac776743b699b50b071b02bea6fe"}, + {file = "orjson-3.10.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3dcfbede6737fdbef3ce9c37af3fb6142e8e1ebc10336daa05872bfb1d87839c"}, + {file = "orjson-3.10.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:11748c135f281203f4ee695b7f80bb1358a82a63905f9f0b794769483ea854ad"}, + {file = "orjson-3.10.7-cp313-none-win32.whl", hash = "sha256:a7e19150d215c7a13f39eb787d84db274298d3f83d85463e61d277bbd7f401d2"}, + {file = "orjson-3.10.7-cp313-none-win_amd64.whl", hash = "sha256:eef44224729e9525d5261cc8d28d6b11cafc90e6bd0be2157bde69a52ec83024"}, + {file = "orjson-3.10.7-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:6ea2b2258eff652c82652d5e0f02bd5e0463a6a52abb78e49ac288827aaa1469"}, + {file = "orjson-3.10.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:430ee4d85841e1483d487e7b81401785a5dfd69db5de01314538f31f8fbf7ee1"}, + {file = "orjson-3.10.7-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4b6146e439af4c2472c56f8540d799a67a81226e11992008cb47e1267a9b3225"}, + {file = "orjson-3.10.7-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:084e537806b458911137f76097e53ce7bf5806dda33ddf6aaa66a028f8d43a23"}, + {file = "orjson-3.10.7-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4829cf2195838e3f93b70fd3b4292156fc5e097aac3739859ac0dcc722b27ac0"}, + {file = "orjson-3.10.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1193b2416cbad1a769f868b1749535d5da47626ac29445803dae7cc64b3f5c98"}, + {file = "orjson-3.10.7-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:4e6c3da13e5a57e4b3dca2de059f243ebec705857522f188f0180ae88badd354"}, + {file = "orjson-3.10.7-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:c31008598424dfbe52ce8c5b47e0752dca918a4fdc4a2a32004efd9fab41d866"}, + {file = "orjson-3.10.7-cp38-none-win32.whl", hash = "sha256:7122a99831f9e7fe977dc45784d3b2edc821c172d545e6420c375e5a935f5a1c"}, + {file = "orjson-3.10.7-cp38-none-win_amd64.whl", hash = "sha256:a763bc0e58504cc803739e7df040685816145a6f3c8a589787084b54ebc9f16e"}, + {file = "orjson-3.10.7-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:e76be12658a6fa376fcd331b1ea4e58f5a06fd0220653450f0d415b8fd0fbe20"}, + {file = "orjson-3.10.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed350d6978d28b92939bfeb1a0570c523f6170efc3f0a0ef1f1df287cd4f4960"}, + {file = "orjson-3.10.7-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:144888c76f8520e39bfa121b31fd637e18d4cc2f115727865fdf9fa325b10412"}, + {file = "orjson-3.10.7-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09b2d92fd95ad2402188cf51573acde57eb269eddabaa60f69ea0d733e789fe9"}, + {file = "orjson-3.10.7-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5b24a579123fa884f3a3caadaed7b75eb5715ee2b17ab5c66ac97d29b18fe57f"}, + {file = "orjson-3.10.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e72591bcfe7512353bd609875ab38050efe3d55e18934e2f18950c108334b4ff"}, + {file = "orjson-3.10.7-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:f4db56635b58cd1a200b0a23744ff44206ee6aa428185e2b6c4a65b3197abdcd"}, + {file = "orjson-3.10.7-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:0fa5886854673222618638c6df7718ea7fe2f3f2384c452c9ccedc70b4a510a5"}, + {file = "orjson-3.10.7-cp39-none-win32.whl", hash = "sha256:8272527d08450ab16eb405f47e0f4ef0e5ff5981c3d82afe0efd25dcbef2bcd2"}, + {file = "orjson-3.10.7-cp39-none-win_amd64.whl", hash = "sha256:974683d4618c0c7dbf4f69c95a979734bf183d0658611760017f6e70a145af58"}, + {file = "orjson-3.10.7.tar.gz", hash = "sha256:75ef0640403f945f3a1f9f6400686560dbfb0fb5b16589ad62cd477043c4eee3"}, ] [[package]] @@ -1106,6 +1359,78 @@ files = [ {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, ] +[[package]] +name = "pandas" +version = "2.2.2" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pandas-2.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:90c6fca2acf139569e74e8781709dccb6fe25940488755716d1d354d6bc58bce"}, + {file = "pandas-2.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c7adfc142dac335d8c1e0dcbd37eb8617eac386596eb9e1a1b77791cf2498238"}, + {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4abfe0be0d7221be4f12552995e58723c7422c80a659da13ca382697de830c08"}, + {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8635c16bf3d99040fdf3ca3db669a7250ddf49c55dc4aa8fe0ae0fa8d6dcc1f0"}, + {file = "pandas-2.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:40ae1dffb3967a52203105a077415a86044a2bea011b5f321c6aa64b379a3f51"}, + {file = "pandas-2.2.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8e5a0b00e1e56a842f922e7fae8ae4077aee4af0acb5ae3622bd4b4c30aedf99"}, + {file = "pandas-2.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:ddf818e4e6c7c6f4f7c8a12709696d193976b591cc7dc50588d3d1a6b5dc8772"}, + {file = "pandas-2.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:696039430f7a562b74fa45f540aca068ea85fa34c244d0deee539cb6d70aa288"}, + {file = "pandas-2.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8e90497254aacacbc4ea6ae5e7a8cd75629d6ad2b30025a4a8b09aa4faf55151"}, + {file = "pandas-2.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58b84b91b0b9f4bafac2a0ac55002280c094dfc6402402332c0913a59654ab2b"}, + {file = "pandas-2.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2123dc9ad6a814bcdea0f099885276b31b24f7edf40f6cdbc0912672e22eee"}, + {file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2925720037f06e89af896c70bca73459d7e6a4be96f9de79e2d440bd499fe0db"}, + {file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0cace394b6ea70c01ca1595f839cf193df35d1575986e484ad35c4aeae7266c1"}, + {file = "pandas-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:873d13d177501a28b2756375d59816c365e42ed8417b41665f346289adc68d24"}, + {file = "pandas-2.2.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef"}, + {file = "pandas-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e9b79011ff7a0f4b1d6da6a61aa1aa604fb312d6647de5bad20013682d1429ce"}, + {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cb51fe389360f3b5a4d57dbd2848a5f033350336ca3b340d1c53a1fad33bcad"}, + {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eee3a87076c0756de40b05c5e9a6069c035ba43e8dd71c379e68cab2c20f16ad"}, + {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3e374f59e440d4ab45ca2fffde54b81ac3834cf5ae2cdfa69c90bc03bde04d76"}, + {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:43498c0bdb43d55cb162cdc8c06fac328ccb5d2eabe3cadeb3529ae6f0517c32"}, + {file = "pandas-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23"}, + {file = "pandas-2.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0ca6377b8fca51815f382bd0b697a0814c8bda55115678cbc94c30aacbb6eff2"}, + {file = "pandas-2.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9057e6aa78a584bc93a13f0a9bf7e753a5e9770a30b4d758b8d5f2a62a9433cd"}, + {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:001910ad31abc7bf06f49dcc903755d2f7f3a9186c0c040b827e522e9cef0863"}, + {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66b479b0bd07204e37583c191535505410daa8df638fd8e75ae1b383851fe921"}, + {file = "pandas-2.2.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a77e9d1c386196879aa5eb712e77461aaee433e54c68cf253053a73b7e49c33a"}, + {file = "pandas-2.2.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:92fd6b027924a7e178ac202cfbe25e53368db90d56872d20ffae94b96c7acc57"}, + {file = "pandas-2.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:640cef9aa381b60e296db324337a554aeeb883ead99dc8f6c18e81a93942f5f4"}, + {file = "pandas-2.2.2.tar.gz", hash = "sha256:9e79019aba43cb4fda9e4d983f8e88ca0373adbb697ae9c6c43093218de28b54"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.22.4", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, +] +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.7" + +[package.extras] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] +aws = ["s3fs (>=2022.11.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] +compression = ["zstandard (>=0.19.0)"] +computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] +feather = ["pyarrow (>=10.0.1)"] +fss = ["fsspec (>=2022.11.0)"] +gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] +hdf5 = ["tables (>=3.8.0)"] +html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] +mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=10.0.1)"] +performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] +plot = ["matplotlib (>=3.6.3)"] +postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] +pyarrow = ["pyarrow (>=10.0.1)"] +spss = ["pyreadstat (>=1.2.0)"] +sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.9.2)"] + [[package]] name = "pendulum" version = "2.1.2" @@ -1142,19 +1467,19 @@ pytzdata = ">=2020.1" [[package]] name = "platformdirs" -version = "4.2.2" +version = "4.3.6" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.8" files = [ - {file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"}, - {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"}, + {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"}, + {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"}, ] [package.extras] -docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] -type = ["mypy (>=1.8)"] +docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.2)", "pytest-cov (>=5)", "pytest-mock (>=3.14)"] +type = ["mypy (>=1.11.2)"] [[package]] name = "pluggy" @@ -1171,17 +1496,6 @@ files = [ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] -[[package]] -name = "py" -version = "1.11.0" -description = "library with cross-python path, ini-parsing, io, code, log facilities" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -files = [ - {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, - {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, -] - [[package]] name = "pycountry" version = "24.6.1" @@ -1206,119 +1520,120 @@ files = [ [[package]] name = "pydantic" -version = "2.8.2" +version = "2.9.2" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic-2.8.2-py3-none-any.whl", hash = "sha256:73ee9fddd406dc318b885c7a2eab8a6472b68b8fb5ba8150949fc3db939f23c8"}, - {file = "pydantic-2.8.2.tar.gz", hash = "sha256:6f62c13d067b0755ad1c21a34bdd06c0c12625a22b0fc09c6b149816604f7c2a"}, + {file = "pydantic-2.9.2-py3-none-any.whl", hash = "sha256:f048cec7b26778210e28a0459867920654d48e5e62db0958433636cde4254f12"}, + {file = "pydantic-2.9.2.tar.gz", hash = "sha256:d155cef71265d1e9807ed1c32b4c8deec042a44a50a4188b25ac67ecd81a9c0f"}, ] [package.dependencies] -annotated-types = ">=0.4.0" -pydantic-core = "2.20.1" +annotated-types = ">=0.6.0" +pydantic-core = "2.23.4" typing-extensions = {version = ">=4.6.1", markers = "python_version < \"3.13\""} [package.extras] email = ["email-validator (>=2.0.0)"] +timezone = ["tzdata"] [[package]] name = "pydantic-core" -version = "2.20.1" +version = "2.23.4" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_core-2.20.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3acae97ffd19bf091c72df4d726d552c473f3576409b2a7ca36b2f535ffff4a3"}, - {file = "pydantic_core-2.20.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:41f4c96227a67a013e7de5ff8f20fb496ce573893b7f4f2707d065907bffdbd6"}, - {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f239eb799a2081495ea659d8d4a43a8f42cd1fe9ff2e7e436295c38a10c286a"}, - {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:53e431da3fc53360db73eedf6f7124d1076e1b4ee4276b36fb25514544ceb4a3"}, - {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f1f62b2413c3a0e846c3b838b2ecd6c7a19ec6793b2a522745b0869e37ab5bc1"}, - {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d41e6daee2813ecceea8eda38062d69e280b39df793f5a942fa515b8ed67953"}, - {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d482efec8b7dc6bfaedc0f166b2ce349df0011f5d2f1f25537ced4cfc34fd98"}, - {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e93e1a4b4b33daed65d781a57a522ff153dcf748dee70b40c7258c5861e1768a"}, - {file = "pydantic_core-2.20.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e7c4ea22b6739b162c9ecaaa41d718dfad48a244909fe7ef4b54c0b530effc5a"}, - {file = "pydantic_core-2.20.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4f2790949cf385d985a31984907fecb3896999329103df4e4983a4a41e13e840"}, - {file = "pydantic_core-2.20.1-cp310-none-win32.whl", hash = "sha256:5e999ba8dd90e93d57410c5e67ebb67ffcaadcea0ad973240fdfd3a135506250"}, - {file = "pydantic_core-2.20.1-cp310-none-win_amd64.whl", hash = "sha256:512ecfbefef6dac7bc5eaaf46177b2de58cdf7acac8793fe033b24ece0b9566c"}, - {file = "pydantic_core-2.20.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:d2a8fa9d6d6f891f3deec72f5cc668e6f66b188ab14bb1ab52422fe8e644f312"}, - {file = "pydantic_core-2.20.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:175873691124f3d0da55aeea1d90660a6ea7a3cfea137c38afa0a5ffabe37b88"}, - {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37eee5b638f0e0dcd18d21f59b679686bbd18917b87db0193ae36f9c23c355fc"}, - {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25e9185e2d06c16ee438ed39bf62935ec436474a6ac4f9358524220f1b236e43"}, - {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:150906b40ff188a3260cbee25380e7494ee85048584998c1e66df0c7a11c17a6"}, - {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ad4aeb3e9a97286573c03df758fc7627aecdd02f1da04516a86dc159bf70121"}, - {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3f3ed29cd9f978c604708511a1f9c2fdcb6c38b9aae36a51905b8811ee5cbf1"}, - {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b0dae11d8f5ded51699c74d9548dcc5938e0804cc8298ec0aa0da95c21fff57b"}, - {file = "pydantic_core-2.20.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:faa6b09ee09433b87992fb5a2859efd1c264ddc37280d2dd5db502126d0e7f27"}, - {file = "pydantic_core-2.20.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9dc1b507c12eb0481d071f3c1808f0529ad41dc415d0ca11f7ebfc666e66a18b"}, - {file = "pydantic_core-2.20.1-cp311-none-win32.whl", hash = "sha256:fa2fddcb7107e0d1808086ca306dcade7df60a13a6c347a7acf1ec139aa6789a"}, - {file = "pydantic_core-2.20.1-cp311-none-win_amd64.whl", hash = "sha256:40a783fb7ee353c50bd3853e626f15677ea527ae556429453685ae32280c19c2"}, - {file = "pydantic_core-2.20.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:595ba5be69b35777474fa07f80fc260ea71255656191adb22a8c53aba4479231"}, - {file = "pydantic_core-2.20.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a4f55095ad087474999ee28d3398bae183a66be4823f753cd7d67dd0153427c9"}, - {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9aa05d09ecf4c75157197f27cdc9cfaeb7c5f15021c6373932bf3e124af029f"}, - {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e97fdf088d4b31ff4ba35db26d9cc472ac7ef4a2ff2badeabf8d727b3377fc52"}, - {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bc633a9fe1eb87e250b5c57d389cf28998e4292336926b0b6cdaee353f89a237"}, - {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d573faf8eb7e6b1cbbcb4f5b247c60ca8be39fe2c674495df0eb4318303137fe"}, - {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26dc97754b57d2fd00ac2b24dfa341abffc380b823211994c4efac7f13b9e90e"}, - {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:33499e85e739a4b60c9dac710c20a08dc73cb3240c9a0e22325e671b27b70d24"}, - {file = "pydantic_core-2.20.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bebb4d6715c814597f85297c332297c6ce81e29436125ca59d1159b07f423eb1"}, - {file = "pydantic_core-2.20.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:516d9227919612425c8ef1c9b869bbbee249bc91912c8aaffb66116c0b447ebd"}, - {file = "pydantic_core-2.20.1-cp312-none-win32.whl", hash = "sha256:469f29f9093c9d834432034d33f5fe45699e664f12a13bf38c04967ce233d688"}, - {file = "pydantic_core-2.20.1-cp312-none-win_amd64.whl", hash = "sha256:035ede2e16da7281041f0e626459bcae33ed998cca6a0a007a5ebb73414ac72d"}, - {file = "pydantic_core-2.20.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:0827505a5c87e8aa285dc31e9ec7f4a17c81a813d45f70b1d9164e03a813a686"}, - {file = "pydantic_core-2.20.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19c0fa39fa154e7e0b7f82f88ef85faa2a4c23cc65aae2f5aea625e3c13c735a"}, - {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa223cd1e36b642092c326d694d8bf59b71ddddc94cdb752bbbb1c5c91d833b"}, - {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c336a6d235522a62fef872c6295a42ecb0c4e1d0f1a3e500fe949415761b8a19"}, - {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7eb6a0587eded33aeefea9f916899d42b1799b7b14b8f8ff2753c0ac1741edac"}, - {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:70c8daf4faca8da5a6d655f9af86faf6ec2e1768f4b8b9d0226c02f3d6209703"}, - {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9fa4c9bf273ca41f940bceb86922a7667cd5bf90e95dbb157cbb8441008482c"}, - {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:11b71d67b4725e7e2a9f6e9c0ac1239bbc0c48cce3dc59f98635efc57d6dac83"}, - {file = "pydantic_core-2.20.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:270755f15174fb983890c49881e93f8f1b80f0b5e3a3cc1394a255706cabd203"}, - {file = "pydantic_core-2.20.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c81131869240e3e568916ef4c307f8b99583efaa60a8112ef27a366eefba8ef0"}, - {file = "pydantic_core-2.20.1-cp313-none-win32.whl", hash = "sha256:b91ced227c41aa29c672814f50dbb05ec93536abf8f43cd14ec9521ea09afe4e"}, - {file = "pydantic_core-2.20.1-cp313-none-win_amd64.whl", hash = "sha256:65db0f2eefcaad1a3950f498aabb4875c8890438bc80b19362cf633b87a8ab20"}, - {file = "pydantic_core-2.20.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:4745f4ac52cc6686390c40eaa01d48b18997cb130833154801a442323cc78f91"}, - {file = "pydantic_core-2.20.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a8ad4c766d3f33ba8fd692f9aa297c9058970530a32c728a2c4bfd2616d3358b"}, - {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41e81317dd6a0127cabce83c0c9c3fbecceae981c8391e6f1dec88a77c8a569a"}, - {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:04024d270cf63f586ad41fff13fde4311c4fc13ea74676962c876d9577bcc78f"}, - {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eaad4ff2de1c3823fddf82f41121bdf453d922e9a238642b1dedb33c4e4f98ad"}, - {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:26ab812fa0c845df815e506be30337e2df27e88399b985d0bb4e3ecfe72df31c"}, - {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c5ebac750d9d5f2706654c638c041635c385596caf68f81342011ddfa1e5598"}, - {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2aafc5a503855ea5885559eae883978c9b6d8c8993d67766ee73d82e841300dd"}, - {file = "pydantic_core-2.20.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4868f6bd7c9d98904b748a2653031fc9c2f85b6237009d475b1008bfaeb0a5aa"}, - {file = "pydantic_core-2.20.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:aa2f457b4af386254372dfa78a2eda2563680d982422641a85f271c859df1987"}, - {file = "pydantic_core-2.20.1-cp38-none-win32.whl", hash = "sha256:225b67a1f6d602de0ce7f6c1c3ae89a4aa25d3de9be857999e9124f15dab486a"}, - {file = "pydantic_core-2.20.1-cp38-none-win_amd64.whl", hash = "sha256:6b507132dcfc0dea440cce23ee2182c0ce7aba7054576efc65634f080dbe9434"}, - {file = "pydantic_core-2.20.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b03f7941783b4c4a26051846dea594628b38f6940a2fdc0df00b221aed39314c"}, - {file = "pydantic_core-2.20.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1eedfeb6089ed3fad42e81a67755846ad4dcc14d73698c120a82e4ccf0f1f9f6"}, - {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:635fee4e041ab9c479e31edda27fcf966ea9614fff1317e280d99eb3e5ab6fe2"}, - {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:77bf3ac639c1ff567ae3b47f8d4cc3dc20f9966a2a6dd2311dcc055d3d04fb8a"}, - {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ed1b0132f24beeec5a78b67d9388656d03e6a7c837394f99257e2d55b461611"}, - {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c6514f963b023aeee506678a1cf821fe31159b925c4b76fe2afa94cc70b3222b"}, - {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10d4204d8ca33146e761c79f83cc861df20e7ae9f6487ca290a97702daf56006"}, - {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2d036c7187b9422ae5b262badb87a20a49eb6c5238b2004e96d4da1231badef1"}, - {file = "pydantic_core-2.20.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9ebfef07dbe1d93efb94b4700f2d278494e9162565a54f124c404a5656d7ff09"}, - {file = "pydantic_core-2.20.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6b9d9bb600328a1ce523ab4f454859e9d439150abb0906c5a1983c146580ebab"}, - {file = "pydantic_core-2.20.1-cp39-none-win32.whl", hash = "sha256:784c1214cb6dd1e3b15dd8b91b9a53852aed16671cc3fbe4786f4f1db07089e2"}, - {file = "pydantic_core-2.20.1-cp39-none-win_amd64.whl", hash = "sha256:d2fe69c5434391727efa54b47a1e7986bb0186e72a41b203df8f5b0a19a4f669"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a45f84b09ac9c3d35dfcf6a27fd0634d30d183205230a0ebe8373a0e8cfa0906"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d02a72df14dfdbaf228424573a07af10637bd490f0901cee872c4f434a735b94"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d2b27e6af28f07e2f195552b37d7d66b150adbaa39a6d327766ffd695799780f"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:084659fac3c83fd674596612aeff6041a18402f1e1bc19ca39e417d554468482"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:242b8feb3c493ab78be289c034a1f659e8826e2233786e36f2893a950a719bb6"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:38cf1c40a921d05c5edc61a785c0ddb4bed67827069f535d794ce6bcded919fc"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e0bbdd76ce9aa5d4209d65f2b27fc6e5ef1312ae6c5333c26db3f5ade53a1e99"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:254ec27fdb5b1ee60684f91683be95e5133c994cc54e86a0b0963afa25c8f8a6"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:407653af5617f0757261ae249d3fba09504d7a71ab36ac057c938572d1bc9331"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:c693e916709c2465b02ca0ad7b387c4f8423d1db7b4649c551f27a529181c5ad"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b5ff4911aea936a47d9376fd3ab17e970cc543d1b68921886e7f64bd28308d1"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:177f55a886d74f1808763976ac4efd29b7ed15c69f4d838bbd74d9d09cf6fa86"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:964faa8a861d2664f0c7ab0c181af0bea66098b1919439815ca8803ef136fc4e"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4dd484681c15e6b9a977c785a345d3e378d72678fd5f1f3c0509608da24f2ac0"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f6d6cff3538391e8486a431569b77921adfcdef14eb18fbf19b7c0a5294d4e6a"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a6d511cc297ff0883bc3708b465ff82d7560193169a8b93260f74ecb0a5e08a7"}, - {file = "pydantic_core-2.20.1.tar.gz", hash = "sha256:26ca695eeee5f9f1aeeb211ffc12f10bcb6f71e2989988fda61dabd65db878d4"}, + {file = "pydantic_core-2.23.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:b10bd51f823d891193d4717448fab065733958bdb6a6b351967bd349d48d5c9b"}, + {file = "pydantic_core-2.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fc714bdbfb534f94034efaa6eadd74e5b93c8fa6315565a222f7b6f42ca1166"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63e46b3169866bd62849936de036f901a9356e36376079b05efa83caeaa02ceb"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed1a53de42fbe34853ba90513cea21673481cd81ed1be739f7f2efb931b24916"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cfdd16ab5e59fc31b5e906d1a3f666571abc367598e3e02c83403acabc092e07"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:255a8ef062cbf6674450e668482456abac99a5583bbafb73f9ad469540a3a232"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a7cd62e831afe623fbb7aabbb4fe583212115b3ef38a9f6b71869ba644624a2"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f09e2ff1f17c2b51f2bc76d1cc33da96298f0a036a137f5440ab3ec5360b624f"}, + {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e38e63e6f3d1cec5a27e0afe90a085af8b6806ee208b33030e65b6516353f1a3"}, + {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0dbd8dbed2085ed23b5c04afa29d8fd2771674223135dc9bc937f3c09284d071"}, + {file = "pydantic_core-2.23.4-cp310-none-win32.whl", hash = "sha256:6531b7ca5f951d663c339002e91aaebda765ec7d61b7d1e3991051906ddde119"}, + {file = "pydantic_core-2.23.4-cp310-none-win_amd64.whl", hash = "sha256:7c9129eb40958b3d4500fa2467e6a83356b3b61bfff1b414c7361d9220f9ae8f"}, + {file = "pydantic_core-2.23.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:77733e3892bb0a7fa797826361ce8a9184d25c8dffaec60b7ffe928153680ba8"}, + {file = "pydantic_core-2.23.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b84d168f6c48fabd1f2027a3d1bdfe62f92cade1fb273a5d68e621da0e44e6d"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df49e7a0861a8c36d089c1ed57d308623d60416dab2647a4a17fe050ba85de0e"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ff02b6d461a6de369f07ec15e465a88895f3223eb75073ffea56b84d9331f607"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:996a38a83508c54c78a5f41456b0103c30508fed9abcad0a59b876d7398f25fd"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d97683ddee4723ae8c95d1eddac7c192e8c552da0c73a925a89fa8649bf13eea"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:216f9b2d7713eb98cb83c80b9c794de1f6b7e3145eef40400c62e86cee5f4e1e"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6f783e0ec4803c787bcea93e13e9932edab72068f68ecffdf86a99fd5918878b"}, + {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d0776dea117cf5272382634bd2a5c1b6eb16767c223c6a5317cd3e2a757c61a0"}, + {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d5f7a395a8cf1621939692dba2a6b6a830efa6b3cee787d82c7de1ad2930de64"}, + {file = "pydantic_core-2.23.4-cp311-none-win32.whl", hash = "sha256:74b9127ffea03643e998e0c5ad9bd3811d3dac8c676e47db17b0ee7c3c3bf35f"}, + {file = "pydantic_core-2.23.4-cp311-none-win_amd64.whl", hash = "sha256:98d134c954828488b153d88ba1f34e14259284f256180ce659e8d83e9c05eaa3"}, + {file = "pydantic_core-2.23.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f3e0da4ebaef65158d4dfd7d3678aad692f7666877df0002b8a522cdf088f231"}, + {file = "pydantic_core-2.23.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f69a8e0b033b747bb3e36a44e7732f0c99f7edd5cea723d45bc0d6e95377ffee"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:723314c1d51722ab28bfcd5240d858512ffd3116449c557a1336cbe3919beb87"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb2802e667b7051a1bebbfe93684841cc9351004e2badbd6411bf357ab8d5ac8"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d18ca8148bebe1b0a382a27a8ee60350091a6ddaf475fa05ef50dc35b5df6327"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33e3d65a85a2a4a0dc3b092b938a4062b1a05f3a9abde65ea93b233bca0e03f2"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:128585782e5bfa515c590ccee4b727fb76925dd04a98864182b22e89a4e6ed36"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:68665f4c17edcceecc112dfed5dbe6f92261fb9d6054b47d01bf6371a6196126"}, + {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:20152074317d9bed6b7a95ade3b7d6054845d70584216160860425f4fbd5ee9e"}, + {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9261d3ce84fa1d38ed649c3638feefeae23d32ba9182963e465d58d62203bd24"}, + {file = "pydantic_core-2.23.4-cp312-none-win32.whl", hash = "sha256:4ba762ed58e8d68657fc1281e9bb72e1c3e79cc5d464be146e260c541ec12d84"}, + {file = "pydantic_core-2.23.4-cp312-none-win_amd64.whl", hash = "sha256:97df63000f4fea395b2824da80e169731088656d1818a11b95f3b173747b6cd9"}, + {file = "pydantic_core-2.23.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7530e201d10d7d14abce4fb54cfe5b94a0aefc87da539d0346a484ead376c3cc"}, + {file = "pydantic_core-2.23.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:df933278128ea1cd77772673c73954e53a1c95a4fdf41eef97c2b779271bd0bd"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cb3da3fd1b6a5d0279a01877713dbda118a2a4fc6f0d821a57da2e464793f05"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42c6dcb030aefb668a2b7009c85b27f90e51e6a3b4d5c9bc4c57631292015b0d"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:696dd8d674d6ce621ab9d45b205df149399e4bb9aa34102c970b721554828510"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2971bb5ffe72cc0f555c13e19b23c85b654dd2a8f7ab493c262071377bfce9f6"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8394d940e5d400d04cad4f75c0598665cbb81aecefaca82ca85bd28264af7f9b"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0dff76e0602ca7d4cdaacc1ac4c005e0ce0dcfe095d5b5259163a80d3a10d327"}, + {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7d32706badfe136888bdea71c0def994644e09fff0bfe47441deaed8e96fdbc6"}, + {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ed541d70698978a20eb63d8c5d72f2cc6d7079d9d90f6b50bad07826f1320f5f"}, + {file = "pydantic_core-2.23.4-cp313-none-win32.whl", hash = "sha256:3d5639516376dce1940ea36edf408c554475369f5da2abd45d44621cb616f769"}, + {file = "pydantic_core-2.23.4-cp313-none-win_amd64.whl", hash = "sha256:5a1504ad17ba4210df3a045132a7baeeba5a200e930f57512ee02909fc5c4cb5"}, + {file = "pydantic_core-2.23.4-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d4488a93b071c04dc20f5cecc3631fc78b9789dd72483ba15d423b5b3689b555"}, + {file = "pydantic_core-2.23.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:81965a16b675b35e1d09dd14df53f190f9129c0202356ed44ab2728b1c905658"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ffa2ebd4c8530079140dd2d7f794a9d9a73cbb8e9d59ffe24c63436efa8f271"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:61817945f2fe7d166e75fbfb28004034b48e44878177fc54d81688e7b85a3665"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29d2c342c4bc01b88402d60189f3df065fb0dda3654744d5a165a5288a657368"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5e11661ce0fd30a6790e8bcdf263b9ec5988e95e63cf901972107efc49218b13"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d18368b137c6295db49ce7218b1a9ba15c5bc254c96d7c9f9e924a9bc7825ad"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ec4e55f79b1c4ffb2eecd8a0cfba9955a2588497d96851f4c8f99aa4a1d39b12"}, + {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:374a5e5049eda9e0a44c696c7ade3ff355f06b1fe0bb945ea3cac2bc336478a2"}, + {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5c364564d17da23db1106787675fc7af45f2f7b58b4173bfdd105564e132e6fb"}, + {file = "pydantic_core-2.23.4-cp38-none-win32.whl", hash = "sha256:d7a80d21d613eec45e3d41eb22f8f94ddc758a6c4720842dc74c0581f54993d6"}, + {file = "pydantic_core-2.23.4-cp38-none-win_amd64.whl", hash = "sha256:5f5ff8d839f4566a474a969508fe1c5e59c31c80d9e140566f9a37bba7b8d556"}, + {file = "pydantic_core-2.23.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:a4fa4fc04dff799089689f4fd502ce7d59de529fc2f40a2c8836886c03e0175a"}, + {file = "pydantic_core-2.23.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0a7df63886be5e270da67e0966cf4afbae86069501d35c8c1b3b6c168f42cb36"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dcedcd19a557e182628afa1d553c3895a9f825b936415d0dbd3cd0bbcfd29b4b"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f54b118ce5de9ac21c363d9b3caa6c800341e8c47a508787e5868c6b79c9323"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86d2f57d3e1379a9525c5ab067b27dbb8a0642fb5d454e17a9ac434f9ce523e3"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de6d1d1b9e5101508cb37ab0d972357cac5235f5c6533d1071964c47139257df"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1278e0d324f6908e872730c9102b0112477a7f7cf88b308e4fc36ce1bdb6d58c"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a6b5099eeec78827553827f4c6b8615978bb4b6a88e5d9b93eddf8bb6790f55"}, + {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e55541f756f9b3ee346b840103f32779c695a19826a4c442b7954550a0972040"}, + {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a5c7ba8ffb6d6f8f2ab08743be203654bb1aaa8c9dcb09f82ddd34eadb695605"}, + {file = "pydantic_core-2.23.4-cp39-none-win32.whl", hash = "sha256:37b0fe330e4a58d3c58b24d91d1eb102aeec675a3db4c292ec3928ecd892a9a6"}, + {file = "pydantic_core-2.23.4-cp39-none-win_amd64.whl", hash = "sha256:1498bec4c05c9c787bde9125cfdcc63a41004ff167f495063191b863399b1a29"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f455ee30a9d61d3e1a15abd5068827773d6e4dc513e795f380cdd59932c782d5"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1e90d2e3bd2c3863d48525d297cd143fe541be8bbf6f579504b9712cb6b643ec"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e203fdf807ac7e12ab59ca2bfcabb38c7cf0b33c41efeb00f8e5da1d86af480"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e08277a400de01bc72436a0ccd02bdf596631411f592ad985dcee21445bd0068"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f220b0eea5965dec25480b6333c788fb72ce5f9129e8759ef876a1d805d00801"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d06b0c8da4f16d1d1e352134427cb194a0a6e19ad5db9161bf32b2113409e728"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ba1a0996f6c2773bd83e63f18914c1de3c9dd26d55f4ac302a7efe93fb8e7433"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:9a5bce9d23aac8f0cf0836ecfc033896aa8443b501c58d0602dbfd5bd5b37753"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:78ddaaa81421a29574a682b3179d4cf9e6d405a09b99d93ddcf7e5239c742e21"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:883a91b5dd7d26492ff2f04f40fbb652de40fcc0afe07e8129e8ae779c2110eb"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88ad334a15b32a791ea935af224b9de1bf99bcd62fabf745d5f3442199d86d59"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:233710f069d251feb12a56da21e14cca67994eab08362207785cf8c598e74577"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:19442362866a753485ba5e4be408964644dd6a09123d9416c54cd49171f50744"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:624e278a7d29b6445e4e813af92af37820fafb6dcc55c012c834f9e26f9aaaef"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f5ef8f42bec47f21d07668a043f077d507e5bf4e668d5c6dfe6aaba89de1a5b8"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:aea443fffa9fbe3af1a9ba721a87f926fe548d32cab71d188a6ede77d0ff244e"}, + {file = "pydantic_core-2.23.4.tar.gz", hash = "sha256:2584f7cf844ac4d970fba483a717dbe10c1c1c96a969bf65d61ffe94df1b2863"}, ] [package.dependencies] @@ -1326,19 +1641,19 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" [[package]] name = "pyjwt" -version = "2.8.0" +version = "2.9.0" description = "JSON Web Token implementation in Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, - {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, + {file = "PyJWT-2.9.0-py3-none-any.whl", hash = "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850"}, + {file = "pyjwt-2.9.0.tar.gz", hash = "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c"}, ] [package.extras] crypto = ["cryptography (>=3.4.0)"] -dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] -docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] [[package]] @@ -1399,27 +1714,25 @@ files = [ [[package]] name = "pytest" -version = "6.2.5" +version = "7.4.4" description = "pytest: simple powerful testing with Python" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, - {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, + {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, + {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, ] [package.dependencies] -atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} -attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" packaging = "*" pluggy = ">=0.12,<2.0" -py = ">=1.8.2" -toml = "*" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] name = "pytest-mock" @@ -1476,62 +1789,167 @@ files = [ [[package]] name = "pyyaml" -version = "6.0.1" +version = "6.0.2" description = "YAML parser and emitter for Python" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, - {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, - {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, - {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, - {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, - {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, - {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, - {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, - {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, - {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, - {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, - {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, - {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, - {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"}, + {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"}, + {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"}, + {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"}, + {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"}, + {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"}, + {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"}, + {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"}, + {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"}, + {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"}, + {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"}, + {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"}, + {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"}, + {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"}, + {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, + {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, +] + +[[package]] +name = "regex" +version = "2024.9.11" +description = "Alternative regular expression module, to replace re." +optional = false +python-versions = ">=3.8" +files = [ + {file = "regex-2024.9.11-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1494fa8725c285a81d01dc8c06b55287a1ee5e0e382d8413adc0a9197aac6408"}, + {file = "regex-2024.9.11-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0e12c481ad92d129c78f13a2a3662317e46ee7ef96c94fd332e1c29131875b7d"}, + {file = "regex-2024.9.11-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:16e13a7929791ac1216afde26f712802e3df7bf0360b32e4914dca3ab8baeea5"}, + {file = "regex-2024.9.11-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:46989629904bad940bbec2106528140a218b4a36bb3042d8406980be1941429c"}, + {file = "regex-2024.9.11-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a906ed5e47a0ce5f04b2c981af1c9acf9e8696066900bf03b9d7879a6f679fc8"}, + {file = "regex-2024.9.11-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e9a091b0550b3b0207784a7d6d0f1a00d1d1c8a11699c1a4d93db3fbefc3ad35"}, + {file = "regex-2024.9.11-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ddcd9a179c0a6fa8add279a4444015acddcd7f232a49071ae57fa6e278f1f71"}, + {file = "regex-2024.9.11-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6b41e1adc61fa347662b09398e31ad446afadff932a24807d3ceb955ed865cc8"}, + {file = "regex-2024.9.11-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ced479f601cd2f8ca1fd7b23925a7e0ad512a56d6e9476f79b8f381d9d37090a"}, + {file = "regex-2024.9.11-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:635a1d96665f84b292e401c3d62775851aedc31d4f8784117b3c68c4fcd4118d"}, + {file = "regex-2024.9.11-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:c0256beda696edcf7d97ef16b2a33a8e5a875affd6fa6567b54f7c577b30a137"}, + {file = "regex-2024.9.11-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:3ce4f1185db3fbde8ed8aa223fc9620f276c58de8b0d4f8cc86fd1360829edb6"}, + {file = "regex-2024.9.11-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:09d77559e80dcc9d24570da3745ab859a9cf91953062e4ab126ba9d5993688ca"}, + {file = "regex-2024.9.11-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7a22ccefd4db3f12b526eccb129390942fe874a3a9fdbdd24cf55773a1faab1a"}, + {file = "regex-2024.9.11-cp310-cp310-win32.whl", hash = "sha256:f745ec09bc1b0bd15cfc73df6fa4f726dcc26bb16c23a03f9e3367d357eeedd0"}, + {file = "regex-2024.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:01c2acb51f8a7d6494c8c5eafe3d8e06d76563d8a8a4643b37e9b2dd8a2ff623"}, + {file = "regex-2024.9.11-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2cce2449e5927a0bf084d346da6cd5eb016b2beca10d0013ab50e3c226ffc0df"}, + {file = "regex-2024.9.11-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b37fa423beefa44919e009745ccbf353d8c981516e807995b2bd11c2c77d268"}, + {file = "regex-2024.9.11-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:64ce2799bd75039b480cc0360907c4fb2f50022f030bf9e7a8705b636e408fad"}, + {file = "regex-2024.9.11-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a4cc92bb6db56ab0c1cbd17294e14f5e9224f0cc6521167ef388332604e92679"}, + {file = "regex-2024.9.11-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d05ac6fa06959c4172eccd99a222e1fbf17b5670c4d596cb1e5cde99600674c4"}, + {file = "regex-2024.9.11-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040562757795eeea356394a7fb13076ad4f99d3c62ab0f8bdfb21f99a1f85664"}, + {file = "regex-2024.9.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6113c008a7780792efc80f9dfe10ba0cd043cbf8dc9a76ef757850f51b4edc50"}, + {file = "regex-2024.9.11-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8e5fb5f77c8745a60105403a774fe2c1759b71d3e7b4ca237a5e67ad066c7199"}, + {file = "regex-2024.9.11-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:54d9ff35d4515debf14bc27f1e3b38bfc453eff3220f5bce159642fa762fe5d4"}, + {file = "regex-2024.9.11-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:df5cbb1fbc74a8305b6065d4ade43b993be03dbe0f8b30032cced0d7740994bd"}, + {file = "regex-2024.9.11-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7fb89ee5d106e4a7a51bce305ac4efb981536301895f7bdcf93ec92ae0d91c7f"}, + {file = "regex-2024.9.11-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:a738b937d512b30bf75995c0159c0ddf9eec0775c9d72ac0202076c72f24aa96"}, + {file = "regex-2024.9.11-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e28f9faeb14b6f23ac55bfbbfd3643f5c7c18ede093977f1df249f73fd22c7b1"}, + {file = "regex-2024.9.11-cp311-cp311-win32.whl", hash = "sha256:18e707ce6c92d7282dfce370cd205098384b8ee21544e7cb29b8aab955b66fa9"}, + {file = "regex-2024.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:313ea15e5ff2a8cbbad96ccef6be638393041b0a7863183c2d31e0c6116688cf"}, + {file = "regex-2024.9.11-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b0d0a6c64fcc4ef9c69bd5b3b3626cc3776520a1637d8abaa62b9edc147a58f7"}, + {file = "regex-2024.9.11-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:49b0e06786ea663f933f3710a51e9385ce0cba0ea56b67107fd841a55d56a231"}, + {file = "regex-2024.9.11-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5b513b6997a0b2f10e4fd3a1313568e373926e8c252bd76c960f96fd039cd28d"}, + {file = "regex-2024.9.11-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee439691d8c23e76f9802c42a95cfeebf9d47cf4ffd06f18489122dbb0a7ad64"}, + {file = "regex-2024.9.11-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a8f877c89719d759e52783f7fe6e1c67121076b87b40542966c02de5503ace42"}, + {file = "regex-2024.9.11-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:23b30c62d0f16827f2ae9f2bb87619bc4fba2044911e2e6c2eb1af0161cdb766"}, + {file = "regex-2024.9.11-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85ab7824093d8f10d44330fe1e6493f756f252d145323dd17ab6b48733ff6c0a"}, + {file = "regex-2024.9.11-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8dee5b4810a89447151999428fe096977346cf2f29f4d5e29609d2e19e0199c9"}, + {file = "regex-2024.9.11-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:98eeee2f2e63edae2181c886d7911ce502e1292794f4c5ee71e60e23e8d26b5d"}, + {file = "regex-2024.9.11-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:57fdd2e0b2694ce6fc2e5ccf189789c3e2962916fb38779d3e3521ff8fe7a822"}, + {file = "regex-2024.9.11-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:d552c78411f60b1fdaafd117a1fca2f02e562e309223b9d44b7de8be451ec5e0"}, + {file = "regex-2024.9.11-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a0b2b80321c2ed3fcf0385ec9e51a12253c50f146fddb2abbb10f033fe3d049a"}, + {file = "regex-2024.9.11-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:18406efb2f5a0e57e3a5881cd9354c1512d3bb4f5c45d96d110a66114d84d23a"}, + {file = "regex-2024.9.11-cp312-cp312-win32.whl", hash = "sha256:e464b467f1588e2c42d26814231edecbcfe77f5ac414d92cbf4e7b55b2c2a776"}, + {file = "regex-2024.9.11-cp312-cp312-win_amd64.whl", hash = "sha256:9e8719792ca63c6b8340380352c24dcb8cd7ec49dae36e963742a275dfae6009"}, + {file = "regex-2024.9.11-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:c157bb447303070f256e084668b702073db99bbb61d44f85d811025fcf38f784"}, + {file = "regex-2024.9.11-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4db21ece84dfeefc5d8a3863f101995de646c6cb0536952c321a2650aa202c36"}, + {file = "regex-2024.9.11-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:220e92a30b426daf23bb67a7962900ed4613589bab80382be09b48896d211e92"}, + {file = "regex-2024.9.11-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb1ae19e64c14c7ec1995f40bd932448713d3c73509e82d8cd7744dc00e29e86"}, + {file = "regex-2024.9.11-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f47cd43a5bfa48f86925fe26fbdd0a488ff15b62468abb5d2a1e092a4fb10e85"}, + {file = "regex-2024.9.11-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9d4a76b96f398697fe01117093613166e6aa8195d63f1b4ec3f21ab637632963"}, + {file = "regex-2024.9.11-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ea51dcc0835eea2ea31d66456210a4e01a076d820e9039b04ae8d17ac11dee6"}, + {file = "regex-2024.9.11-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7aaa315101c6567a9a45d2839322c51c8d6e81f67683d529512f5bcfb99c802"}, + {file = "regex-2024.9.11-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c57d08ad67aba97af57a7263c2d9006d5c404d721c5f7542f077f109ec2a4a29"}, + {file = "regex-2024.9.11-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f8404bf61298bb6f8224bb9176c1424548ee1181130818fcd2cbffddc768bed8"}, + {file = "regex-2024.9.11-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:dd4490a33eb909ef5078ab20f5f000087afa2a4daa27b4c072ccb3cb3050ad84"}, + {file = "regex-2024.9.11-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:eee9130eaad130649fd73e5cd92f60e55708952260ede70da64de420cdcad554"}, + {file = "regex-2024.9.11-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6a2644a93da36c784e546de579ec1806bfd2763ef47babc1b03d765fe560c9f8"}, + {file = "regex-2024.9.11-cp313-cp313-win32.whl", hash = "sha256:e997fd30430c57138adc06bba4c7c2968fb13d101e57dd5bb9355bf8ce3fa7e8"}, + {file = "regex-2024.9.11-cp313-cp313-win_amd64.whl", hash = "sha256:042c55879cfeb21a8adacc84ea347721d3d83a159da6acdf1116859e2427c43f"}, + {file = "regex-2024.9.11-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:35f4a6f96aa6cb3f2f7247027b07b15a374f0d5b912c0001418d1d55024d5cb4"}, + {file = "regex-2024.9.11-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:55b96e7ce3a69a8449a66984c268062fbaa0d8ae437b285428e12797baefce7e"}, + {file = "regex-2024.9.11-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cb130fccd1a37ed894824b8c046321540263013da72745d755f2d35114b81a60"}, + {file = "regex-2024.9.11-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:323c1f04be6b2968944d730e5c2091c8c89767903ecaa135203eec4565ed2b2b"}, + {file = "regex-2024.9.11-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be1c8ed48c4c4065ecb19d882a0ce1afe0745dfad8ce48c49586b90a55f02366"}, + {file = "regex-2024.9.11-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b5b029322e6e7b94fff16cd120ab35a253236a5f99a79fb04fda7ae71ca20ae8"}, + {file = "regex-2024.9.11-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6fff13ef6b5f29221d6904aa816c34701462956aa72a77f1f151a8ec4f56aeb"}, + {file = "regex-2024.9.11-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:587d4af3979376652010e400accc30404e6c16b7df574048ab1f581af82065e4"}, + {file = "regex-2024.9.11-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:079400a8269544b955ffa9e31f186f01d96829110a3bf79dc338e9910f794fca"}, + {file = "regex-2024.9.11-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:f9268774428ec173654985ce55fc6caf4c6d11ade0f6f914d48ef4719eb05ebb"}, + {file = "regex-2024.9.11-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:23f9985c8784e544d53fc2930fc1ac1a7319f5d5332d228437acc9f418f2f168"}, + {file = "regex-2024.9.11-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:ae2941333154baff9838e88aa71c1d84f4438189ecc6021a12c7573728b5838e"}, + {file = "regex-2024.9.11-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:e93f1c331ca8e86fe877a48ad64e77882c0c4da0097f2212873a69bbfea95d0c"}, + {file = "regex-2024.9.11-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:846bc79ee753acf93aef4184c040d709940c9d001029ceb7b7a52747b80ed2dd"}, + {file = "regex-2024.9.11-cp38-cp38-win32.whl", hash = "sha256:c94bb0a9f1db10a1d16c00880bdebd5f9faf267273b8f5bd1878126e0fbde771"}, + {file = "regex-2024.9.11-cp38-cp38-win_amd64.whl", hash = "sha256:2b08fce89fbd45664d3df6ad93e554b6c16933ffa9d55cb7e01182baaf971508"}, + {file = "regex-2024.9.11-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:07f45f287469039ffc2c53caf6803cd506eb5f5f637f1d4acb37a738f71dd066"}, + {file = "regex-2024.9.11-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4838e24ee015101d9f901988001038f7f0d90dc0c3b115541a1365fb439add62"}, + {file = "regex-2024.9.11-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6edd623bae6a737f10ce853ea076f56f507fd7726bee96a41ee3d68d347e4d16"}, + {file = "regex-2024.9.11-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c69ada171c2d0e97a4b5aa78fbb835e0ffbb6b13fc5da968c09811346564f0d3"}, + {file = "regex-2024.9.11-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:02087ea0a03b4af1ed6ebab2c54d7118127fee8d71b26398e8e4b05b78963199"}, + {file = "regex-2024.9.11-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:69dee6a020693d12a3cf892aba4808fe168d2a4cef368eb9bf74f5398bfd4ee8"}, + {file = "regex-2024.9.11-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:297f54910247508e6e5cae669f2bc308985c60540a4edd1c77203ef19bfa63ca"}, + {file = "regex-2024.9.11-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ecea58b43a67b1b79805f1a0255730edaf5191ecef84dbc4cc85eb30bc8b63b9"}, + {file = "regex-2024.9.11-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:eab4bb380f15e189d1313195b062a6aa908f5bd687a0ceccd47c8211e9cf0d4a"}, + {file = "regex-2024.9.11-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0cbff728659ce4bbf4c30b2a1be040faafaa9eca6ecde40aaff86f7889f4ab39"}, + {file = "regex-2024.9.11-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:54c4a097b8bc5bb0dfc83ae498061d53ad7b5762e00f4adaa23bee22b012e6ba"}, + {file = "regex-2024.9.11-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:73d6d2f64f4d894c96626a75578b0bf7d9e56dcda8c3d037a2118fdfe9b1c664"}, + {file = "regex-2024.9.11-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:e53b5fbab5d675aec9f0c501274c467c0f9a5d23696cfc94247e1fb56501ed89"}, + {file = "regex-2024.9.11-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:0ffbcf9221e04502fc35e54d1ce9567541979c3fdfb93d2c554f0ca583a19b35"}, + {file = "regex-2024.9.11-cp39-cp39-win32.whl", hash = "sha256:e4c22e1ac1f1ec1e09f72e6c44d8f2244173db7eb9629cc3a346a8d7ccc31142"}, + {file = "regex-2024.9.11-cp39-cp39-win_amd64.whl", hash = "sha256:faa3c142464efec496967359ca99696c896c591c56c53506bac1ad465f66e919"}, + {file = "regex-2024.9.11.tar.gz", hash = "sha256:6c188c307e8433bcb63dc1915022deb553b4203a70722fc542c363bf120a01fd"}, ] [[package]] @@ -1602,20 +2020,93 @@ requests = ">=2.22,<3" [package.extras] fixture = ["fixtures"] +[[package]] +name = "requests-toolbelt" +version = "1.0.0" +description = "A utility belt for advanced users of python-requests" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6"}, + {file = "requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06"}, +] + +[package.dependencies] +requests = ">=2.0.1,<3.0.0" + +[[package]] +name = "serpyco-rs" +version = "1.11.0" +description = "" +optional = false +python-versions = ">=3.9" +files = [ + {file = "serpyco_rs-1.11.0-cp310-cp310-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:4b2bd933539bd8c84315e2fb5ae52ef7a58ace5a6dfe3f8b73f74dc71216779e"}, + {file = "serpyco_rs-1.11.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:627f957889ff73c4d2269fc7b6bba93212381befe03633e7cb5495de66ba9a33"}, + {file = "serpyco_rs-1.11.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b0933620abc01434023e0e3e22255b7e4ab9b427b5a9a5ee00834656d792377a"}, + {file = "serpyco_rs-1.11.0-cp310-cp310-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:9ce46683d92e34abb20304817fc5ac6cb141a06fc7468dedb1d8865a8a9682f6"}, + {file = "serpyco_rs-1.11.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bda437d86e8859bf91c189c1f4650899822f6d6d7b02b48f5729da904eb7bb7d"}, + {file = "serpyco_rs-1.11.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5a72bfbd282af17ebe76d122639013e802c09902543fdbbd828fb2159ec9755e"}, + {file = "serpyco_rs-1.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d4808df5384e3e8581e31a90ba7a1fa501c0837b1f174284bb8a4555b6864ea"}, + {file = "serpyco_rs-1.11.0-cp310-none-win_amd64.whl", hash = "sha256:c7b60aef4c16d68efb0d6241f05d0a434d873d98449cbb4366b0d385f0a7172b"}, + {file = "serpyco_rs-1.11.0-cp311-cp311-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:8d47ee577cf4d69b53917615cb031ad8708eb2f59fe78194b1968c13130fc2f7"}, + {file = "serpyco_rs-1.11.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6090d9a1487237cdd4e9362a823eede23249602019b917e7bd57846179286e79"}, + {file = "serpyco_rs-1.11.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7192eb3df576386fefd595ea31ae25c62522841ffec7e7aeb37a80b55bdc3213"}, + {file = "serpyco_rs-1.11.0-cp311-cp311-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:b52ef8affb7e71b9b98a7d5216d6a7ad03b04e990acb147cd9211c8b931c5487"}, + {file = "serpyco_rs-1.11.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3480e09e473560c60e74aaa789e6b4d079637371aae0a98235440111464bbba7"}, + {file = "serpyco_rs-1.11.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c92e36b0ab6fe866601c2331f7e99c809a126d21963c03d8a5c29331526deed"}, + {file = "serpyco_rs-1.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84f497361952d4566bc1f77e9e15a84a2614f593cc671fbf0a0fa80046f9c3d7"}, + {file = "serpyco_rs-1.11.0-cp311-none-win_amd64.whl", hash = "sha256:37fc1cf192bef9784fbf1f4e03cec21750b9e704bef55cc0442f71a715eee920"}, + {file = "serpyco_rs-1.11.0-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:3ea93d485f03dc8b0cfb0d477f0ad2e86e78f0461b53010656ab5b4db1b41fb0"}, + {file = "serpyco_rs-1.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7772410d15694b03f9c5500a2c47d62eed76e191bea4087ad042250346b1a38e"}, + {file = "serpyco_rs-1.11.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42118463c1679846cffd2f06f47744c9b9eb33c5d0448afd88ea19e1a81a8ddd"}, + {file = "serpyco_rs-1.11.0-cp312-cp312-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:79481a455b76cc56021dc55bb6d5bdda1b2b32bcb6a1ee711b597140d112e9b1"}, + {file = "serpyco_rs-1.11.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c8fd79051f9af9591fc03cf7d3033ff180416301f6a4fd3d1e3d92ebd2d68697"}, + {file = "serpyco_rs-1.11.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d29c8f9aeed734a3b51f7349d04ec9063516ffa4e10b632d75e9b1309e4930e4"}, + {file = "serpyco_rs-1.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15609158b0d9591ffa118302cd9d0039970cb3faf91dce32975f7d276e7411d5"}, + {file = "serpyco_rs-1.11.0-cp312-none-win_amd64.whl", hash = "sha256:00081eae77fbf4c5d88371c5586317ab02ccb293a330b460869a283edf2b7b69"}, + {file = "serpyco_rs-1.11.0-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:3028893366a1985adcedb13fa8f6f98c087c185efc427f94c2ccdafa40f45832"}, + {file = "serpyco_rs-1.11.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c18bf511316f3abf648a68ee62ef88617bec57d3fcde69466b4361102715ae5"}, + {file = "serpyco_rs-1.11.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7dde9ef09cdfaf7c62378186b9e29f54ec76114be4c347be6a06dd559c5681e"}, + {file = "serpyco_rs-1.11.0-cp313-cp313-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:18500ebc5e75285841e35585a238629a990b709e14f68933233640d15ca17d5f"}, + {file = "serpyco_rs-1.11.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f47c23132d4e03982703a7630aa09877b41e499722142f76b6153f6619b612f3"}, + {file = "serpyco_rs-1.11.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5f8e6ba499f6a0825bee0d8f8764569d367af871b563fc6512c171474e8e5383"}, + {file = "serpyco_rs-1.11.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15438a076047c34cff6601a977df54948e8d39d1a86f89d05c48bc60f4c12a61"}, + {file = "serpyco_rs-1.11.0-cp313-none-win_amd64.whl", hash = "sha256:84ee2c109415bd81904fc9abb9aec86a5dd13166808c21142cf23ec639f683bd"}, + {file = "serpyco_rs-1.11.0-cp39-cp39-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:5c97c16c865261577fac4effeccc7ef5e0a1e8e35e7a3ee6c90c77c3a4cd7ff9"}, + {file = "serpyco_rs-1.11.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47825e70f86fd6ef7c4a835dea3d6e8eef4fee354ed7b39ced99f31aba74a86e"}, + {file = "serpyco_rs-1.11.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:24d220220365110edba2f778f41ab3cf396883da0f26e1361a3ada9bd0227f73"}, + {file = "serpyco_rs-1.11.0-cp39-cp39-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:3a46f334af5a9d77acc6e1e58f355ae497900a2798929371f0545e274f6e6166"}, + {file = "serpyco_rs-1.11.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29d72b748acce4b4e3c7c9724e1eb33d033a1c26b08a698b393e0288060e0901"}, + {file = "serpyco_rs-1.11.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e2b8b6f205e8cc038d4d30dd0e70eece7bbecc816eb2f3787c330dc2218e232d"}, + {file = "serpyco_rs-1.11.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:038d748bfff31f150f0c3edab2766b8843edb952cb1bd3bf547886beb0912dae"}, + {file = "serpyco_rs-1.11.0-cp39-none-win_amd64.whl", hash = "sha256:0fee1c89ec2cb013dc232e4ebef88e2844357ce8631063b56639dbfb83762f20"}, + {file = "serpyco_rs-1.11.0.tar.gz", hash = "sha256:70a844615ffb229e6e89c204b3ab7404aacaf2838911814c7d847969b8da2e3a"}, +] + +[package.dependencies] +attributes-doc = "*" +typing-extensions = "*" + [[package]] name = "setuptools" -version = "70.3.0" +version = "75.1.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-70.3.0-py3-none-any.whl", hash = "sha256:fe384da74336c398e0d956d1cae0669bc02eed936cdb1d49b57de1990dc11ffc"}, - {file = "setuptools-70.3.0.tar.gz", hash = "sha256:f171bab1dfbc86b132997f26a119f6056a57950d058587841a0082e8830f9dc5"}, + {file = "setuptools-75.1.0-py3-none-any.whl", hash = "sha256:35ab7fd3bcd95e6b7fd704e4a1539513edad446c097797f2985e0e4b960772f2"}, + {file = "setuptools-75.1.0.tar.gz", hash = "sha256:d59a21b17a275fb872a9c3dae73963160ae079f1049ed956880cd7c09b120538"}, ] [package.extras] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.10.0)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.5.2)"] +core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.collections", "jaraco.functools", "jaraco.text (>=3.7)", "more-itertools", "more-itertools (>=8.8)", "packaging", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib-metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.11.*)", "pytest-mypy"] [[package]] name = "six" @@ -1628,6 +2119,17 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +[[package]] +name = "sniffio" +version = "1.3.1" +description = "Sniff out which async library your code is running under" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, + {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, +] + [[package]] name = "tenacity" version = "8.5.0" @@ -1644,16 +2146,36 @@ doc = ["reno", "sphinx"] test = ["pytest", "tornado (>=4.5)", "typeguard"] [[package]] -name = "toml" -version = "0.10.2" -description = "Python Library for Tom's Obvious, Minimal Language" +name = "tomli" +version = "2.0.2" +description = "A lil' TOML parser" optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +python-versions = ">=3.8" files = [ - {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, - {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, + {file = "tomli-2.0.2-py3-none-any.whl", hash = "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38"}, + {file = "tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed"}, ] +[[package]] +name = "tqdm" +version = "4.66.5" +description = "Fast, Extensible Progress Meter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tqdm-4.66.5-py3-none-any.whl", hash = "sha256:90279a3770753eafc9194a0364852159802111925aa30eb3f9d85b0e805ac7cd"}, + {file = "tqdm-4.66.5.tar.gz", hash = "sha256:e1020aef2e5096702d8a025ac7d16b1577279c9d63f8375b63083e9a5f0fcbad"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + [[package]] name = "typing-extensions" version = "4.12.2" @@ -1665,6 +2187,17 @@ files = [ {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] +[[package]] +name = "tzdata" +version = "2024.2" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"}, + {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"}, +] + [[package]] name = "url-normalize" version = "1.4.3" @@ -1681,13 +2214,13 @@ six = "*" [[package]] name = "urllib3" -version = "2.2.2" +version = "2.2.3" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.8" files = [ - {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, - {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, + {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"}, + {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"}, ] [package.extras] @@ -1791,101 +2324,103 @@ files = [ [[package]] name = "yarl" -version = "1.9.4" +version = "1.13.1" description = "Yet another URL library" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"}, - {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2"}, - {file = "yarl-1.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66"}, - {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234"}, - {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392"}, - {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551"}, - {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455"}, - {file = "yarl-1.9.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c"}, - {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53"}, - {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385"}, - {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863"}, - {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b"}, - {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541"}, - {file = "yarl-1.9.4-cp310-cp310-win32.whl", hash = "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d"}, - {file = "yarl-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b"}, - {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099"}, - {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c"}, - {file = "yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0"}, - {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525"}, - {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8"}, - {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9"}, - {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42"}, - {file = "yarl-1.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe"}, - {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce"}, - {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9"}, - {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572"}, - {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958"}, - {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98"}, - {file = "yarl-1.9.4-cp311-cp311-win32.whl", hash = "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31"}, - {file = "yarl-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1"}, - {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81"}, - {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142"}, - {file = "yarl-1.9.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074"}, - {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129"}, - {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2"}, - {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78"}, - {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4"}, - {file = "yarl-1.9.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0"}, - {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51"}, - {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff"}, - {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7"}, - {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc"}, - {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10"}, - {file = "yarl-1.9.4-cp312-cp312-win32.whl", hash = "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7"}, - {file = "yarl-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984"}, - {file = "yarl-1.9.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f"}, - {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17"}, - {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14"}, - {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5"}, - {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd"}, - {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7"}, - {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e"}, - {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"}, - {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c"}, - {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead"}, - {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434"}, - {file = "yarl-1.9.4-cp37-cp37m-win32.whl", hash = "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749"}, - {file = "yarl-1.9.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2"}, - {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be"}, - {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f"}, - {file = "yarl-1.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf"}, - {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1"}, - {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57"}, - {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa"}, - {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130"}, - {file = "yarl-1.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559"}, - {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23"}, - {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec"}, - {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78"}, - {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be"}, - {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3"}, - {file = "yarl-1.9.4-cp38-cp38-win32.whl", hash = "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece"}, - {file = "yarl-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b"}, - {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27"}, - {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1"}, - {file = "yarl-1.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91"}, - {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b"}, - {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5"}, - {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34"}, - {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136"}, - {file = "yarl-1.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7"}, - {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e"}, - {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4"}, - {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec"}, - {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c"}, - {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0"}, - {file = "yarl-1.9.4-cp39-cp39-win32.whl", hash = "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575"}, - {file = "yarl-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15"}, - {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"}, - {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"}, + {file = "yarl-1.13.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:82e692fb325013a18a5b73a4fed5a1edaa7c58144dc67ad9ef3d604eccd451ad"}, + {file = "yarl-1.13.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df4e82e68f43a07735ae70a2d84c0353e58e20add20ec0af611f32cd5ba43fb4"}, + {file = "yarl-1.13.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ec9dd328016d8d25702a24ee274932aebf6be9787ed1c28d021945d264235b3c"}, + {file = "yarl-1.13.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5820bd4178e6a639b3ef1db8b18500a82ceab6d8b89309e121a6859f56585b05"}, + {file = "yarl-1.13.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86c438ce920e089c8c2388c7dcc8ab30dfe13c09b8af3d306bcabb46a053d6f7"}, + {file = "yarl-1.13.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3de86547c820e4f4da4606d1c8ab5765dd633189791f15247706a2eeabc783ae"}, + {file = "yarl-1.13.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ca53632007c69ddcdefe1e8cbc3920dd88825e618153795b57e6ebcc92e752a"}, + {file = "yarl-1.13.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d4ee1d240b84e2f213565f0ec08caef27a0e657d4c42859809155cf3a29d1735"}, + {file = "yarl-1.13.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c49f3e379177f4477f929097f7ed4b0622a586b0aa40c07ac8c0f8e40659a1ac"}, + {file = "yarl-1.13.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5c5e32fef09ce101fe14acd0f498232b5710effe13abac14cd95de9c274e689e"}, + {file = "yarl-1.13.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ab9524e45ee809a083338a749af3b53cc7efec458c3ad084361c1dbf7aaf82a2"}, + {file = "yarl-1.13.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:b1481c048fe787f65e34cb06f7d6824376d5d99f1231eae4778bbe5c3831076d"}, + {file = "yarl-1.13.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:31497aefd68036d8e31bfbacef915826ca2e741dbb97a8d6c7eac66deda3b606"}, + {file = "yarl-1.13.1-cp310-cp310-win32.whl", hash = "sha256:1fa56f34b2236f5192cb5fceba7bbb09620e5337e0b6dfe2ea0ddbd19dd5b154"}, + {file = "yarl-1.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:1bbb418f46c7f7355084833051701b2301092e4611d9e392360c3ba2e3e69f88"}, + {file = "yarl-1.13.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:216a6785f296169ed52cd7dcdc2612f82c20f8c9634bf7446327f50398732a51"}, + {file = "yarl-1.13.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:40c6e73c03a6befb85b72da213638b8aaa80fe4136ec8691560cf98b11b8ae6e"}, + {file = "yarl-1.13.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2430cf996113abe5aee387d39ee19529327205cda975d2b82c0e7e96e5fdabdc"}, + {file = "yarl-1.13.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fb4134cc6e005b99fa29dbc86f1ea0a298440ab6b07c6b3ee09232a3b48f495"}, + {file = "yarl-1.13.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:309c104ecf67626c033845b860d31594a41343766a46fa58c3309c538a1e22b2"}, + {file = "yarl-1.13.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f90575e9fe3aae2c1e686393a9689c724cd00045275407f71771ae5d690ccf38"}, + {file = "yarl-1.13.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d2e1626be8712333a9f71270366f4a132f476ffbe83b689dd6dc0d114796c74"}, + {file = "yarl-1.13.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b66c87da3c6da8f8e8b648878903ca54589038a0b1e08dde2c86d9cd92d4ac9"}, + {file = "yarl-1.13.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cf1ad338620249f8dd6d4b6a91a69d1f265387df3697ad5dc996305cf6c26fb2"}, + {file = "yarl-1.13.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:9915300fe5a0aa663c01363db37e4ae8e7c15996ebe2c6cce995e7033ff6457f"}, + {file = "yarl-1.13.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:703b0f584fcf157ef87816a3c0ff868e8c9f3c370009a8b23b56255885528f10"}, + {file = "yarl-1.13.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:1d8e3ca29f643dd121f264a7c89f329f0fcb2e4461833f02de6e39fef80f89da"}, + {file = "yarl-1.13.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7055bbade838d68af73aea13f8c86588e4bcc00c2235b4b6d6edb0dbd174e246"}, + {file = "yarl-1.13.1-cp311-cp311-win32.whl", hash = "sha256:a3442c31c11088e462d44a644a454d48110f0588de830921fd201060ff19612a"}, + {file = "yarl-1.13.1-cp311-cp311-win_amd64.whl", hash = "sha256:81bad32c8f8b5897c909bf3468bf601f1b855d12f53b6af0271963ee67fff0d2"}, + {file = "yarl-1.13.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:f452cc1436151387d3d50533523291d5f77c6bc7913c116eb985304abdbd9ec9"}, + {file = "yarl-1.13.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9cec42a20eae8bebf81e9ce23fb0d0c729fc54cf00643eb251ce7c0215ad49fe"}, + {file = "yarl-1.13.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d959fe96e5c2712c1876d69af0507d98f0b0e8d81bee14cfb3f6737470205419"}, + {file = "yarl-1.13.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8c837ab90c455f3ea8e68bee143472ee87828bff19ba19776e16ff961425b57"}, + {file = "yarl-1.13.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:94a993f976cdcb2dc1b855d8b89b792893220db8862d1a619efa7451817c836b"}, + {file = "yarl-1.13.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b2442a415a5f4c55ced0fade7b72123210d579f7d950e0b5527fc598866e62c"}, + {file = "yarl-1.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3fdbf0418489525231723cdb6c79e7738b3cbacbaed2b750cb033e4ea208f220"}, + {file = "yarl-1.13.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6b7f6e699304717fdc265a7e1922561b02a93ceffdaefdc877acaf9b9f3080b8"}, + {file = "yarl-1.13.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bcd5bf4132e6a8d3eb54b8d56885f3d3a38ecd7ecae8426ecf7d9673b270de43"}, + {file = "yarl-1.13.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:2a93a4557f7fc74a38ca5a404abb443a242217b91cd0c4840b1ebedaad8919d4"}, + {file = "yarl-1.13.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:22b739f99c7e4787922903f27a892744189482125cc7b95b747f04dd5c83aa9f"}, + {file = "yarl-1.13.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:2db874dd1d22d4c2c657807562411ffdfabec38ce4c5ce48b4c654be552759dc"}, + {file = "yarl-1.13.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4feaaa4742517eaceafcbe74595ed335a494c84634d33961214b278126ec1485"}, + {file = "yarl-1.13.1-cp312-cp312-win32.whl", hash = "sha256:bbf9c2a589be7414ac4a534d54e4517d03f1cbb142c0041191b729c2fa23f320"}, + {file = "yarl-1.13.1-cp312-cp312-win_amd64.whl", hash = "sha256:d07b52c8c450f9366c34aa205754355e933922c79135125541daae6cbf31c799"}, + {file = "yarl-1.13.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:95c6737f28069153c399d875317f226bbdea939fd48a6349a3b03da6829fb550"}, + {file = "yarl-1.13.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:cd66152561632ed4b2a9192e7f8e5a1d41e28f58120b4761622e0355f0fe034c"}, + {file = "yarl-1.13.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6a2acde25be0cf9be23a8f6cbd31734536a264723fca860af3ae5e89d771cd71"}, + {file = "yarl-1.13.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a18595e6a2ee0826bf7dfdee823b6ab55c9b70e8f80f8b77c37e694288f5de1"}, + {file = "yarl-1.13.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a31d21089894942f7d9a8df166b495101b7258ff11ae0abec58e32daf8088813"}, + {file = "yarl-1.13.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:45f209fb4bbfe8630e3d2e2052535ca5b53d4ce2d2026bed4d0637b0416830da"}, + {file = "yarl-1.13.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f722f30366474a99745533cc4015b1781ee54b08de73260b2bbe13316079851"}, + {file = "yarl-1.13.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3bf60444269345d712838bb11cc4eadaf51ff1a364ae39ce87a5ca8ad3bb2c8"}, + {file = "yarl-1.13.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:942c80a832a79c3707cca46bd12ab8aa58fddb34b1626d42b05aa8f0bcefc206"}, + {file = "yarl-1.13.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:44b07e1690f010c3c01d353b5790ec73b2f59b4eae5b0000593199766b3f7a5c"}, + {file = "yarl-1.13.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:396e59b8de7e4d59ff5507fb4322d2329865b909f29a7ed7ca37e63ade7f835c"}, + {file = "yarl-1.13.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3bb83a0f12701c0b91112a11148b5217617982e1e466069d0555be9b372f2734"}, + {file = "yarl-1.13.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c92b89bffc660f1274779cb6fbb290ec1f90d6dfe14492523a0667f10170de26"}, + {file = "yarl-1.13.1-cp313-cp313-win32.whl", hash = "sha256:269c201bbc01d2cbba5b86997a1e0f73ba5e2f471cfa6e226bcaa7fd664b598d"}, + {file = "yarl-1.13.1-cp313-cp313-win_amd64.whl", hash = "sha256:1d0828e17fa701b557c6eaed5edbd9098eb62d8838344486248489ff233998b8"}, + {file = "yarl-1.13.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:8be8cdfe20787e6a5fcbd010f8066227e2bb9058331a4eccddec6c0db2bb85b2"}, + {file = "yarl-1.13.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:08d7148ff11cb8e886d86dadbfd2e466a76d5dd38c7ea8ebd9b0e07946e76e4b"}, + {file = "yarl-1.13.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4afdf84610ca44dcffe8b6c22c68f309aff96be55f5ea2fa31c0c225d6b83e23"}, + {file = "yarl-1.13.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0d12fe78dcf60efa205e9a63f395b5d343e801cf31e5e1dda0d2c1fb618073d"}, + {file = "yarl-1.13.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:298c1eecfd3257aa16c0cb0bdffb54411e3e831351cd69e6b0739be16b1bdaa8"}, + {file = "yarl-1.13.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c14c16831b565707149c742d87a6203eb5597f4329278446d5c0ae7a1a43928e"}, + {file = "yarl-1.13.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a9bacedbb99685a75ad033fd4de37129449e69808e50e08034034c0bf063f99"}, + {file = "yarl-1.13.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:658e8449b84b92a4373f99305de042b6bd0d19bf2080c093881e0516557474a5"}, + {file = "yarl-1.13.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:373f16f38721c680316a6a00ae21cc178e3a8ef43c0227f88356a24c5193abd6"}, + {file = "yarl-1.13.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:45d23c4668d4925688e2ea251b53f36a498e9ea860913ce43b52d9605d3d8177"}, + {file = "yarl-1.13.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:f7917697bcaa3bc3e83db91aa3a0e448bf5cde43c84b7fc1ae2427d2417c0224"}, + {file = "yarl-1.13.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:5989a38ba1281e43e4663931a53fbf356f78a0325251fd6af09dd03b1d676a09"}, + {file = "yarl-1.13.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:11b3ca8b42a024513adce810385fcabdd682772411d95bbbda3b9ed1a4257644"}, + {file = "yarl-1.13.1-cp38-cp38-win32.whl", hash = "sha256:dcaef817e13eafa547cdfdc5284fe77970b891f731266545aae08d6cce52161e"}, + {file = "yarl-1.13.1-cp38-cp38-win_amd64.whl", hash = "sha256:7addd26594e588503bdef03908fc207206adac5bd90b6d4bc3e3cf33a829f57d"}, + {file = "yarl-1.13.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a0ae6637b173d0c40b9c1462e12a7a2000a71a3258fa88756a34c7d38926911c"}, + {file = "yarl-1.13.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:576365c9f7469e1f6124d67b001639b77113cfd05e85ce0310f5f318fd02fe85"}, + {file = "yarl-1.13.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:78f271722423b2d4851cf1f4fa1a1c4833a128d020062721ba35e1a87154a049"}, + {file = "yarl-1.13.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d74f3c335cfe9c21ea78988e67f18eb9822f5d31f88b41aec3a1ec5ecd32da5"}, + {file = "yarl-1.13.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1891d69a6ba16e89473909665cd355d783a8a31bc84720902c5911dbb6373465"}, + {file = "yarl-1.13.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fb382fd7b4377363cc9f13ba7c819c3c78ed97c36a82f16f3f92f108c787cbbf"}, + {file = "yarl-1.13.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c8854b9f80693d20cec797d8e48a848c2fb273eb6f2587b57763ccba3f3bd4b"}, + {file = "yarl-1.13.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bbf2c3f04ff50f16404ce70f822cdc59760e5e2d7965905f0e700270feb2bbfc"}, + {file = "yarl-1.13.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fb9f59f3848edf186a76446eb8bcf4c900fe147cb756fbbd730ef43b2e67c6a7"}, + {file = "yarl-1.13.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:ef9b85fa1bc91c4db24407e7c4da93a5822a73dd4513d67b454ca7064e8dc6a3"}, + {file = "yarl-1.13.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:098b870c18f1341786f290b4d699504e18f1cd050ed179af8123fd8232513424"}, + {file = "yarl-1.13.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:8c723c91c94a3bc8033dd2696a0f53e5d5f8496186013167bddc3fb5d9df46a3"}, + {file = "yarl-1.13.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:44a4c40a6f84e4d5955b63462a0e2a988f8982fba245cf885ce3be7618f6aa7d"}, + {file = "yarl-1.13.1-cp39-cp39-win32.whl", hash = "sha256:84bbcdcf393139f0abc9f642bf03f00cac31010f3034faa03224a9ef0bb74323"}, + {file = "yarl-1.13.1-cp39-cp39-win_amd64.whl", hash = "sha256:fc2931ac9ce9c61c9968989ec831d3a5e6fcaaff9474e7cfa8de80b7aff5a093"}, + {file = "yarl-1.13.1-py3-none-any.whl", hash = "sha256:6a5185ad722ab4dd52d5fb1f30dcc73282eb1ed494906a92d1a228d3f89607b0"}, + {file = "yarl-1.13.1.tar.gz", hash = "sha256:ec8cfe2295f3e5e44c51f57272afbd69414ae629ec7c6b27f5a410efc78b70a0"}, ] [package.dependencies] @@ -1894,5 +2429,5 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" -python-versions = "^3.9,<3.12" -content-hash = "01051dcaec1e59d3d3ba4d15da3361f55856298595801ed584814fa158d604f4" +python-versions = "^3.10,<3.12" +content-hash = "63a8c6eb542461130d3832d496ecf41dab9564677d0a2017229b8822e013dfbf" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml b/airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml index 2c2d02cdd729..09bd949227db 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml +++ b/airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "3.3.15" +version = "3.3.16" name = "source-facebook-marketing" description = "Source implementation for Facebook Marketing." authors = [ "Airbyte ",] @@ -16,8 +16,8 @@ repository = "https://github.com/airbytehq/airbyte" include = "source_facebook_marketing" [tool.poetry.dependencies] -python = "^3.9,<3.12" -airbyte-cdk = "^3.5.0" +python = "^3.10,<3.12" +airbyte-cdk = "^5" facebook-business = "19.0.0" cached-property = "==1.5.2" @@ -27,5 +27,5 @@ source-facebook-marketing = "source_facebook_marketing.run:run" [tool.poetry.group.dev.dependencies] pytest-mock = "^3.6" freezegun = "^1.4.0" -pytest = "^6.1" +pytest = "^7" requests-mock = "^1.9.3" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py index 67d54a939428..da5b8dc1c285 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py @@ -6,7 +6,7 @@ import logging from typing import Any, List, Mapping -from airbyte_cdk.config_observation import create_connector_config_control_message +from airbyte_cdk import emit_configuration_as_airbyte_control_message from airbyte_cdk.entrypoint import AirbyteEntrypoint from airbyte_cdk.sources import Source from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository @@ -56,14 +56,6 @@ def modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, # return modified config return migrated_config - @classmethod - def emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: - # add the Airbyte Control Message to message repo - cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) - # emit the Airbyte Control Message from message queue to stdout - for message in cls.message_repository._message_queue: - print(message.json(exclude_unset=True)) - @classmethod def migrate(cls, args: List[str], source: Source) -> None: """ @@ -78,9 +70,7 @@ def migrate(cls, args: List[str], source: Source) -> None: config = source.read_config(config_path) # migration check if cls.should_migrate(config): - cls.emit_control_message( - cls.modify_and_save(config_path, source, config), - ) + emit_configuration_as_airbyte_control_message(cls.modify_and_save(config_path, source, config)) class MigrateIncludeDeletedToStatusFilters(MigrateAccountIdToArray): @@ -156,9 +146,7 @@ def migrate(cls, args: List[str], source: Source) -> None: config = source.read_config(config_path) # migration check if cls._should_migrate(config): - cls._emit_control_message( - cls._modify_and_save(config_path, source, config), - ) + emit_configuration_as_airbyte_control_message(cls._modify_and_save(config_path, source, config)) @classmethod def _transform(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: @@ -186,8 +174,3 @@ def _modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, source.write_config(migrated_config, config_path) # return modified config return migrated_config - - @classmethod - def _emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: - # add the Airbyte Control Message to message repo - print(create_connector_config_control_message(migrated_config).json(exclude_unset=True)) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index e27ba6a032d4..b6eb3cb30c1e 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -61,7 +61,7 @@ class SourceFacebookMarketing(AbstractSource): # Skip exceptions on missing streams - raise_exception_on_missing_stream = False + raise_exception_on_missing_stream = True def _validate_and_transform(self, config: Mapping[str, Any]): config.setdefault("action_breakdowns_allow_empty", False) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py index 363aa4169155..b33b9278eeb7 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py @@ -3,7 +3,7 @@ # import logging -from functools import cache +from functools import cache, cached_property from typing import Any, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Union import airbyte_cdk.sources.utils.casing as casing @@ -102,7 +102,7 @@ def __init__( self._next_cursor_values = self._get_start_date() self._completed_slices = {account_id: set() for account_id in self._account_ids} - @property + @cached_property def name(self) -> str: """We override stream name to let the user change it via configuration.""" name = self._new_class_name or self.__class__.__name__ @@ -189,13 +189,13 @@ def state(self) -> MutableMapping[str, Any]: if account_id in self._cursor_values and self._cursor_values[account_id]: new_state[account_id] = {self.cursor_field: self._cursor_values[account_id].isoformat()} - new_state[account_id]["slices"] = {d.isoformat() for d in self._completed_slices[account_id]} + new_state[account_id]["slices"] = sorted(list({d.isoformat() for d in self._completed_slices[account_id]})) new_state["time_increment"] = self.time_increment return new_state if self._completed_slices: for account_id in self._account_ids: - new_state[account_id]["slices"] = {d.isoformat() for d in self._completed_slices[account_id]} + new_state[account_id]["slices"] = sorted(list({d.isoformat() for d in self._completed_slices[account_id]})) new_state["time_increment"] = self.time_increment return new_state diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py index 2b726c4aadfc..ed4bc29aef34 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py @@ -23,6 +23,7 @@ from source_facebook_marketing.api import API logger = logging.getLogger("airbyte") +from airbyte_cdk.sources.streams import CheckpointMixin class FBMarketingStream(Stream, ABC): @@ -240,7 +241,7 @@ def _filter_all_statuses(self) -> MutableMapping[str, Any]: ) -class FBMarketingIncrementalStream(FBMarketingStream, ABC): +class FBMarketingIncrementalStream(FBMarketingStream, CheckpointMixin, ABC): """Base class for incremental streams""" cursor_field = "updated_time" @@ -249,8 +250,17 @@ def __init__(self, start_date: Optional[datetime], end_date: Optional[datetime], super().__init__(**kwargs) self._start_date = pendulum.instance(start_date) if start_date else None self._end_date = pendulum.instance(end_date) if end_date else None + self._state = {} - def get_updated_state( + @property + def state(self): + return self._state + + @state.setter + def state(self, value: Mapping[str, Any]): + self._state.update(**value) + + def _get_updated_state( self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any], @@ -313,6 +323,17 @@ def _state_filter(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]: ], } + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Mapping[str, Any]]: + for record in super().read_records(sync_mode, cursor_field, stream_slice, stream_state): + self.state = self._get_updated_state(self.state, record) + yield record + class FBMarketingReversedIncrementalStream(FBMarketingIncrementalStream, ABC): """The base class for streams that don't support filtering and return records sorted desc by cursor_value""" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py index 2cfbe863503a..2a290cd48cbc 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_ads_insights_action_product_id.py @@ -11,6 +11,7 @@ import freezegun import pendulum +from airbyte_cdk.models import AirbyteStateMessage, AirbyteStreamStateSerializer, StreamDescriptor, SyncMode from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import ( @@ -22,7 +23,6 @@ create_response_builder, find_template, ) -from airbyte_protocol.models import AirbyteStateMessage, StreamDescriptor, SyncMode from source_facebook_marketing.streams.async_job import Status from .config import ACCESS_TOKEN, ACCOUNT_ID, DATE_FORMAT, END_DATE, NOW, START_DATE, ConfigBuilder @@ -467,7 +467,7 @@ def test_when_read_then_state_message_produced_and_state_match_start_interval(se ) output = self._read(config().with_account_ids([account_id]).with_start_date(start_date).with_end_date(end_date)) - cursor_value_from_state_message = output.most_recent_state.stream_state.dict().get(account_id, {}).get(_CURSOR_FIELD) + cursor_value_from_state_message = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id, {}).get(_CURSOR_FIELD) assert output.most_recent_state.stream_descriptor == StreamDescriptor(name=_STREAM_NAME) assert cursor_value_from_state_message == start_date.strftime(DATE_FORMAT) @@ -511,8 +511,8 @@ def test_given_multiple_account_ids_when_read_then_state_produced_by_account_id_ ) output = self._read(config().with_account_ids([account_id_1, account_id_2]).with_start_date(start_date).with_end_date(end_date)) - cursor_value_from_state_account_1 = output.most_recent_state.stream_state.dict().get(account_id_1, {}).get(_CURSOR_FIELD) - cursor_value_from_state_account_2 = output.most_recent_state.stream_state.dict().get(account_id_2, {}).get(_CURSOR_FIELD) + cursor_value_from_state_account_1 = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_1, {}).get(_CURSOR_FIELD) + cursor_value_from_state_account_2 = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_2, {}).get(_CURSOR_FIELD) expected_cursor_value = start_date.strftime(DATE_FORMAT) assert output.most_recent_state.stream_descriptor == StreamDescriptor(name=_STREAM_NAME) assert cursor_value_from_state_account_1 == expected_cursor_value diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_include_deleted.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_include_deleted.py index 8a59796832a4..cd6702add9f1 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_include_deleted.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_include_deleted.py @@ -2,6 +2,7 @@ from unittest import TestCase +from airbyte_cdk.models import AirbyteStreamStateSerializer, SyncMode from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import ( @@ -12,7 +13,6 @@ create_response_builder, find_template, ) -from airbyte_protocol.models import SyncMode from .config import ACCOUNT_ID, ConfigBuilder from .request_builder import get_account_request, get_ad_sets_request, get_ads_request, get_campaigns_request @@ -90,7 +90,7 @@ def test_ads_stream(self, http_mocker: HttpMocker): output = self._read(config().with_ad_statuses(self.statuses), "ads") assert len(output.records) == 1 - account_state = output.most_recent_state.dict()["stream_state"][self.account_id] + account_state = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state")[self.account_id] assert self.filter_statuses_flag in account_state, f"State should include `filter_statuses` flag to track new records in the past." assert account_state == {"filter_statuses": self.statuses, "updated_time": "2023-03-21T22:41:46-0700"} @@ -140,7 +140,7 @@ def test_campaigns_stream(self, http_mocker: HttpMocker): output = self._read(config().with_campaign_statuses(self.statuses), "campaigns") assert len(output.records) == 1 - account_state = output.most_recent_state.dict()["stream_state"][self.account_id] + account_state = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state")[self.account_id] assert self.filter_statuses_flag in account_state, f"State should include `filter_statuses` flag to track new records in the past." assert account_state == {"filter_statuses": self.statuses, "updated_time": "2024-03-12T15:02:47-0700"} @@ -184,6 +184,6 @@ def test_ad_sets_stream(self, http_mocker: HttpMocker): output = self._read(config().with_ad_set_statuses(self.statuses), "ad_sets") assert len(output.records) == 1 - account_state = output.most_recent_state.dict()["stream_state"][self.account_id] + account_state = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state")[self.account_id] assert self.filter_statuses_flag in account_state, f"State should include `filter_statuses` flag to track new records in the past." assert account_state == {"filter_statuses": self.statuses, "updated_time": "2024-03-02T15:02:47-0700"} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py index ed1384d952b5..d437566f1d48 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/test_videos.py @@ -7,6 +7,7 @@ from unittest import TestCase import freezegun +from airbyte_cdk.models import AirbyteStateMessage, AirbyteStreamStateSerializer, SyncMode from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.mock_http import HttpMocker from airbyte_cdk.test.mock_http.response_builder import ( @@ -18,7 +19,6 @@ find_template, ) from airbyte_cdk.test.state_builder import StateBuilder -from airbyte_protocol.models import AirbyteStateMessage, SyncMode from .config import ACCESS_TOKEN, ACCOUNT_ID, NOW, ConfigBuilder from .pagination import NEXT_PAGE_TOKEN, FacebookMarketingPaginationStrategy @@ -244,7 +244,7 @@ def test_when_read_then_state_message_produced_and_state_match_latest_record(sel ) output = self._read(config().with_account_ids([account_id])) - cursor_value_from_state_message = output.most_recent_state.stream_state.dict().get(account_id, {}).get(_CURSOR_FIELD) + cursor_value_from_state_message = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id, {}).get(_CURSOR_FIELD) assert cursor_value_from_state_message == max_cursor_value @HttpMocker() @@ -276,8 +276,8 @@ def test_given_multiple_account_ids_when_read_then_state_produced_by_account_id_ ) output = self._read(config().with_account_ids([account_id_1, account_id_2])) - cursor_value_from_state_account_1 = output.most_recent_state.stream_state.dict().get(account_id_1, {}).get(_CURSOR_FIELD) - cursor_value_from_state_account_2 = output.most_recent_state.stream_state.dict().get(account_id_2, {}).get(_CURSOR_FIELD) + cursor_value_from_state_account_1 = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_1, {}).get(_CURSOR_FIELD) + cursor_value_from_state_account_2 = AirbyteStreamStateSerializer.dump(output.most_recent_state).get("stream_state").get(account_id_2, {}).get(_CURSOR_FIELD) assert cursor_value_from_state_account_1 == max_cursor_value_account_id_1 assert cursor_value_from_state_account_2 == max_cursor_value_account_id_2 diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/utils.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/utils.py index 994b56316ba0..2686186e1840 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/utils.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/integration/utils.py @@ -6,9 +6,9 @@ from typing import Any, Dict, List, Optional from urllib.parse import urlencode +from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.test.catalog_builder import ConfiguredAirbyteStreamBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read -from airbyte_protocol.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, SyncMode from facebook_business.api import _top_level_param_json_encode from source_facebook_marketing import SourceFacebookMarketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py index 2c4601d48e7f..9b9a0e2a1f9f 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py @@ -217,10 +217,10 @@ def test_read_records_add_account_id(self, mocker, api, some_config): { "unknown_account": { AdsInsights.cursor_field: "2010-10-03", - "slices": { + "slices": [ "2010-01-01", "2010-01-02", - }, + ], }, "time_increment": 1, }, @@ -244,10 +244,10 @@ def test_read_records_add_account_id(self, mocker, api, some_config): }, { "unknown_account": { - "slices": { + "slices": [ "2010-01-01", "2010-01-02", - } + ] } }, ), @@ -256,10 +256,10 @@ def test_read_records_add_account_id(self, mocker, api, some_config): { "unknown_account": { AdsInsights.cursor_field: "2010-10-03", - "slices": { + "slices": [ "2010-01-01", "2010-01-02", - }, + ], }, "time_increment": 1, }, @@ -276,10 +276,10 @@ def test_read_records_add_account_id(self, mocker, api, some_config): ( { "unknown_account": { - "slices": { + "slices": [ "2010-01-01", "2010-01-02", - } + ] } }, None, @@ -298,14 +298,14 @@ def test_state(self, api, state, result_state, some_config): assert stream.state == { "time_increment": 1, - "unknown_account": {"slices": set()}, + "unknown_account": {"slices": []}, } stream.state = state actual_state = stream.state result_state = state if not result_state else result_state - result_state[some_config["account_ids"][0]]["slices"] = result_state[some_config["account_ids"][0]].get("slices", set()) + result_state[some_config["account_ids"][0]]["slices"] = result_state[some_config["account_ids"][0]].get("slices", []) result_state["time_increment"] = 1 assert actual_state == result_state diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_streams.py index dd1cfdff690c..b5ed2233fa43 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_streams.py @@ -258,5 +258,5 @@ def test_get_updated_state( # Set the instance's filter_statuses incremental_class_instance._filter_statuses = instance_filter_statuses - new_state = incremental_class_instance.get_updated_state(current_stream_state, latest_record) + new_state = incremental_class_instance._get_updated_state(current_stream_state, latest_record) assert new_state == expected_state diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py index 6ece8a3c5ee0..885560c92d3a 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py @@ -6,9 +6,8 @@ import pendulum import pytest -from airbyte_cdk.models import SyncMode +from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.utils import AirbyteTracedException -from airbyte_protocol.models import FailureType from facebook_business import FacebookAdsApi, FacebookSession from facebook_business.exceptions import FacebookRequestError from source_facebook_marketing.streams import Activities, AdAccount, AdCreatives, Campaigns, Videos diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py index 54bb48f02a1b..68229d5923a1 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py @@ -50,7 +50,7 @@ def revert_migration(config_path: str = TEST_CONFIG_PATH) -> None: config = json.dumps(config) updated_config.write(config) - def test_migrate_config(self): + def test_migrate_config(self, capsys): migration_instance = MigrateAccountIdToArray() original_config = load_config(self.TEST_CONFIG_PATH) # migrate the test_config @@ -68,15 +68,15 @@ def test_migrate_config(self): # load the old custom reports VS migrated assert [original_config["account_id"]] == test_migrated_config["account_ids"] # test CONTROL MESSAGE was emitted - control_msg = migration_instance.message_repository._message_queue[0] - assert control_msg.type == Type.CONTROL - assert control_msg.control.type == OrchestratorType.CONNECTOR_CONFIG + control_msg = json.loads(capsys.readouterr().out) + assert control_msg["type"] == Type.CONTROL.value + assert control_msg["control"]["type"] == OrchestratorType.CONNECTOR_CONFIG.value # old custom_reports are stil type(str) - assert isinstance(control_msg.control.connectorConfig.config["account_id"], str) + assert isinstance(control_msg["control"]["connectorConfig"]["config"]["account_id"], str) # new custom_reports are type(list) - assert isinstance(control_msg.control.connectorConfig.config["account_ids"], list) + assert isinstance(control_msg["control"]["connectorConfig"]["config"]["account_ids"], list) # check the migrated values - assert control_msg.control.connectorConfig.config["account_ids"] == ["01234567890"] + assert control_msg["control"]["connectorConfig"]["config"]["account_ids"] == ["01234567890"] # revert the test_config to the starting point self.revert_migration() @@ -131,7 +131,7 @@ def revert_migration(self, config_path: str) -> None: "old_config_path, new_config_path, include_deleted", [(OLD_TEST1_CONFIG_PATH, NEW_TEST1_CONFIG_PATH, False), (OLD_TEST2_CONFIG_PATH, NEW_TEST2_CONFIG_PATH, True)], ) - def test_migrate_config(self, old_config_path, new_config_path, include_deleted): + def test_migrate_config(self, old_config_path, new_config_path, include_deleted, capsys): migration_instance = MigrateIncludeDeletedToStatusFilters() # migrate the test_config migration_instance.migrate([CMD, "--config", old_config_path], SOURCE) @@ -151,9 +151,9 @@ def test_migrate_config(self, old_config_path, new_config_path, include_deleted) assert not migration_instance.should_migrate(test_migrated_config) if include_deleted: # test CONTROL MESSAGE was emitted - control_msg = migration_instance.message_repository._message_queue[0] - assert control_msg.type == Type.CONTROL - assert control_msg.control.type == OrchestratorType.CONNECTOR_CONFIG + control_msg = json.loads(capsys.readouterr().out) + assert control_msg["type"] == Type.CONTROL.value + assert control_msg["control"]["type"] == OrchestratorType.CONNECTOR_CONFIG.value # revert the test_config to the starting point self.revert_migration(old_config_path) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py index 70d765315294..774f83c1c9e5 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py @@ -7,6 +7,7 @@ from unittest.mock import call import pytest +from airbyte_cdk import AirbyteTracedException from airbyte_cdk.models import ( AirbyteConnectionStatus, AirbyteStream, @@ -206,10 +207,8 @@ def test_read_missing_stream(self, config, api, logger_mock, fb_marketing): ] ) - try: + with pytest.raises(AirbyteTracedException): list(fb_marketing.read(logger_mock, config=config, catalog=catalog)) - except KeyError as error: - pytest.fail(str(error)) def test_check_config(config_gen, requests_mock, fb_marketing): diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/utils.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/utils.py index bb97c9a1ae00..7619b4f60339 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/utils.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/utils.py @@ -5,7 +5,7 @@ from unittest import mock -from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification +from airbyte_cdk.models import ConnectorSpecification from airbyte_cdk.sources import Source from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index 068f32817d77..505b4a419081 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -269,44 +269,45 @@ This response indicates that the Facebook Graph API requires you to reduce the f | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 3.3.15 | 2024-07-15 | [42562](https://github.com/airbytehq/airbyte/pull/42562) | Add friendly messages for "reduce fields" and "start date" errors | -| 3.3.14 | 2024-07-15 | [41958](https://github.com/airbytehq/airbyte/pull/41958) | Update cdk to filter invalid fields from configured catalog | -| 3.3.13 | 2024-07-13 | [41732](https://github.com/airbytehq/airbyte/pull/41732) | Update dependencies | -| 3.3.12 | 2024-07-11 | [41644](https://github.com/airbytehq/airbyte/pull/41644) | Remove discriminator with missing schemas | -| 3.3.11 | 2024-07-10 | [41039](https://github.com/airbytehq/airbyte/pull/41039) | Pick request fields from configured json schema properties if present | -| 3.3.10 | 2024-07-10 | [41458](https://github.com/airbytehq/airbyte/pull/41458) | Update dependencies | -| 3.3.9 | 2024-07-09 | [41106](https://github.com/airbytehq/airbyte/pull/41106) | Update dependencies | -| 3.3.8 | 2024-07-06 | [40934](https://github.com/airbytehq/airbyte/pull/40934) | Update dependencies | -| 3.3.7 | 2024-07-01 | [40645](https://github.com/airbytehq/airbyte/pull/40645) | Use latest `CDK` version possible | -| 3.3.6 | 2024-06-24 | [40241](https://github.com/airbytehq/airbyte/pull/40241) | Update AdsInsights fields - removed `adset_start` | -| 3.3.5 | 2024-06-26 | [40545](https://github.com/airbytehq/airbyte/pull/40545) | Fixed issue when the `STATE` is literal `None` (RFR) | -| 3.3.4 | 2024-06-25 | [40485](https://github.com/airbytehq/airbyte/pull/40485) | Update dependencies | -| 3.3.3 | 2024-06-22 | [40191](https://github.com/airbytehq/airbyte/pull/40191) | Update dependencies | -| 3.3.2 | 2024-06-06 | [39174](https://github.com/airbytehq/airbyte/pull/39174) | [autopull] Upgrade base image to v1.2.2 | -| 3.3.1 | 2024-06-15 | [39511](https://github.com/airbytehq/airbyte/pull/39511) | Fix validation of the spec `custom_insights.time_increment` field | -| 3.3.0 | 2024-06-30 | [33648](https://github.com/airbytehq/airbyte/pull/33648) | Add support to field `source_instagram_media_id` to `ad_creatives` report | -| 3.2.0 | 2024-06-05 | [37625](https://github.com/airbytehq/airbyte/pull/37625) | Source Facebook-Marketing: Add Selectable Auth | -| 3.1.0 | 2024-06-01 | [38845](https://github.com/airbytehq/airbyte/pull/38845) | Update AdsInsights fields - removed `cost_per_conversion_lead` and `conversion_lead_rate` | -| 3.0.0 | 2024-04-30 | [36608](https://github.com/airbytehq/airbyte/pull/36608) | Update `body_asset, call_to_action_asset, description_asset, image_asset, link_url_asset, title_asset, video_asset` breakdowns schema. | -| 2.1.9 | 2024-05-17 | [38301](https://github.com/airbytehq/airbyte/pull/38301) | Fix data inaccuracies when `wish_bid` is requested | -| 2.1.8 | 2024-05-07 | [37771](https://github.com/airbytehq/airbyte/pull/37771) | Handle errors without API error codes/messages | -| 2.1.7 | 2024-04-24 | [36634](https://github.com/airbytehq/airbyte/pull/36634) | Update to CDK 0.80.0 | -| 2.1.6 | 2024-04-24 | [36634](https://github.com/airbytehq/airbyte/pull/36634) | Schema descriptions | -| 2.1.5 | 2024-04-17 | [37341](https://github.com/airbytehq/airbyte/pull/37341) | Move rate limit errors to transient errors. | -| 2.1.4 | 2024-04-16 | [37367](https://github.com/airbytehq/airbyte/pull/37367) | Skip config migration when the legacy account_id field does not exist | -| 2.1.3 | 2024-04-16 | [37320](https://github.com/airbytehq/airbyte/pull/37320) | Add retry for transient error | -| 2.1.2 | 2024-03-29 | [36689](https://github.com/airbytehq/airbyte/pull/36689) | Fix key error `account_id` for custom reports. | -| 2.1.1 | 2024-03-18 | [36025](https://github.com/airbytehq/airbyte/pull/36025) | Fix start_date selection behaviour | -| 2.1.0 | 2024-03-12 | [35978](https://github.com/airbytehq/airbyte/pull/35978) | Upgrade CDK to start emitting record counts with state and full refresh state | -| 2.0.1 | 2024-03-08 | [35913](https://github.com/airbytehq/airbyte/pull/35913) | Fix lookback window | -| 2.0.0 | 2024-03-01 | [35746](https://github.com/airbytehq/airbyte/pull/35746) | Update API to `v19.0` | -| 1.4.2 | 2024-02-22 | [35539](https://github.com/airbytehq/airbyte/pull/35539) | Add missing config migration from `include_deleted` field | -| 1.4.1 | 2024-02-21 | [35467](https://github.com/airbytehq/airbyte/pull/35467) | Fix error with incorrect state transforming in the 1.4.0 version | -| 1.4.0 | 2024-02-20 | [32449](https://github.com/airbytehq/airbyte/pull/32449) | Replace "Include Deleted Campaigns, Ads, and AdSets" option in configuration with specific statuses selection per stream | -| 1.3.3 | 2024-02-15 | [35061](https://github.com/airbytehq/airbyte/pull/35061) | Add integration tests | -| 1.3.2 | 2024-02-12 | [35178](https://github.com/airbytehq/airbyte/pull/35178) | Manage dependencies with Poetry | -| 1.3.1 | 2024-02-05 | [34845](https://github.com/airbytehq/airbyte/pull/34845) | Add missing fields to schemas | -| 1.3.0 | 2024-01-09 | [33538](https://github.com/airbytehq/airbyte/pull/33538) | Updated the `Ad Account ID(s)` property to support multiple IDs | +| 3.3.16 | 2024-07-15 | [46546](https://github.com/airbytehq/airbyte/pull/46546) | Raise exception on missing stream | +| 3.3.15 | 2024-07-15 | [42562](https://github.com/airbytehq/airbyte/pull/42562) | Add friendly messages for "reduce fields" and "start date" errors | +| 3.3.14 | 2024-07-15 | [41958](https://github.com/airbytehq/airbyte/pull/41958) | Update cdk to filter invalid fields from configured catalog | +| 3.3.13 | 2024-07-13 | [41732](https://github.com/airbytehq/airbyte/pull/41732) | Update dependencies | +| 3.3.12 | 2024-07-11 | [41644](https://github.com/airbytehq/airbyte/pull/41644) | Remove discriminator with missing schemas | +| 3.3.11 | 2024-07-10 | [41039](https://github.com/airbytehq/airbyte/pull/41039) | Pick request fields from configured json schema properties if present | +| 3.3.10 | 2024-07-10 | [41458](https://github.com/airbytehq/airbyte/pull/41458) | Update dependencies | +| 3.3.9 | 2024-07-09 | [41106](https://github.com/airbytehq/airbyte/pull/41106) | Update dependencies | +| 3.3.8 | 2024-07-06 | [40934](https://github.com/airbytehq/airbyte/pull/40934) | Update dependencies | +| 3.3.7 | 2024-07-01 | [40645](https://github.com/airbytehq/airbyte/pull/40645) | Use latest `CDK` version possible | +| 3.3.6 | 2024-06-24 | [40241](https://github.com/airbytehq/airbyte/pull/40241) | Update AdsInsights fields - removed `adset_start` | +| 3.3.5 | 2024-06-26 | [40545](https://github.com/airbytehq/airbyte/pull/40545) | Fixed issue when the `STATE` is literal `None` (RFR) | +| 3.3.4 | 2024-06-25 | [40485](https://github.com/airbytehq/airbyte/pull/40485) | Update dependencies | +| 3.3.3 | 2024-06-22 | [40191](https://github.com/airbytehq/airbyte/pull/40191) | Update dependencies | +| 3.3.2 | 2024-06-06 | [39174](https://github.com/airbytehq/airbyte/pull/39174) | [autopull] Upgrade base image to v1.2.2 | +| 3.3.1 | 2024-06-15 | [39511](https://github.com/airbytehq/airbyte/pull/39511) | Fix validation of the spec `custom_insights.time_increment` field | +| 3.3.0 | 2024-06-30 | [33648](https://github.com/airbytehq/airbyte/pull/33648) | Add support to field `source_instagram_media_id` to `ad_creatives` report | +| 3.2.0 | 2024-06-05 | [37625](https://github.com/airbytehq/airbyte/pull/37625) | Source Facebook-Marketing: Add Selectable Auth | +| 3.1.0 | 2024-06-01 | [38845](https://github.com/airbytehq/airbyte/pull/38845) | Update AdsInsights fields - removed `cost_per_conversion_lead` and `conversion_lead_rate` | +| 3.0.0 | 2024-04-30 | [36608](https://github.com/airbytehq/airbyte/pull/36608) | Update `body_asset, call_to_action_asset, description_asset, image_asset, link_url_asset, title_asset, video_asset` breakdowns schema. | +| 2.1.9 | 2024-05-17 | [38301](https://github.com/airbytehq/airbyte/pull/38301) | Fix data inaccuracies when `wish_bid` is requested | +| 2.1.8 | 2024-05-07 | [37771](https://github.com/airbytehq/airbyte/pull/37771) | Handle errors without API error codes/messages | +| 2.1.7 | 2024-04-24 | [36634](https://github.com/airbytehq/airbyte/pull/36634) | Update to CDK 0.80.0 | +| 2.1.6 | 2024-04-24 | [36634](https://github.com/airbytehq/airbyte/pull/36634) | Schema descriptions | +| 2.1.5 | 2024-04-17 | [37341](https://github.com/airbytehq/airbyte/pull/37341) | Move rate limit errors to transient errors. | +| 2.1.4 | 2024-04-16 | [37367](https://github.com/airbytehq/airbyte/pull/37367) | Skip config migration when the legacy account_id field does not exist | +| 2.1.3 | 2024-04-16 | [37320](https://github.com/airbytehq/airbyte/pull/37320) | Add retry for transient error | +| 2.1.2 | 2024-03-29 | [36689](https://github.com/airbytehq/airbyte/pull/36689) | Fix key error `account_id` for custom reports. | +| 2.1.1 | 2024-03-18 | [36025](https://github.com/airbytehq/airbyte/pull/36025) | Fix start_date selection behaviour | +| 2.1.0 | 2024-03-12 | [35978](https://github.com/airbytehq/airbyte/pull/35978) | Upgrade CDK to start emitting record counts with state and full refresh state | +| 2.0.1 | 2024-03-08 | [35913](https://github.com/airbytehq/airbyte/pull/35913) | Fix lookback window | +| 2.0.0 | 2024-03-01 | [35746](https://github.com/airbytehq/airbyte/pull/35746) | Update API to `v19.0` | +| 1.4.2 | 2024-02-22 | [35539](https://github.com/airbytehq/airbyte/pull/35539) | Add missing config migration from `include_deleted` field | +| 1.4.1 | 2024-02-21 | [35467](https://github.com/airbytehq/airbyte/pull/35467) | Fix error with incorrect state transforming in the 1.4.0 version | +| 1.4.0 | 2024-02-20 | [32449](https://github.com/airbytehq/airbyte/pull/32449) | Replace "Include Deleted Campaigns, Ads, and AdSets" option in configuration with specific statuses selection per stream | +| 1.3.3 | 2024-02-15 | [35061](https://github.com/airbytehq/airbyte/pull/35061) | Add integration tests | +| 1.3.2 | 2024-02-12 | [35178](https://github.com/airbytehq/airbyte/pull/35178) | Manage dependencies with Poetry | +| 1.3.1 | 2024-02-05 | [34845](https://github.com/airbytehq/airbyte/pull/34845) | Add missing fields to schemas | +| 1.3.0 | 2024-01-09 | [33538](https://github.com/airbytehq/airbyte/pull/33538) | Updated the `Ad Account ID(s)` property to support multiple IDs | | 1.2.3 | 2024-01-04 | [33934](https://github.com/airbytehq/airbyte/pull/33828) | Make ready for airbyte-lib | | 1.2.2 | 2024-01-02 | [33828](https://github.com/airbytehq/airbyte/pull/33828) | Add insights job timeout to be an option, so a user can specify their own value | | 1.2.1 | 2023-11-22 | [32731](https://github.com/airbytehq/airbyte/pull/32731) | Removed validation that blocked personal ad accounts during `check` | From 569ed5c45a19175929cb319d3ce0e85352790e9f Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk <35109939+tolik0@users.noreply.github.com> Date: Fri, 18 Oct 2024 16:19:29 +0300 Subject: [PATCH 10/18] fix(airbyte-cdk): Fix yielding parent records in SubstreamPartitionRouter (#46918) --- .../sources/declarative/declarative_stream.py | 43 +++++++++++- .../substream_partition_router.py | 62 +++++------------ .../streams/checkpoint/checkpoint_reader.py | 11 +-- .../test_parent_state_stream.py | 59 ++++++++++++++++ .../test_substream_partition_router.py | 68 +++++++++++-------- 5 files changed, 161 insertions(+), 82 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py index d46c9e422ce1..d30d833c8d5b 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py @@ -1,18 +1,19 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +import logging from dataclasses import InitVar, dataclass, field from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.declarative.incremental import GlobalSubstreamCursor, PerPartitionCursor from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever from airbyte_cdk.sources.declarative.schema import DefaultSchemaLoader from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader -from airbyte_cdk.sources.streams.checkpoint import Cursor +from airbyte_cdk.sources.streams.checkpoint import CheckpointMode, CheckpointReader, Cursor, CursorBasedCheckpointReader from airbyte_cdk.sources.streams.core import Stream from airbyte_cdk.sources.types import Config, StreamSlice @@ -133,7 +134,7 @@ def read_records( stream_slice = StreamSlice(partition={}, cursor_slice={}) if not isinstance(stream_slice, StreamSlice): raise ValueError(f"DeclarativeStream does not support stream_slices that are not StreamSlice. Got {stream_slice}") - yield from self.retriever.read_records(self.get_json_schema(), stream_slice) + yield from self.retriever.read_records(self.get_json_schema(), stream_slice) # type: ignore # records are of the correct type def get_json_schema(self) -> Mapping[str, Any]: # type: ignore """ @@ -172,3 +173,39 @@ def get_cursor(self) -> Optional[Cursor]: if self.retriever and isinstance(self.retriever, SimpleRetriever): return self.retriever.cursor return None + + def _get_checkpoint_reader( + self, + logger: logging.Logger, + cursor_field: Optional[List[str]], + sync_mode: SyncMode, + stream_state: MutableMapping[str, Any], + ) -> CheckpointReader: + """ + This method is overridden to prevent issues with stream slice classification for incremental streams that have parent streams. + + The classification logic, when used with `itertools.tee`, creates a copy of the stream slices. When `stream_slices` is called + the second time, the parent records generated during the classification phase are lost. This occurs because `itertools.tee` + only buffers the results, meaning the logic in `simple_retriever` that observes and updates the cursor isn't executed again. + + By overriding this method, we ensure that the stream slices are processed correctly and parent records are not lost, + allowing the cursor to function as expected. + """ + mappings_or_slices = self.stream_slices( + cursor_field=cursor_field, + sync_mode=sync_mode, # todo: change this interface to no longer rely on sync_mode for behavior + stream_state=stream_state, + ) + + cursor = self.get_cursor() + checkpoint_mode = self._checkpoint_mode + + if isinstance(cursor, (GlobalSubstreamCursor, PerPartitionCursor)): + self.has_multiple_slices = True + return CursorBasedCheckpointReader( + stream_slices=mappings_or_slices, + cursor=cursor, + read_state_from_cursor=checkpoint_mode == CheckpointMode.RESUMABLE_FULL_REFRESH, + ) + + return super()._get_checkpoint_reader(logger, cursor_field, sync_mode, stream_state) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py index e3db1f369d86..9eac1f6bb66e 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/partition_routers/substream_partition_router.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import copy import logging from dataclasses import InitVar, dataclass from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Mapping, Optional, Union @@ -145,14 +146,10 @@ def stream_slices(self) -> Iterable[StreamSlice]: incremental_dependency = parent_stream_config.incremental_dependency - stream_slices_for_parent = [] - previous_associated_slice = None - # read_stateless() assumes the parent is not concurrent. This is currently okay since the concurrent CDK does # not support either substreams or RFR, but something that needs to be considered once we do for parent_record in parent_stream.read_only_records(): parent_partition = None - parent_associated_slice = None # Skip non-records (eg AirbyteLogMessage) if isinstance(parent_record, AirbyteMessage): self.logger.warning( @@ -164,7 +161,6 @@ def stream_slices(self) -> Iterable[StreamSlice]: continue elif isinstance(parent_record, Record): parent_partition = parent_record.associated_slice.partition if parent_record.associated_slice else {} - parent_associated_slice = parent_record.associated_slice parent_record = parent_record.data elif not isinstance(parent_record, Mapping): # The parent_record should only take the form of a Record, AirbyteMessage, or Mapping. Anything else is invalid @@ -172,48 +168,23 @@ def stream_slices(self) -> Iterable[StreamSlice]: try: partition_value = dpath.get(parent_record, parent_field) except KeyError: - pass - else: - if incremental_dependency: - if previous_associated_slice is None: - previous_associated_slice = parent_associated_slice - elif previous_associated_slice != parent_associated_slice: - # Update the parent state, as parent stream read all record for current slice and state - # is already updated. - # - # When the associated slice of the current record of the parent stream changes, this - # indicates the parent stream has finished processing the current slice and has moved onto - # the next. When this happens, we should update the partition router's current state and - # flush the previous set of collected records and start a new set - # - # Note: One tricky aspect to take note of here is that parent_stream.state will actually - # fetch state of the stream of the previous record's slice NOT the current record's slice. - # This is because in the retriever, we only update stream state after yielding all the - # records. And since we are in the middle of the current slice, parent_stream.state is - # still set to the previous state. - self._parent_state[parent_stream.name] = parent_stream.state - yield from stream_slices_for_parent - - # Reset stream_slices_for_parent after we've flushed parent records for the previous parent slice - stream_slices_for_parent = [] - previous_associated_slice = parent_associated_slice - - # Add extra fields - extracted_extra_fields = self._extract_extra_fields(parent_record, extra_fields) - - stream_slices_for_parent.append( - StreamSlice( - partition={partition_field: partition_value, "parent_slice": parent_partition or {}}, - cursor_slice={}, - extra_fields=extracted_extra_fields, - ) - ) + continue + + # Add extra fields + extracted_extra_fields = self._extract_extra_fields(parent_record, extra_fields) + + yield StreamSlice( + partition={partition_field: partition_value, "parent_slice": parent_partition or {}}, + cursor_slice={}, + extra_fields=extracted_extra_fields, + ) + + if incremental_dependency: + self._parent_state[parent_stream.name] = copy.deepcopy(parent_stream.state) # A final parent state update and yield of records is needed, so we don't skip records for the final parent slice if incremental_dependency: - self._parent_state[parent_stream.name] = parent_stream.state - - yield from stream_slices_for_parent + self._parent_state[parent_stream.name] = copy.deepcopy(parent_stream.state) def _extract_extra_fields( self, parent_record: Mapping[str, Any] | AirbyteMessage, extra_fields: Optional[List[List[str]]] = None @@ -271,6 +242,7 @@ def set_initial_state(self, stream_state: StreamState) -> None: for parent_config in self.parent_stream_configs: if parent_config.incremental_dependency: parent_config.stream.state = parent_state.get(parent_config.stream.name, {}) + self._parent_state[parent_config.stream.name] = parent_config.stream.state def get_stream_state(self) -> Optional[Mapping[str, StreamState]]: """ @@ -289,7 +261,7 @@ def get_stream_state(self) -> Optional[Mapping[str, StreamState]]: } } """ - return self._parent_state + return copy.deepcopy(self._parent_state) @property def logger(self) -> logging.Logger: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py index fcfe960ffcc7..1b6d63247206 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/checkpoint_reader.py @@ -95,6 +95,7 @@ def __init__(self, cursor: Cursor, stream_slices: Iterable[Optional[Mapping[str, self._read_state_from_cursor = read_state_from_cursor self._current_slice: Optional[StreamSlice] = None self._finished_sync = False + self._previous_state: Optional[Mapping[str, Any]] = None def next(self) -> Optional[Mapping[str, Any]]: try: @@ -110,11 +111,11 @@ def observe(self, new_state: Mapping[str, Any]) -> None: pass def get_checkpoint(self) -> Optional[Mapping[str, Any]]: - # This is used to avoid sending a duplicate state message at the end of a sync since the stream has already - # emitted state at the end of each slice. We only emit state if _current_slice is None which indicates we had no - # slices and emitted no record or are currently in the process of emitting records. - if self.current_slice is None or not self._finished_sync: - return self._cursor.get_stream_state() + # This is used to avoid sending a duplicate state messages + new_state = self._cursor.get_stream_state() + if new_state != self._previous_state: + self._previous_state = new_state + return new_state else: return None diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_parent_state_stream.py b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_parent_state_stream.py index a99909fefa0d..d7f35475a716 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_parent_state_stream.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_parent_state_stream.py @@ -335,6 +335,31 @@ def _run_read( "https://api.example.com/community/posts/3/comments/30/votes?per_page=100", {"votes": [{"id": 300, "comment_id": 30, "created_at": "2024-01-10T00:00:00Z"}]}, ), + # Requests with intermediate states + # Fetch votes for comment 10 of post 1 + ( + "https://api.example.com/community/posts/1/comments/10/votes?per_page=100&start_time=2024-01-15T00:00:00Z", + { + "votes": [{"id": 100, "comment_id": 10, "created_at": "2024-01-15T00:00:00Z"}], + }, + ), + # Fetch votes for comment 11 of post 1 + ( + "https://api.example.com/community/posts/1/comments/11/votes?per_page=100&start_time=2024-01-13T00:00:00Z", + { + "votes": [{"id": 102, "comment_id": 11, "created_at": "2024-01-13T00:00:00Z"}], + }, + ), + # Fetch votes for comment 20 of post 2 + ( + "https://api.example.com/community/posts/2/comments/20/votes?per_page=100&start_time=2024-01-12T00:00:00Z", + {"votes": [{"id": 200, "comment_id": 20, "created_at": "2024-01-12T00:00:00Z"}]}, + ), + # Fetch votes for comment 21 of post 2 + ( + "https://api.example.com/community/posts/2/comments/21/votes?per_page=100&start_time=2024-01-12T00:00:15Z", + {"votes": [{"id": 201, "comment_id": 21, "created_at": "2024-01-12T00:00:15Z"}]}, + ), ], # Expected records [ @@ -422,10 +447,44 @@ def test_incremental_parent_state(test_name, manifest, mock_requests, expected_r for url, response in mock_requests: m.get(url, json=response) + # Run the initial read output = _run_read(manifest, config, _stream_name, initial_state) output_data = [message.record.data for message in output if message.record] + # Assert that output_data equals expected_records assert output_data == expected_records + + # Collect the intermediate states and records produced before each state + cumulative_records = [] + intermediate_states = [] + for message in output: + if message.type.value == "RECORD": + record_data = message.record.data + cumulative_records.append(record_data) + elif message.type.value == "STATE": + # Record the state and the records produced before this state + state = message.state + records_before_state = cumulative_records.copy() + intermediate_states.append((state, records_before_state)) + + # For each intermediate state, perform another read starting from that state + for state, records_before_state in intermediate_states[:-1]: + output_intermediate = _run_read(manifest, config, _stream_name, [state]) + records_from_state = [message.record.data for message in output_intermediate if message.record] + + # Combine records produced before the state with records from the new read + cumulative_records_state = records_before_state + records_from_state + + # Duplicates may occur because the state matches the cursor of the last record, causing it to be re-emitted in the next sync. + cumulative_records_state_deduped = list({orjson.dumps(record): record for record in cumulative_records_state}.values()) + + # Compare the cumulative records with the expected records + expected_records_set = list({orjson.dumps(record): record for record in expected_records}.values()) + assert sorted(cumulative_records_state_deduped, key=lambda x: orjson.dumps(x)) == sorted( + expected_records_set, key=lambda x: orjson.dumps(x) + ), f"Records mismatch with intermediate state {state}. Expected {expected_records}, got {cumulative_records_state_deduped}" + + # Assert that the final state matches the expected state final_state = [orjson.loads(orjson.dumps(message.state.stream.stream_state)) for message in output if message.state] assert final_state[-1] == expected_state diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py index 82caa956e3f3..8e362048de9c 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py @@ -622,10 +622,11 @@ def test_substream_checkpoints_after_each_parent_partition(): ] expected_parent_state = [ - {"start_time": "2024-04-27", "end_time": "2024-05-27"}, - {"start_time": "2024-04-27", "end_time": "2024-05-27"}, - {"start_time": "2024-05-27", "end_time": "2024-06-27"}, - {"start_time": "2024-05-27", "end_time": "2024-06-27"}, + {}, + {"first_stream": {}}, + {"first_stream": {}}, + {"first_stream": {"start_time": "2024-04-27", "end_time": "2024-05-27"}}, + {"first_stream": {"start_time": "2024-05-27", "end_time": "2024-06-27"}}, ] partition_router = SubstreamPartitionRouter( @@ -655,8 +656,9 @@ def test_substream_checkpoints_after_each_parent_partition(): expected_counter = 0 for actual_slice in partition_router.stream_slices(): assert actual_slice == expected_slices[expected_counter] - assert partition_router._parent_state["first_stream"] == expected_parent_state[expected_counter] + assert partition_router._parent_state == expected_parent_state[expected_counter] expected_counter += 1 + assert partition_router._parent_state == expected_parent_state[expected_counter] @pytest.mark.parametrize( @@ -683,12 +685,13 @@ def test_substream_using_resumable_full_refresh_parent_stream(use_incremental_de ] expected_parent_state = [ - {"next_page_token": 2}, - {"next_page_token": 2}, - {"next_page_token": 3}, - {"next_page_token": 3}, - {"__ab_full_refresh_sync_complete": True}, - {"__ab_full_refresh_sync_complete": True}, + {}, + {"persona_3_characters": {}}, + {"persona_3_characters": {}}, + {"persona_3_characters": {"next_page_token": 2}}, + {"persona_3_characters": {"next_page_token": 2}}, + {"persona_3_characters": {"next_page_token": 3}}, + {"persona_3_characters": {"__ab_full_refresh_sync_complete": True}}, ] partition_router = SubstreamPartitionRouter( @@ -728,8 +731,10 @@ def test_substream_using_resumable_full_refresh_parent_stream(use_incremental_de for actual_slice in partition_router.stream_slices(): assert actual_slice == expected_slices[expected_counter] if use_incremental_dependency: - assert partition_router._parent_state["persona_3_characters"] == expected_parent_state[expected_counter] + assert partition_router._parent_state == expected_parent_state[expected_counter] expected_counter += 1 + if use_incremental_dependency: + assert partition_router._parent_state == expected_parent_state[expected_counter] @pytest.mark.parametrize( @@ -756,12 +761,13 @@ def test_substream_using_resumable_full_refresh_parent_stream_slices(use_increme ] expected_parent_state = [ - {"next_page_token": 2}, - {"next_page_token": 2}, - {"next_page_token": 3}, - {"next_page_token": 3}, - {"__ab_full_refresh_sync_complete": True}, - {"__ab_full_refresh_sync_complete": True}, + {}, + {"persona_3_characters": {}}, + {"persona_3_characters": {}}, + {"persona_3_characters": {"next_page_token": 2}}, + {"persona_3_characters": {"next_page_token": 2}}, + {"persona_3_characters": {"next_page_token": 3}}, + {"persona_3_characters": {"__ab_full_refresh_sync_complete": True}}, ] expected_substream_state = { @@ -822,10 +828,10 @@ def test_substream_using_resumable_full_refresh_parent_stream_slices(use_increme assert actual_slice == expected_parent_slices[expected_counter] # check for parent state if use_incremental_dependency: - assert ( - substream_cursor_slicer._partition_router._parent_state["persona_3_characters"] == expected_parent_state[expected_counter] - ) + assert substream_cursor_slicer._partition_router._parent_state == expected_parent_state[expected_counter] expected_counter += 1 + if use_incremental_dependency: + assert substream_cursor_slicer._partition_router._parent_state == expected_parent_state[expected_counter] # validate final state for closed substream slices final_state = substream_cursor_slicer.get_stream_state() @@ -841,7 +847,14 @@ def test_substream_using_resumable_full_refresh_parent_stream_slices(use_increme ( [ ParentStreamConfig( - stream=MockStream([{}], [{"id": 1, "field_1": "value_1", "field_2": {"nested_field": "nested_value_1"}}, {"id": 2, "field_1": "value_2", "field_2": {"nested_field": "nested_value_2"}}], "first_stream"), + stream=MockStream( + [{}], + [ + {"id": 1, "field_1": "value_1", "field_2": {"nested_field": "nested_value_1"}}, + {"id": 2, "field_1": "value_2", "field_2": {"nested_field": "nested_value_2"}}, + ], + "first_stream", + ), parent_key="id", partition_field="first_stream_id", extra_fields=[["field_1"], ["field_2", "nested_field"]], @@ -851,7 +864,7 @@ def test_substream_using_resumable_full_refresh_parent_stream_slices(use_increme ], [ {"field_1": "value_1", "field_2.nested_field": "nested_value_1"}, - {"field_1": "value_2", "field_2.nested_field": "nested_value_2"} + {"field_1": "value_2", "field_2.nested_field": "nested_value_2"}, ], ), ( @@ -865,16 +878,13 @@ def test_substream_using_resumable_full_refresh_parent_stream_slices(use_increme config={}, ) ], - [ - {"field_1": "value_1"}, - {"field_1": "value_2"} - ], - ) + [{"field_1": "value_1"}, {"field_1": "value_2"}], + ), ], ids=[ "test_with_nested_extra_keys", "test_with_single_extra_key", - ] + ], ) def test_substream_partition_router_with_extra_keys(parent_stream_configs, expected_slices): partition_router = SubstreamPartitionRouter(parent_stream_configs=parent_stream_configs, parameters={}, config={}) From c9670af5b896c1b26a343a466af63f9e077c0305 Mon Sep 17 00:00:00 2001 From: tolik0 Date: Fri, 18 Oct 2024 13:24:42 +0000 Subject: [PATCH 11/18] =?UTF-8?q?=F0=9F=A4=96=20minor=20bump=20Python=20CD?= =?UTF-8?q?K=20to=20version=205.14.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-cdk/python/CHANGELOG.md | 3 +++ airbyte-cdk/python/pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 4c8ed93434ac..4119f5981414 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 5.14.0 +Fix yielding parent records in SubstreamPartitionRouter + ## 5.13.0 Add extra fields to StreamSlice diff --git a/airbyte-cdk/python/pyproject.toml b/airbyte-cdk/python/pyproject.toml index 1340e72af506..220088491944 100644 --- a/airbyte-cdk/python/pyproject.toml +++ b/airbyte-cdk/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "airbyte-cdk" -version = "5.13.0" +version = "5.14.0" description = "A framework for writing Airbyte Connectors." authors = ["Airbyte "] license = "MIT" From 0a33fb6761ec524723c2a598e13cee6b565a24ca Mon Sep 17 00:00:00 2001 From: tolik0 Date: Fri, 18 Oct 2024 13:30:58 +0000 Subject: [PATCH 12/18] =?UTF-8?q?=F0=9F=A4=96=20Cut=20version=205.14.0=20o?= =?UTF-8?q?f=20source-declarative-manifest?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../source-declarative-manifest/metadata.yaml | 2 +- .../connectors/source-declarative-manifest/poetry.lock | 10 +++++----- .../source-declarative-manifest/pyproject.toml | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-declarative-manifest/metadata.yaml b/airbyte-integrations/connectors/source-declarative-manifest/metadata.yaml index 233f418de115..dce6836c2071 100644 --- a/airbyte-integrations/connectors/source-declarative-manifest/metadata.yaml +++ b/airbyte-integrations/connectors/source-declarative-manifest/metadata.yaml @@ -8,7 +8,7 @@ data: connectorType: source definitionId: 64a2f99c-542f-4af8-9a6f-355f1217b436 # This version should not be updated manually - it is updated by the CDK release workflow. - dockerImageTag: 5.13.0 + dockerImageTag: 5.14.0 dockerRepository: airbyte/source-declarative-manifest # This page is hidden from the docs for now, since the connector is not in any Airbyte registries. documentationUrl: https://docs.airbyte.com/integrations/sources/low-code diff --git a/airbyte-integrations/connectors/source-declarative-manifest/poetry.lock b/airbyte-integrations/connectors/source-declarative-manifest/poetry.lock index 8c2ef89ba8cf..4cc0adc6912b 100644 --- a/airbyte-integrations/connectors/source-declarative-manifest/poetry.lock +++ b/airbyte-integrations/connectors/source-declarative-manifest/poetry.lock @@ -1,14 +1,14 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. [[package]] name = "airbyte-cdk" -version = "5.13.0" +version = "5.14.0" description = "A framework for writing Airbyte Connectors." optional = false python-versions = "<4.0,>=3.10" files = [ - {file = "airbyte_cdk-5.13.0-py3-none-any.whl", hash = "sha256:b26c99631e797c0bdb8373a6a05c81bf62b7d7397ca41b6ee05702d1013bd389"}, - {file = "airbyte_cdk-5.13.0.tar.gz", hash = "sha256:ab5799a238366dff61a8cded347b02deb63818513af4e61e2d1a51608a2280df"}, + {file = "airbyte_cdk-5.14.0-py3-none-any.whl", hash = "sha256:2791b684ffdb4ea7a877bcde5b0a461dbfa4ac6b0e619d5a973bd63dd09c0aa1"}, + {file = "airbyte_cdk-5.14.0.tar.gz", hash = "sha256:d6953dc5a69293d3d9cf36443522ad07c7d351c942a859047a7bdb0807eeaa58"}, ] [package.dependencies] @@ -1746,4 +1746,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.10,<3.12" -content-hash = "b4c5b44380eaf01b9b396b1ea4f3e7a6ce03ce0eb40393cc8d4e2868a5a7a4f8" +content-hash = "baff24abe6e7ba61b0d758887ed230ffb116199271e52672e88567fe5ee11512" diff --git a/airbyte-integrations/connectors/source-declarative-manifest/pyproject.toml b/airbyte-integrations/connectors/source-declarative-manifest/pyproject.toml index 828f4bdcca68..a56aa91cd527 100644 --- a/airbyte-integrations/connectors/source-declarative-manifest/pyproject.toml +++ b/airbyte-integrations/connectors/source-declarative-manifest/pyproject.toml @@ -3,7 +3,7 @@ requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "5.13.0" +version = "5.14.0" name = "source-declarative-manifest" description = "Base source implementation for low-code sources." authors = ["Airbyte "] @@ -17,7 +17,7 @@ include = "source_declarative_manifest" [tool.poetry.dependencies] python = "^3.10,<3.12" -airbyte-cdk = "5.13.0" +airbyte-cdk = "5.14.0" [tool.poetry.scripts] source-declarative-manifest = "source_declarative_manifest.run:run" From 2ca40255c473df70212c015f450885bbc04ccf86 Mon Sep 17 00:00:00 2001 From: letiescanciano <45267095+letiescanciano@users.noreply.github.com> Date: Fri, 18 Oct 2024 16:43:42 +0200 Subject: [PATCH 13/18] fix: pinned dompurify version (#46985) --- docusaurus/package.json | 3 +++ docusaurus/pnpm-lock.yaml | 11 +++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docusaurus/package.json b/docusaurus/package.json index 164bfdace1d6..9636f93a0330 100644 --- a/docusaurus/package.json +++ b/docusaurus/package.json @@ -136,6 +136,9 @@ "webpack-dev-server": "4.9.2", "yaml-loader": "^0.8.0" }, + "resolutions": { + "dompurify": "3.1.6" + }, "browserslist": { "production": [ ">0.5%", diff --git a/docusaurus/pnpm-lock.yaml b/docusaurus/pnpm-lock.yaml index b78fb89a1c63..a45c7ff3e8be 100644 --- a/docusaurus/pnpm-lock.yaml +++ b/docusaurus/pnpm-lock.yaml @@ -4,6 +4,9 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false +overrides: + dompurify: 3.1.6 + importers: .: @@ -2880,8 +2883,8 @@ packages: resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==} engines: {node: '>= 4'} - dompurify@3.1.7: - resolution: {integrity: sha512-VaTstWtsneJY8xzy7DekmYWEOZcmzIe3Qb3zPd4STve1OBTa+e+WmS1ITQec1fZYXI3HCsOZZiSMpG6oxoWMWQ==} + dompurify@3.1.6: + resolution: {integrity: sha512-cTOAhc36AalkjtBpfG6O8JimdTMWNXjiePT2xQH/ppBGi/4uIpmj8eKyIkMJErXWARyINV/sB38yf8JCLF5pbQ==} domutils@2.8.0: resolution: {integrity: sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==} @@ -9391,7 +9394,7 @@ snapshots: dependencies: domelementtype: 2.3.0 - dompurify@3.1.7: {} + dompurify@3.1.6: {} domutils@2.8.0: dependencies: @@ -10672,7 +10675,7 @@ snapshots: d3-sankey: 0.12.3 dagre-d3-es: 7.0.10 dayjs: 1.11.11 - dompurify: 3.1.7 + dompurify: 3.1.6 elkjs: 0.8.2 khroma: 2.1.0 lodash-es: 4.17.21 From 26f00100ceed1c4c1d608c4a6e08193d67179148 Mon Sep 17 00:00:00 2001 From: Aazam Thakur <59562284+aazam-gh@users.noreply.github.com> Date: Fri, 18 Oct 2024 22:25:10 +0530 Subject: [PATCH 14/18] source-teamtailor contribution from aazam-gh (#46895) Co-authored-by: Marcos Marx --- .../connectors/source-teamtailor/README.md | 41 + .../acceptance-test-config.yml | 17 + .../connectors/source-teamtailor/icon.svg | 1 + .../source-teamtailor/manifest.yaml | 3349 +++++++++++++++++ .../source-teamtailor/metadata.yaml | 35 + docs/integrations/sources/teamtailor.md | 50 + 6 files changed, 3493 insertions(+) create mode 100644 airbyte-integrations/connectors/source-teamtailor/README.md create mode 100644 airbyte-integrations/connectors/source-teamtailor/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-teamtailor/icon.svg create mode 100644 airbyte-integrations/connectors/source-teamtailor/manifest.yaml create mode 100644 airbyte-integrations/connectors/source-teamtailor/metadata.yaml create mode 100644 docs/integrations/sources/teamtailor.md diff --git a/airbyte-integrations/connectors/source-teamtailor/README.md b/airbyte-integrations/connectors/source-teamtailor/README.md new file mode 100644 index 000000000000..91b9713027fd --- /dev/null +++ b/airbyte-integrations/connectors/source-teamtailor/README.md @@ -0,0 +1,41 @@ +# Teamtailor +This directory contains the manifest-only connector for `source-teamtailor`. + +This is the setup for the Teamtailor source that ingests data from the teamtailor API. + +Teamtailor is a recruitment software, provding a new way to attract and hire top talent https://www.teamtailor.com/ + +In order to use this source, you must first create an account on teamtailor. + +Navigate to your organisation settings -> API Key to create the required API token. You must also specify a version number and can use today's date as X-Api-Version to always get the latest version of the API. + +Make sure to have the add-ons installed in your account for using the `nps-response` and `job-offers` endpoints. + +## Usage +There are multiple ways to use this connector: +- You can use this connector as any other connector in Airbyte Marketplace. +- You can load this connector in `pyairbyte` using `get_source`! +- You can open this connector in Connector Builder, edit it, and publish to your workspaces. + +Please refer to the manifest-only connector documentation for more details. + +## Local Development +We recommend you use the Connector Builder to edit this connector. + +But, if you want to develop this connector locally, you can use the following steps. + +### Environment Setup +You will need `airbyte-ci` installed. You can find the documentation [here](airbyte-ci). + +### Build +This will create a dev image (`source-teamtailor:dev`) that you can use to test the connector locally. +```bash +airbyte-ci connectors --name=source-teamtailor build +``` + +### Test +This will run the acceptance tests for the connector. +```bash +airbyte-ci connectors --name=source-teamtailor test +``` + diff --git a/airbyte-integrations/connectors/source-teamtailor/acceptance-test-config.yml b/airbyte-integrations/connectors/source-teamtailor/acceptance-test-config.yml new file mode 100644 index 000000000000..4c42c085af15 --- /dev/null +++ b/airbyte-integrations/connectors/source-teamtailor/acceptance-test-config.yml @@ -0,0 +1,17 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-teamtailor:dev +acceptance_tests: + spec: + tests: + - spec_path: "manifest.yaml" + connection: + bypass_reason: "This is a builder contribution, and we do not have secrets at this time" + discovery: + bypass_reason: "This is a builder contribution, and we do not have secrets at this time" + basic_read: + bypass_reason: "This is a builder contribution, and we do not have secrets at this time" + incremental: + bypass_reason: "This is a builder contribution, and we do not have secrets at this time" + full_refresh: + bypass_reason: "This is a builder contribution, and we do not have secrets at this time" diff --git a/airbyte-integrations/connectors/source-teamtailor/icon.svg b/airbyte-integrations/connectors/source-teamtailor/icon.svg new file mode 100644 index 000000000000..8a207ce59a8c --- /dev/null +++ b/airbyte-integrations/connectors/source-teamtailor/icon.svg @@ -0,0 +1 @@ + diff --git a/airbyte-integrations/connectors/source-teamtailor/manifest.yaml b/airbyte-integrations/connectors/source-teamtailor/manifest.yaml new file mode 100644 index 000000000000..231abe805abc --- /dev/null +++ b/airbyte-integrations/connectors/source-teamtailor/manifest.yaml @@ -0,0 +1,3349 @@ +version: 5.12.0 + +type: DeclarativeSource + +description: >- + This is the setup for the Teamtailor source that ingests data from the + teamtailor API. + + + Teamtailor is a recruitment software, provding a new way to attract and hire + top talent https://www.teamtailor.com/ + + + In order to use this source, you must first create an account on teamtailor. + + + Navigate to your organisation settings -> API Key to create the required API + token. You must also specify a version number and can use today's date as + X-Api-Version to always get the latest version of the API. + + + Make sure to have the add-ons installed in your account for using the + `nps-response` and `job-offers` endpoints. + +check: + type: CheckStream + stream_names: + - candidates + +definitions: + streams: + candidates: + type: DeclarativeStream + name: candidates + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: candidates + http_method: GET + request_headers: + Authorization: Token token={{ config["api"] }} + X-Api-Version: "{{ config[\"x_api_version\"] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page[number] + page_size_option: + type: RequestOption + field_name: page[size] + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 30 + start_from_page: 1 + inject_on_first_request: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/candidates" + custom-fields: + type: DeclarativeStream + name: custom-fields + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: custom-fields + http_method: GET + request_headers: + Authorization: Token token={{ config["api"] }} + X-Api-Version: "{{ config[\"x_api_version\"] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page[number] + page_size_option: + type: RequestOption + field_name: page[size] + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 30 + start_from_page: 1 + inject_on_first_request: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/custom-fields" + departments: + type: DeclarativeStream + name: departments + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: departments + http_method: GET + request_headers: + Authorization: Token token={{ config["api"] }} + X-Api-Version: "{{ config[\"x_api_version\"] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page[number] + page_size_option: + type: RequestOption + field_name: page[size] + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 30 + start_from_page: 1 + inject_on_first_request: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/departments" + jobs: + type: DeclarativeStream + name: jobs + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: jobs + http_method: GET + request_headers: + Authorization: Token token={{ config["api"] }} + X-Api-Version: "{{ config[\"x_api_version\"] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page[number] + page_size_option: + type: RequestOption + field_name: page[size] + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 30 + start_from_page: 1 + inject_on_first_request: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/jobs" + job-applications: + type: DeclarativeStream + name: job-applications + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: job-applications + http_method: GET + request_headers: + Authorization: Token token={{ config["api"] }} + X-Api-Version: "{{ config[\"x_api_version\"] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page[number] + page_size_option: + type: RequestOption + field_name: page[size] + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 30 + start_from_page: 1 + inject_on_first_request: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/job-applications" + job-offers: + type: DeclarativeStream + name: job-offers + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: job-offers + http_method: GET + request_headers: + Authorization: Token token={{ config["api"] }} + X-Api-Version: "{{ config[\"x_api_version\"] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page[number] + page_size_option: + type: RequestOption + field_name: page[size] + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 30 + start_from_page: 1 + inject_on_first_request: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/job-offers" + locations: + type: DeclarativeStream + name: locations + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: locations + http_method: GET + request_headers: + Authorization: Token token={{ config["api"] }} + X-Api-Version: "{{ config[\"x_api_version\"] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page[number] + page_size_option: + type: RequestOption + field_name: page[size] + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 30 + start_from_page: 1 + inject_on_first_request: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/locations" + users: + type: DeclarativeStream + name: users + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: users + http_method: GET + request_headers: + Authorization: Token token={{ config["api"] }} + X-Api-Version: "{{ config[\"x_api_version\"] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page[number] + page_size_option: + type: RequestOption + field_name: page[size] + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 30 + start_from_page: 1 + inject_on_first_request: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/users" + todos: + type: DeclarativeStream + name: todos + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: todos + http_method: GET + request_headers: + Authorization: Token token={{ config["api"] }} + X-Api-Version: "{{ config[\"x_api_version\"] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page[number] + page_size_option: + type: RequestOption + field_name: page[size] + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 30 + start_from_page: 1 + inject_on_first_request: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/todos" + teams: + type: DeclarativeStream + name: teams + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: teams + http_method: GET + request_headers: + Authorization: Token token={{ config["api"] }} + X-Api-Version: "{{ config[\"x_api_version\"] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page[number] + page_size_option: + type: RequestOption + field_name: page[size] + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 30 + start_from_page: 1 + inject_on_first_request: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/teams" + team_memberships: + type: DeclarativeStream + name: team_memberships + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: team-memberships + http_method: GET + request_headers: + Authorization: Token token={{ config["api"] }} + X-Api-Version: "{{ config[\"x_api_version\"] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page[number] + page_size_option: + type: RequestOption + field_name: page[size] + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 30 + start_from_page: 1 + inject_on_first_request: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/team_memberships" + stages: + type: DeclarativeStream + name: stages + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: stages + http_method: GET + request_headers: + Authorization: Token token={{ config["api"] }} + X-Api-Version: "{{ config[\"x_api_version\"] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page[number] + page_size_option: + type: RequestOption + field_name: page[size] + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 30 + start_from_page: 1 + inject_on_first_request: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/stages" + roles: + type: DeclarativeStream + name: roles + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: roles + http_method: GET + request_headers: + Authorization: Token token={{ config["api"] }} + X-Api-Version: "{{ config[\"x_api_version\"] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page[number] + page_size_option: + type: RequestOption + field_name: page[size] + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 30 + start_from_page: 1 + inject_on_first_request: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/roles" + regions: + type: DeclarativeStream + name: regions + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: regions + http_method: GET + request_headers: + Authorization: Token token={{ config["api"] }} + X-Api-Version: "{{ config[\"x_api_version\"] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page[number] + page_size_option: + type: RequestOption + field_name: page[size] + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 30 + start_from_page: 1 + inject_on_first_request: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/regions" + referrals: + type: DeclarativeStream + name: referrals + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: referrals + http_method: GET + request_headers: + Authorization: Token token={{ config["api"] }} + X-Api-Version: "{{ config[\"x_api_version\"] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page[number] + page_size_option: + type: RequestOption + field_name: page[size] + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 30 + start_from_page: 1 + inject_on_first_request: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/referrals" + questions: + type: DeclarativeStream + name: questions + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: questions + http_method: GET + request_headers: + Authorization: Token token={{ config["api"] }} + X-Api-Version: "{{ config[\"x_api_version\"] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page[number] + page_size_option: + type: RequestOption + field_name: page[size] + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 30 + start_from_page: 1 + inject_on_first_request: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/questions" + notes: + type: DeclarativeStream + name: notes + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: notes + http_method: GET + request_headers: + Authorization: Token token={{ config["api"] }} + X-Api-Version: "{{ config[\"x_api_version\"] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page[number] + page_size_option: + type: RequestOption + field_name: page[size] + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 30 + start_from_page: 1 + inject_on_first_request: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/notes" + nps_responses: + type: DeclarativeStream + name: nps_responses + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: nps-responses + http_method: GET + request_headers: + Authorization: Token token={{ config["api"] }} + X-Api-Version: "{{ config[\"x_api_version\"] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page[number] + page_size_option: + type: RequestOption + field_name: page[size] + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 30 + start_from_page: 1 + inject_on_first_request: true + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/nps_responses" + base_requester: + type: HttpRequester + url_base: https://api.teamtailor.com/v1/ + +streams: + - $ref: "#/definitions/streams/candidates" + - $ref: "#/definitions/streams/custom-fields" + - $ref: "#/definitions/streams/departments" + - $ref: "#/definitions/streams/jobs" + - $ref: "#/definitions/streams/job-applications" + - $ref: "#/definitions/streams/job-offers" + - $ref: "#/definitions/streams/locations" + - $ref: "#/definitions/streams/users" + - $ref: "#/definitions/streams/todos" + - $ref: "#/definitions/streams/teams" + - $ref: "#/definitions/streams/team_memberships" + - $ref: "#/definitions/streams/stages" + - $ref: "#/definitions/streams/roles" + - $ref: "#/definitions/streams/regions" + - $ref: "#/definitions/streams/referrals" + - $ref: "#/definitions/streams/questions" + - $ref: "#/definitions/streams/notes" + - $ref: "#/definitions/streams/nps_responses" + +spec: + type: Spec + connection_specification: + type: object + $schema: http://json-schema.org/draft-07/schema# + required: + - x_api_version + - api + properties: + x_api_version: + type: string + description: The version of the API + order: 0 + title: X-Api-Version + api: + type: string + order: 1 + title: api + additionalProperties: true + +metadata: + autoImportSchema: + candidates: true + custom-fields: true + departments: true + jobs: true + job-applications: true + job-offers: true + locations: true + users: true + todos: true + teams: true + team_memberships: true + stages: true + roles: true + regions: true + referrals: true + questions: true + notes: true + nps_responses: true + testedStreams: + candidates: + hasRecords: true + streamHash: 294171311b855ec48747b8ae5f6036470043370e + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + custom-fields: + hasRecords: true + streamHash: 30418d4c6f972a22adfd7a6f524fa533c26e39d3 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + departments: + hasRecords: true + streamHash: 33b2f7986aacf0f7604bd7fc7290cd83e85bada1 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + jobs: + hasRecords: true + streamHash: 457712f32b3510edabcf0efd38cbb5f5d4bbffa2 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + job-applications: + hasRecords: true + streamHash: 0cf9be09b4f1437eabc9dc1d1f0d07ad9018eb1f + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + job-offers: + hasRecords: true + streamHash: cc4b3c3ce824eb818f66088a97de961e19d98513 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + locations: + hasRecords: true + streamHash: 07af1d1e90e0f8c0886f998b0b5b1246a3325ca9 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + users: + hasRecords: true + streamHash: b954e57104785b82ec078292d291532a3e0f7110 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + todos: + hasRecords: true + streamHash: 3841f5c44be77d30034661426aa88565da35cfec + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + teams: + hasRecords: true + streamHash: 5a1ac77f6258a77628c746dcafeaacdb78a9a074 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + team_memberships: + hasRecords: true + streamHash: 933dce91658322430be7cb39f224daa5cb29e79e + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + stages: + hasRecords: true + streamHash: 8dbdcccf39e95d9e38379f6e7a50aed65f562fad + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + roles: + hasRecords: true + streamHash: 41705cd6d5f22c45731d83a932d81182e8f7d98f + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + regions: + hasRecords: true + streamHash: 55006304ae0acb6f3b70d8354a565acc0eb84e26 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + referrals: + hasRecords: true + streamHash: 634ff22519b0424a79fdf05a0b91806fa365153e + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + questions: + hasRecords: true + streamHash: 651fc262e6e479f921143842f4baeb145656057a + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + notes: + hasRecords: true + streamHash: 138aabee6f26e1177983be4b1af92a2b915592a5 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + nps_responses: + streamHash: c1b259faa89d6166816f864b8770804f16df3edc + hasResponse: true + responsesAreSuccessful: true + hasRecords: true + primaryKeysArePresent: true + primaryKeysAreUnique: true + assist: + docsUrl: https://docs.teamtailor.com/ + +schemas: + candidates: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + attributes: + type: + - object + - "null" + properties: + connected: + type: + - boolean + - "null" + created-at: + type: + - string + - "null" + email: + type: + - string + - "null" + first-name: + type: + - string + - "null" + internal: + type: + - boolean + - "null" + last-name: + type: + - string + - "null" + original-resume: + type: + - string + - "null" + phone: + type: + - string + - "null" + referred: + type: + - boolean + - "null" + resume: + type: + - string + - "null" + sourced: + type: + - boolean + - "null" + tags: + type: + - array + - "null" + unsubscribed: + type: + - boolean + - "null" + updated-at: + type: + - string + - "null" + id: + type: string + links: + type: + - object + - "null" + properties: + self: + type: + - string + - "null" + relationships: + type: + - object + - "null" + properties: + activities: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + answers: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + custom-field-values: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + department: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + job-applications: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + locations: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + nps-responses: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + partner-results: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + questions: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + regions: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + role: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + uploads: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + required: + - id + custom-fields: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + attributes: + type: + - object + - "null" + properties: + api-name: + type: + - string + - "null" + created-at: + type: + - string + - "null" + field-type: + type: + - string + - "null" + is-external: + type: + - boolean + - "null" + is-featured: + type: + - boolean + - "null" + is-hidden: + type: + - boolean + - "null" + is-private: + type: + - boolean + - "null" + is-required: + type: + - boolean + - "null" + is-searchable: + type: + - boolean + - "null" + name: + type: + - string + - "null" + owner-type: + type: + - string + - "null" + updated-at: + type: + - string + - "null" + id: + type: string + links: + type: + - object + - "null" + properties: + self: + type: + - string + - "null" + relationships: + type: + - object + - "null" + properties: + custom-field-values: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + required: + - id + departments: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + attributes: + type: + - object + - "null" + properties: + name: + type: + - string + - "null" + id: + type: string + links: + type: + - object + - "null" + properties: + self: + type: + - string + - "null" + relationships: + type: + - object + - "null" + properties: + manager: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + roles: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + teams: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + required: + - id + jobs: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + attributes: + type: + - object + - "null" + properties: + additional-files-requirement: + type: + - string + - "null" + body: + type: + - string + - "null" + cover-letter-requirement: + type: + - string + - "null" + created-at: + type: + - string + - "null" + human-status: + type: + - string + - "null" + internal: + type: + - boolean + - "null" + language-code: + type: + - string + - "null" + mailbox: + type: + - string + - "null" + name-requirement: + type: + - string + - "null" + phone-requirement: + type: + - string + - "null" + pinned: + type: + - boolean + - "null" + recruiter-email: + type: + - string + - "null" + remote-status: + type: + - string + - "null" + resume-requirement: + type: + - string + - "null" + sharing-image-layout: + type: + - string + - "null" + status: + type: + - string + - "null" + tags: + type: + - array + - "null" + title: + type: + - string + - "null" + updated-at: + type: + - string + - "null" + id: + type: string + links: + type: + - object + - "null" + properties: + careersite-job-apply-iframe-url: + type: + - string + - "null" + careersite-job-apply-url: + type: + - string + - "null" + careersite-job-url: + type: + - string + - "null" + self: + type: + - string + - "null" + relationships: + type: + - object + - "null" + properties: + activities: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + candidates: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + colleagues: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + custom-field-values: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + custom-fields: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + department: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + location: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + locations: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + picked-questions: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + questions: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + regions: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + requisition: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + role: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + stages: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + team: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + team-memberships: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + user: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + required: + - id + job-applications: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + attributes: + type: + - object + - "null" + properties: + changed-stage-at: + type: + - string + - "null" + created-at: + type: + - string + - "null" + row-order: + type: + - number + - "null" + sourced: + type: + - boolean + - "null" + updated-at: + type: + - string + - "null" + id: + type: string + links: + type: + - object + - "null" + properties: + self: + type: + - string + - "null" + relationships: + type: + - object + - "null" + properties: + candidate: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + job: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + nps-responses: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + reject-reason: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + stage: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + required: + - id + job-offers: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + attributes: + type: + - object + - "null" + properties: + answered-at: + type: + - string + - "null" + created-at: + type: + - string + - "null" + details: + type: + - object + - "null" + properties: + acceptance-message: + type: + - string + - "null" + rejection-message: + type: + - string + - "null" + salary: + type: + - string + - "null" + response: + type: + - string + - "null" + sent-at: + type: + - string + - "null" + status: + type: + - string + - "null" + id: + type: string + links: + type: + - object + - "null" + properties: + self: + type: + - string + - "null" + relationships: + type: + - object + - "null" + properties: + job-application: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + user: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + required: + - id + locations: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + attributes: + type: + - object + - "null" + properties: + address: + type: + - string + - "null" + city: + type: + - string + - "null" + country: + type: + - string + - "null" + email: + type: + - string + - "null" + headquarters: + type: + - boolean + - "null" + lat: + type: + - string + - "null" + long: + type: + - string + - "null" + name: + type: + - string + - "null" + phone: + type: + - string + - "null" + zip: + type: + - string + - "null" + id: + type: string + links: + type: + - object + - "null" + properties: + self: + type: + - string + - "null" + relationships: + type: + - object + - "null" + properties: + teams: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + required: + - id + users: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + attributes: + type: + - object + - "null" + properties: + login-email: + type: + - string + - "null" + role: + type: + - string + - "null" + role-addons: + type: + - array + - "null" + signature: + type: + - string + - "null" + username: + type: + - string + - "null" + id: + type: string + links: + type: + - object + - "null" + properties: + self: + type: + - string + - "null" + relationships: + type: + - object + - "null" + properties: + activities: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + department: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + jobs: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + location: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + notification-settings: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + team-memberships: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + teams: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + required: + - id + todos: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + attributes: + type: + - object + - "null" + properties: + created-at: + type: + - string + - "null" + value: + type: + - string + - "null" + id: + type: string + links: + type: + - object + - "null" + properties: + self: + type: + - string + - "null" + relationships: + type: + - object + - "null" + properties: + assignee: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + assigner: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + candidate: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + user: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + required: + - id + teams: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + attributes: + type: + - object + - "null" + properties: + emoji: + type: + - string + - "null" + name: + type: + - string + - "null" + no-department: + type: + - boolean + - "null" + id: + type: string + links: + type: + - object + - "null" + properties: + self: + type: + - string + - "null" + relationships: + type: + - object + - "null" + properties: + departments: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + jobs: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + locations: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + roles: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + user: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + users: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + required: + - id + team_memberships: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + id: + type: string + links: + type: + - object + - "null" + properties: + self: + type: + - string + - "null" + relationships: + type: + - object + - "null" + properties: + team: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + user: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + required: + - id + stages: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + attributes: + type: + - object + - "null" + properties: + active-job-applications-count: + type: + - number + - "null" + created-at: + type: + - string + - "null" + legacy-stage-type-name: + type: + - string + - "null" + name: + type: + - string + - "null" + rejected-job-applications-count: + type: + - number + - "null" + row-order: + type: + - number + - "null" + updated-at: + type: + - string + - "null" + id: + type: string + links: + type: + - object + - "null" + properties: + self: + type: + - string + - "null" + relationships: + type: + - object + - "null" + properties: + job: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + job-applications: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + stage-type: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + triggers: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + required: + - id + roles: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + attributes: + type: + - object + - "null" + properties: + name: + type: + - string + - "null" + id: + type: string + links: + type: + - object + - "null" + properties: + self: + type: + - string + - "null" + relationships: + type: + - object + - "null" + properties: + department: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + teams: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + required: + - id + regions: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + attributes: + type: + - object + - "null" + properties: + created-at: + type: + - string + - "null" + name: + type: + - string + - "null" + id: + type: string + links: + type: + - object + - "null" + properties: + self: + type: + - string + - "null" + relationships: + type: + - object + - "null" + properties: + locations: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + required: + - id + referrals: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + attributes: + type: + - object + - "null" + properties: + comment: + type: + - string + - "null" + created-at: + type: + - string + - "null" + email: + type: + - string + - "null" + name: + type: + - string + - "null" + phone: + type: + - string + - "null" + id: + type: string + links: + type: + - object + - "null" + properties: + self: + type: + - string + - "null" + relationships: + type: + - object + - "null" + properties: + candidate: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + department: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + job: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + role: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + user: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + required: + - id + questions: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + attributes: + type: + - object + - "null" + properties: + description: + type: + - string + - "null" + question-type: + type: + - string + - "null" + single-line: + type: + - boolean + - "null" + title: + type: + - string + - "null" + id: + type: string + links: + type: + - object + - "null" + properties: + self: + type: + - string + - "null" + required: + - id + notes: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + attributes: + type: + - object + - "null" + properties: + candidate-id: + type: + - number + - "null" + created-at: + type: + - string + - "null" + created-by-trigger: + type: + - boolean + - "null" + mentioned-user-ids: + type: + - array + - "null" + note: + type: + - string + - "null" + private-note: + type: + - boolean + - "null" + updated-at: + type: + - string + - "null" + user-id: + type: + - number + - "null" + id: + type: string + links: + type: + - object + - "null" + properties: + self: + type: + - string + - "null" + relationships: + type: + - object + - "null" + properties: + candidate: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + user: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + required: + - id + nps_responses: + type: object + $schema: http://json-schema.org/schema# + additionalProperties: true + properties: + type: + type: + - string + - "null" + attributes: + type: + - object + - "null" + properties: + created-at: + type: + - string + - "null" + feedback: + type: + - string + - "null" + on-reject: + type: + - boolean + - "null" + score: + type: + - number + - "null" + scored-at: + type: + - string + - "null" + sent-at: + type: + - string + - "null" + updated-at: + type: + - string + - "null" + id: + type: string + links: + type: + - object + - "null" + properties: + self: + type: + - string + - "null" + relationships: + type: + - object + - "null" + properties: + candidate: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + job-application: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + stage: + type: + - object + - "null" + properties: + links: + type: + - object + - "null" + properties: + related: + type: + - string + - "null" + self: + type: + - string + - "null" + required: + - id diff --git a/airbyte-integrations/connectors/source-teamtailor/metadata.yaml b/airbyte-integrations/connectors/source-teamtailor/metadata.yaml new file mode 100644 index 000000000000..34d2578c762d --- /dev/null +++ b/airbyte-integrations/connectors/source-teamtailor/metadata.yaml @@ -0,0 +1,35 @@ +metadataSpecVersion: "1.0" +data: + allowedHosts: + hosts: + - "api.teamtailor.com" + registryOverrides: + oss: + enabled: true + cloud: + enabled: true + remoteRegistries: + pypi: + enabled: false + packageName: airbyte-source-teamtailor + connectorBuildOptions: + baseImage: docker.io/airbyte/source-declarative-manifest:5.13.0@sha256:ffc5977f59e1f38bf3f5dd70b6fa0520c2450ebf85153c5a8df315b8c918d5c3 + connectorSubtype: api + connectorType: source + definitionId: 6d811b1b-5b94-4d5a-a74a-c2e46e5cb87c + dockerImageTag: 0.0.1 + dockerRepository: airbyte/source-teamtailor + githubIssueLabel: source-teamtailor + icon: icon.svg + license: MIT + name: Teamtailor + releaseDate: 2024-10-14 + releaseStage: alpha + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/sources/teamtailor + tags: + - language:manifest-only + - cdk:low-code + ab_internal: + ql: 100 + sl: 100 diff --git a/docs/integrations/sources/teamtailor.md b/docs/integrations/sources/teamtailor.md new file mode 100644 index 000000000000..2323c7981c0d --- /dev/null +++ b/docs/integrations/sources/teamtailor.md @@ -0,0 +1,50 @@ +# Teamtailor +This is the setup for the Teamtailor source that ingests data from the teamtailor API. + +Teamtailor is a recruitment software, provding a new way to attract and hire top talent https://www.teamtailor.com/ + +In order to use this source, you must first create an account on teamtailor. + +Navigate to your organisation Settings Page -> API Key to create the required API token. You must also specify a version number and can use today's date as X-Api-Version to always get the latest version of the API. + +Make sure to have the add-ons installed in your account for using the `nps-response` and `job-offers` endpoints. + +## Configuration + +| Input | Type | Description | Default Value | +|-------|------|-------------|---------------| +| `x_api_version` | `string` | X-Api-Version. The version of the API | | +| `api` | `string` | api. | | + +## Streams +| Stream Name | Primary Key | Pagination | Supports Full Sync | Supports Incremental | +|-------------|-------------|------------|---------------------|----------------------| +| candidates | id | DefaultPaginator | ✅ | ❌ | +| custom-fields | id | DefaultPaginator | ✅ | ❌ | +| departments | id | DefaultPaginator | ✅ | ❌ | +| jobs | id | DefaultPaginator | ✅ | ❌ | +| job-applications | id | DefaultPaginator | ✅ | ❌ | +| job-offers | id | DefaultPaginator | ✅ | ❌ | +| locations | id | DefaultPaginator | ✅ | ❌ | +| users | id | DefaultPaginator | ✅ | ❌ | +| todos | id | DefaultPaginator | ✅ | ❌ | +| teams | id | DefaultPaginator | ✅ | ❌ | +| team_memberships | id | DefaultPaginator | ✅ | ❌ | +| stages | id | DefaultPaginator | ✅ | ❌ | +| roles | id | DefaultPaginator | ✅ | ❌ | +| regions | id | DefaultPaginator | ✅ | ❌ | +| referrals | id | DefaultPaginator | ✅ | ❌ | +| questions | id | DefaultPaginator | ✅ | ❌ | +| notes | id | DefaultPaginator | ✅ | ❌ | +| nps_responses | id | DefaultPaginator | ✅ | ❌ | + +## Changelog + +
+ Expand to review + +| Version | Date | Pull Request | Subject | +|------------------|-------------------|--------------|----------------| +| 0.0.1 | 2024-10-14 | | Initial release by [@aazam-gh](https://github.com/aazam-gh) via Connector Builder | + +
From 58d01c1825f317aa03afc6ac55a73aa5c01d611a Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Fri, 18 Oct 2024 12:45:16 -0700 Subject: [PATCH 15/18] Bulk load CDK: test runner not micronaut, fix concurrent execution (#47006) --- .../cdk/load/test/util/IntegrationTest.kt | 11 +------- .../destination_process/DestinationProcess.kt | 27 +++++++++++++++++++ .../main/groovy/airbyte-bulk-connector.gradle | 16 +++++++---- 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/IntegrationTest.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/IntegrationTest.kt index 7c15388a4e8d..bb615153af3a 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/IntegrationTest.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/IntegrationTest.kt @@ -15,13 +15,11 @@ import io.airbyte.protocol.models.v0.AirbyteStreamStatusTraceMessage import io.airbyte.protocol.models.v0.AirbyteStreamStatusTraceMessage.AirbyteStreamStatus import io.airbyte.protocol.models.v0.AirbyteTraceMessage import io.airbyte.protocol.models.v0.StreamDescriptor -import io.micronaut.test.extensions.junit5.annotation.MicronautTest import java.time.Instant import java.time.LocalDateTime import java.time.ZoneOffset import java.time.format.DateTimeFormatter import java.util.concurrent.atomic.AtomicBoolean -import javax.inject.Inject import kotlin.test.fail import kotlinx.coroutines.async import kotlinx.coroutines.runBlocking @@ -36,13 +34,6 @@ import uk.org.webcompere.systemstubs.environment.EnvironmentVariables import uk.org.webcompere.systemstubs.jupiter.SystemStub import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension -@MicronautTest( - // Manually add metadata.yaml as a property source so that we can use its - // values as injectable properties. - // This is _infinitely_ easier than trying to wire up - // MetadataYamlPropertySource to be available at test time. - propertySources = ["classpath:metadata.yaml"] -) @Execution(ExecutionMode.CONCURRENT) // Spotbugs doesn't let you suppress the actual lateinit property, // so we have to suppress the entire class. @@ -70,7 +61,7 @@ abstract class IntegrationTest( // Intentionally don't inject the actual destination process - we need a full factory // because some tests want to run multiple syncs, so we need to run the destination // multiple times. - @Inject lateinit var destinationProcessFactory: DestinationProcessFactory + val destinationProcessFactory = DestinationProcessFactory.get() @Suppress("DEPRECATION") private val randomSuffix = RandomStringUtils.randomAlphabetic(4) private val timestampString = diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DestinationProcess.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DestinationProcess.kt index 42ec5c0d3218..de7960f952fe 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DestinationProcess.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DestinationProcess.kt @@ -10,6 +10,9 @@ import io.airbyte.cdk.command.FeatureFlag import io.airbyte.cdk.load.test.util.IntegrationTest import io.airbyte.protocol.models.v0.AirbyteMessage import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog +import io.micronaut.context.env.yaml.YamlPropertySourceLoader +import java.nio.file.Files +import java.nio.file.Path const val DOCKERIZED_TEST_ENV = "DOCKERIZED_INTEGRATION_TEST" @@ -56,4 +59,28 @@ abstract class DestinationProcessFactory { catalog: ConfiguredAirbyteCatalog? = null, vararg featureFlags: FeatureFlag, ): DestinationProcess + + companion object { + fun get(): DestinationProcessFactory = + when (val runner = System.getenv("AIRBYTE_CONNECTOR_INTEGRATION_TEST_RUNNER")) { + null, + "non-docker" -> NonDockerizedDestinationFactory() + "docker" -> { + val rawProperties: Map = + YamlPropertySourceLoader() + .read( + "irrelevant", + Files.readAllBytes(Path.of("metadata.yaml")), + ) + DockerizedDestinationFactory( + rawProperties["data.dockerRepository"] as String, + "dev" + ) + } + else -> + throw IllegalArgumentException( + "Unknown AIRBYTE_CONNECTOR_INTEGRATION_TEST_RUNNER environment variable: $runner" + ) + } + } } diff --git a/buildSrc/src/main/groovy/airbyte-bulk-connector.gradle b/buildSrc/src/main/groovy/airbyte-bulk-connector.gradle index 3fdb5ac4ada8..7eaf90335941 100644 --- a/buildSrc/src/main/groovy/airbyte-bulk-connector.gradle +++ b/buildSrc/src/main/groovy/airbyte-bulk-connector.gradle @@ -152,10 +152,6 @@ class AirbyteBulkConnectorPlugin implements Plugin { resources { srcDir 'src/test-integration/resources' } - resources { - srcDir '.' - include 'metadata.yaml' - } } // This source set should only be used for tests based on the old CDK's test classes, // in particular DestinationAcceptanceTest / BaseTypingDedupingTest. @@ -178,6 +174,11 @@ class AirbyteBulkConnectorPlugin implements Plugin { testClassesDirs = project.sourceSets.integrationTest.output.classesDirs classpath = project.sourceSets.integrationTest.runtimeClasspath useJUnitPlatform() + + jvmArgs = project.test.jvmArgs + systemProperties = project.test.systemProperties + maxParallelForks = project.test.maxParallelForks + maxHeapSize = project.test.maxHeapSize } // For historical reasons (i.e. airbyte-ci), this task is called integrationTestJava. @@ -193,7 +194,12 @@ class AirbyteBulkConnectorPlugin implements Plugin { useJUnitPlatform() // We need a docker image to run this task, so depend on assemble dependsOn project.tasks.assemble - environment "MICRONAUT_ENVIRONMENTS", "DOCKERIZED_INTEGRATION_TEST" + environment "AIRBYTE_CONNECTOR_INTEGRATION_TEST_RUNNER", "docker" + + jvmArgs = project.test.jvmArgs + systemProperties = project.test.systemProperties + maxParallelForks = project.test.maxParallelForks + maxHeapSize = project.test.maxHeapSize } project.dependencies { From 738655ded64105fedf6ae18003baf0fa540f06e7 Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Fri, 18 Oct 2024 14:48:27 -0700 Subject: [PATCH 16/18] Bulk load CDK: misc DestinationProcess improvements (#46922) --- .../kotlin/io/airbyte/cdk/load/test/util/IntegrationTest.kt | 5 ++--- .../test/util/destination_process/DestinationProcess.kt | 6 ++++-- .../test/util/destination_process/DockerizedDestination.kt | 6 +++--- .../util/destination_process/NonDockerizedDestination.kt | 4 ++++ 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/IntegrationTest.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/IntegrationTest.kt index bb615153af3a..cb682fabfc8e 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/IntegrationTest.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/IntegrationTest.kt @@ -21,7 +21,7 @@ import java.time.ZoneOffset import java.time.format.DateTimeFormatter import java.util.concurrent.atomic.AtomicBoolean import kotlin.test.fail -import kotlinx.coroutines.async +import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import org.apache.commons.lang3.RandomStringUtils import org.junit.jupiter.api.AfterEach @@ -141,7 +141,7 @@ abstract class IntegrationTest( catalog.asProtocolObject(), ) return runBlocking { - val destinationCompletion = async { destination.run() } + launch { destination.run() } messages.forEach { destination.sendMessage(it.asProtocolMessage()) } if (streamStatus != null) { catalog.streams.forEach { @@ -166,7 +166,6 @@ abstract class IntegrationTest( } } destination.shutdown() - destinationCompletion.await() destination.readMessages() } } diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DestinationProcess.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DestinationProcess.kt index de7960f952fe..2ec5e6de67ef 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DestinationProcess.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DestinationProcess.kt @@ -33,13 +33,15 @@ interface DestinationProcess { suspend fun run() fun sendMessage(message: AirbyteMessage) + fun sendMessages(vararg messages: AirbyteMessage) { + messages.forEach { sendMessage(it) } + } /** Return all messages the destination emitted since the last call to [readMessages]. */ fun readMessages(): List /** - * Wait for the destination to terminate, then return all messages it emitted since the last - * call to [readMessages]. + * Signal the destination to exit (i.e. close its stdin stream), then wait for it to terminate. */ suspend fun shutdown() } diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DockerizedDestination.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DockerizedDestination.kt index 35eafc396068..7e31ac39f26f 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DockerizedDestination.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/DockerizedDestination.kt @@ -199,13 +199,13 @@ class DockerizedDestination( override suspend fun shutdown() { withContext(Dispatchers.IO) { destinationStdin.close() - // The old cdk had a 1-minute timeout here. That seems... weird? - // We can just rely on the junit timeout, presumably? - process.waitFor() // Wait for ourselves to drain stdout/stderr. Otherwise we won't capture // all the destination output (logs/trace messages). stdoutDrained.join() stderrDrained.join() + // The old cdk had a 1-minute timeout here. That seems... weird? + // We can just rely on the junit timeout, presumably? + process.waitFor() val exitCode = process.exitValue() if (exitCode != 0) { throw DestinationUncleanExitException.of(exitCode, destinationOutput.traces()) diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/NonDockerizedDestination.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/NonDockerizedDestination.kt index 7908e7268f29..e4054c60259a 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/NonDockerizedDestination.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/test/util/destination_process/NonDockerizedDestination.kt @@ -17,6 +17,7 @@ import java.io.PipedInputStream import java.io.PipedOutputStream import java.io.PrintWriter import javax.inject.Singleton +import kotlinx.coroutines.CompletableDeferred class NonDockerizedDestination( command: String, @@ -26,6 +27,7 @@ class NonDockerizedDestination( ) : DestinationProcess { private val destinationStdinPipe: PrintWriter private val destination: CliRunnable + private val destinationComplete = CompletableDeferred() init { val destinationStdin = PipedInputStream() @@ -53,6 +55,7 @@ class NonDockerizedDestination( } catch (e: ConnectorUncleanExitException) { throw DestinationUncleanExitException.of(e.exitCode, destination.results.traces()) } + destinationComplete.complete(Unit) } override fun sendMessage(message: AirbyteMessage) { @@ -63,6 +66,7 @@ class NonDockerizedDestination( override suspend fun shutdown() { destinationStdinPipe.close() + destinationComplete.join() } } From 8f2c6f9e2618abe13f39d690d7399db38fea8be3 Mon Sep 17 00:00:00 2001 From: Johnny Schmidt Date: Fri, 18 Oct 2024 14:48:45 -0700 Subject: [PATCH 17/18] Bulk Load CDK: GZip Compression and S3V2 Usage (#46980) --- ...DestinationRecordToAirbyteValueWithMeta.kt | 2 + .../airbyte/cdk/load/file/StreamProcessor.kt | 26 ++++ .../toolkits/load-object-storage/build.gradle | 2 + .../ObjectStorageCompressionSpecification.kt | 86 ++++++++++++ .../ObjectStorageFormatSpecification.kt | 51 ++++++- .../object_storage/ObjectStorageClient.kt | 11 +- .../ObjectStoragePathFactory.kt | 18 ++- .../cdk/load/ObjectStorageDataDumper.kt | 62 +++++++++ .../bulk/toolkits/load-s3/build.gradle | 1 + .../load/command/s3/S3PathSpecification.kt | 9 ++ .../io/airbyte/cdk/load/file/s3/S3Client.kt | 100 +++++--------- .../cdk/load/file/s3/S3MultipartUpload.kt | 83 ++++++++++++ .../destination-s3-v2/metadata.yaml | 7 +- .../src/main/kotlin/S3V2Checker.kt | 11 +- .../src/main/kotlin/S3V2Configuration.kt | 25 ++-- .../src/main/kotlin/S3V2Writer.kt | 8 +- .../destination/s3_v2/S3V2CheckTest.kt | 6 +- .../destination/s3_v2/S3V2DataDumper.kt | 32 +++++ .../destination/s3_v2/S3V2TestUtils.kt | 7 +- .../destination/s3_v2/S3V2WriteTest.kt | 65 ++------- .../resources/expected-spec-cloud.json | 128 +++++++++++++----- .../resources/expected-spec-oss.json | 128 +++++++++++++----- 22 files changed, 646 insertions(+), 222 deletions(-) create mode 100644 airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/file/StreamProcessor.kt create mode 100644 airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/command/object_storage/ObjectStorageCompressionSpecification.kt create mode 100644 airbyte-cdk/bulk/toolkits/load-object-storage/src/testFixtures/kotlin/io/airbyte/cdk/load/ObjectStorageDataDumper.kt create mode 100644 airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3MultipartUpload.kt create mode 100644 airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2DataDumper.kt diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/DestinationRecordToAirbyteValueWithMeta.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/DestinationRecordToAirbyteValueWithMeta.kt index 0c80d9b6f1e9..84392b1503c3 100644 --- a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/DestinationRecordToAirbyteValueWithMeta.kt +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/data/DestinationRecordToAirbyteValueWithMeta.kt @@ -7,10 +7,12 @@ package io.airbyte.cdk.load.data import io.airbyte.cdk.load.command.DestinationCatalog import io.airbyte.cdk.load.message.DestinationRecord import io.airbyte.cdk.load.message.DestinationRecord.Meta +import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton import java.util.* @Singleton +@Secondary class DestinationRecordToAirbyteValueWithMeta(private val catalog: DestinationCatalog) { fun decorate(record: DestinationRecord): ObjectValue { val streamActual = catalog.getStream(record.stream.name, record.stream.namespace) diff --git a/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/file/StreamProcessor.kt b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/file/StreamProcessor.kt new file mode 100644 index 000000000000..1edc10a2c366 --- /dev/null +++ b/airbyte-cdk/bulk/core/load/src/main/kotlin/io/airbyte/cdk/load/file/StreamProcessor.kt @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.load.file + +import java.io.ByteArrayOutputStream +import java.util.zip.GZIPOutputStream + +interface StreamProcessor { + val wrapper: (ByteArrayOutputStream) -> T + val partFinisher: T.() -> Unit + val extension: String? +} + +data object NoopProcessor : StreamProcessor { + override val wrapper: (ByteArrayOutputStream) -> ByteArrayOutputStream = { it } + override val partFinisher: ByteArrayOutputStream.() -> Unit = {} + override val extension: String? = null +} + +data object GZIPProcessor : StreamProcessor { + override val wrapper: (ByteArrayOutputStream) -> GZIPOutputStream = { GZIPOutputStream(it) } + override val partFinisher: GZIPOutputStream.() -> Unit = { finish() } + override val extension: String = "gz" +} diff --git a/airbyte-cdk/bulk/toolkits/load-object-storage/build.gradle b/airbyte-cdk/bulk/toolkits/load-object-storage/build.gradle index 32b69385afe4..889fd7a5852f 100644 --- a/airbyte-cdk/bulk/toolkits/load-object-storage/build.gradle +++ b/airbyte-cdk/bulk/toolkits/load-object-storage/build.gradle @@ -1,4 +1,6 @@ dependencies { implementation project(':airbyte-cdk:bulk:core:bulk-cdk-core-base') implementation project(':airbyte-cdk:bulk:core:bulk-cdk-core-load') + + testFixturesImplementation testFixtures(project(":airbyte-cdk:bulk:core:bulk-cdk-core-load")) } diff --git a/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/command/object_storage/ObjectStorageCompressionSpecification.kt b/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/command/object_storage/ObjectStorageCompressionSpecification.kt new file mode 100644 index 000000000000..e1a5e6c64f52 --- /dev/null +++ b/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/command/object_storage/ObjectStorageCompressionSpecification.kt @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.load.command.object_storage + +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.annotation.JsonPropertyDescription +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonTypeInfo +import com.fasterxml.jackson.annotation.JsonValue +import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle +import io.airbyte.cdk.load.file.GZIPProcessor +import io.airbyte.cdk.load.file.NoopProcessor +import io.airbyte.cdk.load.file.StreamProcessor +import java.io.ByteArrayOutputStream +import java.io.OutputStream + +/** + * Mix-in to provide file format configuration. + * + * The specification is intended to be applied to file formats that are compatible with file-level + * compression (csv, jsonl) and does not need to be added to the destination spec directly. The + * [ObjectStorageCompressionConfigurationProvider] can be added to the top-level + * [io.airbyte.cdk.load.command.DestinationConfiguration] and initialized directly with + * [ObjectStorageFormatSpecificationProvider.toObjectStorageFormatConfiguration]. (See the comments + * on [io.airbyte.cdk.load.command.DestinationConfiguration] for more details.) + */ +interface ObjectStorageCompressionSpecificationProvider { + @get:JsonSchemaTitle("Compression") + @get:JsonPropertyDescription( + "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".jsonl.gz\").", + ) + @get:JsonProperty("compression") + val compression: ObjectStorageCompressionSpecification + + fun toCompressionConfiguration(): ObjectStorageCompressionConfiguration<*> { + return when (compression) { + is NoCompressionSpecification -> ObjectStorageCompressionConfiguration(NoopProcessor) + is GZIPCompressionSpecification -> ObjectStorageCompressionConfiguration(GZIPProcessor) + } + } + + companion object { + fun getNoCompressionConfiguration(): + ObjectStorageCompressionConfiguration { + return ObjectStorageCompressionConfiguration(NoopProcessor) + } + } +} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.EXISTING_PROPERTY, + property = "compression_type", +) +@JsonSubTypes( + JsonSubTypes.Type(value = NoCompressionSpecification::class, name = "No Compression"), + JsonSubTypes.Type(value = GZIPCompressionSpecification::class, name = "GZIP"), +) +sealed class ObjectStorageCompressionSpecification( + @JsonProperty("compression_type") open val compressionType: Type +) { + enum class Type(@get:JsonValue val typeName: String) { + NoCompression("No Compression"), + GZIP("GZIP"), + } +} + +@JsonSchemaTitle("No Compression") +class NoCompressionSpecification( + @JsonProperty("compression_type") override val compressionType: Type = Type.NoCompression +) : ObjectStorageCompressionSpecification(compressionType) + +@JsonSchemaTitle("GZIP") +class GZIPCompressionSpecification( + @JsonProperty("compression_type") override val compressionType: Type = Type.GZIP +) : ObjectStorageCompressionSpecification(compressionType) + +data class ObjectStorageCompressionConfiguration( + val compressor: StreamProcessor +) + +interface ObjectStorageCompressionConfigurationProvider { + val objectStorageCompressionConfiguration: ObjectStorageCompressionConfiguration +} diff --git a/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/command/object_storage/ObjectStorageFormatSpecification.kt b/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/command/object_storage/ObjectStorageFormatSpecification.kt index 9f51099e8d23..f8e57af85562 100644 --- a/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/command/object_storage/ObjectStorageFormatSpecification.kt +++ b/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/command/object_storage/ObjectStorageFormatSpecification.kt @@ -8,13 +8,24 @@ import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.annotation.JsonPropertyDescription import com.fasterxml.jackson.annotation.JsonSubTypes import com.fasterxml.jackson.annotation.JsonTypeInfo +import com.fasterxml.jackson.annotation.JsonValue import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle +/** + * Mix-in to provide file format configuration. + * + * NOTE: This assumes a fixed set of file formats. If you need to support a different set, clone the + * [ObjectStorageFormatSpecification] class with a new set of enums. + * + * See [io.airbyte.cdk.load.command.DestinationConfiguration] for more details on how to use this + * interface. + */ interface ObjectStorageFormatSpecificationProvider { @get:JsonSchemaTitle("Output Format") @get:JsonPropertyDescription( "Format of the data output. See here for more details", ) + @get:JsonProperty("format") val format: ObjectStorageFormatSpecification fun toObjectStorageFormatConfiguration(): ObjectStorageFormatConfiguration { @@ -25,6 +36,15 @@ interface ObjectStorageFormatSpecificationProvider { is ParquetFormatSpecification -> ParquetFormatConfiguration() } } + + fun toCompressionConfiguration(): ObjectStorageCompressionConfiguration<*> { + return when (format) { + is ObjectStorageCompressionSpecificationProvider -> + (format as ObjectStorageCompressionSpecificationProvider) + .toCompressionConfiguration() + else -> ObjectStorageCompressionSpecificationProvider.getNoCompressionConfiguration() + } + } } @JsonTypeInfo( @@ -38,21 +58,40 @@ interface ObjectStorageFormatSpecificationProvider { JsonSubTypes.Type(value = AvroFormatSpecification::class, name = "AVRO"), JsonSubTypes.Type(value = ParquetFormatSpecification::class, name = "PARQUET") ) -sealed class ObjectStorageFormatSpecification { - @JsonSchemaTitle("Format Type") @JsonProperty("format_type") val formatType: String = "JSONL" +sealed class ObjectStorageFormatSpecification( + @JsonSchemaTitle("Format Type") @JsonProperty("format_type") open val formatType: Type +) { + enum class Type(@get:JsonValue val typeName: String) { + JSONL("JSONL"), + CSV("CSV"), + AVRO("AVRO"), + PARQUET("PARQUET") + } } @JsonSchemaTitle("JSON Lines: Newline-delimited JSON") -class JsonFormatSpecification : ObjectStorageFormatSpecification() +class JsonFormatSpecification( + @JsonProperty("format_type") override val formatType: Type = Type.JSONL +) : ObjectStorageFormatSpecification(formatType), ObjectStorageCompressionSpecificationProvider { + override val compression: ObjectStorageCompressionSpecification = NoCompressionSpecification() +} @JsonSchemaTitle("CSV: Comma-Separated Values") -class CSVFormatSpecification : ObjectStorageFormatSpecification() +class CSVFormatSpecification( + @JsonProperty("format_type") override val formatType: Type = Type.CSV +) : ObjectStorageFormatSpecification(formatType), ObjectStorageCompressionSpecificationProvider { + override val compression: ObjectStorageCompressionSpecification = NoCompressionSpecification() +} @JsonSchemaTitle("Avro: Apache Avro") -class AvroFormatSpecification : ObjectStorageFormatSpecification() +class AvroFormatSpecification( + @JsonProperty("format_type") override val formatType: Type = Type.AVRO +) : ObjectStorageFormatSpecification(formatType) @JsonSchemaTitle("Parquet: Columnar Storage") -class ParquetFormatSpecification : ObjectStorageFormatSpecification() +class ParquetFormatSpecification( + @JsonProperty("format_type") override val formatType: Type = Type.PARQUET +) : ObjectStorageFormatSpecification(formatType) interface OutputFormatConfigurationProvider { val outputFormat: ObjectStorageFormatConfiguration diff --git a/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/file/object_storage/ObjectStorageClient.kt b/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/file/object_storage/ObjectStorageClient.kt index 7ddaa5e2b583..1d2e7faa66de 100644 --- a/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/file/object_storage/ObjectStorageClient.kt +++ b/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/file/object_storage/ObjectStorageClient.kt @@ -4,7 +4,10 @@ package io.airbyte.cdk.load.file.object_storage +import io.airbyte.cdk.load.file.NoopProcessor +import io.airbyte.cdk.load.file.StreamProcessor import java.io.InputStream +import java.io.OutputStream import kotlinx.coroutines.flow.Flow interface ObjectStorageClient, U : ObjectStorageStreamingUploadWriter> { @@ -13,7 +16,13 @@ interface ObjectStorageClient, U : ObjectStorageStreamingUpl suspend fun get(key: String, block: (InputStream) -> U): U suspend fun put(key: String, bytes: ByteArray): T suspend fun delete(remoteObject: T) - suspend fun streamingUpload(key: String, collector: suspend U.() -> Unit): T + suspend fun streamingUpload(key: String, block: suspend (U) -> Unit): T = + streamingUpload(key, NoopProcessor, block) + suspend fun streamingUpload( + key: String, + streamProcessor: StreamProcessor, + block: suspend (U) -> Unit + ): T } interface ObjectStorageStreamingUploadWriter { diff --git a/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/file/object_storage/ObjectStoragePathFactory.kt b/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/file/object_storage/ObjectStoragePathFactory.kt index 1d5d93cca17e..a40a0b8ea44e 100644 --- a/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/file/object_storage/ObjectStoragePathFactory.kt +++ b/airbyte-cdk/bulk/toolkits/load-object-storage/src/main/kotlin/io/airbyte/cdk/load/file/object_storage/ObjectStoragePathFactory.kt @@ -5,6 +5,7 @@ package io.airbyte.cdk.load.file.object_storage import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.command.object_storage.ObjectStorageCompressionConfigurationProvider import io.airbyte.cdk.load.command.object_storage.ObjectStorageFormatConfigurationProvider import io.airbyte.cdk.load.command.object_storage.ObjectStoragePathConfigurationProvider import io.airbyte.cdk.load.file.TimeProvider @@ -23,11 +24,21 @@ import java.util.* class ObjectStoragePathFactory( pathConfigProvider: ObjectStoragePathConfigurationProvider, formatConfigProvider: ObjectStorageFormatConfigurationProvider? = null, + compressionConfigProvider: ObjectStorageCompressionConfigurationProvider<*>? = null, timeProvider: TimeProvider? = null, ) { private val loadedAt = timeProvider?.let { Instant.ofEpochMilli(it.currentTimeMillis()) } private val pathConfig = pathConfigProvider.objectStoragePathConfiguration - private val defaultExtension = formatConfigProvider?.objectStorageFormatConfiguration?.extension + private val fileFormatExtension = + formatConfigProvider?.objectStorageFormatConfiguration?.extension + private val compressionExtension = + compressionConfigProvider?.objectStorageCompressionConfiguration?.compressor?.extension + private val defaultExtension = + if (fileFormatExtension != null && compressionExtension != null) { + "$fileFormatExtension.$compressionExtension" + } else { + fileFormatExtension ?: compressionExtension + } inner class VariableContext( val stream: DestinationStream, @@ -143,8 +154,9 @@ class ObjectStoragePathFactory( fun from(config: T, timeProvider: TimeProvider? = null): ObjectStoragePathFactory where T : ObjectStoragePathConfigurationProvider, - T : ObjectStorageFormatConfigurationProvider { - return ObjectStoragePathFactory(config, config, timeProvider) + T : ObjectStorageFormatConfigurationProvider, + T : ObjectStorageCompressionConfigurationProvider<*> { + return ObjectStoragePathFactory(config, config, config, timeProvider) } } diff --git a/airbyte-cdk/bulk/toolkits/load-object-storage/src/testFixtures/kotlin/io/airbyte/cdk/load/ObjectStorageDataDumper.kt b/airbyte-cdk/bulk/toolkits/load-object-storage/src/testFixtures/kotlin/io/airbyte/cdk/load/ObjectStorageDataDumper.kt new file mode 100644 index 000000000000..82396476119f --- /dev/null +++ b/airbyte-cdk/bulk/toolkits/load-object-storage/src/testFixtures/kotlin/io/airbyte/cdk/load/ObjectStorageDataDumper.kt @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.load + +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.command.object_storage.ObjectStorageCompressionConfiguration +import io.airbyte.cdk.load.data.toAirbyteValue +import io.airbyte.cdk.load.file.GZIPProcessor +import io.airbyte.cdk.load.file.NoopProcessor +import io.airbyte.cdk.load.file.object_storage.ObjectStorageClient +import io.airbyte.cdk.load.file.object_storage.ObjectStoragePathFactory +import io.airbyte.cdk.load.file.object_storage.RemoteObject +import io.airbyte.cdk.load.test.util.OutputRecord +import io.airbyte.cdk.load.test.util.toOutputRecord +import io.airbyte.cdk.load.util.deserializeToNode +import java.io.InputStream +import java.util.zip.GZIPInputStream +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext + +class ObjectStorageDataDumper( + private val stream: DestinationStream, + private val client: ObjectStorageClient<*, *>, + private val pathFactory: ObjectStoragePathFactory, + private val compressionConfig: ObjectStorageCompressionConfiguration<*>? = null +) { + fun dump(): List { + val prefix = pathFactory.getFinalDirectory(stream).toString() + return runBlocking { + withContext(Dispatchers.IO) { + client + .list(prefix) + .map { listedObject: RemoteObject<*> -> + client.get(listedObject.key) { objectData: InputStream -> + when (compressionConfig?.compressor) { + is GZIPProcessor -> GZIPInputStream(objectData) + is NoopProcessor, + null -> objectData + else -> error("Unsupported compressor") + } + .bufferedReader() + .lineSequence() + .map { line -> + line + .deserializeToNode() + .toAirbyteValue(stream.schemaWithMeta) + .toOutputRecord() + } + .toList() + } + } + .toList() + .flatten() + } + } + } +} diff --git a/airbyte-cdk/bulk/toolkits/load-s3/build.gradle b/airbyte-cdk/bulk/toolkits/load-s3/build.gradle index eb4e0b973098..00336f267d49 100644 --- a/airbyte-cdk/bulk/toolkits/load-s3/build.gradle +++ b/airbyte-cdk/bulk/toolkits/load-s3/build.gradle @@ -4,5 +4,6 @@ dependencies { api project(':airbyte-cdk:bulk:toolkits:bulk-cdk-toolkit-load-aws') api project(':airbyte-cdk:bulk:toolkits:bulk-cdk-toolkit-load-object-storage') + testFixturesApi(testFixtures(project(":airbyte-cdk:bulk:toolkits:bulk-cdk-toolkit-load-object-storage"))) implementation("aws.sdk.kotlin:s3:1.0.0") } diff --git a/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/command/s3/S3PathSpecification.kt b/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/command/s3/S3PathSpecification.kt index 743cfe46d2b0..b7a76991a3d3 100644 --- a/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/command/s3/S3PathSpecification.kt +++ b/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/command/s3/S3PathSpecification.kt @@ -10,6 +10,15 @@ import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaInject import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle import io.airbyte.cdk.load.command.object_storage.ObjectStoragePathConfiguration +/** + * Mix-in to provide S3 path configuration fields as properties. + * + * NOTE: For legacy reasons, this is unnecessarily s3-specific. Future cloud storage solutions + * should create a single generic version of this in the `object-storage` toolkit and use that. + * + * See [io.airbyte.cdk.load.command.DestinationConfiguration] for more details on how to use this + * interface. + */ interface S3PathSpecification { @get:JsonSchemaTitle("S3 Path Format") @get:JsonPropertyDescription( diff --git a/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt b/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt index 41390bedd5fa..bd5235c0eba5 100644 --- a/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt +++ b/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3Client.kt @@ -5,27 +5,25 @@ package io.airbyte.cdk.load.file.s3 import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider -import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest -import aws.sdk.kotlin.services.s3.model.CompletedMultipartUpload -import aws.sdk.kotlin.services.s3.model.CompletedPart import aws.sdk.kotlin.services.s3.model.CopyObjectRequest import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadRequest -import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadResponse import aws.sdk.kotlin.services.s3.model.DeleteObjectRequest import aws.sdk.kotlin.services.s3.model.GetObjectRequest import aws.sdk.kotlin.services.s3.model.ListObjectsRequest import aws.sdk.kotlin.services.s3.model.PutObjectRequest -import aws.sdk.kotlin.services.s3.model.UploadPartRequest import aws.smithy.kotlin.runtime.content.ByteStream import aws.smithy.kotlin.runtime.content.toInputStream import edu.umd.cs.findbugs.annotations.SuppressFBWarnings import io.airbyte.cdk.load.command.aws.AWSAccessKeyConfigurationProvider +import io.airbyte.cdk.load.command.object_storage.ObjectStorageCompressionConfiguration +import io.airbyte.cdk.load.command.object_storage.ObjectStorageCompressionConfigurationProvider import io.airbyte.cdk.load.command.object_storage.ObjectStorageUploadConfiguration import io.airbyte.cdk.load.command.object_storage.ObjectStorageUploadConfigurationProvider import io.airbyte.cdk.load.command.s3.S3BucketConfiguration import io.airbyte.cdk.load.command.s3.S3BucketConfigurationProvider +import io.airbyte.cdk.load.file.NoopProcessor +import io.airbyte.cdk.load.file.StreamProcessor import io.airbyte.cdk.load.file.object_storage.ObjectStorageClient -import io.airbyte.cdk.load.file.object_storage.ObjectStorageStreamingUploadWriter import io.airbyte.cdk.load.file.object_storage.RemoteObject import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Factory @@ -33,6 +31,7 @@ import io.micronaut.context.annotation.Secondary import jakarta.inject.Singleton import java.io.ByteArrayOutputStream import java.io.InputStream +import java.io.OutputStream import kotlinx.coroutines.flow.flow data class S3Object(override val key: String, override val storageConfig: S3BucketConfiguration) : @@ -46,7 +45,8 @@ class S3Client( private val client: aws.sdk.kotlin.services.s3.S3Client, val bucketConfig: S3BucketConfiguration, private val uploadConfig: ObjectStorageUploadConfiguration?, -) : ObjectStorageClient { + private val compressionConfig: ObjectStorageCompressionConfiguration<*>? = null, +) : ObjectStorageClient.Writer> { private val log = KotlinLogging.logger {} override suspend fun list(prefix: String) = flow { @@ -114,75 +114,35 @@ class S3Client( override suspend fun streamingUpload( key: String, - collector: suspend S3MultipartUpload.Writer.() -> Unit + block: suspend (S3MultipartUpload<*>.Writer) -> Unit + ): S3Object { + return streamingUpload(key, compressionConfig?.compressor ?: NoopProcessor, block) + } + + override suspend fun streamingUpload( + key: String, + streamProcessor: StreamProcessor, + block: suspend (S3MultipartUpload<*>.Writer) -> Unit ): S3Object { val request = CreateMultipartUploadRequest { this.bucket = bucketConfig.s3BucketName this.key = key } val response = client.createMultipartUpload(request) - S3MultipartUpload(response).upload(collector) - return S3Object(key, bucketConfig) - } - - inner class S3MultipartUpload(private val response: CreateMultipartUploadResponse) { - private val uploadedParts = mutableListOf() - private var inputBuffer = ByteArrayOutputStream() - private val partSize = - uploadConfig?.streamingUploadPartSize - ?: throw IllegalStateException("Streaming upload part size is not configured") - - suspend fun upload(collector: suspend S3MultipartUpload.Writer.() -> Unit) { - log.info { - "Starting multipart upload to ${response.bucket}/${response.key} (${response.uploadId}" - } - collector.invoke(Writer()) - complete() - } - - inner class Writer : ObjectStorageStreamingUploadWriter { - override suspend fun write(bytes: ByteArray) { - inputBuffer.write(bytes) - if (inputBuffer.size() >= partSize) { - uploadPart() - } - } - } - - private suspend fun uploadPart() { - val partNumber = uploadedParts.size + 1 - val request = UploadPartRequest { - uploadId = response.uploadId - bucket = response.bucket - key = response.key - body = ByteStream.fromBytes(inputBuffer.toByteArray()) - this.partNumber = partNumber - } - val uploadResponse = client.uploadPart(request) - uploadedParts.add( - CompletedPart { - this.partNumber = partNumber - this.eTag = uploadResponse.eTag - } + val upload = + S3MultipartUpload( + client, + response, + ByteArrayOutputStream(), + streamProcessor, + uploadConfig ) - inputBuffer.reset() - } - - private suspend fun complete() { - if (inputBuffer.size() > 0) { - uploadPart() - } - log.info { - "Completing multipart upload to ${response.bucket}/${response.key} (${response.uploadId}" - } - val request = CompleteMultipartUploadRequest { - uploadId = response.uploadId - bucket = response.bucket - key = response.key - this.multipartUpload = CompletedMultipartUpload { parts = uploadedParts } - } - client.completeMultipartUpload(request) + log.info { + "Starting multipart upload to ${response.bucket}/${response.key} (${response.uploadId}" } + block(upload.Writer()) + upload.complete() + return S3Object(key, bucketConfig) } } @@ -191,6 +151,7 @@ class S3ClientFactory( private val keyConfig: AWSAccessKeyConfigurationProvider, private val bucketConfig: S3BucketConfigurationProvider, private val uploadConifg: ObjectStorageUploadConfigurationProvider? = null, + private val compressionConfig: ObjectStorageCompressionConfigurationProvider<*>? = null, ) { companion object { fun make(config: T) where @@ -217,7 +178,8 @@ class S3ClientFactory( return S3Client( client, bucketConfig.s3BucketConfiguration, - uploadConifg?.objectStorageUploadConfiguration + uploadConifg?.objectStorageUploadConfiguration, + compressionConfig?.objectStorageCompressionConfiguration, ) } } diff --git a/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3MultipartUpload.kt b/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3MultipartUpload.kt new file mode 100644 index 000000000000..87f62c29ab90 --- /dev/null +++ b/airbyte-cdk/bulk/toolkits/load-s3/src/main/kotlin/io/airbyte/cdk/load/file/s3/S3MultipartUpload.kt @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.load.file.s3 + +import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.CompletedMultipartUpload +import aws.sdk.kotlin.services.s3.model.CompletedPart +import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadResponse +import aws.sdk.kotlin.services.s3.model.UploadPartRequest +import aws.smithy.kotlin.runtime.content.ByteStream +import io.airbyte.cdk.load.command.object_storage.ObjectStorageUploadConfiguration +import io.airbyte.cdk.load.file.StreamProcessor +import io.airbyte.cdk.load.file.object_storage.ObjectStorageStreamingUploadWriter +import io.github.oshai.kotlinlogging.KotlinLogging +import java.io.ByteArrayOutputStream +import java.io.OutputStream + +class S3MultipartUpload( + private val client: aws.sdk.kotlin.services.s3.S3Client, + private val response: CreateMultipartUploadResponse, + private val underlyingBuffer: ByteArrayOutputStream, + private val streamProcessor: StreamProcessor, + uploadConfig: ObjectStorageUploadConfiguration?, +) { + private val log = KotlinLogging.logger {} + + private val uploadedParts = mutableListOf() + private val partSize = + uploadConfig?.streamingUploadPartSize + ?: throw IllegalStateException("Streaming upload part size is not configured") + private val wrappingBuffer = streamProcessor.wrapper(underlyingBuffer) + + inner class Writer : ObjectStorageStreamingUploadWriter { + override suspend fun write(bytes: ByteArray) { + wrappingBuffer.write(bytes) + if (underlyingBuffer.size() >= partSize) { + uploadPart() + } + } + + override suspend fun write(string: String) { + write(string.toByteArray(Charsets.UTF_8)) + } + } + + private suspend fun uploadPart() { + streamProcessor.partFinisher.invoke(wrappingBuffer) + val partNumber = uploadedParts.size + 1 + val request = UploadPartRequest { + uploadId = response.uploadId + bucket = response.bucket + key = response.key + body = ByteStream.fromBytes(underlyingBuffer.toByteArray()) + this.partNumber = partNumber + } + val uploadResponse = client.uploadPart(request) + uploadedParts.add( + CompletedPart { + this.partNumber = partNumber + this.eTag = uploadResponse.eTag + } + ) + underlyingBuffer.reset() + } + + suspend fun complete() { + if (underlyingBuffer.size() > 0) { + uploadPart() + } + log.info { + "Completing multipart upload to ${response.bucket}/${response.key} (${response.uploadId}" + } + val request = CompleteMultipartUploadRequest { + uploadId = response.uploadId + bucket = response.bucket + key = response.key + this.multipartUpload = CompletedMultipartUpload { parts = uploadedParts } + } + client.completeMultipartUpload(request) + } +} diff --git a/airbyte-integrations/connectors/destination-s3-v2/metadata.yaml b/airbyte-integrations/connectors/destination-s3-v2/metadata.yaml index 744f472028ae..62211564e565 100644 --- a/airbyte-integrations/connectors/destination-s3-v2/metadata.yaml +++ b/airbyte-integrations/connectors/destination-s3-v2/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: file connectorType: destination definitionId: d6116991-e809-4c7c-ae09-c64712df5b66 - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.5 dockerRepository: airbyte/destination-s3-v2 githubIssueLabel: destination-s3-v2 icon: s3.svg @@ -31,4 +31,9 @@ data: secretStore: type: GSM alias: airbyte-connector-testing-secret-store + - name: SECRET_DESTINATION-S3-V2-JSONL-GZIP + fileName: s3_dest_v2_jsonl_gzip_config.json + secretStore: + type: GSM + alias: airbyte-connector-testing-secret-store metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Checker.kt b/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Checker.kt index 773184d10216..4bb161d6d5a9 100644 --- a/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Checker.kt +++ b/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Checker.kt @@ -13,14 +13,16 @@ import io.airbyte.cdk.load.file.s3.S3Object import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.exceptions.ConfigurationException import jakarta.inject.Singleton +import java.io.OutputStream import kotlinx.coroutines.flow.toList import kotlinx.coroutines.runBlocking @Singleton -class S3V2Checker(private val timeProvider: TimeProvider) : DestinationChecker { +class S3V2Checker(private val timeProvider: TimeProvider) : + DestinationChecker> { private val log = KotlinLogging.logger {} - override fun check(config: S3V2Configuration) { + override fun check(config: S3V2Configuration) { runBlocking { if (config.objectStorageFormatConfiguration !is JsonFormatConfiguration) { throw ConfigurationException("Currently only JSON format is supported") @@ -28,11 +30,12 @@ class S3V2Checker(private val timeProvider: TimeProvider) : DestinationChecker( // Client-facing configuration override val awsAccessKeyConfiguration: AWSAccessKeyConfiguration, override val s3BucketConfiguration: S3BucketConfiguration, override val objectStoragePathConfiguration: ObjectStoragePathConfiguration, override val objectStorageFormatConfiguration: ObjectStorageFormatConfiguration, + override val objectStorageCompressionConfiguration: ObjectStorageCompressionConfiguration, // Internal configuration override val objectStorageUploadConfiguration: ObjectStorageUploadConfiguration = ObjectStorageUploadConfiguration(5L * 1024 * 1024), - override val recordBatchSizeBytes: Long = 200L * 1024 * 1024 + override val recordBatchSizeBytes: Long = 200L * 1024 * 1024, ) : DestinationConfiguration(), AWSAccessKeyConfigurationProvider, S3BucketConfigurationProvider, ObjectStoragePathConfigurationProvider, ObjectStorageFormatConfigurationProvider, - ObjectStorageUploadConfigurationProvider + ObjectStorageUploadConfigurationProvider, + ObjectStorageCompressionConfigurationProvider @Singleton class S3V2ConfigurationFactory : - DestinationConfigurationFactory { - override fun makeWithoutExceptionHandling(pojo: S3V2Specification): S3V2Configuration { + DestinationConfigurationFactory> { + override fun makeWithoutExceptionHandling(pojo: S3V2Specification): S3V2Configuration<*> { return S3V2Configuration( awsAccessKeyConfiguration = pojo.toAWSAccessKeyConfiguration(), s3BucketConfiguration = pojo.toS3BucketConfiguration(), objectStoragePathConfiguration = pojo.toObjectStoragePathConfiguration(), - objectStorageFormatConfiguration = pojo.toObjectStorageFormatConfiguration() + objectStorageFormatConfiguration = pojo.toObjectStorageFormatConfiguration(), + objectStorageCompressionConfiguration = pojo.toCompressionConfiguration() ) } } +@Suppress("UNCHECKED_CAST") @Factory -class S3V2ConfigurationProvider(private val config: DestinationConfiguration) { +class S3V2ConfigurationProvider(private val config: DestinationConfiguration) { @Singleton - fun get(): S3V2Configuration { - return config as S3V2Configuration + fun get(): S3V2Configuration { + return config as S3V2Configuration } } diff --git a/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Writer.kt b/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Writer.kt index 2c8df1b0af31..aa8da9da1a98 100644 --- a/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Writer.kt +++ b/airbyte-integrations/connectors/destination-s3-v2/src/main/kotlin/S3V2Writer.kt @@ -23,7 +23,7 @@ import java.util.concurrent.atomic.AtomicLong class S3V2Writer( private val s3Client: S3Client, private val pathFactory: ObjectStoragePathFactory, - private val recordDecorator: DestinationRecordToAirbyteValueWithMeta + private val recordDecorator: DestinationRecordToAirbyteValueWithMeta, ) : DestinationWriter { sealed interface S3V2Batch : Batch data class StagedObject( @@ -51,11 +51,11 @@ class S3V2Writer( val partNumber = partNumber.getAndIncrement() val key = pathFactory.getPathToFile(stream, partNumber, isStaging = true).toString() val s3Object = - s3Client.streamingUpload(key) { + s3Client.streamingUpload(key) { writer -> records.forEach { val serialized = recordDecorator.decorate(it).toJson().serializeToString() - write(serialized) - write("\n") + writer.write(serialized) + writer.write("\n") } } return StagedObject(s3Object = s3Object, partNumber = partNumber) diff --git a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2CheckTest.kt b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2CheckTest.kt index 3bac6fb726f9..c322111f70ae 100644 --- a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2CheckTest.kt +++ b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2CheckTest.kt @@ -15,8 +15,12 @@ class S3V2CheckTest : successConfigFilenames = listOf( CheckTestConfig( - S3V2TestUtils.MINIMAL_CONFIG_PATH, + S3V2TestUtils.JSON_UNCOMPRESSED_CONFIG_PATH, setOf(FeatureFlag.AIRBYTE_CLOUD_DEPLOYMENT) + ), + CheckTestConfig( + S3V2TestUtils.JSON_GZIP_CONFIG_PATH, + setOf(FeatureFlag.AIRBYTE_CLOUD_DEPLOYMENT), ) ), failConfigFilenamesAndFailureReasons = emptyMap() diff --git a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2DataDumper.kt b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2DataDumper.kt new file mode 100644 index 000000000000..77a9809a882e --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2DataDumper.kt @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.s3_v2 + +import io.airbyte.cdk.command.ConfigurationSpecification +import io.airbyte.cdk.load.ObjectStorageDataDumper +import io.airbyte.cdk.load.command.DestinationStream +import io.airbyte.cdk.load.file.object_storage.ObjectStoragePathFactory +import io.airbyte.cdk.load.file.s3.S3ClientFactory +import io.airbyte.cdk.load.test.util.DestinationDataDumper +import io.airbyte.cdk.load.test.util.OutputRecord + +object S3V2DataDumper : DestinationDataDumper { + override fun dumpRecords( + spec: ConfigurationSpecification, + stream: DestinationStream + ): List { + val config = + S3V2ConfigurationFactory().makeWithoutExceptionHandling(spec as S3V2Specification) + val s3Client = S3ClientFactory.make(config) + val pathFactory = ObjectStoragePathFactory.from(config) + return ObjectStorageDataDumper( + stream, + s3Client, + pathFactory, + config.objectStorageCompressionConfiguration + ) + .dump() + } +} diff --git a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2TestUtils.kt b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2TestUtils.kt index 2d72b272c73f..98bd860a6b53 100644 --- a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2TestUtils.kt +++ b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2TestUtils.kt @@ -9,10 +9,11 @@ import java.nio.file.Files import java.nio.file.Path object S3V2TestUtils { - const val MINIMAL_CONFIG_PATH = "secrets/s3_dest_v2_minimal_required_config.json" - val minimalConfig: S3V2Specification = + const val JSON_UNCOMPRESSED_CONFIG_PATH = "secrets/s3_dest_v2_minimal_required_config.json" + const val JSON_GZIP_CONFIG_PATH = "secrets/s3_dest_v2_jsonl_gzip_config.json" + fun getConfig(configPath: String): S3V2Specification = ValidatedJsonUtils.parseOne( S3V2Specification::class.java, - Files.readString(Path.of(MINIMAL_CONFIG_PATH)), + Files.readString(Path.of(configPath)), ) } diff --git a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2WriteTest.kt b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2WriteTest.kt index 666d5a12782a..cbf7e23747e8 100644 --- a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2WriteTest.kt +++ b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2WriteTest.kt @@ -4,30 +4,14 @@ package io.airbyte.integrations.destination.s3_v2 -import io.airbyte.cdk.command.ConfigurationSpecification -import io.airbyte.cdk.load.command.DestinationStream -import io.airbyte.cdk.load.data.toAirbyteValue -import io.airbyte.cdk.load.file.object_storage.ObjectStoragePathFactory -import io.airbyte.cdk.load.file.s3.S3ClientFactory -import io.airbyte.cdk.load.file.s3.S3Object -import io.airbyte.cdk.load.test.util.DestinationDataDumper import io.airbyte.cdk.load.test.util.NoopDestinationCleaner import io.airbyte.cdk.load.test.util.NoopExpectedRecordMapper -import io.airbyte.cdk.load.test.util.OutputRecord -import io.airbyte.cdk.load.test.util.toOutputRecord -import io.airbyte.cdk.load.util.deserializeToNode import io.airbyte.cdk.load.write.BasicFunctionalityIntegrationTest -import java.io.InputStream -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.flow.map -import kotlinx.coroutines.flow.toList -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.withContext import org.junit.jupiter.api.Test -class S3V2WriteTest : +class S3V2WriteTestJsonUncompressed : BasicFunctionalityIntegrationTest( - S3V2TestUtils.minimalConfig, + S3V2TestUtils.getConfig(S3V2TestUtils.JSON_UNCOMPRESSED_CONFIG_PATH), S3V2DataDumper, NoopDestinationCleaner, NoopExpectedRecordMapper, @@ -38,40 +22,15 @@ class S3V2WriteTest : } } -object S3V2DataDumper : DestinationDataDumper { - override fun dumpRecords( - spec: ConfigurationSpecification, - stream: DestinationStream - ): List { - val config = - S3V2ConfigurationFactory().makeWithoutExceptionHandling(spec as S3V2Specification) - val s3Client = S3ClientFactory.make(config) - // Note: because we cannot mock wall time in docker, this - // path code cannot contain time-based macros. - // TODO: add pattern matching to the path factory. - val pathFactory = ObjectStoragePathFactory.from(config) - val prefix = pathFactory.getFinalDirectory(stream).toString() - return runBlocking { - withContext(Dispatchers.IO) { - s3Client - .list(prefix) - .map { listedObject: S3Object -> - s3Client.get(listedObject.key) { objectData: InputStream -> - objectData - .bufferedReader() - .lineSequence() - .map { line -> - line - .deserializeToNode() - .toAirbyteValue(stream.schemaWithMeta) - .toOutputRecord() - } - .toList() - } - } - .toList() - .flatten() - } - } +class S3V2WriteTestJsonGzip : + BasicFunctionalityIntegrationTest( + S3V2TestUtils.getConfig(S3V2TestUtils.JSON_GZIP_CONFIG_PATH), + S3V2DataDumper, + NoopDestinationCleaner, + NoopExpectedRecordMapper, + ) { + @Test + override fun testBasicWrite() { + super.testBasicWrite() } } diff --git a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/resources/expected-spec-cloud.json b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/resources/expected-spec-cloud.json index 2b93394a99a8..f713da3d7210 100644 --- a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/resources/expected-spec-cloud.json +++ b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/resources/expected-spec-cloud.json @@ -6,6 +6,37 @@ "type" : "object", "additionalProperties" : true, "properties" : { + "access_key_id" : { + "type" : "string", + "description" : "The access key ID to access the S3 bucket. Airbyte requires Read and Write permissions to the given bucket. Read more here.", + "title" : "S3 Key ID", + "examples" : [ "A012345678910EXAMPLE" ] + }, + "secret_access_key" : { + "type" : "string", + "description" : "The corresponding secret to the access key ID. Read more here", + "title" : "S3 Access Key", + "examples" : [ "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY" ] + }, + "s3_bucket_name" : { + "type" : "string", + "description" : "The name of the S3 bucket. Read more here.", + "title" : "S3 Bucket Name", + "examples" : [ "airbyte_sync" ] + }, + "s3_bucket_path" : { + "type" : "string", + "description" : "Directory under the S3 bucket where data will be written. Read more here", + "title" : "S3 Bucket Path", + "examples" : [ "data_sync/test" ] + }, + "s3_bucket_region" : { + "type" : "string", + "enum" : [ "af-south-1", "ap-east-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-south-2", "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", "ca-central-1", "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", "eu-central-2", "eu-north-1", "eu-south-1", "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", "il-central-1", "me-central-1", "me-south-1", "sa-east-1", "us-east-1", "us-east-2", "us-gov-east-1", "us-gov-west-1", "us-west-1", "us-west-2" ], + "description" : "The region of the S3 bucket. See here for all region codes.", + "title" : "S3 Bucket Region", + "examples" : [ "us-east-1" ] + }, "format" : { "oneOf" : [ { "title" : "JSON Lines: Newline-delimited JSON", @@ -16,9 +47,39 @@ "type" : "string", "enum" : [ "JSONL" ], "default" : "JSONL" + }, + "compression" : { + "oneOf" : [ { + "title" : "No Compression", + "type" : "object", + "additionalProperties" : true, + "properties" : { + "compression_type" : { + "type" : "string", + "enum" : [ "No Compression" ], + "default" : "No Compression" + } + }, + "required" : [ "compression_type" ] + }, { + "title" : "GZIP", + "type" : "object", + "additionalProperties" : true, + "properties" : { + "compression_type" : { + "type" : "string", + "enum" : [ "GZIP" ], + "default" : "GZIP" + } + }, + "required" : [ "compression_type" ] + } ], + "description" : "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".jsonl.gz\").", + "title" : "Compression", + "type" : "object" } }, - "required" : [ "format_type" ] + "required" : [ "format_type", "compression" ] }, { "title" : "CSV: Comma-Separated Values", "type" : "object", @@ -28,9 +89,39 @@ "type" : "string", "enum" : [ "CSV" ], "default" : "CSV" + }, + "compression" : { + "oneOf" : [ { + "title" : "No Compression", + "type" : "object", + "additionalProperties" : true, + "properties" : { + "compression_type" : { + "type" : "string", + "enum" : [ "No Compression" ], + "default" : "No Compression" + } + }, + "required" : [ "compression_type" ] + }, { + "title" : "GZIP", + "type" : "object", + "additionalProperties" : true, + "properties" : { + "compression_type" : { + "type" : "string", + "enum" : [ "GZIP" ], + "default" : "GZIP" + } + }, + "required" : [ "compression_type" ] + } ], + "description" : "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".jsonl.gz\").", + "title" : "Compression", + "type" : "object" } }, - "required" : [ "format_type" ] + "required" : [ "format_type", "compression" ] }, { "title" : "Avro: Apache Avro", "type" : "object", @@ -60,37 +151,6 @@ "title" : "Output Format", "type" : "object" }, - "access_key_id" : { - "type" : "string", - "description" : "The access key ID to access the S3 bucket. Airbyte requires Read and Write permissions to the given bucket. Read more here.", - "title" : "S3 Key ID", - "examples" : [ "A012345678910EXAMPLE" ] - }, - "secret_access_key" : { - "type" : "string", - "description" : "The corresponding secret to the access key ID. Read more here", - "title" : "S3 Access Key", - "examples" : [ "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY" ] - }, - "s3_bucket_name" : { - "type" : "string", - "description" : "The name of the S3 bucket. Read more here.", - "title" : "S3 Bucket Name", - "examples" : [ "airbyte_sync" ] - }, - "s3_bucket_path" : { - "type" : "string", - "description" : "Directory under the S3 bucket where data will be written. Read more here", - "title" : "S3 Bucket Path", - "examples" : [ "data_sync/test" ] - }, - "s3_bucket_region" : { - "type" : "string", - "enum" : [ "af-south-1", "ap-east-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-south-2", "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", "ca-central-1", "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", "eu-central-2", "eu-north-1", "eu-south-1", "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", "il-central-1", "me-central-1", "me-south-1", "sa-east-1", "us-east-1", "us-east-2", "us-gov-east-1", "us-gov-west-1", "us-west-1", "us-west-2" ], - "description" : "The region of the S3 bucket. See here for all region codes.", - "title" : "S3 Bucket Region", - "examples" : [ "us-east-1" ] - }, "s3_endpoint" : { "type" : "string", "description" : "Your S3 endpoint url. Read more here", @@ -117,7 +177,7 @@ "examples" : [ "__staging/data_sync/test" ] } }, - "required" : [ "format", "access_key_id", "secret_access_key", "s3_bucket_name", "s3_bucket_path", "s3_bucket_region" ] + "required" : [ "access_key_id", "secret_access_key", "s3_bucket_name", "s3_bucket_path", "s3_bucket_region", "format" ] }, "supportsIncremental" : true, "supportsNormalization" : false, diff --git a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/resources/expected-spec-oss.json b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/resources/expected-spec-oss.json index 2b93394a99a8..f713da3d7210 100644 --- a/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/resources/expected-spec-oss.json +++ b/airbyte-integrations/connectors/destination-s3-v2/src/test-integration/resources/expected-spec-oss.json @@ -6,6 +6,37 @@ "type" : "object", "additionalProperties" : true, "properties" : { + "access_key_id" : { + "type" : "string", + "description" : "The access key ID to access the S3 bucket. Airbyte requires Read and Write permissions to the given bucket. Read more here.", + "title" : "S3 Key ID", + "examples" : [ "A012345678910EXAMPLE" ] + }, + "secret_access_key" : { + "type" : "string", + "description" : "The corresponding secret to the access key ID. Read more here", + "title" : "S3 Access Key", + "examples" : [ "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY" ] + }, + "s3_bucket_name" : { + "type" : "string", + "description" : "The name of the S3 bucket. Read more here.", + "title" : "S3 Bucket Name", + "examples" : [ "airbyte_sync" ] + }, + "s3_bucket_path" : { + "type" : "string", + "description" : "Directory under the S3 bucket where data will be written. Read more here", + "title" : "S3 Bucket Path", + "examples" : [ "data_sync/test" ] + }, + "s3_bucket_region" : { + "type" : "string", + "enum" : [ "af-south-1", "ap-east-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-south-2", "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", "ca-central-1", "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", "eu-central-2", "eu-north-1", "eu-south-1", "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", "il-central-1", "me-central-1", "me-south-1", "sa-east-1", "us-east-1", "us-east-2", "us-gov-east-1", "us-gov-west-1", "us-west-1", "us-west-2" ], + "description" : "The region of the S3 bucket. See here for all region codes.", + "title" : "S3 Bucket Region", + "examples" : [ "us-east-1" ] + }, "format" : { "oneOf" : [ { "title" : "JSON Lines: Newline-delimited JSON", @@ -16,9 +47,39 @@ "type" : "string", "enum" : [ "JSONL" ], "default" : "JSONL" + }, + "compression" : { + "oneOf" : [ { + "title" : "No Compression", + "type" : "object", + "additionalProperties" : true, + "properties" : { + "compression_type" : { + "type" : "string", + "enum" : [ "No Compression" ], + "default" : "No Compression" + } + }, + "required" : [ "compression_type" ] + }, { + "title" : "GZIP", + "type" : "object", + "additionalProperties" : true, + "properties" : { + "compression_type" : { + "type" : "string", + "enum" : [ "GZIP" ], + "default" : "GZIP" + } + }, + "required" : [ "compression_type" ] + } ], + "description" : "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".jsonl.gz\").", + "title" : "Compression", + "type" : "object" } }, - "required" : [ "format_type" ] + "required" : [ "format_type", "compression" ] }, { "title" : "CSV: Comma-Separated Values", "type" : "object", @@ -28,9 +89,39 @@ "type" : "string", "enum" : [ "CSV" ], "default" : "CSV" + }, + "compression" : { + "oneOf" : [ { + "title" : "No Compression", + "type" : "object", + "additionalProperties" : true, + "properties" : { + "compression_type" : { + "type" : "string", + "enum" : [ "No Compression" ], + "default" : "No Compression" + } + }, + "required" : [ "compression_type" ] + }, { + "title" : "GZIP", + "type" : "object", + "additionalProperties" : true, + "properties" : { + "compression_type" : { + "type" : "string", + "enum" : [ "GZIP" ], + "default" : "GZIP" + } + }, + "required" : [ "compression_type" ] + } ], + "description" : "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".jsonl.gz\").", + "title" : "Compression", + "type" : "object" } }, - "required" : [ "format_type" ] + "required" : [ "format_type", "compression" ] }, { "title" : "Avro: Apache Avro", "type" : "object", @@ -60,37 +151,6 @@ "title" : "Output Format", "type" : "object" }, - "access_key_id" : { - "type" : "string", - "description" : "The access key ID to access the S3 bucket. Airbyte requires Read and Write permissions to the given bucket. Read more here.", - "title" : "S3 Key ID", - "examples" : [ "A012345678910EXAMPLE" ] - }, - "secret_access_key" : { - "type" : "string", - "description" : "The corresponding secret to the access key ID. Read more here", - "title" : "S3 Access Key", - "examples" : [ "a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY" ] - }, - "s3_bucket_name" : { - "type" : "string", - "description" : "The name of the S3 bucket. Read more here.", - "title" : "S3 Bucket Name", - "examples" : [ "airbyte_sync" ] - }, - "s3_bucket_path" : { - "type" : "string", - "description" : "Directory under the S3 bucket where data will be written. Read more here", - "title" : "S3 Bucket Path", - "examples" : [ "data_sync/test" ] - }, - "s3_bucket_region" : { - "type" : "string", - "enum" : [ "af-south-1", "ap-east-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-south-2", "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4", "ca-central-1", "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", "eu-central-2", "eu-north-1", "eu-south-1", "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", "il-central-1", "me-central-1", "me-south-1", "sa-east-1", "us-east-1", "us-east-2", "us-gov-east-1", "us-gov-west-1", "us-west-1", "us-west-2" ], - "description" : "The region of the S3 bucket. See here for all region codes.", - "title" : "S3 Bucket Region", - "examples" : [ "us-east-1" ] - }, "s3_endpoint" : { "type" : "string", "description" : "Your S3 endpoint url. Read more here", @@ -117,7 +177,7 @@ "examples" : [ "__staging/data_sync/test" ] } }, - "required" : [ "format", "access_key_id", "secret_access_key", "s3_bucket_name", "s3_bucket_path", "s3_bucket_region" ] + "required" : [ "access_key_id", "secret_access_key", "s3_bucket_name", "s3_bucket_path", "s3_bucket_region", "format" ] }, "supportsIncremental" : true, "supportsNormalization" : false, From 1cc7f2cc0e222f386fac80901cf8a683ba56eaaa Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Fri, 18 Oct 2024 14:52:15 -0700 Subject: [PATCH 18/18] Bulk load cdk: add checkpointing test (#46749) --- .../MockBasicFunctionalityIntegrationTest.kt | 5 + .../MockDestinationConfiguration.kt | 5 +- .../BasicFunctionalityIntegrationTest.kt | 177 +++++++++++++++++- ...evNullBasicFunctionalityIntegrationTest.kt | 5 + 4 files changed, 189 insertions(+), 3 deletions(-) diff --git a/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockBasicFunctionalityIntegrationTest.kt b/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockBasicFunctionalityIntegrationTest.kt index ae93391b9a1d..326dca6c0587 100644 --- a/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockBasicFunctionalityIntegrationTest.kt +++ b/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockBasicFunctionalityIntegrationTest.kt @@ -22,4 +22,9 @@ class MockBasicFunctionalityIntegrationTest : override fun testBasicWrite() { super.testBasicWrite() } + + @Test + override fun testMidSyncCheckpointingStreamState() { + super.testMidSyncCheckpointingStreamState() + } } diff --git a/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockDestinationConfiguration.kt b/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockDestinationConfiguration.kt index c582e5e463df..b3906b3dc712 100644 --- a/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockDestinationConfiguration.kt +++ b/airbyte-cdk/bulk/core/load/src/integrationTest/kotlin/io/airbyte/cdk/load/mock_integration_test/MockDestinationConfiguration.kt @@ -10,7 +10,10 @@ import io.airbyte.cdk.load.command.DestinationConfigurationFactory import io.micronaut.context.annotation.Factory import jakarta.inject.Singleton -class MockDestinationConfiguration : DestinationConfiguration() +class MockDestinationConfiguration : DestinationConfiguration() { + // override to 10KB instead of 200MB + override val recordBatchSizeBytes = 10 * 1024L +} @Singleton class MockDestinationSpecification : ConfigurationSpecification() diff --git a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/write/BasicFunctionalityIntegrationTest.kt b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/write/BasicFunctionalityIntegrationTest.kt index c881602a9df8..02225c9738ac 100644 --- a/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/write/BasicFunctionalityIntegrationTest.kt +++ b/airbyte-cdk/bulk/core/load/src/testFixtures/kotlin/io/airbyte/cdk/load/write/BasicFunctionalityIntegrationTest.kt @@ -6,9 +6,13 @@ package io.airbyte.cdk.load.write import io.airbyte.cdk.command.ConfigurationSpecification import io.airbyte.cdk.load.command.Append +import io.airbyte.cdk.load.command.DestinationCatalog import io.airbyte.cdk.load.command.DestinationStream -import io.airbyte.cdk.load.data.ObjectTypeWithoutSchema +import io.airbyte.cdk.load.data.FieldType +import io.airbyte.cdk.load.data.IntegerType +import io.airbyte.cdk.load.data.ObjectType import io.airbyte.cdk.load.message.DestinationRecord +import io.airbyte.cdk.load.message.DestinationStreamComplete import io.airbyte.cdk.load.message.StreamCheckpoint import io.airbyte.cdk.load.test.util.DestinationCleaner import io.airbyte.cdk.load.test.util.DestinationDataDumper @@ -18,9 +22,15 @@ import io.airbyte.cdk.load.test.util.NameMapper import io.airbyte.cdk.load.test.util.NoopExpectedRecordMapper import io.airbyte.cdk.load.test.util.NoopNameMapper import io.airbyte.cdk.load.test.util.OutputRecord +import io.airbyte.cdk.util.Jsons import io.airbyte.protocol.models.v0.AirbyteMessage import io.airbyte.protocol.models.v0.AirbyteRecordMessageMetaChange +import io.airbyte.protocol.models.v0.AirbyteStateMessage import kotlin.test.assertEquals +import kotlin.test.assertTrue +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertAll @@ -42,7 +52,7 @@ abstract class BasicFunctionalityIntegrationTest( DestinationStream( DestinationStream.Descriptor(randomizedNamespace, "test_stream"), Append, - ObjectTypeWithoutSchema, + ObjectType(linkedMapOf("id" to FieldType(IntegerType, nullable = true))), generationId = 0, minimumGenerationId = 0, syncId = 42, @@ -132,4 +142,167 @@ abstract class BasicFunctionalityIntegrationTest( }, ) } + + @Test + open fun testMidSyncCheckpointingStreamState() = + runBlocking(Dispatchers.IO) { + fun makeStream(name: String) = + DestinationStream( + DestinationStream.Descriptor(randomizedNamespace, name), + Append, + ObjectType(linkedMapOf("id" to FieldType(IntegerType, nullable = true))), + generationId = 0, + minimumGenerationId = 0, + syncId = 42, + ) + val destination = + destinationProcessFactory.createDestinationProcess( + "write", + config, + DestinationCatalog( + listOf( + makeStream("test_stream1"), + makeStream("test_stream2"), + ) + ) + .asProtocolObject(), + ) + launch { destination.run() } + + // Send one record+state to each stream + destination.sendMessages( + DestinationRecord( + namespace = randomizedNamespace, + name = "test_stream1", + data = """{"id": 12}""", + emittedAtMs = 1234, + ) + .asProtocolMessage(), + StreamCheckpoint( + streamNamespace = randomizedNamespace, + streamName = "test_stream1", + blob = """{"foo": "bar1"}""", + sourceRecordCount = 1 + ) + .asProtocolMessage(), + DestinationRecord( + namespace = randomizedNamespace, + name = "test_stream2", + data = """{"id": 34}""", + emittedAtMs = 1234, + ) + .asProtocolMessage(), + StreamCheckpoint( + streamNamespace = randomizedNamespace, + streamName = "test_stream2", + blob = """{"foo": "bar2"}""", + sourceRecordCount = 1 + ) + .asProtocolMessage() + ) + // Send records to stream1 until we get a state message back. + // Generally, we expect that that state message will belong to stream1. + val stateMessages: List + var i = 0 + while (true) { + destination.sendMessage( + DestinationRecord( + namespace = randomizedNamespace, + name = "test_stream1", + data = """{"id": 56}""", + emittedAtMs = 1234, + ) + .asProtocolMessage() + ) + val returnedMessages = destination.readMessages() + if (returnedMessages.any { it.type == AirbyteMessage.Type.STATE }) { + stateMessages = + returnedMessages + .filter { it.type == AirbyteMessage.Type.STATE } + .map { it.state } + break + } + i++ + } + + // for each state message, verify that it's a valid state, + // and that we actually wrote the data + stateMessages.forEach { stateMessage -> + val streamName = stateMessage.stream.streamDescriptor.name + val streamNamespace = stateMessage.stream.streamDescriptor.namespace + // basic state message checks - this is mostly just exercising the CDK itself, + // but is cheap and easy to do. + assertAll( + { assertEquals(randomizedNamespace, streamNamespace) }, + { + assertTrue( + streamName == "test_stream1" || streamName == "test_stream2", + "Expected stream name to be test_stream1 or test_stream2, got $streamName" + ) + }, + { + assertEquals( + 1.0, + stateMessage.destinationStats.recordCount, + "Expected destination stats to show 1 record" + ) + }, + { + when (streamName) { + "test_stream1" -> { + assertEquals( + Jsons.readTree("""{"foo": "bar1"}"""), + stateMessage.stream.streamState, + ) + } + "test_stream2" -> { + assertEquals( + Jsons.readTree("""{"foo": "bar2"}"""), + stateMessage.stream.streamState + ) + } + else -> + throw IllegalStateException("Unexpected stream name: $streamName") + } + } + ) + if (verifyDataWriting) { + val records = dataDumper.dumpRecords(config, makeStream(streamName)) + val expectedId = + when (streamName) { + "test_stream1" -> 12 + "test_stream2" -> 34 + else -> + throw IllegalStateException("Unexpected stream name: $streamName") + } + val expectedRecord = + recordMangler.mapRecord( + OutputRecord( + extractedAt = 1234, + generationId = 0, + data = mapOf("id" to expectedId), + airbyteMeta = OutputRecord.Meta(changes = listOf(), syncId = 42) + ) + ) + + assertTrue("Expected the first record to be present in the dumped records.") { + records.any { actualRecord -> expectedRecord.data == actualRecord.data } + } + } + } + + destination.sendMessages( + DestinationStreamComplete( + DestinationStream.Descriptor(randomizedNamespace, "test_stream1"), + System.currentTimeMillis() + ) + .asProtocolMessage(), + DestinationStreamComplete( + DestinationStream.Descriptor(randomizedNamespace, "test_stream2"), + System.currentTimeMillis() + ) + .asProtocolMessage() + ) + destination.shutdown() + } } diff --git a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullBasicFunctionalityIntegrationTest.kt b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullBasicFunctionalityIntegrationTest.kt index dc246a21bd4d..ea71b530c007 100644 --- a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullBasicFunctionalityIntegrationTest.kt +++ b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/kotlin/io/airbyte/integrations/destination/dev_null/DevNullBasicFunctionalityIntegrationTest.kt @@ -21,4 +21,9 @@ class DevNullBasicFunctionalityIntegrationTest : override fun testBasicWrite() { super.testBasicWrite() } + + @Test + override fun testMidSyncCheckpointingStreamState() { + super.testMidSyncCheckpointingStreamState() + } }