From 21cfb2a083a746f165e0e98562948b81b86abf9c Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk <35109939+tolik0@users.noreply.github.com> Date: Fri, 10 Nov 2023 12:33:38 +0200 Subject: [PATCH 001/124] =?UTF-8?q?=E2=9C=A8=20Source=20S3:=20Add=20HTTPS?= =?UTF-8?q?=20validation=20for=20S3=20endpoint=20(#32109)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../connectors/source-s3/.coveragerc | 6 + .../source-s3/acceptance-test-config.yml | 2 + .../integration_tests/cloud_spec.json | 594 ++++++++++++++++++ .../source-s3/integration_tests/spec.json | 1 + .../connectors/source-s3/metadata.yaml | 2 +- .../source-s3/source_s3/v4/config.py | 14 +- .../source-s3/source_s3/v4/source.py | 10 + .../source-s3/unit_tests/v4/test_config.py | 18 +- .../unit_tests/v4/test_stream_reader.py | 2 +- docs/integrations/sources/s3.md | 8 +- 10 files changed, 645 insertions(+), 12 deletions(-) create mode 100644 airbyte-integrations/connectors/source-s3/.coveragerc create mode 100644 airbyte-integrations/connectors/source-s3/integration_tests/cloud_spec.json diff --git a/airbyte-integrations/connectors/source-s3/.coveragerc b/airbyte-integrations/connectors/source-s3/.coveragerc new file mode 100644 index 000000000000..0c7476c81073 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/.coveragerc @@ -0,0 +1,6 @@ +[run] +omit = + source_s3/exceptions.py + source_s3/stream.py + source_s3/utils.py + source_s3/source_files_abstract/source.py \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml b/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml index 98c394999c01..252460fff68d 100644 --- a/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml @@ -286,5 +286,7 @@ acceptance_tests: - spec_path: integration_tests/spec.json backward_compatibility_tests_config: disable_for_version: "4.0.3" # removing the `streams.*.file_type` field which was redundant with `streams.*.format` + - spec_path: integration_tests/cloud_spec.json + deployment_mode: "cloud" connector_image: airbyte/source-s3:dev test_strictness_level: high diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/cloud_spec.json b/airbyte-integrations/connectors/source-s3/integration_tests/cloud_spec.json new file mode 100644 index 000000000000..a6a87e6d431c --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/cloud_spec.json @@ -0,0 +1,594 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/s3", + "connectionSpecification": { + "title": "Config", + "description": "NOTE: When this Spec is changed, legacy_config_transformer.py must also be modified to uptake the changes\nbecause it is responsible for converting legacy S3 v3 configs into v4 configs using the File-Based CDK.", + "type": "object", + "properties": { + "start_date": { + "title": "Start Date", + "description": "UTC date and time in the format 2017-01-25T00:00:00.000000Z. Any file modified before this date will not be replicated.", + "examples": ["2021-01-01T00:00:00.000000Z"], + "format": "date-time", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}Z$", + "pattern_descriptor": "YYYY-MM-DDTHH:mm:ss.SSSSSSZ", + "order": 1, + "type": "string" + }, + "streams": { + "title": "The list of streams to sync", + "description": "Each instance of this configuration defines a stream. Use this to define which files belong in the stream, their format, and how they should be parsed and validated. When sending data to warehouse destination such as Snowflake or BigQuery, each stream is a separate table.", + "order": 10, + "type": "array", + "items": { + "title": "FileBasedStreamConfig", + "type": "object", + "properties": { + "name": { + "title": "Name", + "description": "The name of the stream.", + "type": "string" + }, + "globs": { + "title": "Globs", + "description": "The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look here.", + "type": "array", + "items": { + "type": "string" + } + }, + "legacy_prefix": { + "title": "Legacy Prefix", + "description": "The path prefix configured in v3 versions of the S3 connector. This option is deprecated in favor of a single glob.", + "airbyte_hidden": true, + "type": "string" + }, + "validation_policy": { + "title": "Validation Policy", + "description": "The name of the validation policy that dictates sync behavior when a record does not adhere to the stream schema.", + "default": "Emit Record", + "enum": ["Emit Record", "Skip Record", "Wait for Discover"] + }, + "input_schema": { + "title": "Input Schema", + "description": "The schema that will be used to validate records extracted from the file. This will override the stream schema that is auto-detected from incoming files.", + "type": "string" + }, + "primary_key": { + "title": "Primary Key", + "description": "The column or columns (for a composite key) that serves as the unique identifier of a record.", + "type": "string" + }, + "days_to_sync_if_history_is_full": { + "title": "Days To Sync If History Is Full", + "description": "When the state history of the file store is full, syncs will only read files that were last modified in the provided day range.", + "default": 3, + "type": "integer" + }, + "format": { + "title": "Format", + "description": "The configuration options that are used to alter how to read incoming files that deviate from the standard formatting.", + "type": "object", + "oneOf": [ + { + "title": "Avro Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "avro", + "const": "avro", + "type": "string" + }, + "double_as_string": { + "title": "Convert Double Fields to Strings", + "description": "Whether to convert double fields to strings. This is recommended if you have decimal numbers with a high degree of precision because there can be a loss precision when handling floating point numbers.", + "default": false, + "type": "boolean" + } + } + }, + { + "title": "CSV Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "csv", + "const": "csv", + "type": "string" + }, + "delimiter": { + "title": "Delimiter", + "description": "The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.", + "default": ",", + "type": "string" + }, + "quote_char": { + "title": "Quote Character", + "description": "The character used for quoting CSV values. To disallow quoting, make this field blank.", + "default": "\"", + "type": "string" + }, + "escape_char": { + "title": "Escape Character", + "description": "The character used for escaping special characters. To disallow escaping, leave this field blank.", + "type": "string" + }, + "encoding": { + "title": "Encoding", + "description": "The character encoding of the CSV data. Leave blank to default to UTF8. See list of python encodings for allowable options.", + "default": "utf8", + "type": "string" + }, + "double_quote": { + "title": "Double Quote", + "description": "Whether two quotes in a quoted CSV value denote a single quote in the data.", + "default": true, + "type": "boolean" + }, + "null_values": { + "title": "Null Values", + "description": "A set of case-sensitive strings that should be interpreted as null values. For example, if the value 'NA' should be interpreted as null, enter 'NA' in this field.", + "default": [], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "strings_can_be_null": { + "title": "Strings Can Be Null", + "description": "Whether strings can be interpreted as null values. If true, strings that match the null_values set will be interpreted as null. If false, strings that match the null_values set will be interpreted as the string itself.", + "default": true, + "type": "boolean" + }, + "skip_rows_before_header": { + "title": "Skip Rows Before Header", + "description": "The number of rows to skip before the header row. For example, if the header row is on the 3rd row, enter 2 in this field.", + "default": 0, + "type": "integer" + }, + "skip_rows_after_header": { + "title": "Skip Rows After Header", + "description": "The number of rows to skip after the header row.", + "default": 0, + "type": "integer" + }, + "header_definition": { + "title": "CSV Header Definition", + "description": "How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can skip rows.", + "default": { + "header_definition_type": "From CSV" + }, + "oneOf": [ + { + "title": "From CSV", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "From CSV", + "const": "From CSV", + "type": "string" + } + } + }, + { + "title": "Autogenerated", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "Autogenerated", + "const": "Autogenerated", + "type": "string" + } + } + }, + { + "title": "User Provided", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "User Provided", + "const": "User Provided", + "type": "string" + }, + "column_names": { + "title": "Column Names", + "description": "The column names that will be used while emitting the CSV records", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": ["column_names"] + } + ], + "type": "object" + }, + "true_values": { + "title": "True Values", + "description": "A set of case-sensitive strings that should be interpreted as true values.", + "default": ["y", "yes", "t", "true", "on", "1"], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "false_values": { + "title": "False Values", + "description": "A set of case-sensitive strings that should be interpreted as false values.", + "default": ["n", "no", "f", "false", "off", "0"], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "inference_type": { + "title": "Inference Type", + "description": "How to infer the types of the columns. If none, inference default to strings.", + "default": "None", + "airbyte_hidden": true, + "enum": ["None", "Primitive Types Only"] + } + } + }, + { + "title": "Jsonl Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "jsonl", + "const": "jsonl", + "type": "string" + } + } + }, + { + "title": "Parquet Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "parquet", + "const": "parquet", + "type": "string" + }, + "decimal_as_float": { + "title": "Convert Decimal Fields to Floats", + "description": "Whether to convert decimal fields to floats. There is a loss of precision when converting decimals to floats, so this is not recommended.", + "default": false, + "type": "boolean" + } + } + }, + { + "title": "Document File Type Format (Experimental)", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "unstructured", + "const": "unstructured", + "type": "string" + } + }, + "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file." + } + ] + }, + "schemaless": { + "title": "Schemaless", + "description": "When enabled, syncs will not validate or structure records against the stream's schema.", + "default": false, + "type": "boolean" + } + }, + "required": ["name", "format"] + } + }, + "bucket": { + "title": "Bucket", + "description": "Name of the S3 bucket where the file(s) exist.", + "order": 0, + "type": "string" + }, + "aws_access_key_id": { + "title": "AWS Access Key ID", + "description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.", + "airbyte_secret": true, + "order": 2, + "type": "string" + }, + "aws_secret_access_key": { + "title": "AWS Secret Access Key", + "description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.", + "airbyte_secret": true, + "order": 3, + "type": "string" + }, + "endpoint": { + "title": "Endpoint", + "description": "Endpoint to an S3 compatible service. Leave empty to use AWS. The custom endpoint must be secure, but the 'https' prefix is not required.", + "default": "", + "examples": ["my-s3-endpoint.com", "https://my-s3-endpoint.com"], + "pattern": "^(?!http://).*$", + "order": 4, + "type": "string" + }, + "dataset": { + "title": "Output Stream Name", + "description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.name instead. The name of the stream you would like this source to output. Can contain letters, numbers, or underscores.", + "pattern": "^([A-Za-z0-9-_]+)$", + "order": 100, + "type": "string", + "airbyte_hidden": true + }, + "path_pattern": { + "title": "Pattern of files to replicate", + "description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.globs instead. A regular expression which tells the connector which files to replicate. All files which match this pattern will be replicated. Use | to separate multiple patterns. See this page to understand pattern syntax (GLOBSTAR and SPLIT flags are enabled). Use pattern ** to pick up all files.", + "examples": [ + "**", + "myFolder/myTableFiles/*.csv|myFolder/myOtherTableFiles/*.csv" + ], + "order": 110, + "type": "string", + "airbyte_hidden": true + }, + "format": { + "title": "File Format", + "description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.format instead. The format of the files you'd like to replicate", + "default": "csv", + "order": 120, + "type": "object", + "oneOf": [ + { + "title": "CSV", + "description": "This connector utilises PyArrow (Apache Arrow) for CSV parsing.", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "csv", + "const": "csv", + "type": "string" + }, + "delimiter": { + "title": "Delimiter", + "description": "The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.", + "default": ",", + "minLength": 1, + "order": 0, + "type": "string" + }, + "infer_datatypes": { + "title": "Infer Datatypes", + "description": "Configures whether a schema for the source should be inferred from the current data or not. If set to false and a custom schema is set, then the manually enforced schema is used. If a schema is not manually set, and this is set to false, then all fields will be read as strings", + "default": true, + "order": 1, + "type": "boolean" + }, + "quote_char": { + "title": "Quote Character", + "description": "The character used for quoting CSV values. To disallow quoting, make this field blank.", + "default": "\"", + "order": 2, + "type": "string" + }, + "escape_char": { + "title": "Escape Character", + "description": "The character used for escaping special characters. To disallow escaping, leave this field blank.", + "order": 3, + "type": "string" + }, + "encoding": { + "title": "Encoding", + "description": "The character encoding of the CSV data. Leave blank to default to UTF8. See list of python encodings for allowable options.", + "default": "utf8", + "order": 4, + "type": "string" + }, + "double_quote": { + "title": "Double Quote", + "description": "Whether two quotes in a quoted CSV value denote a single quote in the data.", + "default": true, + "order": 5, + "type": "boolean" + }, + "newlines_in_values": { + "title": "Allow newlines in values", + "description": "Whether newline characters are allowed in CSV values. Turning this on may affect performance. Leave blank to default to False.", + "default": false, + "order": 6, + "type": "boolean" + }, + "additional_reader_options": { + "title": "Additional Reader Options", + "description": "Optionally add a valid JSON string here to provide additional options to the csv reader. Mappings must correspond to options detailed here. 'column_types' is used internally to handle schema so overriding that would likely cause problems.", + "examples": [ + "{\"timestamp_parsers\": [\"%m/%d/%Y %H:%M\", \"%Y/%m/%d %H:%M\"], \"strings_can_be_null\": true, \"null_values\": [\"NA\", \"NULL\"]}" + ], + "order": 7, + "type": "string" + }, + "advanced_options": { + "title": "Advanced Options", + "description": "Optionally add a valid JSON string here to provide additional Pyarrow ReadOptions. Specify 'column_names' here if your CSV doesn't have header, or if you want to use custom column names. 'block_size' and 'encoding' are already used above, specify them again here will override the values above.", + "examples": ["{\"column_names\": [\"column1\", \"column2\"]}"], + "order": 8, + "type": "string" + }, + "block_size": { + "title": "Block Size", + "description": "The chunk size in bytes to process at a time in memory from each file. If your data is particularly wide and failing during schema detection, increasing this should solve it. Beware of raising this too high as you could hit OOM errors.", + "default": 10000, + "minimum": 1, + "maximum": 2147483647, + "order": 9, + "type": "integer" + } + } + }, + { + "title": "Parquet", + "description": "This connector utilises PyArrow (Apache Arrow) for Parquet parsing.", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "parquet", + "const": "parquet", + "type": "string" + }, + "columns": { + "title": "Selected Columns", + "description": "If you only want to sync a subset of the columns from the file(s), add the columns you want here as a comma-delimited list. Leave it empty to sync all columns.", + "order": 0, + "type": "array", + "items": { + "type": "string" + } + }, + "batch_size": { + "title": "Record batch size", + "description": "Maximum number of records per batch read from the input files. Batches may be smaller if there aren’t enough rows in the file. This option can help avoid out-of-memory errors if your data is particularly wide.", + "default": 65536, + "order": 1, + "type": "integer" + }, + "buffer_size": { + "title": "Buffer Size", + "description": "Perform read buffering when deserializing individual column chunks. By default every group column will be loaded fully to memory. This option can help avoid out-of-memory errors if your data is particularly wide.", + "default": 2, + "type": "integer" + } + } + }, + { + "title": "Avro", + "description": "This connector utilises fastavro for Avro parsing.", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "avro", + "const": "avro", + "type": "string" + } + } + }, + { + "title": "Jsonl", + "description": "This connector uses PyArrow for JSON Lines (jsonl) file parsing.", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "jsonl", + "const": "jsonl", + "type": "string" + }, + "newlines_in_values": { + "title": "Allow newlines in values", + "description": "Whether newline characters are allowed in JSON values. Turning this on may affect performance. Leave blank to default to False.", + "default": false, + "order": 0, + "type": "boolean" + }, + "unexpected_field_behavior": { + "title": "Unexpected field behavior", + "description": "How JSON fields outside of explicit_schema (if given) are treated. Check PyArrow documentation for details", + "default": "infer", + "examples": ["ignore", "infer", "error"], + "order": 1, + "enum": ["ignore", "infer", "error"] + }, + "block_size": { + "title": "Block Size", + "description": "The chunk size in bytes to process at a time in memory from each file. If your data is particularly wide and failing during schema detection, increasing this should solve it. Beware of raising this too high as you could hit OOM errors.", + "default": 0, + "order": 2, + "type": "integer" + } + } + } + ], + "airbyte_hidden": true + }, + "schema": { + "title": "Manually enforced data schema", + "description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.input_schema instead. Optionally provide a schema to enforce, as a valid JSON string. Ensure this is a mapping of { \"column\" : \"type\" }, where types are valid JSON Schema datatypes. Leave as {} to auto-infer the schema.", + "default": "{}", + "examples": [ + "{\"column_1\": \"number\", \"column_2\": \"string\", \"column_3\": \"array\", \"column_4\": \"object\", \"column_5\": \"boolean\"}" + ], + "order": 130, + "type": "string", + "airbyte_hidden": true + }, + "provider": { + "title": "S3: Amazon Web Services", + "type": "object", + "properties": { + "bucket": { + "title": "Bucket", + "description": "Name of the S3 bucket where the file(s) exist.", + "order": 0, + "type": "string" + }, + "aws_access_key_id": { + "title": "AWS Access Key ID", + "description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.", + "airbyte_secret": true, + "always_show": true, + "order": 1, + "type": "string" + }, + "aws_secret_access_key": { + "title": "AWS Secret Access Key", + "description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.", + "airbyte_secret": true, + "always_show": true, + "order": 2, + "type": "string" + }, + "path_prefix": { + "title": "Path Prefix", + "description": "By providing a path-like prefix (e.g. myFolder/thisTable/) under which all the relevant files sit, we can optimize finding these in S3. This is optional but recommended if your bucket contains many folders/files which you don't need to replicate.", + "default": "", + "order": 3, + "type": "string" + }, + "endpoint": { + "title": "Endpoint", + "description": "Endpoint to an S3 compatible service. Leave empty to use AWS.", + "default": "", + "order": 4, + "type": "string" + }, + "start_date": { + "title": "Start Date", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any file modified before this date will not be replicated.", + "examples": ["2021-01-01T00:00:00Z"], + "format": "date-time", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "order": 5, + "type": "string" + } + }, + "required": [], + "order": 111, + "description": "Deprecated and will be removed soon. Please do not use this field anymore and use bucket, aws_access_key_id, aws_secret_access_key and endpoint instead. Use this to load files from S3 or S3-compatible services", + "airbyte_hidden": true + } + }, + "required": ["streams", "bucket"] + } +} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/spec.json b/airbyte-integrations/connectors/source-s3/integration_tests/spec.json index c97aa81955f1..525f71b61488 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-s3/integration_tests/spec.json @@ -318,6 +318,7 @@ "title": "Endpoint", "description": "Endpoint to an S3 compatible service. Leave empty to use AWS.", "default": "", + "examples": ["my-s3-endpoint.com", "https://my-s3-endpoint.com"], "order": 4, "type": "string" }, diff --git a/airbyte-integrations/connectors/source-s3/metadata.yaml b/airbyte-integrations/connectors/source-s3/metadata.yaml index 1ce5e178792f..2daa8c6e4f8b 100644 --- a/airbyte-integrations/connectors/source-s3/metadata.yaml +++ b/airbyte-integrations/connectors/source-s3/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: file connectorType: source definitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2 - dockerImageTag: 4.1.4 + dockerImageTag: 4.2.0 dockerRepository: airbyte/source-s3 documentationUrl: https://docs.airbyte.com/integrations/sources/s3 githubIssueLabel: source-s3 diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py index 16a9af413b25..5f0a891c6a61 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py @@ -5,6 +5,7 @@ from typing import Optional from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec +from airbyte_cdk.utils import is_cloud_environment from pydantic import AnyUrl, Field, ValidationError, root_validator @@ -39,7 +40,11 @@ def documentation_url(cls) -> AnyUrl: ) endpoint: Optional[str] = Field( - "", title="Endpoint", description="Endpoint to an S3 compatible service. Leave empty to use AWS.", order=4 + default="", + title="Endpoint", + description="Endpoint to an S3 compatible service. Leave empty to use AWS.", + examples=["my-s3-endpoint.com", "https://my-s3-endpoint.com"], + order=4, ) @root_validator @@ -50,4 +55,11 @@ def validate_optional_args(cls, values): raise ValidationError( "`aws_access_key_id` and `aws_secret_access_key` are both required to authenticate with AWS.", model=Config ) + + if is_cloud_environment(): + endpoint = values.get("endpoint") + if endpoint: + if endpoint.startswith("http://"): # ignore-https-check + raise ValidationError("The endpoint must be a secure HTTPS endpoint.", model=Config) + return values diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py index 780a3bb0cf1f..bcb4ccdffef4 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py @@ -7,6 +7,7 @@ from airbyte_cdk.config_observation import emit_configuration_as_airbyte_control_message from airbyte_cdk.models import ConnectorSpecification from airbyte_cdk.sources.file_based.file_based_source import FileBasedSource +from airbyte_cdk.utils import is_cloud_environment from source_s3.source import SourceS3Spec from source_s3.v4.legacy_config_transformer import LegacyConfigTransformer @@ -51,6 +52,15 @@ def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: ) self._clean_required_fields(s4_spec["properties"][v3_property_key]) + if is_cloud_environment(): + s4_spec["properties"]["endpoint"].update( + { + "description": "Endpoint to an S3 compatible service. Leave empty to use AWS. " + "The custom endpoint must be secure, but the 'https' prefix is not required.", + "pattern": "^(?!http://).*$", # ignore-https-check + } + ) + return ConnectorSpecification( documentationUrl=self.spec_class.documentation_url(), connectionSpecification=s4_spec, diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_config.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_config.py index 250f3344f801..66213120a704 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_config.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_config.py @@ -13,30 +13,36 @@ @pytest.mark.parametrize( - "kwargs,expected_error", + "kwargs, is_cloud, expected_error", [ - pytest.param({"bucket": "test", "streams": []}, None, id="required-fields"), + pytest.param({"bucket": "test", "streams": []}, False, None, id="required-fields"), pytest.param( {"bucket": "test", "streams": [], "aws_access_key_id": "access_key", "aws_secret_access_key": "secret_access_key"}, + True, None, id="config-created-with-aws-info", ), - pytest.param({"bucket": "test", "streams": [], "endpoint": "http://test.com"}, None, id="config-created-with-endpoint"), + pytest.param({"bucket": "test", "streams": [], "endpoint": "https://test.com"}, False, None, id="config-created-with-endpoint"), + pytest.param({"bucket": "test", "streams": [], "endpoint": "http://test.com"}, True, ValidationError, id="http-endpoint-error"), + pytest.param({"bucket": "test", "streams": [], "endpoint": "http://test.com"}, False, None, id="http-endpoint-error"), pytest.param( { "bucket": "test", "streams": [], "aws_access_key_id": "access_key", "aws_secret_access_key": "secret_access_key", - "endpoint": "http://test.com", + "endpoint": "https://test.com", }, + True, None, id="config-created-with-endpoint-and-aws-info", ), - pytest.param({"streams": []}, ValidationError, id="missing-bucket"), + pytest.param({"streams": []}, False, ValidationError, id="missing-bucket"), ], ) -def test_config(kwargs, expected_error): +def test_config(mocker, kwargs, is_cloud, expected_error): + mocker.patch("source_s3.v4.config.is_cloud_environment", lambda: is_cloud) + if expected_error: with pytest.raises(expected_error): Config(**kwargs) diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py index 608e27cd1b0e..05d7f7873be1 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py @@ -22,7 +22,7 @@ logger = logging.Logger("") -endpoint_values = ["http://fake.com", None] +endpoint_values = ["https://fake.com", None] _get_matching_files_cases = [ pytest.param([], [], False, set(), id="no-files-match-if-no-globs"), pytest.param( diff --git a/docs/integrations/sources/s3.md b/docs/integrations/sources/s3.md index f48e22cc557d..9ed244e9e836 100644 --- a/docs/integrations/sources/s3.md +++ b/docs/integrations/sources/s3.md @@ -238,6 +238,7 @@ The Avro parser uses the [Fastavro library](https://fastavro.readthedocs.io/en/l There are currently no options for JSONL parsing. + ### Document File Type Format (Experimental) :::warning @@ -254,9 +255,10 @@ To perform the text extraction from PDF and Docx files, the connector uses the [ ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :-------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------- | -| 4.1.4 | 2023-10-30 | [31904](https://github.com/airbytehq/airbyte/pull/31904) | Update CDK | -| 4.1.3 | 2023-10-25 | [31654](https://github.com/airbytehq/airbyte/pull/31654) | Reduce image size | +|:--------|:-----------|:----------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------| +| 4.2.0 | 2023-11-02 | [32109](https://github.com/airbytehq/airbyte/pull/32109) | Fix docs; add HTTPS validation for S3 endpoint; fix coverage | +| 4.1.4 | 2023-10-30 | [31904](https://github.com/airbytehq/airbyte/pull/31904) | Update CDK | +| 4.1.3 | 2023-10-25 | [31654](https://github.com/airbytehq/airbyte/pull/31654) | Reduce image size | | 4.1.2 | 2023-10-23 | [31383](https://github.com/airbytehq/airbyte/pull/31383) | Add handling NoSuchBucket error | | 4.1.1 | 2023-10-19 | [31601](https://github.com/airbytehq/airbyte/pull/31601) | Base image migration: remove Dockerfile and use the python-connector-base image | | 4.1.0 | 2023-10-17 | [31340](https://github.com/airbytehq/airbyte/pull/31340) | Add reading files inside zip archive | From 4677cd7345071755caa2437573f0e433e1641a29 Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk <35109939+tolik0@users.noreply.github.com> Date: Fri, 10 Nov 2023 14:07:05 +0200 Subject: [PATCH 002/124] =?UTF-8?q?=F0=9F=90=9B=20Source=20Instagram:=20Ad?= =?UTF-8?q?d=20INTERNAL=5FSERVER=5FERROR=20to=20retriable=20errors=20(#322?= =?UTF-8?q?00)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Roman Yermilov --- .../connectors/source-instagram/Dockerfile | 16 ------------ .../connectors/source-instagram/README.md | 26 +++++++++---------- .../acceptance-test-config.yml | 2 ++ .../integration_tests/expected_records.jsonl | 15 +++++------ .../connectors/source-instagram/metadata.yaml | 4 ++- .../source_instagram/common.py | 14 ++++++++++ .../unit_tests/test_streams.py | 7 +++++ docs/integrations/sources/instagram.md | 1 + 8 files changed, 47 insertions(+), 38 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-instagram/Dockerfile diff --git a/airbyte-integrations/connectors/source-instagram/Dockerfile b/airbyte-integrations/connectors/source-instagram/Dockerfile deleted file mode 100644 index 529b56cc22fb..000000000000 --- a/airbyte-integrations/connectors/source-instagram/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY source_instagram ./source_instagram -COPY main.py ./ -COPY setup.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=1.0.11 -LABEL io.airbyte.name=airbyte/source-instagram diff --git a/airbyte-integrations/connectors/source-instagram/README.md b/airbyte-integrations/connectors/source-instagram/README.md index fff295c4497b..8cfb12455f66 100644 --- a/airbyte-integrations/connectors/source-instagram/README.md +++ b/airbyte-integrations/connectors/source-instagram/README.md @@ -1,7 +1,7 @@ -# Salesloft Source +# Instagram Source -This is the repository for the Salesloft source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/salesloft). +This is the repository for the Instagram source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/instagram). ## Local development @@ -30,12 +30,12 @@ If this is mumbo jumbo to you, don't worry about it, just put your deps in `setu should work as you expect. #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/salesloft) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_salesloft/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/instagram) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_instagram/spec.json` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source salesloft test creds` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source instagram test creds` and place them into `secrets/config.json`. ### Locally running the connector @@ -52,23 +52,23 @@ python main.py read --config secrets/config.json --catalog integration_tests/con #### Build **Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** ```bash -airbyte-ci connectors --name source-salesloft build +airbyte-ci connectors --name source-instagram build ``` -An image will be built with the tag `airbyte/source-salesloft:dev`. +An image will be built with the tag `airbyte/source-instagram:dev`. **Via `docker build`:** ```bash -docker build -t airbyte/source-salesloft:dev . +docker build -t airbyte/source-instagram:dev . ``` #### Run Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-salesloft:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-salesloft:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-salesloft:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-salesloft:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +docker run --rm airbyte/source-instagram:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-instagram:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-instagram:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-instagram:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing diff --git a/airbyte-integrations/connectors/source-instagram/acceptance-test-config.yml b/airbyte-integrations/connectors/source-instagram/acceptance-test-config.yml index 2213c736366e..f70680a35b17 100644 --- a/airbyte-integrations/connectors/source-instagram/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-instagram/acceptance-test-config.yml @@ -52,6 +52,8 @@ acceptance_tests: media: - name: media_url bypass_reason: Contains auto generated hash + - name: thumbnail_url + bypass_reason: Contains auto generated hash media_insights: - name: id bypass_reason: For statistic anonymization diff --git a/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl index 925793748eef..78eff4518f29 100644 --- a/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl @@ -1,8 +1,7 @@ -{"stream": "users", "data": {"id": "17841408147298757", "website": "https://www.airbyte.io/", "media_count": 0, "username": "airbytehq", "followers_count": 0, "follows_count": 0, "profile_picture_url": "https://scontent-mrs2-1.xx.fbcdn.net/v/t51.2885-15/153169696_890787328349641_8382928081987798464_n.jpg?_nc_cat=111&_nc_sid=86c713&_nc_ohc=EZf9l6dXYAkAX9UKDmz&_nc_ht=scontent-mrs2-1.xx&edm=AL-3X8kEAAAA&oh=00_AfDy5GVLylNGrfrbuixG1fol12GiBAKo0YkkRKu08nZfpg&oe=6402B9FE", "biography": "Airbyte is the new open-source data integration platform that consolidates your data into your warehouses.", "ig_id": 8070063576, "name": "Airbyte", "page_id": "112704783733939"}, "emitted_at": 1677611621424} -{"stream": "user_lifetime_insights", "data": {"page_id": "112704783733939", "business_account_id": "17841408147298757", "metric": "audience_city"}, "emitted_at": 1675080684688} -{"stream": "user_lifetime_insights", "data": {"page_id": "112704783733939", "business_account_id": "17841408147298757", "metric": "audience_country"}, "emitted_at": 1675080684688} -{"stream": "user_lifetime_insights", "data": {"page_id": "112704783733939", "business_account_id": "17841408147298757", "metric": "audience_gender_age"}, "emitted_at": 1675080684688} -{"stream": "user_lifetime_insights", "data": {"page_id": "112704783733939", "business_account_id": "17841408147298757", "metric": "audience_locale"}, "emitted_at": 1675080684688} -{"stream": "media", "data": {"id": "17922055321172142", "media_url": "https://scontent-mrs2-1.cdninstagram.com/v/t51.2885-15/35575181_234396600491925_6359096645375754240_n.jpg?_nc_cat=109&ccb=1-7&_nc_sid=8ae9d6&_nc_ohc=Ozc82doXbDYAX-5EB14&_nc_ht=scontent-mrs2-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfDtSblYdZs5I_S1Buhk3AUc-qU1EUTtEtmoB41dJ-zlHw&oe=6403D2F5", "timestamp": "2018-06-27T09:22:01+0000", "shortcode": "Bkhd1E4gEyu", "owner": {"id": "17841408147298757"}, "like_count": 1, "is_comment_enabled": true, "permalink": "https://www.instagram.com/p/Bkhd1E4gEyu/", "ig_id": "1810859715903638702", "media_type": "IMAGE", "username": "airbytehq", "caption": "And now we will open the fortune cookies that were made before 05/05!!", "comments_count": 0, "page_id": "112704783733939", "business_account_id": "17841408147298757"}, "emitted_at": 1677610834989} -{"stream": "media", "data": {"id": "17930423551138131", "media_url": "https://scontent-mrs2-2.cdninstagram.com/v/t51.2885-15/35999650_494254794340080_7942251990459875328_n.jpg?_nc_cat=108&ccb=1-7&_nc_sid=8ae9d6&_nc_ohc=p92Ke0D-uSMAX_8M5eN&_nc_ht=scontent-mrs2-2.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfAZ0f80E_esComf-cW1OTocAx5y1IhWudZ6c3sWj9ZJjw&oe=640435B5", "timestamp": "2018-06-26T09:22:01+0000", "shortcode": "Bke5CHUj6RZ", "owner": {"id": "17841408147298757"}, "like_count": 0, "is_comment_enabled": true, "permalink": "https://www.instagram.com/p/Bke5CHUj6RZ/", "ig_id": "1810134934200755289", "media_type": "IMAGE", "username": "airbytehq", "caption": "Every time !!! Come on...we're teasing you...are we \ud83e\udd14?", "comments_count": 0, "page_id": "112704783733939", "business_account_id": "17841408147298757"}, "emitted_at": 1677610834990} -{"stream": "media", "data": {"id": "17915811973199640", "media_url": "https://scontent-mrs2-2.cdninstagram.com/v/t51.2885-15/35574691_612934559075657_3171610615386996736_n.jpg?_nc_cat=108&ccb=1-7&_nc_sid=8ae9d6&_nc_ohc=UPpYhQeKF08AX95XXF6&_nc_ht=scontent-mrs2-2.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfCo5yj_JzcOAxPi61wATijfwkTusLcoZZdnQJosZz8Dbg&oe=64024D5C", "timestamp": "2018-06-25T09:23:03+0000", "shortcode": "BkcUW0-DK2x", "owner": {"id": "17841408147298757"}, "like_count": 0, "is_comment_enabled": true, "permalink": "https://www.instagram.com/p/BkcUW0-DK2x/", "ig_id": "1809410679930400177", "media_type": "IMAGE", "username": "airbytehq", "caption": "\"Can we postpone this?\" \"Nope\" \"Ok...\"", "comments_count": 0, "page_id": "112704783733939", "business_account_id": "17841408147298757"}, "emitted_at": 1677610834990} +{"stream": "users", "data": {"id": "17841408147298757", "website": "https://www.airbyte.io/", "biography": "Airbyte is the new open-source data integration platform that consolidates your data into your warehouses.", "profile_picture_url": "https://scontent-iev1-1.xx.fbcdn.net/v/t51.2885-15/153169696_890787328349641_8382928081987798464_n.jpg?_nc_cat=111&_nc_sid=7d201b&_nc_ohc=_R1t6pmIljcAX_4SCqZ&_nc_ht=scontent-iev1-1.xx&edm=AL-3X8kEAAAA&oh=00_AfBjHz1cywYrZ7Epl5qfAv74F0nHFgsNQ3h6yJqD9A6_5w&oe=6550EE3E", "media_count": 258, "name": "Jean Lafleur", "username": "airbytehq", "followers_count": 1253, "follows_count": 14, "ig_id": 8070063576, "page_id": "144706962067225"}, "emitted_at": 1699532816445} +{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_city", "date": "2023-11-09T08:00:00+0000", "value": {"London, England": 7, "Sydney, New South Wales": 19, "Atlanta, Georgia": 5, "Algiers, Algiers Province": 5, "S\u00e3o Paulo, S\u00e3o Paulo (state)": 13, "Rio de Janeiro, Rio de Janeiro (state)": 5, "Hong Kong, Hong Kong": 4, "Montreal, Quebec": 4, "Berlin, Berlin": 7, "Kolkata, West Bengal": 4, "Tulsa, Oklahoma": 4, "Lagos, Lagos State": 18, "Ahmedabad, Gujarat": 4, "Skopje, Municipality of Centar (Skopje)": 4, "Moscow, Moscow": 5, "Bogot\u00e1, Distrito Especial": 5, "Karachi, Sindh": 5, "Dar es Salaam, Dar es Salaam": 7, "Jakarta, Jakarta": 10, "Accra, Greater Accra Region": 4, "Buenos Aires, Ciudad Aut\u00f3noma de Buenos Aires": 9, "Melbourne, Victoria": 7, "Delhi, Delhi": 5, "Gurugram, Haryana": 6, "Kuala Lumpur, Kuala Lumpur": 4, "Los Angeles, California": 5, "Lima, Lima Region": 5, "Istanbul, Istanbul Province": 9, "Toronto, Ontario": 4, "Chennai, Tamil Nadu": 6, "Abuja, Federal Capital Territory": 6, "Bangkok, Bangkok": 5, "Mexico City, Distrito Federal": 6, "Cape Town, Western Cape": 5, "San Francisco, California": 6, "Tehran, Tehran Province": 4, "New York, New York": 13, "Cairo, Cairo Governorate": 4, "Santiago, Santiago Metropolitan Region": 6, "Dubai, Dubai": 8, "Mumbai, Maharashtra": 9, "Bangalore, Karnataka": 18, "Singapore, Singapore": 6, "Hyderabad, Telangana": 7, "San Diego, California": 6}}, "emitted_at": 1699532816665} +{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_country", "date": "2023-11-09T08:00:00+0000", "value": {"DE": 31, "BD": 4, "TW": 5, "RU": 9, "PT": 4, "TZ": 8, "FR": 9, "SA": 7, "BR": 63, "SE": 6, "MA": 6, "SG": 6, "DZ": 6, "ID": 29, "GB": 46, "CA": 26, "US": 263, "GH": 4, "EG": 10, "AE": 9, "CH": 7, "IN": 125, "ZA": 16, "IQ": 6, "CL": 9, "IR": 12, "GR": 6, "IT": 19, "MX": 23, "MY": 9, "CO": 11, "ES": 12, "VE": 9, "AR": 23, "TH": 7, "AU": 35, "PE": 5, "PH": 7, "NG": 30, "TN": 6, "PK": 10, "PL": 5, "TR": 10, "NL": 13, "LK": 4}}, "emitted_at": 1699532816666} +{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_gender_age", "date": "2023-11-09T08:00:00+0000", "value": {"F.18-24": 11, "F.25-34": 76, "F.35-44": 71, "F.45-54": 17, "F.55-64": 1, "F.65+": 2, "M.13-17": 2, "M.18-24": 49, "M.25-34": 366, "M.35-44": 226, "M.45-54": 83, "M.55-64": 20, "M.65+": 12, "U.18-24": 18, "U.25-34": 67, "U.35-44": 43, "U.45-54": 19, "U.55-64": 5}}, "emitted_at": 1699532816666} +{"stream": "media", "data": {"id": "17884386203808767", "comments_count": 2, "owner": {"id": "17841408147298757"}, "caption": "Terraform Explained Part 1\n.\n.\n.\n#airbyte #dataengineering #tech #terraform #cloud #cloudengineer #coding #reels", "ig_id": "3123724930722523505", "media_type": "VIDEO", "is_comment_enabled": true, "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/B34BFFBB0614049AD69F066D153FDD8C_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=107&vs=986202625710684_1200838240&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC9CMzRCRkZCQjA2MTQwNDlBRDY5RjA2NkQxNTNGREQ4Q192aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dDQm9HQlV3a2JxUWwtY0JBRnZGTnFBUkdQeHpicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJtDf4euHnbtAFQIoAkMzLBdAUBtDlYEGJRgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfDob4qgnW95lOhpSj4VrMWJE0DgJkqaAYcbkQPd3KrQJA&oe=654EAA9A&_nc_sid=1d576d", "permalink": "https://www.instagram.com/reel/CtZs0Y3v2lx/", "timestamp": "2023-06-12T19:20:02+0000", "like_count": 9, "username": "airbytehq", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/353022694_609901831117241_2447211336606431614_n.jpg?_nc_cat=100&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=8xgXGzm1iRIAX-Q2FuK&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfCc0YkR683nBbUGvqI89zTzwegJXwxKVIWCeBb0O4Zjkg&oe=65529A0E", "shortcode": "CtZs0Y3v2lx", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1699532772760} +{"stream": "media", "data": {"id": "17864256500936159", "comments_count": 0, "owner": {"id": "17841408147298757"}, "caption": "When and why you should be using Rust for Data Engineering! \n\n#rust #airbyte #coding #programming #tech #dataengineering #data", "ig_id": "3106359072491902976", "media_type": "VIDEO", "is_comment_enabled": true, "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/BE4F848CC97FBA35A1AE1B1150B989A7_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=110&vs=6290041361087047_1877877688&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC9CRTRGODQ4Q0M5N0ZCQTM1QTFBRTFCMTE1MEI5ODlBN192aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dEaE94aFJJdk1BWGZaWURBQXQyS0FLWWxOSlhicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJrD%2B6LaRwf1AFQIoAkMzLBdARDmZmZmZmhgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfDynJmluL-Qe4y6WoQEQnoAMIDsrp-Gwb8793zMm-K1hA&oe=654EBF26&_nc_sid=1d576d", "permalink": "https://www.instagram.com/reel/CscAR5EsRgA/", "timestamp": "2023-05-19T20:08:33+0000", "like_count": 6, "username": "airbytehq", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/347441626_604256678433845_716271787932876577_n.jpg?_nc_cat=108&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=v0GfB4fQB2wAX_epYcL&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfDXZqVwGSKkoHRF4VI4fVDWGOhSJ2V4d6yHnTX4w3ultg&oe=65515ED9", "shortcode": "CscAR5EsRgA", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1699532772760} +{"stream": "media", "data": {"id": "17964324206288599", "comments_count": 0, "owner": {"id": "17841408147298757"}, "caption": "We've all been there right? \ud83e\udd23\n\n#airbyte #data #dataengineering #datascience #dataanalytics #tech #softwareengineer", "ig_id": "3104241732634871967", "media_type": "VIDEO", "is_comment_enabled": true, "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/274503D36EA0F6E79A7CF3797A8D5985_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNTc2LmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=106&vs=1336282350269744_3931649106&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC8yNzQ1MDNEMzZFQTBGNkU3OUE3Q0YzNzk3QThENTk4NV92aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dQdzNzaFRId3VlSlBFWURBSDFmTjUzcUNhd0JicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJrDwmtqO44lAFQIoAkMzLBdAIewIMSbpeRgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfAEnsrWGqakgnSxAhFmKsqTAukx-v52gXNECxPa7OtZJQ&oe=654E4FB2&_nc_sid=1d576d", "permalink": "https://www.instagram.com/reel/CsUe2iqpQif/", "timestamp": "2023-05-16T22:01:45+0000", "like_count": 13, "username": "airbytehq", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/347429218_1848940842145573_5975413208994727174_n.jpg?_nc_cat=101&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=AGiJHqHavZIAX8DSQPd&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfBNvwjxSb-6D_Vl9QyngS1zEkIASrT2WeQ62gqNfebnCw&oe=65514B7C", "shortcode": "CsUe2iqpQif", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1699532772761} diff --git a/airbyte-integrations/connectors/source-instagram/metadata.yaml b/airbyte-integrations/connectors/source-instagram/metadata.yaml index d15bd17a275f..010093c987a2 100644 --- a/airbyte-integrations/connectors/source-instagram/metadata.yaml +++ b/airbyte-integrations/connectors/source-instagram/metadata.yaml @@ -2,10 +2,12 @@ data: allowedHosts: hosts: - graph.facebook.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 6acf6b55-4f1e-4fca-944e-1a3caef8aba8 - dockerImageTag: 1.0.11 + dockerImageTag: 1.0.12 dockerRepository: airbyte/source-instagram githubIssueLabel: source-instagram icon: instagram.svg diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/common.py b/airbyte-integrations/connectors/source-instagram/source_instagram/common.py index 1899bbead023..25f8673d0ebf 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/common.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/common.py @@ -37,6 +37,20 @@ def should_retry_api_error(exc: FacebookRequestError): if exc.api_error_code() in (4, 17, 32, 613): return True + if ( + exc.http_status() == status_codes.INTERNAL_SERVER_ERROR + and exc.api_error_code() == 1 + and exc.api_error_message() == "Please reduce the amount of data you're asking for, then retry your request" + ): + return True + + if ( + exc.http_status() == status_codes.INTERNAL_SERVER_ERROR + and exc.api_error_code() == 1 + and exc.api_error_message() == "An unknown error occurred" + ): + return True + if exc.http_status() == status_codes.TOO_MANY_REQUESTS: return True diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py index eb76eaaa4ad9..74ca5fd2837d 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py @@ -299,6 +299,11 @@ def test_stories_insights_read(api, requests_mock, user_stories_data, user_media {"json": {"error": {"type": "OAuthException", "code": 1}}}, {"json": {"error": {"code": 4}}}, {"json": {}, "status_code": 429}, + { + "json": {"error": {"code": 1, "message": "Please reduce the amount of data you're asking for, then retry your request"}}, + "status_code": 500, + }, + {"json": {"error": {"code": 1, "message": "An unknown error occurred"}}, "status_code": 500}, {"json": {"error": {"type": "OAuthException", "message": "(#10) Not enough viewers for the media to show insights", "code": 10}}}, {"json": {"error": {"code": 100, "error_subcode": 33}}, "status_code": 400}, {"json": {"error": {"is_transient": True}}}, @@ -308,6 +313,8 @@ def test_stories_insights_read(api, requests_mock, user_stories_data, user_media "oauth_error", "rate_limit_error", "too_many_request_error", + "reduce_amount_of_data_error", + "unknown_error", "viewers_insights_error", "4028_issue_error", "transient_error", diff --git a/docs/integrations/sources/instagram.md b/docs/integrations/sources/instagram.md index cfe5d8b7c9a6..99cf3aa15cb2 100644 --- a/docs/integrations/sources/instagram.md +++ b/docs/integrations/sources/instagram.md @@ -87,6 +87,7 @@ AirbyteRecords are required to conform to the [Airbyte type](https://docs.airbyt | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------ | +| 1.0.12 | 2023-11-07 | [32200](https://github.com/airbytehq/airbyte/pull/32200) | The backoff strategy has been updated to make some errors retriable | | 1.0.11 | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes | | 1.0.10 | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references | | 1.0.9 | 2023-07-01 | [27908](https://github.com/airbytehq/airbyte/pull/27908) | Fix bug when `user_lifetime_insights` stream returns `Key Error (end_time)`, refactored `state` to use `IncrementalMixin` | From b7f8a72be80b8633a85b3b9095e1e8826f8a6905 Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk <35109939+tolik0@users.noreply.github.com> Date: Fri, 10 Nov 2023 17:08:27 +0200 Subject: [PATCH 003/124] =?UTF-8?q?=F0=9F=90=9B=20Source=20Google=20Ads:?= =?UTF-8?q?=20Add=20backoff=20strategy=20for=20read=5Frecords=20method=20(?= =?UTF-8?q?#32414)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../source-google-ads/metadata.yaml | 2 +- .../source_google_ads/streams.py | 95 ++++--------- .../source_google_ads/utils.py | 128 +++++++++++++++++- .../unit_tests/test_utils.py | 78 ++++++++++- docs/integrations/sources/google-ads.md | 1 + 5 files changed, 231 insertions(+), 73 deletions(-) diff --git a/airbyte-integrations/connectors/source-google-ads/metadata.yaml b/airbyte-integrations/connectors/source-google-ads/metadata.yaml index 9df42221f3ff..c69732e864ff 100644 --- a/airbyte-integrations/connectors/source-google-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-ads/metadata.yaml @@ -11,7 +11,7 @@ data: connectorSubtype: api connectorType: source definitionId: 253487c0-2246-43ba-a21f-5116b20a2c50 - dockerImageTag: 2.0.3 + dockerImageTag: 2.0.4 dockerRepository: airbyte/source-google-ads documentationUrl: https://docs.airbyte.com/integrations/sources/google-ads githubIssueLabel: source-google-ads diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py index dca2ebed9fad..af7bf549b25d 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py @@ -3,8 +3,9 @@ # from abc import ABC, abstractmethod -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional +from typing import Any, Iterable, Iterator, List, Mapping, MutableMapping, Optional +import backoff import pendulum from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import IncrementalMixin, Stream @@ -12,75 +13,13 @@ from airbyte_cdk.utils import AirbyteTracedException from airbyte_protocol.models import FailureType from google.ads.googleads.errors import GoogleAdsException -from google.ads.googleads.v11.services.services.google_ads_service.pagers import SearchPager +from google.ads.googleads.v13.services.services.google_ads_service.pagers import SearchPager +from google.ads.googleads.v13.services.types.google_ads_service import SearchGoogleAdsResponse +from google.api_core.exceptions import InternalServerError, ServerError, ServiceUnavailable, TooManyRequests -from .google_ads import GoogleAds +from .google_ads import GoogleAds, logger from .models import CustomerModel -from .utils import ExpiredPageTokenError, get_resource_name, traced_exception - - -def parse_dates(stream_slice): - start_date = pendulum.parse(stream_slice["start_date"]) - end_date = pendulum.parse(stream_slice["end_date"]) - return start_date, end_date - - -def chunk_date_range( - start_date: str, - end_date: str = None, - conversion_window: int = 0, - days_of_data_storage: int = None, - time_zone=None, - time_format="YYYY-MM-DD", - slice_duration: pendulum.Duration = pendulum.duration(days=14), - slice_step: pendulum.Duration = pendulum.duration(days=1), -) -> Iterable[Optional[MutableMapping[str, any]]]: - """ - Splits a date range into smaller chunks based on the provided parameters. - - Args: - start_date (str): The beginning date of the range. - end_date (str, optional): The ending date of the range. Defaults to today's date. - conversion_window (int): Number of days to subtract from the start date. Defaults to 0. - days_of_data_storage (int, optional): Maximum age of data that can be retrieved. Used to adjust the start date. - time_zone: Time zone to be used for date parsing and today's date calculation. If not provided, the default time zone is used. - time_format (str): Format to be used when returning dates. Defaults to 'YYYY-MM-DD'. - slice_duration (pendulum.Duration): Duration of each chunk. Defaults to 14 days. - slice_step (pendulum.Duration): Step size to move to the next chunk. Defaults to 1 day. - - Returns: - Iterable[Optional[MutableMapping[str, any]]]: An iterable of dictionaries containing start and end dates for each chunk. - If the adjusted start date is greater than the end date, returns a list with a None value. - - Notes: - - If the difference between `end_date` and `start_date` is large (e.g., >= 1 month), processing all records might take a long time. - - Tokens for fetching subsequent pages of data might expire after 2 hours, leading to potential errors. - - The function adjusts the start date based on `days_of_data_storage` and `conversion_window` to adhere to certain data retrieval policies, such as Google Ads' policy of only retrieving data not older than a certain number of days. - - The method returns `start_date` and `end_date` with a difference typically spanning 15 days to avoid token expiration issues. - """ - start_date = pendulum.parse(start_date, tz=time_zone) - today = pendulum.today(tz=time_zone) - end_date = pendulum.parse(end_date, tz=time_zone) if end_date else today - - # For some metrics we can only get data not older than N days, it is Google Ads policy - if days_of_data_storage: - start_date = max(start_date, pendulum.now(tz=time_zone).subtract(days=days_of_data_storage - conversion_window)) - - # As in to return some state when state in abnormal - if start_date > end_date: - return [None] - - # applying conversion window - start_date = start_date.subtract(days=conversion_window) - slice_start = start_date - - while slice_start <= end_date: - slice_end = min(end_date, slice_start + slice_duration) - yield { - "start_date": slice_start.format(time_format), - "end_date": slice_end.format(time_format), - } - slice_start = slice_end + slice_step +from .utils import ExpiredPageTokenError, chunk_date_range, generator_backoff, get_resource_name, parse_dates, traced_exception class GoogleAdsStream(Stream, ABC): @@ -111,11 +50,27 @@ def read_records(self, sync_mode, stream_slice: Optional[Mapping[str, Any]] = No customer_id = stream_slice["customer_id"] try: response_records = self.google_ads_client.send_request(self.get_query(stream_slice), customer_id=customer_id) - for response in response_records: - yield from self.parse_response(response, stream_slice) + + yield from self.parse_records_with_backoff(response_records, stream_slice) except GoogleAdsException as exception: traced_exception(exception, customer_id, self.CATCH_CUSTOMER_NOT_ENABLED_ERROR) + @generator_backoff( + wait_gen=backoff.expo, + exception=(InternalServerError, ServerError, ServiceUnavailable, TooManyRequests), + max_tries=5, + max_time=600, + on_backoff=lambda details: logger.info( + f"Caught retryable error {details['exception']} after {details['tries']} tries. Waiting {details['wait']} seconds then retrying..." + ), + factor=5, + ) + def parse_records_with_backoff( + self, response_records: Iterator[SearchGoogleAdsResponse], stream_slice: Optional[Mapping[str, Any]] = None + ) -> Iterable[Mapping[str, Any]]: + for response in response_records: + yield from self.parse_response(response, stream_slice) + class IncrementalGoogleAdsStream(GoogleAdsStream, IncrementalMixin, ABC): primary_key = None diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py index a6f9af66e6bd..1b7f938f7748 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py @@ -3,9 +3,12 @@ # import re +import time from dataclasses import dataclass -from typing import Optional, Tuple +from datetime import datetime +from typing import Any, Callable, Generator, Iterable, MutableMapping, Optional, Tuple, Type, Union +import pendulum from airbyte_cdk.models import FailureType from airbyte_cdk.utils import AirbyteTracedException from google.ads.googleads.errors import GoogleAdsException @@ -117,6 +120,129 @@ def traced_exception(ga_exception: GoogleAdsException, customer_id: str, catch_d raise raise_exception.from_exception(failure_type=failure_type, exc=ga_exception, message=message) from ga_exception +def generator_backoff( + wait_gen: Callable, + exception: Union[Type[Exception], tuple], + max_tries: Optional[int] = None, + max_time: Optional[float] = None, + on_backoff: Optional[Callable] = None, + **wait_gen_kwargs: Any, +): + def decorator(func: Callable) -> Callable: + def wrapper(*args, **kwargs) -> Generator: + tries = 0 + start_time = datetime.now() + wait_times = wait_gen(**wait_gen_kwargs) + next(wait_times) # Skip the first yield which is None + + while True: + try: + yield from func(*args, **kwargs) + return # If the generator completes without error, return + except exception as e: + tries += 1 + elapsed_time = (datetime.now() - start_time).total_seconds() + + if max_time is not None and elapsed_time >= max_time: + print(f"Maximum time of {max_time} seconds exceeded.") + raise + + if max_tries is not None and tries >= max_tries: + print(f"Maximum tries of {max_tries} exceeded.") + raise + + # Get the next wait time from the exponential decay generator + sleep_time = next(wait_times) + + # Adjust sleep time if it exceeds the remaining max_time + if max_time is not None: + time_remaining = max_time - elapsed_time + sleep_time = min(sleep_time, time_remaining) + + if on_backoff: + on_backoff( + { + "target": func, + "args": args, + "kwargs": kwargs, + "tries": tries, + "elapsed": elapsed_time, + "wait": sleep_time, + "exception": e, + } + ) + + time.sleep(sleep_time) + + return wrapper + + return decorator + + +def parse_dates(stream_slice): + start_date = pendulum.parse(stream_slice["start_date"]) + end_date = pendulum.parse(stream_slice["end_date"]) + return start_date, end_date + + +def chunk_date_range( + start_date: str, + end_date: str = None, + conversion_window: int = 0, + days_of_data_storage: int = None, + time_zone=None, + time_format="YYYY-MM-DD", + slice_duration: pendulum.Duration = pendulum.duration(days=14), + slice_step: pendulum.Duration = pendulum.duration(days=1), +) -> Iterable[Optional[MutableMapping[str, any]]]: + """ + Splits a date range into smaller chunks based on the provided parameters. + + Args: + start_date (str): The beginning date of the range. + end_date (str, optional): The ending date of the range. Defaults to today's date. + conversion_window (int): Number of days to subtract from the start date. Defaults to 0. + days_of_data_storage (int, optional): Maximum age of data that can be retrieved. Used to adjust the start date. + time_zone: Time zone to be used for date parsing and today's date calculation. If not provided, the default time zone is used. + time_format (str): Format to be used when returning dates. Defaults to 'YYYY-MM-DD'. + slice_duration (pendulum.Duration): Duration of each chunk. Defaults to 14 days. + slice_step (pendulum.Duration): Step size to move to the next chunk. Defaults to 1 day. + + Returns: + Iterable[Optional[MutableMapping[str, any]]]: An iterable of dictionaries containing start and end dates for each chunk. + If the adjusted start date is greater than the end date, returns a list with a None value. + + Notes: + - If the difference between `end_date` and `start_date` is large (e.g., >= 1 month), processing all records might take a long time. + - Tokens for fetching subsequent pages of data might expire after 2 hours, leading to potential errors. + - The function adjusts the start date based on `days_of_data_storage` and `conversion_window` to adhere to certain data retrieval policies, such as Google Ads' policy of only retrieving data not older than a certain number of days. + - The method returns `start_date` and `end_date` with a difference typically spanning 15 days to avoid token expiration issues. + """ + start_date = pendulum.parse(start_date, tz=time_zone) + today = pendulum.today(tz=time_zone) + end_date = pendulum.parse(end_date, tz=time_zone) if end_date else today + + # For some metrics we can only get data not older than N days, it is Google Ads policy + if days_of_data_storage: + start_date = max(start_date, pendulum.now(tz=time_zone).subtract(days=days_of_data_storage - conversion_window)) + + # As in to return some state when state in abnormal + if start_date > end_date: + return [None] + + # applying conversion window + start_date = start_date.subtract(days=conversion_window) + slice_start = start_date + + while slice_start <= end_date: + slice_end = min(end_date, slice_start + slice_duration) + yield { + "start_date": slice_start.format(time_format), + "end_date": slice_end.format(time_format), + } + slice_start = slice_end + slice_step + + @dataclass(repr=False, eq=False, frozen=True) class GAQL: """ diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py index 4700917853bf..78be0956a1dc 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py @@ -2,10 +2,14 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from datetime import datetime +from unittest.mock import Mock + +import backoff import pytest from airbyte_cdk.utils import AirbyteTracedException from source_google_ads import SourceGoogleAds -from source_google_ads.utils import GAQL +from source_google_ads.utils import GAQL, generator_backoff def test_parse_GAQL_ok(): @@ -124,3 +128,75 @@ def test_parse_GAQL_fail(config): ) def test_get_query_fields(query, fields): assert list(GAQL.parse(query).fields) == fields + + +def test_generator_backoff_retries_until_success(): + tries = 0 + + def flaky_function(): + nonlocal tries # Declare tries as nonlocal to modify it within the function + if tries < 2: + tries += 1 + raise ValueError("Simulated failure") + else: + yield "Success" + + # Mock on_backoff callable + mock_on_backoff = Mock() + + # Apply the decorator to the flaky_function + decorated_flaky_function = generator_backoff( + wait_gen=backoff.expo, + exception=ValueError, + max_tries=4, + max_time=5, + on_backoff=mock_on_backoff, + factor=2, + )(flaky_function) + + # Start the clock + start_time = datetime.now() + + # Run the decorated function and collect results + results = list(decorated_flaky_function()) + + # Check that the function succeeded after retries + assert results == ["Success"] + + # Check that the function was retried the correct number of times + assert mock_on_backoff.call_count == 2 + + # Check that the elapsed time is reasonable + elapsed_time = (datetime.now() - start_time).total_seconds() + # The wait times are 3 and then 2 seconds, so the elapsed time should be at least 5 seconds + assert elapsed_time >= 5 + + # Check that on_backoff was called with the correct parameters + expected_calls = [ + { + "target": flaky_function, + "args": (), + "kwargs": {}, + "tries": 1, + "elapsed": pytest.approx(0.1, abs=0.1), + "wait": pytest.approx(2, abs=0.1), + "exception": "Simulated failure", + }, + { + "target": flaky_function, + "args": (), + "kwargs": {}, + "tries": 2, + "elapsed": pytest.approx(2, abs=0.1), + "wait": pytest.approx(3, abs=0.1), + "exception": "Simulated failure", + }, + ] + + # Convert actual calls to a list of dictionaries + actual_calls = [{**c.args[0], "exception": str(c.args[0]["exception"])} for c in mock_on_backoff.call_args_list] + print(actual_calls) + + # Compare each expected call with the actual call + for expected, actual in zip(expected_calls, actual_calls): + assert expected == actual diff --git a/docs/integrations/sources/google-ads.md b/docs/integrations/sources/google-ads.md index d9085e91de50..318b30508a54 100644 --- a/docs/integrations/sources/google-ads.md +++ b/docs/integrations/sources/google-ads.md @@ -278,6 +278,7 @@ Due to a limitation in the Google Ads API which does not allow getting performan | Version | Date | Pull Request | Subject | |:---------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------| +| `2.0.4` | 2023-11-10 | [32414](https://github.com/airbytehq/airbyte/pull/32414) | Add backoff strategy for read_records method | | `2.0.3` | 2023-11-02 | [32102](https://github.com/airbytehq/airbyte/pull/32102) | Fix incremental events streams | | `2.0.2` | 2023-10-31 | [32001](https://github.com/airbytehq/airbyte/pull/32001) | Added handling (retry) for `InternalServerError` while reading the streams | | `2.0.1` | 2023-10-27 | [31908](https://github.com/airbytehq/airbyte/pull/31908) | Base image migration: remove Dockerfile and use the python-connector-base image | From c09be57b274975b3363e6a8d376cc45ca91746a8 Mon Sep 17 00:00:00 2001 From: Daryna Ishchenko <80129833+darynaishchenko@users.noreply.github.com> Date: Fri, 10 Nov 2023 17:47:16 +0200 Subject: [PATCH 004/124] :sparkles: Source Bing Ads: optional reports start date (#32340) Co-authored-by: darynaishchenko Co-authored-by: Artem Inzhyyants <36314070+artem1205@users.noreply.github.com> Co-authored-by: Artem Inzhyyants --- .../acceptance-test-config.yml | 4 +- .../configured_catalog_full_refresh.json | 270 ++++++++++++++++++ .../connectors/source-bing-ads/metadata.yaml | 2 +- .../source-bing-ads/source_bing_ads/client.py | 7 +- .../source_bing_ads/reports.py | 19 +- .../source-bing-ads/source_bing_ads/spec.json | 12 +- .../source-bing-ads/unit_tests/test_source.py | 17 +- docs/integrations/sources/bing-ads.md | 10 +- 8 files changed, 317 insertions(+), 24 deletions(-) create mode 100644 airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog_full_refresh.json diff --git a/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml index eeb6b054fd3a..cf9c2c522a94 100644 --- a/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml @@ -4,6 +4,8 @@ acceptance_tests: spec: tests: - spec_path: source_bing_ads/spec.json + backward_compatibility_tests_config: + disable_for_version: "1.11.0" discovery: tests: - config_path: secrets/config.json @@ -52,7 +54,7 @@ acceptance_tests: full_refresh: tests: - config_path: secrets/config.json - configured_catalog_path: integration_tests/configured_catalog.json + configured_catalog_path: integration_tests/configured_catalog_full_refresh.json timeout_seconds: 4800 incremental: tests: diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog_full_refresh.json b/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog_full_refresh.json new file mode 100644 index 000000000000..9eb62db8fddc --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog_full_refresh.json @@ -0,0 +1,270 @@ +{ + "streams": [ + { + "stream": { + "name": "ad_groups", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ads", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "campaigns", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "accounts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "account_performance_report_hourly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "account_performance_report_daily", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "account_performance_report_weekly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "account_performance_report_monthly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_group_performance_report_monthly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_performance_report_monthly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "geographic_performance_report_monthly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "budget_summary_report", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "campaign_performance_report_monthly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "campaign_impression_performance_report_monthly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "keyword_performance_report_monthly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "age_gender_audience_report_monthly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "search_query_performance_report_monthly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "user_location_performance_report_monthly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "account_impression_performance_report_monthly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "labels", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["Modified Time"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "app_install_ads", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "app_install_ad_labels", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "campaign_labels", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "keyword_labels", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_group_labels", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "keywords", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_group_impression_performance_report_monthly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml index 13514fb09236..10fa87f954ae 100644 --- a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml @@ -16,7 +16,7 @@ data: connectorSubtype: api connectorType: source definitionId: 47f25999-dd5e-4636-8c39-e7cea2453331 - dockerImageTag: 1.11.0 + dockerImageTag: 1.12.0 dockerRepository: airbyte/source-bing-ads documentationUrl: https://docs.airbyte.com/integrations/sources/bing-ads githubIssueLabel: source-bing-ads diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py index b975d77050e0..c07e4d158f9d 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py @@ -49,10 +49,12 @@ class Client: _download_timeout = 300000 _max_download_timeout = 600000 + reports_start_date = None + def __init__( self, tenant_id: str, - reports_start_date: str, + reports_start_date: str = None, developer_token: str = None, client_id: str = None, client_secret: str = None, @@ -67,7 +69,8 @@ def __init__( self.authentication = self._get_auth_client(client_id, tenant_id, client_secret) self.oauth: OAuthTokens = self._get_access_token() - self.reports_start_date = pendulum.parse(reports_start_date).astimezone(tz=timezone.utc) + if reports_start_date: + self.reports_start_date = pendulum.parse(reports_start_date).astimezone(tz=timezone.utc) def _get_auth_client(self, client_id: str, tenant_id: str, client_secret: str = None) -> OAuthWebAuthCodeGrant: # https://github.com/BingAds/BingAds-Python-SDK/blob/e7b5a618e87a43d0a5e2c79d9aa4626e208797bd/bingads/authorization.py#L390 diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py index 3e88086d361d..731954bad986 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py @@ -148,6 +148,8 @@ class ReportsMixin(ABC): # timeout for reporting download operations in milliseconds timeout: int = 300000 report_file_format: str = "Csv" + # used when reports start date is not provided + time_periods = ["LastYear", "ThisYear"] transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) primary_key: List[str] = ["TimePeriod", "Network", "DeviceType"] @@ -205,16 +207,22 @@ def get_request_date(self, reporting_service: ServiceClient, date: datetime) -> def request_params( self, stream_state: Mapping[str, Any] = None, account_id: str = None, **kwargs: Mapping[str, Any] ) -> Mapping[str, Any]: + stream_slice = kwargs["stream_slice"] start_date = self.get_start_date(stream_state, account_id) reporting_service = self.client.get_service("ReportingService") request_time_zone = reporting_service.factory.create("ReportTimeZone") report_time = reporting_service.factory.create("ReportTime") - report_time.CustomDateRangeStart = self.get_request_date(reporting_service, start_date) - report_time.CustomDateRangeEnd = self.get_request_date(reporting_service, datetime.utcnow()) - report_time.PredefinedTime = None report_time.ReportTimeZone = request_time_zone.GreenwichMeanTimeDublinEdinburghLisbonLondon + if start_date: + report_time.CustomDateRangeStart = self.get_request_date(reporting_service, start_date) + report_time.CustomDateRangeEnd = self.get_request_date(reporting_service, datetime.utcnow()) + report_time.PredefinedTime = None + else: + report_time.CustomDateRangeStart = None + report_time.CustomDateRangeEnd = None + report_time.PredefinedTime = stream_slice["time_period"] report_request = self.get_report_request(account_id, False, False, False, self.report_file_format, False, report_time) @@ -310,7 +318,8 @@ def stream_slices( **kwargs: Mapping[str, Any], ) -> Iterable[Optional[Mapping[str, Any]]]: for account in source_bing_ads.source.Accounts(self.client, self.config).read_records(SyncMode.full_refresh): - yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} + for period in self.time_periods: + yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"], "time_period": period} yield from [] @@ -319,7 +328,7 @@ class PerformanceReportsMixin(ReportsMixin): def get_start_date(self, stream_state: Mapping[str, Any] = None, account_id: str = None): start_date = super().get_start_date(stream_state, account_id) - if self.config.get("lookback_window"): + if self.config.get("lookback_window") and start_date: # Datetime subtract won't work with days = 0 # it'll output an AirbyteError return start_date.subtract(days=self.config["lookback_window"]) diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json index 613e9d2b969d..e7314b86a022 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json @@ -4,12 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Bing Ads Spec", "type": "object", - "required": [ - "developer_token", - "client_id", - "refresh_token", - "reports_start_date" - ], + "required": ["developer_token", "client_id", "refresh_token"], "additionalProperties": true, "properties": { "auth_method": { @@ -57,13 +52,12 @@ "type": "string", "title": "Reports replication start date", "format": "date", - "default": "2020-01-01", - "description": "The start date from which to begin replicating report data. Any data generated before this date will not be replicated in reports. This is a UTC date in YYYY-MM-DD format.", + "description": "The start date from which to begin replicating report data. Any data generated before this date will not be replicated in reports. This is a UTC date in YYYY-MM-DD format. If not set, data from previous and current calendar year will be replicated.", "order": 5 }, "lookback_window": { "title": "Lookback window", - "description": "Also known as attribution or conversion window. How far into the past to look for records (in days). If your conversion window has an hours/minutes granularity, round it up to the number of days exceeding. Used only for performance report streams in incremental mode.", + "description": "Also known as attribution or conversion window. How far into the past to look for records (in days). If your conversion window has an hours/minutes granularity, round it up to the number of days exceeding. Used only for performance report streams in incremental mode without specified Reports Start Date.", "type": "integer", "default": 0, "minimum": 0, diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py index 7531e69f707f..f0f69c63faad 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py @@ -142,7 +142,7 @@ def test_ads_stream_slices(mocked_client, config): def test_AccountPerformanceReportMonthly_request_params(mocked_client, config): accountperformancereportmonthly = AccountPerformanceReportMonthly(mocked_client, config) - request_params = accountperformancereportmonthly.request_params(account_id=180278106) + request_params = accountperformancereportmonthly.request_params(account_id=180278106, stream_slice={"time_period": "ThisYear"}) del request_params["report_request"] assert request_params == { "overwrite_result_file": True, @@ -153,6 +153,21 @@ def test_AccountPerformanceReportMonthly_request_params(mocked_client, config): } +@patch.object(source_bing_ads.source, "Client") +def test_AccountPerformanceReportMonthly_stream_slices(mocked_client, config): + + accountperformancereportmonthly = AccountPerformanceReportMonthly(mocked_client, config) + accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) + with patch.object(Accounts, "read_records", return_value=accounts_read_records): + stream_slice = list(accountperformancereportmonthly.stream_slices()) + assert stream_slice == [ + {"account_id": 180519267, "customer_id": 100, "time_period": "LastYear"}, + {"account_id": 180519267, "customer_id": 100, "time_period": "ThisYear"}, + {"account_id": 180278106, "customer_id": 200, "time_period": "LastYear"}, + {"account_id": 180278106, "customer_id": 200, "time_period": "ThisYear"}, + ] + + @pytest.mark.parametrize( ("stream", "stream_slice"), ( diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md index a2164dc8f938..a269d461261a 100644 --- a/docs/integrations/sources/bing-ads.md +++ b/docs/integrations/sources/bing-ads.md @@ -5,7 +5,6 @@ This page contains the setup guide and reference information for the Bing Ads so ## Prerequisites - Microsoft Advertising account - Microsoft Developer Token -- Reports start date ## Setup guide @@ -52,8 +51,8 @@ The tenant is used in the authentication URL, for example: `https://login.micros 4. Enter a name for your source. 5. For **Tenant ID**, enter the custom tenant or use the common tenant. 6. Add the developer token from [Step 1](#step-1-set-up-bing-ads). -7. For **Replication Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. -8. For **Lookback window** (also known as attribution or conversion window) enter the number of **days** to look into the past. If your conversion window has an hours/minutes granularity, round it up to the number of days exceeding. If you're not using performance report streams in incremental mode, let it with 0 default value. +7. For **Reports Replication Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data from previous and current calendar years. +8. For **Lookback window** (also known as attribution or conversion window) enter the number of **days** to look into the past. If your conversion window has an hours/minutes granularity, round it up to the number of days exceeding. If you're not using performance report streams in incremental mode and Reports Start Date is not provided, let it with 0 default value. 9. Click **Authenticate your Bing Ads account**. 10. Log in and authorize the Bing Ads account. 11. Click **Set up source**. @@ -69,8 +68,8 @@ The tenant is used in the authentication URL, for example: `https://login.micros 4. Enter a name for your source. 5. For **Tenant ID**, enter the custom tenant or use the common tenant. 6. Enter the **Client ID**, **Client Secret**, **Refresh Token**, and **Developer Token** from [Step 1](#step-1-set-up-bing-ads). -7. For **Replication Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. -8. For **Lookback window** (also known as attribution or conversion window) enter the number of **days** to look into the past. If your conversion window has an hours/minutes granularity, round it up to the number of days exceeding. If you're not using performance report streams in incremental mode, let it with 0 default value. +7. For **Reports Replication Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data from previous and current calendar years. +8. For **Lookback window** (also known as attribution or conversion window) enter the number of **days** to look into the past. If your conversion window has an hours/minutes granularity, round it up to the number of days exceeding. If you're not using performance report streams in incremental mode and Reports Start Date is not provided, let it with 0 default value. 9. Click **Set up source**. @@ -169,6 +168,7 @@ The Bing Ads API limits the number of requests for all Microsoft Advertising cli | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------| +| 1.12.0 | 2023-11-09 | [32340](https://github.com/airbytehq/airbyte/pull/32340) | Remove default start date in favor of Time Period - Last Year and This Year, if start date is not provided | | 1.11.0 | 2023-11-06 | [32201](https://github.com/airbytehq/airbyte/pull/32201) | Skip broken CSV report files | | 1.10.0 | 2023-11-06 | [32148](https://github.com/airbytehq/airbyte/pull/32148) | Add new fields to stream Ads: "BusinessName", "CallToAction", "Headline", "Images", "Videos", "Text" | | 1.9.0 | 2023-11-03 | [32131](https://github.com/airbytehq/airbyte/pull/32131) | Add "CampaignId", "AccountId", "CustomerId" fields to Ad Groups, Ads and Campaigns streams. | From 0674105d4f19c182de8e470216a80e5e0b36d133 Mon Sep 17 00:00:00 2001 From: Patrick Nilan Date: Fri, 10 Nov 2023 07:57:47 -0800 Subject: [PATCH 005/124] Docs: Updates Google Sheets Source Connector Docs (Removes inapp page, adds , adds Limitations & Troubleshooting) (#32358) --- .../sources/google-sheets.inapp.md | 45 ----------- docs/integrations/sources/google-sheets.md | 81 ++++++++++++------- 2 files changed, 51 insertions(+), 75 deletions(-) delete mode 100644 docs/integrations/sources/google-sheets.inapp.md diff --git a/docs/integrations/sources/google-sheets.inapp.md b/docs/integrations/sources/google-sheets.inapp.md deleted file mode 100644 index 3e8374aef9d2..000000000000 --- a/docs/integrations/sources/google-sheets.inapp.md +++ /dev/null @@ -1,45 +0,0 @@ -## Prerequisites -- Spreadsheet Link - The link to the Google spreadsheet you want to sync. -- A Google Workspace user with access to the spreadsheet - -:::info -The Google Sheets source connector pulls data from a single Google Sheets spreadsheet. To replicate multiple spreadsheets, set up multiple Google Sheets source connectors in your Airbyte instance. -::: - -## Setup guide - -1. For **Source name**, enter a name to help you identify this source. -2. Select your authentication method: - - - -#### For Airbyte Cloud - -- **(Recommended)** Select **Authenticate via Google (OAuth)** from the Authentication dropdown, click **Sign in with Google** and complete the authentication workflow. - - - - -#### For Airbyte Open Source - -- **(Recommended)** Select **Service Account Key Authentication** from the dropdown and enter your Google Cloud service account key in JSON format: - - ```js - { "type": "service_account", "project_id": "YOUR_PROJECT_ID", "private_key_id": "YOUR_PRIVATE_KEY", ... } - ``` - -- To authenticate your Google account via OAuth, select **Authenticate via Google (OAuth)** from the dropdown and enter your Google application's client ID, client secret, and refresh token. - -For detailed instructions on how to generate a service account key or OAuth credentials, refer to the [full documentation](https://docs.airbyte.io/integrations/sources/google-sheets#setup-guide). - - - -3. For **Spreadsheet Link**, enter the link to the Google spreadsheet. To get the link, go to the Google spreadsheet you want to sync, click **Share** in the top right corner, and click **Copy Link**. -4. (Optional) You may enable the option to **Convert Column Names to SQL-Compliant Format**. Enabling this option will allow the connector to convert column names to a standardized, SQL-friendly format. For example, a column name of `Café Earnings 2022` will be converted to `cafe_earnings_2022`. We recommend enabling this option if your target destination is SQL-based (ie Postgres, MySQL). Set to false by default. -5. Click **Set up source** and wait for the tests to complete. - -### Output schema - -- Airbyte only supports replicating [Grid](https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#SheetType) sheets. - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Google Sheets](https://docs.airbyte.com/integrations/sources/google-sheets/). diff --git a/docs/integrations/sources/google-sheets.md b/docs/integrations/sources/google-sheets.md index 10ea617d0d0c..4fb8abb013c2 100644 --- a/docs/integrations/sources/google-sheets.md +++ b/docs/integrations/sources/google-sheets.md @@ -1,18 +1,22 @@ # Google Sheets + + This page contains the setup guide and reference information for the Google Sheets source connector. + + :::info -The Google Sheets source connector pulls data from a single Google Sheets spreadsheet. Each sheet (tab) within a spreadsheet can be replicated. To replicate multiple spreadsheets, set up multiple Google Sheets source connectors in your Airbyte instance. No other files in your Google Drive are accessed. +The Google Sheets source connector pulls data from a single Google Sheets spreadsheet. Each sheet within a spreadsheet can be replicated. To replicate multiple spreadsheets, set up multiple Google Sheets source connectors in your Airbyte instance. No other files in your Google Drive are accessed. ::: ### Prerequisites - Spreadsheet Link - The link to the Google spreadsheet you want to sync. -- **For Airbyte Cloud** A Google Workspace user with access to the spreadsheet +- **For Airbyte Cloud** A Google Workspace user with access to the spreadsheet -- **For Airbyte Open Source:** +- **For Airbyte Open Source:** - A GCP project - Enable the Google Sheets API in your GCP project - Service Account Key with access to the Spreadsheet you want to replicate @@ -23,19 +27,23 @@ The Google Sheets source connector pulls data from a single Google Sheets spread The Google Sheets source connector supports authentication via either OAuth or Service Account Key Authentication. -For **Airbyte Cloud** users, we highly recommend using OAuth, as it significantly simplifies the setup process and allows you to authenticate [directly from the Airbyte UI](#set-up-the-google-sheets-source-connector-in-airbyte). +**For Airbyte Cloud:** + +We highly recommend using OAuth, as it significantly simplifies the setup process and allows you to authenticate [directly from the Airbyte UI](#set-up-the-google-sheets-source-connector-in-airbyte). -For **Airbyte Open Source** users, we recommend using Service Account Key Authentication. Follow the steps below to create a service account, generate a key, and enable the Google Sheets API. +**For Airbyte Open Source:** + +We recommend using Service Account Key Authentication. Follow the steps below to create a service account, generate a key, and enable the Google Sheets API. :::note If you prefer to use OAuth for authentication with **Airbyte Open Source**, you can follow [Google's OAuth instructions](https://developers.google.com/identity/protocols/oauth2) to create an authentication app. Be sure to set the scopes to `https://www.googleapis.com/auth/spreadsheets.readonly`. You will need to obtain your client ID, client secret, and refresh token for the connector setup. ::: -### Set up the service account key (Airbyte Open Source) +### Set up the service account key #### Create a service account @@ -64,39 +72,36 @@ If your spreadsheet is viewable by anyone with its link, no further action is ne ### Set up the Google Sheets source connector in Airbyte -To set up Google Sheets as a source in Airbyte Cloud: -1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account. + +1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. 3. Find and select **Google Sheets** from the list of available sources. 4. For **Source name**, enter a name to help you identify this source. 5. Select your authentication method: - - -#### For Airbyte Cloud - -- **(Recommended)** Select **Authenticate via Google (OAuth)** from the Authentication dropdown, click **Sign in with Google** and complete the authentication workflow. - + - **For Airbyte Cloud: (Recommended)** Select **Authenticate via Google (OAuth)** from the Authentication dropdown, click **Sign in with Google** and complete the authentication workflow. - -#### For Airbyte Open Source - -- **(Recommended)** Select **Service Account Key Authentication** from the dropdown and enter your Google Cloud service account key in JSON format: - - ```js - { "type": "service_account", "project_id": "YOUR_PROJECT_ID", "private_key_id": "YOUR_PRIVATE_KEY", ... } - ``` - -- To authenticate your Google account via OAuth, select **Authenticate via Google (OAuth)** from the dropdown and enter your Google application's client ID, client secret, and refresh token. - + - **For Airbyte Open Source: (Recommended)** Select **Service Account Key Authentication** from the dropdown and enter your Google Cloud service account key in JSON format: + + ```json + { + "type": "service_account", + "project_id": "YOUR_PROJECT_ID", + "private_key_id": "YOUR_PRIVATE_KEY", + ... + } + ``` + + - To authenticate your Google account via OAuth, select **Authenticate via Google (OAuth)** from the dropdown and enter your Google application's client ID, client secret, and refresh token. - 6. For **Spreadsheet Link**, enter the link to the Google spreadsheet. To get the link, go to the Google spreadsheet you want to sync, click **Share** in the top right corner, and click **Copy Link**. 7. (Optional) You may enable the option to **Convert Column Names to SQL-Compliant Format**. Enabling this option will allow the connector to convert column names to a standardized, SQL-friendly format. For example, a column name of `Café Earnings 2022` will be converted to `cafe_earnings_2022`. We recommend enabling this option if your target destination is SQL-based (ie Postgres, MySQL). Set to false by default. 8. Click **Set up source** and wait for the tests to complete. + + ### Output schema Each sheet in the selected spreadsheet is synced as a separate stream. Each selected column in the sheet is synced as a string field. @@ -116,7 +121,16 @@ The Google Sheets source connector supports the following sync modes: |:-----------------|:-------------|:------| | any type | `string` | | -## Performance consideration +## Limitations & Troubleshooting + +
+ +Expand to see details about Google Sheets connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting The [Google API rate limits](https://developers.google.com/sheets/api/limits) are: @@ -125,8 +139,13 @@ The [Google API rate limits](https://developers.google.com/sheets/api/limits) ar Airbyte batches requests to the API in order to efficiently pull data and respect these rate limits. We recommend not using the same user or service account for more than 3 instances of the Google Sheets source connector to ensure high transfer speeds. -## Troubleshooting -- If your sheet is completely empty(no header rows) or deleted, Airbyte will not delete the table in the destination. If this happens, the sync logs will contain a message saying the sheet has been skipped when syncing the full spreadsheet. +### Troubleshooting + +* If your sheet is completely empty (no header rows) or deleted, Airbyte will not delete the table in the destination. If this happens, the sync logs will contain a message saying the sheet has been skipped when syncing the full spreadsheet. +* Connector setup will fail if the speadsheet is not a Google Sheets file. If the file was saved or imported as another file type the setup could fail. +* Check out common troubleshooting issues for the Google Sheets source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
## Changelog @@ -177,4 +196,6 @@ Airbyte batches requests to the API in order to efficiently pull data and respec | 0.1.7 | 2021-01-21 | [1762](https://github.com/airbytehq/airbyte/pull/1762) | Fix issue large spreadsheet | | 0.1.6 | 2021-01-27 | [1668](https://github.com/airbytehq/airbyte/pull/1668) | Adopt connector best practices | | 0.1.5 | 2020-12-30 | [1438](https://github.com/airbytehq/airbyte/pull/1438) | Implement backoff | -| 0.1.4 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | \ No newline at end of file +| 0.1.4 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | + + \ No newline at end of file From 1ab67daa263deebb9e08a8a503a1206efad4e4f0 Mon Sep 17 00:00:00 2001 From: Patrick Nilan Date: Fri, 10 Nov 2023 07:59:33 -0800 Subject: [PATCH 006/124] Docs: Updates Sendgrid Source Connector Docs (Removes inapp page, adds , adds Limitations & Troubleshooting) (#32373) --- docs/integrations/sources/sendgrid.inapp.md | 23 ------- docs/integrations/sources/sendgrid.md | 75 ++++++++++++--------- 2 files changed, 44 insertions(+), 54 deletions(-) delete mode 100644 docs/integrations/sources/sendgrid.inapp.md diff --git a/docs/integrations/sources/sendgrid.inapp.md b/docs/integrations/sources/sendgrid.inapp.md deleted file mode 100644 index 4c03aca5c0a0..000000000000 --- a/docs/integrations/sources/sendgrid.inapp.md +++ /dev/null @@ -1,23 +0,0 @@ -## Prerequisites - -* [Sendgrid API Key]((https://docs.sendgrid.com/ui/account-and-settings/api-keys#creating-an-api-key)) with - * Read-only access to all resources - * Full access to marketing resources - -## Setup guide - -1. Enter a name for your Sendgridconnector. -2. Enter your `api key`. -3. (Optional) Enter the `start_time` in YYYY-MM-DDTHH:MM:SSZ format. Dataadded on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. -4. Click **Set up source**. - -### (Optional) Create a read-only API key - -While you can set up the Sendgrid connector using any Salesforce user with read permission, we recommend creating a dedicated read-only user for Airbyte. This allows you to granularly control the which resources Airbyte can read. - -The API key should be read-only on all resources except Marketing, where it needs Full Access. - -Sendgrid provides two different kinds of marketing campaigns, "legacy marketing campaigns" and "new marketing campaigns". **Legacy marketing campaigns are not supported by this source connector**. -If you are seeing a `403 FORBIDDEN error message for https://api.sendgrid.com/v3/marketing/campaigns`, it may be because your SendGrid account uses legacy marketing campaigns. - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Sendgrid](https://docs.airbyte.com/integrations/sources/sendgrid). \ No newline at end of file diff --git a/docs/integrations/sources/sendgrid.md b/docs/integrations/sources/sendgrid.md index 34e215d558e9..78bca8f45dcf 100644 --- a/docs/integrations/sources/sendgrid.md +++ b/docs/integrations/sources/sendgrid.md @@ -1,10 +1,14 @@ # Sendgrid -This page contains the setup guide and reference information for the Sendgrid source connector. + + +This page contains the setup guide and reference information for the [Sendgrid](https://sendgrid.com/) source connector. + + ## Prerequisites -* API Key +* [Sendgrid API Key](https://docs.sendgrid.com/ui/account-and-settings/api-keys#creating-an-api-key) ## Setup guide ### Step 1: Set up Sendgrid @@ -14,24 +18,16 @@ This page contains the setup guide and reference information for the Sendgrid so * Read-only access to all resources * Full access to marketing resources -## Step 2: Set up the Sendgrid connector in Airbyte - -### For Airbyte Cloud: +### Step 2: Set up the Sendgrid connector in Airbyte -1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. -2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. +1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account or navigate to the Airbyte Open Source dashboard. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. 3. On the Set up the source page, enter the name for the Sendgrid connector and select **Sendgrid** from the Source type dropdown. 4. Enter your `apikey`. -5. Enter your `start_time`. +5. Enter your `start_time`. 6. Click **Set up source**. -### For Airbyte OSS: - -1. Navigate to the Airbyte Open Source dashboard. -2. Set the name for your source. -3. Enter your `apikey`. -4. Enter your `start_time`. -5. Click **Set up source**. + ## Supported sync modes @@ -43,38 +39,53 @@ The Sendgrid source connector supports the following [sync modes](https://docs.a ## Supported Streams -* [Campaigns](https://docs.sendgrid.com/api-reference/campaigns-api/retrieve-all-campaigns) -* [Lists](https://docs.sendgrid.com/api-reference/lists/get-all-lists) -* [Contacts](https://docs.sendgrid.com/api-reference/contacts/export-contacts) -* [Stats automations](https://docs.sendgrid.com/api-reference/marketing-campaign-stats/get-all-automation-stats) -* [Segments](https://docs.sendgrid.com/api-reference/segmenting-contacts/get-list-of-segments) -* [Single Sends](https://docs.sendgrid.com/api-reference/marketing-campaign-stats/get-all-single-sends-stats) -* [Templates](https://docs.sendgrid.com/api-reference/transactional-templates/retrieve-paged-transactional-templates) +* [Campaigns](https://docs.sendgrid.com/api-reference/campaigns-api/retrieve-all-campaigns) +* [Lists](https://docs.sendgrid.com/api-reference/lists/get-all-lists) +* [Contacts](https://docs.sendgrid.com/api-reference/contacts/export-contacts) +* [Stats automations](https://docs.sendgrid.com/api-reference/marketing-campaign-stats/get-all-automation-stats) +* [Segments](https://docs.sendgrid.com/api-reference/segmenting-contacts/get-list-of-segments) +* [Single Sends](https://docs.sendgrid.com/api-reference/marketing-campaign-stats/get-all-single-sends-stats) +* [Templates](https://docs.sendgrid.com/api-reference/transactional-templates/retrieve-paged-transactional-templates) * [Global suppression](https://docs.sendgrid.com/api-reference/suppressions-global-suppressions/retrieve-all-global-suppressions) \(Incremental\) * [Suppression groups](https://docs.sendgrid.com/api-reference/suppressions-unsubscribe-groups/retrieve-all-suppression-groups-associated-with-the-user) -* [Suppression group members](https://docs.sendgrid.com/api-reference/suppressions-suppressions/retrieve-all-suppressions) +* [Suppression group members](https://docs.sendgrid.com/api-reference/suppressions-suppressions/retrieve-all-suppressions) * [Blocks](https://docs.sendgrid.com/api-reference/blocks-api/retrieve-all-blocks) \(Incremental\) * [Bounces](https://docs.sendgrid.com/api-reference/bounces-api/retrieve-all-bounces) \(Incremental\) * [Invalid emails](https://docs.sendgrid.com/api-reference/invalid-e-mails-api/retrieve-all-invalid-emails) \(Incremental\) * [Spam reports](https://docs.sendgrid.com/api-reference/spam-reports-api/retrieve-all-spam-reports) -## Connector-specific features & highlights, if any +## Create a read-only API key (Optional) + +While you can set up the Sendgrid connector using any Salesforce user with read permission, we recommend creating a dedicated read-only user for Airbyte. This allows you to granularly control the which resources Airbyte can read. + +The API key should be read-only on all resources except Marketing, where it needs Full Access. -We recommend creating a key specifically for Airbyte access. This will allow you to control which resources Airbyte should be able to access. The API key should be read-only on all resources except Marketing, where it needs Full Access. -Sendgrid provides two different kinds of marketing campaigns, "legacy marketing campaigns" and "new marketing campaigns". **Legacy marketing campaigns are not supported by this source connector**. -If you are seeing a `403 FORBIDDEN error message for https://api.sendgrid.com/v3/marketing/campaigns`, it might be because your SendGrid account uses legacy marketing campaigns. +## Limitations & Troubleshooting -## Performance considerations +
+ +Expand to see details about Sendgrid connector limitations and troubleshooting. + -The connector is restricted by normal Sendgrid [requests limitation](https://sendgrid.com/docs/API_Reference/Web_API_v3/How_To_Use_The_Web_API_v3/rate_limits.html). +### Connector limitations + +#### Rate limiting + +The connector is restricted by normal Sendgrid [requests limitation](https://docs.sendgrid.com/api-reference/how-to-use-the-sendgrid-v3-api/rate-limits). + +### Troubleshooting +* **Legacy marketing campaigns are not supported by this source connector**. Sendgrid provides two different kinds of marketing campaigns, "legacy marketing campaigns" and "new marketing campaigns". If you are seeing a `403 FORBIDDEN error message for https://api.sendgrid.com/v3/marketing/campaigns`, it might be because your SendGrid account uses legacy marketing campaigns. +* Check out common troubleshooting issues for the Sendgrid source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
## Changelog | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | 0.4.1 | 2023-10-18 | [31543](https://github.com/airbytehq/airbyte/pull/31543) | Base image migration: remove Dockerfile and use the python-connector-base image | -| 0.4.0 | 2023-05-19 | [23959](https://github.com/airbytehq/airbyte/pull/23959) | Add `unsubscribe_groups`stream +| 0.4.0 | 2023-05-19 | [23959](https://github.com/airbytehq/airbyte/pull/23959) | Add `unsubscribe_groups`stream | 0.3.1 | 2023-01-27 | [21939](https://github.com/airbytehq/airbyte/pull/21939) | Fix contacts missing records; Remove Messages stream | | 0.3.0 | 2023-01-25 | [21587](https://github.com/airbytehq/airbyte/pull/21587) | Make sure spec works as expected in UI - make start_time parameter an ISO string instead of an integer interpreted as timestamp (breaking, update your existing connections and set the start_time parameter to ISO 8601 date time string in UTC) | | 0.2.16 | 2022-11-02 | [18847](https://github.com/airbytehq/airbyte/pull/18847) | Skip the stream on `400, 401 - authorization required` with log message | @@ -87,4 +98,6 @@ The connector is restricted by normal Sendgrid [requests limitation](https://sen | 0.2.9 | 2022-08-11 | [15257](https://github.com/airbytehq/airbyte/pull/15257) | Migrate to config-based framework | | 0.2.8 | 2022-06-07 | [13571](https://github.com/airbytehq/airbyte/pull/13571) | Add Message stream | | 0.2.7 | 2021-09-08 | [5910](https://github.com/airbytehq/airbyte/pull/5910) | Add Single Sends Stats stream | -| 0.2.6 | 2021-07-19 | [4839](https://github.com/airbytehq/airbyte/pull/4839) | Gracefully handle malformed responses from the API | \ No newline at end of file +| 0.2.6 | 2021-07-19 | [4839](https://github.com/airbytehq/airbyte/pull/4839) | Gracefully handle malformed responses from the API | + + \ No newline at end of file From 633cdd1eb6d9c1cfcfe0a903335d332d57e3ea09 Mon Sep 17 00:00:00 2001 From: Pranjal Jain Date: Fri, 10 Nov 2023 22:10:51 +0530 Subject: [PATCH 007/124] =?UTF-8?q?=E2=9C=A8=20Source=20OpsGenie=20-=20Mig?= =?UTF-8?q?rate=20Python=20CDK=20to=20Low=20Code=20CDK=20(#31552)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Marcos Marx Co-authored-by: marcosmarxm Co-authored-by: sajarin --- .../connectors/source-opsgenie/Dockerfile | 2 +- .../connectors/source-opsgenie/README.md | 27 +- .../acceptance-test-config.yml | 20 +- .../connectors/source-opsgenie/metadata.yaml | 4 +- .../source_opsgenie/manifest.yaml | 232 ++++++++ .../source-opsgenie/source_opsgenie/source.py | 58 +- .../source-opsgenie/source_opsgenie/spec.yaml | 2 +- .../source_opsgenie/streams.py | 256 --------- .../source-opsgenie/source_opsgenie/util.py | 12 - .../source-opsgenie/unit_tests/__init__.py | 3 - .../source-opsgenie/unit_tests/test_source.py | 35 -- .../source-opsgenie/unit_tests/test_stream.py | 508 ------------------ .../source-opsgenie/unit_tests/unit_test.py | 16 - docs/integrations/sources/opsgenie.md | 4 +- 14 files changed, 258 insertions(+), 921 deletions(-) create mode 100644 airbyte-integrations/connectors/source-opsgenie/source_opsgenie/manifest.yaml delete mode 100644 airbyte-integrations/connectors/source-opsgenie/source_opsgenie/streams.py delete mode 100644 airbyte-integrations/connectors/source-opsgenie/source_opsgenie/util.py delete mode 100644 airbyte-integrations/connectors/source-opsgenie/unit_tests/__init__.py delete mode 100644 airbyte-integrations/connectors/source-opsgenie/unit_tests/test_source.py delete mode 100644 airbyte-integrations/connectors/source-opsgenie/unit_tests/test_stream.py delete mode 100644 airbyte-integrations/connectors/source-opsgenie/unit_tests/unit_test.py diff --git a/airbyte-integrations/connectors/source-opsgenie/Dockerfile b/airbyte-integrations/connectors/source-opsgenie/Dockerfile index 6a9cd5658038..b8cc08291b97 100644 --- a/airbyte-integrations/connectors/source-opsgenie/Dockerfile +++ b/airbyte-integrations/connectors/source-opsgenie/Dockerfile @@ -34,5 +34,5 @@ COPY source_opsgenie ./source_opsgenie ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/source-opsgenie diff --git a/airbyte-integrations/connectors/source-opsgenie/README.md b/airbyte-integrations/connectors/source-opsgenie/README.md index ff7c5c9ae335..d76b42663d15 100644 --- a/airbyte-integrations/connectors/source-opsgenie/README.md +++ b/airbyte-integrations/connectors/source-opsgenie/README.md @@ -1,34 +1,17 @@ # Opsgenie Source -This is the repository for the Opsgenie source connector, written in Python. +This is the repository for the Opsgenie source connector, written in low-code configuration based source connector. For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/opsgenie). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv +To build using Gradle, from the Airbyte repository root, run: ``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: +./gradlew :airbyte-integrations:connectors:source-opsgenie:build ``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/opsgenie) diff --git a/airbyte-integrations/connectors/source-opsgenie/acceptance-test-config.yml b/airbyte-integrations/connectors/source-opsgenie/acceptance-test-config.yml index 4cd9e6036fb4..bc4e8a53b0c2 100644 --- a/airbyte-integrations/connectors/source-opsgenie/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-opsgenie/acceptance-test-config.yml @@ -14,21 +14,11 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: - [ - "alerts", - "alert_recipients", - "alert_logs", - "incidents", - "integrations", - "teams", - "user_teams", - "services", - ] - incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - # future_state_path: "integration_tests/abnormal_state.json" + empty_streams: ["incidents", "services"] + # incremental: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-opsgenie/metadata.yaml b/airbyte-integrations/connectors/source-opsgenie/metadata.yaml index 0c872187398d..4af87df040a9 100644 --- a/airbyte-integrations/connectors/source-opsgenie/metadata.yaml +++ b/airbyte-integrations/connectors/source-opsgenie/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: 06bdb480-2598-40b8-8b0f-fc2e2d2abdda - dockerImageTag: 0.2.0 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-opsgenie githubIssueLabel: source-opsgenie license: MIT @@ -15,7 +15,7 @@ data: releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/opsgenie tags: - - language:python + - language:lowcode ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/manifest.yaml b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/manifest.yaml new file mode 100644 index 000000000000..a1c6b1e07e72 --- /dev/null +++ b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/manifest.yaml @@ -0,0 +1,232 @@ +version: 0.51.16 +type: DeclarativeSource +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + auth: + type: ApiKeyAuthenticator + api_token: "GenieKey {{ config['api_token'] }}" + inject_into: + type: RequestOption + field_name: Authorization + inject_into: header + on_error: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + backoff_strategies: + - type: WaitUntilTimeFromHeader + header: X-RateLimit-Period-In-Sec + pagination: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next", {}) }}' + + requester: + type: HttpRequester + url_base: "https://{{ config['endpoint'] }}" + http_method: GET + request_parameters: {} + request_headers: + Accept: application/json + authenticator: + $ref: "#/definitions/auth" + error_handler: + $ref: "#/definitions/on_error" + request_body_json: {} + + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/pagination" + + base_stream: + schema_loader: + type: JsonFileSchemaLoader + file_path: "./source_opsgenie/schemas/{{ parameters['name'] }}.json" + retriever: + $ref: "#/definitions/retriever" + + users_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "users" + primary_key: "id" + path: "v2/users" + + teams_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "teams" + primary_key: "id" + path: "v2/teams" + + services_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "services" + primary_key: "id" + path: "v1/services" + + integrations_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "integrations" + primary_key: "id" + path: "v2/integrations" + + incidents_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "incidents" + primary_key: "id" + path: "v1/incidents" + + alerts_stream: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/base_stream/retriever" + requester: + $ref: "#/definitions/requester" + request_parameters: + order: "asc" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updatedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%fZ" + datetime_format: "%s" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + $parameters: + name: "alerts" + primary_key: "id" + path: "v2/alerts" + + user_teams_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: user_teams + primary_key: "id" + retriever: + $ref: "#/definitions/base_stream/retriever" + requester: + $ref: "#/definitions/requester" + path: "v2/users/{{ stream_partition['user_id'] }}/teams" + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: user_id + stream: + $ref: "#/definitions/users_stream" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: ["user_id"] + value: "{{ stream_partition['id'] }}" + + alert_recipients_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "alert_recipients" + primary_key: "user_id" + path: "v2/alerts/{{ record }}/recipients" + retriever: + $ref: "#/definitions/base_stream/retriever" + requester: + $ref: "#/definitions/requester" + path: "v2/alerts/{{ stream_partition['alert_id'] }}/recipients" + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: alert_id + stream: + $ref: "#/definitions/alerts_stream" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: ["alert_id"] + value: "{{ stream_partition['id'] }}" + - type: AddFields + fields: + - type: AddedFieldDefinition + path: ["user_id"] + value: "{{ record['user']['id'] }}" + - type: AddFields + fields: + - type: AddedFieldDefinition + path: ["user_username"] + value: "{{ record['user']['username'] }}" + - type: RemoveFields + field_pointers: + - ["user"] + + alert_logs_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "alert_logs" + primary_key: "offset" + path: "v2/alerts/{{ stream_partition.alert_id }}/logs" + retriever: + $ref: "#/definitions/base_stream/retriever" + requester: + $ref: "#/definitions/requester" + request_parameters: + order: asc + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: alert_id + stream: + $ref: "#/definitions/alerts_stream" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: ["alert_id"] + value: "{{ stream_partition['id'] }}" + +check: + type: CheckStream + stream_names: + - users + # - teams + # - services + # - incidents + # - integrations + # - alerts + # - user_teams + # - alert_recipients + # - alert_logs + +streams: + - "#/definitions/users_stream" + - "#/definitions/teams_stream" + - "#/definitions/services_stream" + - "#/definitions/integrations_stream" + - "#/definitions/incidents_stream" + - "#/definitions/alerts_stream" + - "#/definitions/user_teams_stream" + - "#/definitions/alert_recipients_stream" + - "#/definitions/alert_logs_stream" diff --git a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/source.py b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/source.py index 743694d15b54..c7fca3b2212f 100644 --- a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/source.py +++ b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/source.py @@ -2,54 +2,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from typing import Any, List, Mapping, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from .streams import AlertLogs, AlertRecipients, Alerts, Incidents, Integrations, Services, Teams, Users, UserTeams - - -# Source -class SourceOpsgenie(AbstractSource): - @staticmethod - def get_authenticator(config: Mapping[str, Any]): - return TokenAuthenticator(config["api_token"], auth_method="GenieKey") - - def check_connection(self, logger, config) -> Tuple[bool, any]: - - try: - auth = self.get_authenticator(config) - api_endpoint = f"https://{config['endpoint']}/v2/account" - - response = requests.get( - api_endpoint, - headers=auth.get_auth_header(), - ) - - return response.status_code == requests.codes.ok, None - - except Exception as error: - return False, f"Unable to connect to Opsgenie API with the provided credentials - {repr(error)}" - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - auth = self.get_authenticator(config) - args = {"authenticator": auth, "endpoint": config["endpoint"]} - incremental_args = {**args, "start_date": config.get("start_date", "")} - - users = Users(**args) - alerts = Alerts(**incremental_args) - return [ - alerts, - AlertRecipients(parent_stream=alerts, **args), - AlertLogs(parent_stream=alerts, **args), - Incidents(**incremental_args), - Integrations(**args), - Services(**args), - Teams(**args), - users, - UserTeams(parent_stream=users, **args), - ] +# Declarative Source +class SourceOpsgenie(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/spec.yaml b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/spec.yaml index 82bc07cca77d..b688f8b5b7e3 100644 --- a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/spec.yaml +++ b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/spec.yaml @@ -1,4 +1,4 @@ -documentationUrl: https://docsurl.com +documentationUrl: https://docs.airbyte.com/integrations/sources/opsgenie connectionSpecification: $schema: http://json-schema.org/draft-07/schema# title: Opsgenie Spec diff --git a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/streams.py b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/streams.py deleted file mode 100644 index 6812e8427474..000000000000 --- a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/streams.py +++ /dev/null @@ -1,256 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import time -import urllib.parse as urlparse -from abc import ABC -from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional -from urllib.parse import parse_qs - -import pendulum -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http import HttpStream - - -# Basic full refresh stream -class OpsgenieStream(HttpStream, ABC): - - primary_key = "id" - api_version = "v2" - - flatten_id_keys = [] - flatten_list_keys = [] - - def __init__(self, endpoint: str, **kwargs): - super(OpsgenieStream, self).__init__(**kwargs) - self._endpoint = endpoint - - @property - def url_base(self) -> str: - return f"https://{self._endpoint}/{self.api_version}/" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - params = {} - data = response.json() - if "paging" in data and "next" in data["paging"]: - next_page = data["paging"]["next"] - params = parse_qs(urlparse.urlparse(next_page).query) - - return params - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params: Dict[str, str] = {} - - if next_page_token: - params.update(next_page_token) - - return params - - def request_headers(self, **kwargs) -> Mapping[str, Any]: - return {"Accept": "application/json"} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - response_data = response.json() - data = response_data["data"] - if isinstance(data, list): - for record in data: - yield self.transform(record, **kwargs) - elif isinstance(data, dict): - yield self.transform(data, **kwargs) - else: - Exception(f"Unsupported type of response data for stream {self.name}") - - def transform(self, record: Dict[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs): - for key in self.flatten_id_keys: - self._flatten_id(record, key) - - for key in self.flatten_list_keys: - self._flatten_list(record, key) - - return record - - def _flatten_id(self, record: Dict[str, Any], target: str): - target_value = record.pop(target, None) - record[target + "_id"] = target_value.get("id") if target_value else None - - def _flatten_list(self, record: Dict[str, Any], target: str): - record[target] = [target_data.get("id") for target_data in record.get(target, [])] - - def backoff_time(self, response: requests.Response) -> Optional[float]: - """ - This method is called if we run into the rate limit. - Opsgenie applies rate limits in both requests-per-minute and - requests-per-second buckets. The response will inform which - of these thresholds has been breached by returning a X-RateLimit-Period-In-Sec - header value of 60 and 1 respectively. We take this as the hint for how long - to wait before trying again. - - Rate Limits Docs: https://docs.opsgenie.com/docs/api-rate-limiting - """ - - if "X-RateLimit-Period-In-Sec" in response.headers: - return int(response.headers["X-RateLimit-Period-In-Sec"]) - else: - self.logger.info("X-RateLimit-Period-In-Sec header not found. Using default backoff value") - return 60 - - -class Teams(OpsgenieStream): - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "teams" - - -class Integrations(OpsgenieStream): - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "integrations" - - -class Users(OpsgenieStream): - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "users" - - -class Services(OpsgenieStream): - - api_version = "v1" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "services" - - -class OpsgenieChildStream(OpsgenieStream): - path_list = ["id"] - flatten_parent_id = False - - def __init__(self, parent_stream: OpsgenieStream, **kwargs): - super().__init__(**kwargs) - self.parent_stream = parent_stream - - @property - def path_template(self) -> str: - template = [self.parent_stream.name] + ["{" + path_key + "}" for path_key in self.path_list] - return "/".join(template + [self.name]) - - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - for slice in self.parent_stream.stream_slices(sync_mode=SyncMode.full_refresh): - for record in self.parent_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice): - yield {path_key: record[path_key] for path_key in self.path_list} - - def path(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> str: - return self.path_template.format(**{path_key: stream_slice[path_key] for path_key in self.path_list}) - - def transform(self, record: Dict[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs): - record = super().transform(record, stream_slice, **kwargs) - if self.flatten_parent_id: - record[f"{self.parent_stream.name[:-1]}_id"] = stream_slice["id"] - return record - - -class UserTeams(OpsgenieChildStream): - flatten_parent_id = True - path_template = "users/{id}/teams" - - -# Basic incremental stream -class IncrementalOpsgenieStream(OpsgenieStream, ABC): - def __init__(self, start_date, **kwargs): - super().__init__(**kwargs) - self.start_date = start_date - - state_checkpoint_interval = 100 - cursor_field = "updatedAt" - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - cursor_field = self.cursor_field - latest_record = latest_record.get(self.cursor_field) - - latest_record_date = pendulum.parse(latest_record) - stream_state = current_stream_state.get(cursor_field) - if stream_state: - return {cursor_field: str(max(latest_record_date, pendulum.parse(stream_state)))} - else: - return {cursor_field: str(latest_record_date)} - - def request_params( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, stream_slice, next_page_token) - start_point = self.start_date - start_date_dt = pendulum.parse(start_point) - - state_value = stream_state.get(self.cursor_field) - if state_value: - state_value_dt = pendulum.parse(state_value) - start_date_dt = max(start_date_dt, state_value_dt) - start_date_dt = min(start_date_dt, pendulum.now()) - - dt_timestamp = int(time.mktime(start_date_dt.timetuple()) * 1000) - - params["query"] = [f"{self.cursor_field}>={dt_timestamp}"] - params["order"] = ["asc"] - - return params - - -class Alerts(IncrementalOpsgenieStream): - def path(self, **kwargs) -> str: - return "alerts" - - -class Incidents(IncrementalOpsgenieStream): - - api_version = "v1" - - def path(self, **kwargs) -> str: - return "incidents" - - -class AlertRecipients(OpsgenieChildStream): - flatten_parent_id = True - path_template = "alerts/{id}/recipients" - flatten_id_keys = ["user"] - primary_key = "user_id" - - -class AlertLogs(OpsgenieChildStream): - primary_key = "offset" - cursor_field = "offset" - path_template = "alerts/{id}/logs" - flatten_parent_id = True - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - alert_id = latest_record.get("alert_id") - latest_cursor_value = latest_record.get(self.cursor_field) - current_state = current_stream_state.get(str(alert_id)) - if current_state: - current_state = current_state.get(self.cursor_field) - current_state_value = current_state or latest_cursor_value - max_value = max(current_state_value, latest_cursor_value) - current_stream_state[str(alert_id)] = {self.cursor_field: str(max_value)} - return current_stream_state - - def request_params( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - stream_state = stream_state or {} - params = super().request_params(stream_state, stream_slice, next_page_token) - - state_alert_value = stream_state.get(str(stream_slice["id"])) - if state_alert_value: - state_value = state_alert_value.get(self.cursor_field) - if state_value: - params[self.cursor_field] = [state_value] - params["order"] = ["asc"] - return params diff --git a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/util.py b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/util.py deleted file mode 100644 index 32b939c7a213..000000000000 --- a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/util.py +++ /dev/null @@ -1,12 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from airbyte_cdk.models import SyncMode -from source_opsgenie.streams import OpsgenieStream - - -def read_full_refresh(stream_instance: OpsgenieStream): - slices = stream_instance.stream_slices(sync_mode=SyncMode.full_refresh) - for _slice in slices: - yield from stream_instance.read_records(stream_slice=_slice, sync_mode=SyncMode.full_refresh) diff --git a/airbyte-integrations/connectors/source-opsgenie/unit_tests/__init__.py b/airbyte-integrations/connectors/source-opsgenie/unit_tests/__init__.py deleted file mode 100644 index 1100c1c58cf5..000000000000 --- a/airbyte-integrations/connectors/source-opsgenie/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-opsgenie/unit_tests/test_source.py b/airbyte-integrations/connectors/source-opsgenie/unit_tests/test_source.py deleted file mode 100644 index dfb27745a5c7..000000000000 --- a/airbyte-integrations/connectors/source-opsgenie/unit_tests/test_source.py +++ /dev/null @@ -1,35 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import unittest -from unittest.mock import MagicMock - -import responses -from source_opsgenie.source import SourceOpsgenie - - -class SourceOpsgenieTest(unittest.TestCase): - def test_stream_count(self): - source = SourceOpsgenie() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 9 - self.assertEqual(len(streams), expected_streams_number) - - @responses.activate - def test_check_connection(self): - log_mock, _ = MagicMock(), MagicMock() - source = SourceOpsgenie() - - sample_account = { - "data": {"name": "opsgenie", "userCount": 1450, "plan": {"maxUserCount": 1500, "name": "Enterprise", "isYearly": True}}, - "took": 0.084, - "requestId": "e5122017-f5c5-4681-88ec-84e2898a61ad", - } - - responses.add("GET", "https://api.opsgenie.com/v2/account", json=sample_account) - - (success, err) = source.check_connection(log_mock, {"endpoint": "api.opsgenie.com", "api_token": "123"}) - self.assertTrue(success) - self.assertIsNone(err) diff --git a/airbyte-integrations/connectors/source-opsgenie/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-opsgenie/unit_tests/test_stream.py deleted file mode 100644 index 32003b46a58c..000000000000 --- a/airbyte-integrations/connectors/source-opsgenie/unit_tests/test_stream.py +++ /dev/null @@ -1,508 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import unittest - -import responses -from source_opsgenie.streams import AlertLogs, AlertRecipients, Alerts, Incidents, Integrations, Services, Teams, Users, UserTeams -from source_opsgenie.util import read_full_refresh - - -class TeamsStreamTestCase(unittest.TestCase): - @responses.activate - def test_teams_list(self): - config = {"endpoint": "api.opsgenie.com"} - stream = Teams(**config) - responses.add( - "GET", - "https://api.opsgenie.com/v2/teams", - json={ - "data": [ - {"id": "90098alp9-f0e3-41d3-a060-0ea895027630", "name": "ops_team", "description": ""}, - {"id": "a30alp45-65bf-422f-9d41-67b10a67282a", "name": "TeamName2", "description": "Description"}, - {"id": "c569c016-alp9-4e20-8a28-bd5dc33b798e", "name": "TeamName", "description": ""}, - ], - "took": 1.08, - "requestId": "9cbfalp7-53f5-41ef-a360-be01277a903d", - }, - ) - - records = list(read_full_refresh(stream)) - self.assertEqual(3, len(records)) - - -class UsersStreamTestCase(unittest.TestCase): - @responses.activate - def test_users_list(self): - config = {"endpoint": "api.opsgenie.com"} - stream = Users(**config) - responses.add( - "GET", - "https://api.opsgenie.com/v2/users", - json={ - "totalCount": 8, - "data": [ - { - "blocked": False, - "verified": False, - "id": "b5b92115-bfe7-43eb-8c2a-e467f2e5ddc4", - "username": "john.doe@opsgenie.com", - "fullName": "john doe", - "role": {"id": "Admin", "name": "Admin"}, - "timeZone": "Europe/Kirov", - "locale": "en_US", - "userAddress": {"country": "", "state": "", "city": "", "line": "", "zipCode": ""}, - "createdAt": "2017-05-12T08:34:30.283Z", - }, - { - "blocked": False, - "verified": False, - "id": "e07c63f0-dd8c-4ad4-983e-4ee7dc600463", - "username": "jane.doe@opsgenie.com", - "fullName": "jane doe", - "role": {"id": "Admin", "name": "Admin"}, - "timeZone": "Europe/Moscow", - "locale": "en_GB", - "tags": ["tag1", "tag3"], - "userAddress": { - "country": "US", - "state": "Indiana", - "city": "Terre Haute", - "line": "567 Stratford Park", - "zipCode": "47802", - }, - "details": {"detail1key": ["detail1dvalue1", "detail1value2"], "detail2key": ["detail2value"]}, - "createdAt": "2017-05-12T09:39:14.41Z", - }, - ], - "took": 0.261, - "requestId": "d2c50d0c-1c44-4fa5-99d4-20d1e7ca9938", - }, - ) - - records = list(read_full_refresh(stream)) - self.assertEqual(2, len(records)) - - -class ServicesStreamTestCase(unittest.TestCase): - @responses.activate - def test_services_list(self): - config = {"endpoint": "api.opsgenie.com"} - stream = Services(**config) - responses.add( - "GET", - "https://api.opsgenie.com/v1/services", - json={ - "data": [ - { - "teamId": "2e3c4c13-51e7-4cf6-a353-34c6f75494c7", - "name": "Service API Test Service - Updated", - "description": "Service API Test Service Description [Updated]", - "id": "6aa85159-9e2e-4e54-8088-546f9c15d513", - "tags": [], - "isExternal": False, - }, - { - "teamId": "2e3c4c13-51e7-4cf6-a353-34c6f75494c7", - "name": "Service API Test Service 2 - Updated", - "description": "Service API Test Service 2 Description [Updated]", - "id": "6aa85159-9e2e-4e54-8088-546f9c15d513", - "tags": [], - "isExternal": False, - }, - ], - "requestId": "656cfb15-e19f-11e7-ac88-af7c98633ff2", - }, - ) - - records = list(read_full_refresh(stream)) - self.assertEqual(2, len(records)) - - -class IntegrationsStreamTestCase(unittest.TestCase): - @responses.activate - def test_services_list(self): - config = {"endpoint": "api.opsgenie.com"} - stream = Integrations(**config) - responses.add( - "GET", - "https://api.opsgenie.com/v2/integrations", - json={ - "code": 200, - "data": [ - {"id": "055082dc-9427-48dd-85e0-f93a76e5f4a2", "name": "Signal Sciences", "enabled": True, "type": "SignalSciences"}, - {"id": "073e8e6a-a481-4b9b-8619-5c31d9a6e5da", "name": "Default API", "enabled": True, "type": "API"}, - {"id": "3163a9f9-5950-4e73-b99f-92562956e39c", "name": "Datadog", "enabled": False, "type": "Datadog"}, - {"id": "55e405e3-a130-4c7a-9866-664e498f39a9", "name": "Observium2", "enabled": False, "type": "ObserviumV2"}, - {"id": "72f6f51b-1ea9-4efd-be1b-4f29d1f593c6", "name": "Solarwinds", "enabled": False, "type": "Solarwinds"}, - {"id": "733388de-2ac1-4d70-8a2e-82834cb679d6", "name": "Webhook", "enabled": False, "type": "Webhook"}, - { - "id": "8418d193-2dab-4490-b331-8c02cdd196b7", - "name": "Marid", - "enabled": False, - "type": "Marid", - "teamId": "87311c02-edda-11eb-9a03-0242ac130003", - }, - ], - "took": 0, - "requestId": "9ceeb66b-9890-4687-9dbb-a38abc71eda3", - }, - ) - - records = list(read_full_refresh(stream)) - self.assertEqual(7, len(records)) - - -class UserTeamsStreamTestCase(unittest.TestCase): - @responses.activate - def test_user_teams_list(self): - config = {"endpoint": "api.opsgenie.com"} - users = Users(**config) - stream = UserTeams(parent_stream=users, **config) - responses.add( - "GET", - "https://api.opsgenie.com/v2/users", - json={ - "totalCount": 8, - "data": [ - { - "blocked": False, - "verified": False, - "id": "b5b92115-bfe7-43eb-8c2a-e467f2e5ddc4", - "username": "john.doe@opsgenie.com", - "fullName": "john doe", - "role": {"id": "Admin", "name": "Admin"}, - "timeZone": "Europe/Kirov", - "locale": "en_US", - "userAddress": {"country": "", "state": "", "city": "", "line": "", "zipCode": ""}, - "createdAt": "2017-05-12T08:34:30.283Z", - }, - { - "blocked": False, - "verified": False, - "id": "e07c63f0-dd8c-4ad4-983e-4ee7dc600463", - "username": "jane.doe@opsgenie.com", - "fullName": "jane doe", - "role": {"id": "Admin", "name": "Admin"}, - "timeZone": "Europe/Moscow", - "locale": "en_GB", - "tags": ["tag1", "tag3"], - "userAddress": { - "country": "US", - "state": "Indiana", - "city": "Terre Haute", - "line": "567 Stratford Park", - "zipCode": "47802", - }, - "details": {"detail1key": ["detail1dvalue1", "detail1value2"], "detail2key": ["detail2value"]}, - "createdAt": "2017-05-12T09:39:14.41Z", - }, - ], - "took": 0.261, - "requestId": "d2c50d0c-1c44-4fa5-99d4-20d1e7ca9938", - }, - ) - - responses.add( - "GET", - "https://api.opsgenie.com/v2/users/b5b92115-bfe7-43eb-8c2a-e467f2e5ddc4/teams", - json={ - "data": [{"id": "6fa6848c-8cac-4cea-8a98-ad9ff23d9b16", "name": "TeamName"}], - "took": 0.023, - "requestId": "bc40b7ad-11ee-4dcd-ae5f-6d75dbc16261", - }, - ) - - responses.add( - "GET", - "https://api.opsgenie.com/v2/users/e07c63f0-dd8c-4ad4-983e-4ee7dc600463/teams", - json={ - "data": [ - {"id": "6fa6848c-8cac-4cea-8a98-ad9ff23d9b16", "name": "TeamName"}, - {"id": "bc40b7ad-11ee-4dcd-ae5f-6d75dbc16261", "name": "TeamName"}, - ], - "took": 0.023, - "requestId": "bc40b7ad-11ee-4dcd-ae5f-6d75dbc16261", - }, - ) - - records = list(read_full_refresh(stream)) - self.assertEqual(3, len(records)) - - -class AlertsStreamTestCase(unittest.TestCase): - @responses.activate - def test_alerts_list(self): - config = {"endpoint": "api.opsgenie.com", "start_date": "2022-07-01T00:00:00Z"} - stream = Alerts(**config) - responses.add( - "GET", - "https://api.opsgenie.com/v2/alerts", - json={ - "data": [ - { - "id": "70413a06-38d6-4c85-92b8-5ebc900d42e2", - "tinyId": "1791", - "alias": "event_573", - "message": "Our servers are in danger", - "status": "closed", - "acknowledged": False, - "isSeen": True, - "tags": ["OverwriteQuietHours", "Critical"], - "snoozed": True, - "snoozedUntil": "2017-04-03T20:32:35.143Z", - "count": 79, - "lastOccurredAt": "2017-04-03T20:05:50.894Z", - "createdAt": "2017-03-21T20:32:52.353Z", - "updatedAt": "2017-04-03T20:32:57.301Z", - "source": "Isengard", - "owner": "morpheus@opsgenie.com", - "priority": "P4", - "responders": [ - {"id": "4513b7ea-3b91-438f-b7e4-e3e54af9147c", "type": "team"}, - {"id": "bb4d9938-c3c2-455d-aaab-727aa701c0d8", "type": "user"}, - {"id": "aee8a0de-c80f-4515-a232-501c0bc9d715", "type": "escalation"}, - {"id": "80564037-1984-4f38-b98e-8a1f662df552", "type": "schedule"}, - ], - "integration": {"id": "4513b7ea-3b91-438f-b7e4-e3e54af9147c", "name": "Nebuchadnezzar", "type": "API"}, - "report": { - "ackTime": 15702, - "closeTime": 60503, - "acknowledgedBy": "agent_smith@opsgenie.com", - "closedBy": "neo@opsgenie.com", - }, - }, - { - "id": "70413a06-38d6-4c85-92b8-5ebc900d42e2", - "tinyId": "1791", - "alias": "event_573", - "message": "Sample Message", - "status": "open", - "acknowledged": False, - "isSeen": False, - "tags": ["RandomTag"], - "snoozed": False, - "count": 1, - "lastOccurredAt": "2017-03-21T20:32:52.353Z", - "createdAt": "2017-03-21T20:32:52.353Z", - "updatedAt": "2017-04-03T20:32:57.301Z", - "source": "Zion", - "owner": "", - "priority": "P5", - "responders": [], - "integration": {"id": "4513b7ea-3b91-b7e4-438f-e3e54af9147c", "name": "My_Lovely_Amazon", "type": "CloudWatch"}, - }, - ], - "took": 0.605, - "requestId": "9ae63dd7-ed00-4c81-86f0-c4ffd33142c9", - }, - ) - - records = list(read_full_refresh(stream)) - self.assertEqual(2, len(records)) - - -class IncidentsStreamTestCase(unittest.TestCase): - @responses.activate - def test_alerts_list(self): - config = {"endpoint": "api.opsgenie.com", "start_date": "2022-07-01T00:00:00Z"} - stream = Incidents(**config) - responses.add( - "GET", - "https://api.opsgenie.com/v1/incidents", - json={ - "data": [ - { - "id": "70413a06-38d6-4c85-92b8-5ebc900d42e2", - "tinyId": "1791", - "message": "Our servers are in danger", - "status": "closed", - "tags": ["OverwriteQuietHours", "Critical"], - "createdAt": "2017-03-21T20:32:52.353Z", - "updatedAt": "2017-04-03T20:32:57.301Z", - "priority": "P4", - "responders": [ - {"type": "team", "id": "fc1448b7-46b2-401d-9df8-c02675958e3b"}, - {"type": "team", "id": "fe954a67-813e-4356-87dc-afed1eec6b66"}, - ], - "impactedServices": ["df635094-efd3-48e4-b73a-b8bdfbf1178f", "b6868288-02c7-440b-a693-0a5cf20576f5"], - }, - { - "id": "70413a06-38d6-4c85-92b8-5ebc900d42e2", - "tinyId": "1791", - "message": "Sample Message", - "status": "open", - "tags": ["RandomTag"], - "createdAt": "2017-03-21T20:32:52.353Z", - "updatedAt": "2017-04-03T20:32:57.301Z", - "priority": "P5", - "responders": [{"type": "team", "id": "fc1448b7-46b2-401d-9df8-c02675958e3b"}], - "impactedServices": [], - }, - ], - "took": 0.605, - "requestId": "9ae63dd7-ed00-4c81-86f0-c4ffd33142c9", - }, - ) - - records = list(read_full_refresh(stream)) - self.assertEqual(2, len(records)) - - -class AlertRecipientsStreamTestCase(unittest.TestCase): - @responses.activate - def test_alerts_list(self): - alerts_config = {"endpoint": "api.opsgenie.com", "start_date": "2022-07-01T00:00:00Z"} - config = { - "endpoint": "api.opsgenie.com", - } - stream = AlertRecipients(parent_stream=Alerts(**alerts_config), **config) - responses.add( - "GET", - "https://api.opsgenie.com/v2/alerts", - json={ - "data": [ - { - "id": "70413a06-38d6-4c85-92b8-5ebc900d42e2", - "tinyId": "1791", - "alias": "event_573", - "message": "Our servers are in danger", - "status": "closed", - "acknowledged": False, - "isSeen": True, - "tags": ["OverwriteQuietHours", "Critical"], - "snoozed": True, - "snoozedUntil": "2017-04-03T20:32:35.143Z", - "count": 79, - "lastOccurredAt": "2017-04-03T20:05:50.894Z", - "createdAt": "2017-03-21T20:32:52.353Z", - "updatedAt": "2017-04-03T20:32:57.301Z", - "source": "Isengard", - "owner": "morpheus@opsgenie.com", - "priority": "P4", - "responders": [ - {"id": "4513b7ea-3b91-438f-b7e4-e3e54af9147c", "type": "team"}, - {"id": "bb4d9938-c3c2-455d-aaab-727aa701c0d8", "type": "user"}, - {"id": "aee8a0de-c80f-4515-a232-501c0bc9d715", "type": "escalation"}, - {"id": "80564037-1984-4f38-b98e-8a1f662df552", "type": "schedule"}, - ], - "integration": {"id": "4513b7ea-3b91-438f-b7e4-e3e54af9147c", "name": "Nebuchadnezzar", "type": "API"}, - "report": { - "ackTime": 15702, - "closeTime": 60503, - "acknowledgedBy": "agent_smith@opsgenie.com", - "closedBy": "neo@opsgenie.com", - }, - } - ], - "took": 0.605, - "requestId": "9ae63dd7-ed00-4c81-86f0-c4ffd33142c9", - }, - ) - responses.add( - "GET", - "https://api.opsgenie.com/v2/alerts/70413a06-38d6-4c85-92b8-5ebc900d42e2/recipients", - json={ - "data": [ - { - "user": {"id": "2503a523-8ba5-4158-a4bd-7850074b5cca", "username": "neo@opsgenie.com"}, - "state": "action", - "method": "Acknowledge", - "createdAt": "2017-04-12T12:27:28.52Z", - "updatedAt": "2017-04-12T12:27:52.86Z", - }, - { - "user": {"id": "0966cfd8-fc9a-4f5c-a013-7d1f9318aef8", "username": "trinity@opsgenie.com"}, - "state": "notactive", - "method": "", - "createdAt": "2017-04-12T12:27:28.571Z", - "updatedAt": "2017-04-12T12:27:28.589Z", - }, - ], - "took": 0.605, - "requestId": "9ae63dd7-ed00-4c81-86f0-c4ffd33142c9", - }, - ) - - records = list(read_full_refresh(stream)) - self.assertEqual(2, len(records)) - - -class AlertLogsStreamTestCase(unittest.TestCase): - @responses.activate - def test_alerts_list(self): - alerts_config = {"endpoint": "api.opsgenie.com", "start_date": "2022-07-01T00:00:00Z"} - config = { - "endpoint": "api.opsgenie.com", - } - stream = AlertLogs(parent_stream=Alerts(**alerts_config), **config) - responses.add( - "GET", - "https://api.opsgenie.com/v2/alerts", - json={ - "data": [ - { - "id": "70413a06-38d6-4c85-92b8-5ebc900d42e2", - "tinyId": "1791", - "alias": "event_573", - "message": "Our servers are in danger", - "status": "closed", - "acknowledged": False, - "isSeen": True, - "tags": ["OverwriteQuietHours", "Critical"], - "snoozed": True, - "snoozedUntil": "2017-04-03T20:32:35.143Z", - "count": 79, - "lastOccurredAt": "2017-04-03T20:05:50.894Z", - "createdAt": "2017-03-21T20:32:52.353Z", - "updatedAt": "2017-04-03T20:32:57.301Z", - "source": "Isengard", - "owner": "morpheus@opsgenie.com", - "priority": "P4", - "responders": [ - {"id": "4513b7ea-3b91-438f-b7e4-e3e54af9147c", "type": "team"}, - {"id": "bb4d9938-c3c2-455d-aaab-727aa701c0d8", "type": "user"}, - {"id": "aee8a0de-c80f-4515-a232-501c0bc9d715", "type": "escalation"}, - {"id": "80564037-1984-4f38-b98e-8a1f662df552", "type": "schedule"}, - ], - "integration": {"id": "4513b7ea-3b91-438f-b7e4-e3e54af9147c", "name": "Nebuchadnezzar", "type": "API"}, - "report": { - "ackTime": 15702, - "closeTime": 60503, - "acknowledgedBy": "agent_smith@opsgenie.com", - "closedBy": "neo@opsgenie.com", - }, - } - ], - "took": 0.605, - "requestId": "9ae63dd7-ed00-4c81-86f0-c4ffd33142c9", - }, - ) - responses.add( - "GET", - "https://api.opsgenie.com/v2/alerts/70413a06-38d6-4c85-92b8-5ebc900d42e2/logs", - json={ - "data": [ - { - "log": "Alert acknowledged via web", - "type": "system", - "owner": "neo@opsgenie.com", - "createdAt": "2017-04-12T12:27:52.838Z", - "offset": "1492000072838_1492000072838234593", - }, - { - "log": "Viewed on [web]", - "type": "alertRecipient", - "owner": "trinity@opsgenie.com", - "createdAt": "2017-04-12T12:27:46.379Z", - "offset": "1492000066378_1492000066379000127", - }, - ], - "took": 0.605, - "requestId": "9ae63dd7-ed00-4c81-86f0-c4ffd33142c9", - }, - ) - - records = list(read_full_refresh(stream)) - self.assertEqual(2, len(records)) diff --git a/airbyte-integrations/connectors/source-opsgenie/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-opsgenie/unit_tests/unit_test.py deleted file mode 100644 index 9042065c8925..000000000000 --- a/airbyte-integrations/connectors/source-opsgenie/unit_tests/unit_test.py +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import unittest - -from airbyte_cdk.sources.streams.http.requests_native_auth.token import TokenAuthenticator -from source_opsgenie import SourceOpsgenie - - -class AuthenticatorTestCase(unittest.TestCase): - def test_token(self): - authenticator = SourceOpsgenie.get_authenticator({"api_token": "123"}) - self.assertIsInstance(authenticator, TokenAuthenticator) - self.assertEqual("GenieKey 123", authenticator.token) - self.assertEqual("Authorization", authenticator.auth_header) diff --git a/docs/integrations/sources/opsgenie.md b/docs/integrations/sources/opsgenie.md index 7c9abcfac282..088d27f21988 100644 --- a/docs/integrations/sources/opsgenie.md +++ b/docs/integrations/sources/opsgenie.md @@ -51,6 +51,6 @@ The Opsgenie connector uses the most recent API version for each source of data. | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------| :--- | -| 0.2.0 | 2023-10-24 | [16768](https://github.com/airbytehq/airbyte/pull/16768) | Fix schema | +| 0.3.0 | 2023-10-19 | [31552](https://github.com/airbytehq/airbyte/pull/31552) | Migrated to Low Code | +| 0.2.0 | 2023-10-24 | [31777](https://github.com/airbytehq/airbyte/pull/31777) | Fix schema | | 0.1.0 | 2022-09-14 | [16768](https://github.com/airbytehq/airbyte/pull/16768) | Initial Release | - From a7f692852805e2b378a7897019c8207552e26753 Mon Sep 17 00:00:00 2001 From: Tope Folorunso <66448986+topefolorunso@users.noreply.github.com> Date: Fri, 10 Nov 2023 19:57:49 +0100 Subject: [PATCH 008/124] :bug:Source Close.com: Fix Email Activities Stream Pagination (#31534) Co-authored-by: Marcos Marx Co-authored-by: marcosmarxm --- .../connectors/source-close-com/Dockerfile | 2 +- .../acceptance-test-config.yml | 2 - .../integration_tests/expected_records.jsonl | 51 +++++++++++++++---- .../connectors/source-close-com/metadata.yaml | 2 +- .../source_close_com/manifest.yaml | 5 +- .../schemas/activity_custom_fields.json | 12 +++++ .../schemas/call_activities.json | 24 +++++++++ .../schemas/contact_custom_fields.json | 9 ++++ .../schemas/created_activities.json | 3 ++ .../schemas/email_followup_tasks.json | 6 +++ .../source_close_com/schemas/events.json | 6 +++ .../schemas/lead_custom_fields.json | 9 ++++ .../schemas/meeting_activities.json | 51 +++++++++++++++++++ .../schemas/opportunity_custom_fields.json | 9 ++++ .../opportunity_status_change_activities.json | 21 ++++++++ .../source_close_com/schemas/pipelines.json | 3 ++ .../schemas/sms_activities.json | 30 +++++++++++ .../schemas/task_completed_activities.json | 15 ++++++ docs/integrations/sources/close-com.md | 1 + 19 files changed, 246 insertions(+), 15 deletions(-) diff --git a/airbyte-integrations/connectors/source-close-com/Dockerfile b/airbyte-integrations/connectors/source-close-com/Dockerfile index 4deb69dd321d..e77535415cb0 100644 --- a/airbyte-integrations/connectors/source-close-com/Dockerfile +++ b/airbyte-integrations/connectors/source-close-com/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.4.2 +LABEL io.airbyte.version=0.4.3 LABEL io.airbyte.name=airbyte/source-close-com diff --git a/airbyte-integrations/connectors/source-close-com/acceptance-test-config.yml b/airbyte-integrations/connectors/source-close-com/acceptance-test-config.yml index 98a1285dad97..41c2635332b7 100644 --- a/airbyte-integrations/connectors/source-close-com/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-close-com/acceptance-test-config.yml @@ -18,8 +18,6 @@ acceptance_tests: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - expect_records: - path: "integration_tests/expected_records.jsonl" empty_streams: - name: missed_call_tasks bypass_reason: "unable to populate" diff --git a/airbyte-integrations/connectors/source-close-com/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-close-com/integration_tests/expected_records.jsonl index bfcc3040ef80..072d820d710f 100644 --- a/airbyte-integrations/connectors/source-close-com/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-close-com/integration_tests/expected_records.jsonl @@ -13,10 +13,15 @@ {"stream": "sms_activities", "data": {"local_country_iso": "US", "activity_at": "2021-08-11T18:14:32.750000+00:00", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_phone_formatted": "+1 202-555-0186", "sequence_subscription_id": null, "date_scheduled": null, "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "_type": "SMS", "updated_by_name": "Airbyte Team", "local_phone": "+14154445555", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "id": "acti_GIVSys3F0wFeA519lDa5QKRfYOPgskyqKj2aXiCMSEO", "attachments": [], "sequence_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_sent": null, "local_phone_formatted": "+1 415-444-5555", "created_by_name": "Airbyte Team", "date_created": "2021-08-11T18:14:32.750000+00:00", "cost": null, "text": "Hi! This is a reminder that we have a call scheduled for 12pm PT today.", "direction": "outbound", "error_message": null, "date_updated": "2021-08-11T18:14:32.750000+00:00", "source": "Close.io", "template_id": null, "sequence_id": null, "remote_country_iso": "US", "status": "draft", "remote_phone": "+12025550186", "user_name": "Airbyte Team"}, "emitted_at": 1691417093006} {"stream": "sms_activities", "data": {"lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "attachments": [], "remote_phone": "+14156236785", "sequence_id": null, "remote_country_iso": "US", "sequence_subscription_id": null, "cost": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_phone_formatted": "+1 415-623-6785", "created_by_name": "Airbyte Team", "date_updated": "2022-11-09T14:02:24.943000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "direction": "outbound", "local_phone_formatted": "+1 415-625-1293", "text": "Hi!", "template_id": "smstmpl_6zpaSGDsZyhBhrQH0jmZK9", "date_created": "2022-11-09T14:02:18.756000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "local_phone": "+14156251293", "source": "Close.io", "_type": "SMS", "local_country_iso": "US", "updated_by_name": "Airbyte Team", "date_scheduled": null, "error_message": null, "id": "acti_CXrlrlHc8QpP5MBXqCfcJKpctDqBREuG3Whj0JQODu4", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "activity_at": "2022-11-09T14:02:18.756000+00:00", "user_name": "Airbyte Team", "sequence_name": null, "status": "draft", "date_sent": null, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417093912} {"stream": "sms_activities", "data": {"lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "attachments": [{"media_id": "media_580CfGelcIvP5BzvIYTlhX", "url": "https://app.close.com/go/sms/acti_eQ2xwn1RN5sdEXkGpln8Vag8rwPsYfU1qbyeN7dBPsP/media/media_580CfGelcIvP5BzvIYTlhX/", "thumbnail_url": "https://app.close.com/go/sms/acti_eQ2xwn1RN5sdEXkGpln8Vag8rwPsYfU1qbyeN7dBPsP/media/media_580CfGelcIvP5BzvIYTlhX/thumbnail/", "content_type": "image/png", "size": 6132, "filename": "Airbyte_logo_75x75.png"}], "remote_phone": "+14156236785", "sequence_id": null, "remote_country_iso": "US", "sequence_subscription_id": null, "cost": "3", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_phone_formatted": "+1 415-623-6785", "created_by_name": "Airbyte Team", "date_updated": "2022-11-09T12:54:44.405000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "direction": "outbound", "local_phone_formatted": "+1 415-625-1293", "text": "Hi!", "template_id": "smstmpl_6zpaSGDsZyhBhrQH0jmZK9", "date_created": "2022-11-09T12:54:15.456000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "local_phone": "+14156251293", "source": "Close.io", "_type": "SMS", "local_country_iso": "US", "updated_by_name": "Airbyte Team", "date_scheduled": null, "error_message": null, "id": "acti_eQ2xwn1RN5sdEXkGpln8Vag8rwPsYfU1qbyeN7dBPsP", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "activity_at": "2022-11-09T12:54:44.404000+00:00", "user_name": "Airbyte Team", "sequence_name": null, "status": "sent", "date_sent": "2022-11-09T12:54:44.404000+00:00", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417093915} -{"stream": "call_activities", "data": {"local_phone_formatted": null, "voicemail_url": "https://api.close.com/call/acti_NsDheeFBzEAmjRfpBmclxdzKWqLnGZaIvU3ZvvyrDJt/voicemail/", "transferred_to_user_id": null, "forwarded_to": null, "remote_country_iso": "US", "date_answered": null, "voicemail_duration": 28, "_type": "Call", "disposition": "vm-left", "status": "completed", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "updated_by_name": null, "date_created": "2021-07-16T00:00:12.646000+00:00", "recording_expires_at": null, "remote_phone_formatted": "+1 650-517-6539", "cost": null, "phone": "+16505176539", "is_joinable": false, "created_by_name": null, "note": null, "sequence_id": null, "recording_url": null, "local_phone": null, "sequence_subscription_id": null, "transferred_from_user_id": null, "has_recording": false, "source": "Close.io", "transferred_from": null, "direction": "inbound", "sequence_name": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "dialer_saved_search_id": null, "created_by": null, "remote_phone": "+16505176539", "updated_by": null, "duration": 0, "call_method": "regular", "id": "acti_NsDheeFBzEAmjRfpBmclxdzKWqLnGZaIvU3ZvvyrDJt", "coach_legs": [], "is_to_group_number": false, "date_updated": "2021-07-16T00:00:12.646000+00:00", "contact_id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "users": [], "user_name": null, "note_html": null, "is_forwarded": false, "transferred_to": null, "dialer_id": null, "local_country_iso": "", "user_id": null, "activity_at": "2021-07-16T00:00:12.646000+00:00"}, "emitted_at": 1691417095367} -{"stream": "call_activities", "data": {"local_phone_formatted": null, "voicemail_url": null, "transferred_to_user_id": null, "forwarded_to": null, "remote_country_iso": "US", "date_answered": null, "voicemail_duration": 0, "_type": "Call", "disposition": null, "status": "no-answer", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "updated_by_name": "Airbyte Team", "date_created": "2021-07-05T11:39:00.850000+00:00", "recording_expires_at": null, "remote_phone_formatted": "+1 202-555-0186", "cost": null, "phone": "+12025550186", "is_joinable": false, "created_by_name": "Airbyte Team", "note": "Gob never answered.", "sequence_id": null, "recording_url": null, "local_phone": null, "sequence_subscription_id": null, "transferred_from_user_id": null, "has_recording": false, "source": "Close.io", "transferred_from": null, "direction": "outbound", "sequence_name": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "dialer_saved_search_id": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_phone": "+12025550186", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "duration": 0, "call_method": "regular", "id": "acti_wszWUd92D7wNYSbn5gKWXCf55NeyU0jc1Vguv7DfUiH", "coach_legs": [], "is_to_group_number": false, "date_updated": "2021-07-13T11:39:04.536000+00:00", "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "users": [], "user_name": "Airbyte Team", "note_html": "

Gob never answered.

", "is_forwarded": false, "transferred_to": null, "dialer_id": null, "local_country_iso": "", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "activity_at": "2021-07-05T11:39:00.850000+00:00"}, "emitted_at": 1691417095372} -{"stream": "call_activities", "data": {"activity_at": "2022-11-09T13:57:14.751000+00:00", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "sequence_name": null, "users": [], "recording_url": null, "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "transferred_to": null, "remote_phone": "+14156236785", "remote_country_iso": "US", "cost": "2", "_type": "Call", "local_country_iso": "US", "duration": 17, "created_by_name": "Airbyte Team", "direction": "outbound", "has_recording": false, "voicemail_url": null, "transferred_to_user_id": null, "date_answered": "2022-11-09T13:57:22.063000+00:00", "transferred_from": null, "coach_legs": [], "phone": "+14156236785", "dialer_saved_search_id": null, "voicemail_duration": 0, "updated_by_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "dialer_id": null, "is_forwarded": false, "is_joinable": false, "transferred_from_user_id": null, "local_phone": "+14156251293", "date_updated": "2022-11-09T13:57:40.167000+00:00", "recording_expires_at": null, "forwarded_to": null, "sequence_subscription_id": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status": "completed", "source": "Close.io", "date_created": "2022-11-09T13:57:14.751000+00:00", "local_phone_formatted": "+1 415-625-1293", "note_html": "

", "is_to_group_number": false, "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "disposition": "answered", "remote_phone_formatted": "+1 415-623-6785", "user_name": "Airbyte Team", "note": "", "call_method": "regular", "id": "acti_ZgqHg31m0XXDZPwaxUVUNOhnEoLSG3fY8rMn9iIS3cN", "sequence_id": null}, "emitted_at": 1691417095961} -{"stream": "meeting_activities", "data": {"starts_at": "2022-11-12T18:00:00+00:00", "user_note": null, "updated_by_name": "Airbyte Team", "notetaker_id": null, "_type": "Meeting", "id": "acti_AlRbqNk15jdt7Eq2phHXuV49qSpkLBB7af3mwvJkk1z", "duration": 3600, "ends_at": "2022-11-12T19:00:00+00:00", "title": "Test meeting 2", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status": "completed", "activity_at": "2022-11-12T18:00:00+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "connected_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "user_name": "Airbyte Team", "user_note_html": null, "date_updated": "2022-11-12T19:00:03.639000+00:00", "is_recurring": false, "contact_id": null, "attendees": [{"contact_id": null, "is_organizer": false, "email": "irina.grankova@gmail.com", "name": null, "status": "yes", "user_id": null}, {"contact_id": null, "is_organizer": true, "email": "iryna.grankova@airbyte.io", "name": null, "status": "yes", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}], "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "users": ["user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"], "calendar_event_link": "https://www.google.com/calendar/event?eid=MWRlcDU1cDZtamY0MGxnYnJ2OXI5ajlocG8gaXJ5bmEuZ3JhbmtvdmFAYWlyYnl0ZS5pbw", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "location": null, "integrations": [], "created_by_name": "Airbyte Team", "source": "calendar", "note": "", "date_created": "2022-11-12T18:00:00+00:00", "calendar_event_uids": ["1dep55p6mjf40lgbrv9r9j9hpo"]}, "emitted_at": 1691417097896} +{"stream": "call_activities", "data": {"has_recording": false, "note": null, "remote_phone_formatted": "+1 650-517-6539", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "is_forwarded": false, "sequence_subscription_id": null, "dialer_id": null, "date_answered": null, "transferred_from_user_id": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by_name": null, "transferred_to": null, "updated_by_name": null, "dialer_saved_search_id": null, "call_method": "regular", "parent_meeting_id": null, "local_phone_formatted": null, "updated_by": null, "remote_country_iso": "US", "is_to_group_number": false, "date_created": "2021-07-16T00:00:12.646000+00:00", "voicemail_url": "https://api.close.com/call/acti_NsDheeFBzEAmjRfpBmclxdzKWqLnGZaIvU3ZvvyrDJt/voicemail/", "duration": 0, "forwarded_to": null, "recording_duration": null, "phone": "+16505176539", "voicemail_duration": 28, "cost": null, "sequence_name": null, "created_by": null, "note_html": null, "local_country_iso": "", "transferred_from": null, "source": "Close.io", "users": [], "transferred_to_user_id": null, "recording_url": null, "id": "acti_NsDheeFBzEAmjRfpBmclxdzKWqLnGZaIvU3ZvvyrDJt", "status": "completed", "contact_id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "_type": "Call", "disposition": "vm-left", "local_phone": null, "direction": "inbound", "is_joinable": false, "coach_legs": [], "date_updated": "2021-07-16T00:00:12.646000+00:00", "recording_expires_at": null, "user_id": null, "sequence_id": null, "user_name": null, "activity_at": "2021-07-16T00:00:12.646000+00:00", "remote_phone": "+16505176539"}, "emitted_at": 1699627822986} +{"stream": "call_activities", "data": {"has_recording": false, "note": "Gob never answered.", "remote_phone_formatted": "+1 202-555-0186", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "is_forwarded": false, "sequence_subscription_id": null, "dialer_id": null, "date_answered": null, "transferred_from_user_id": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by_name": "Airbyte Team", "transferred_to": null, "updated_by_name": "Airbyte Team", "dialer_saved_search_id": null, "call_method": "regular", "parent_meeting_id": null, "local_phone_formatted": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_country_iso": "US", "is_to_group_number": false, "date_created": "2021-07-05T11:39:00.850000+00:00", "voicemail_url": null, "duration": 0, "forwarded_to": null, "recording_duration": null, "phone": "+12025550186", "voicemail_duration": 0, "cost": null, "sequence_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "note_html": "

Gob never answered.

", "local_country_iso": "", "transferred_from": null, "source": "Close.io", "users": [], "transferred_to_user_id": null, "recording_url": null, "id": "acti_wszWUd92D7wNYSbn5gKWXCf55NeyU0jc1Vguv7DfUiH", "status": "no-answer", "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "_type": "Call", "disposition": null, "local_phone": null, "direction": "outbound", "is_joinable": false, "coach_legs": [], "date_updated": "2021-07-13T11:39:04.536000+00:00", "recording_expires_at": null, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "sequence_id": null, "user_name": "Airbyte Team", "activity_at": "2021-07-05T11:39:00.850000+00:00", "remote_phone": "+12025550186"}, "emitted_at": 1699627822992} +{"stream": "call_activities", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_country_iso": "US", "date_created": "2022-11-09T13:57:14.751000+00:00", "has_recording": false, "voicemail_url": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "duration": 17, "parent_meeting_id": null, "recording_duration": null, "activity_at": "2022-11-09T13:57:14.751000+00:00", "dialer_saved_search_id": null, "dialer_id": null, "sequence_name": null, "voicemail_duration": 0, "direction": "outbound", "updated_by_name": "Airbyte Team", "recording_url": null, "call_method": "regular", "date_updated": "2022-11-09T13:57:40.167000+00:00", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "transferred_to_user_id": null, "recording_expires_at": null, "note": "", "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "status": "completed", "user_name": "Airbyte Team", "local_country_iso": "US", "transferred_from": null, "cost": "2", "transferred_to": null, "sequence_id": null, "source": "Close.io", "local_phone": "+14156251293", "disposition": "answered", "coach_legs": [], "phone": "+14156236785", "transferred_from_user_id": null, "created_by_name": "Airbyte Team", "sequence_subscription_id": null, "is_forwarded": false, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "note_html": "

", "is_to_group_number": false, "forwarded_to": null, "remote_phone_formatted": "+1 415-623-6785", "local_phone_formatted": "+1 415-625-1293", "id": "acti_ZgqHg31m0XXDZPwaxUVUNOhnEoLSG3fY8rMn9iIS3cN", "remote_phone": "+14156236785", "date_answered": "2022-11-09T13:57:22.063000+00:00", "is_joinable": false, "users": [], "_type": "Call"}, "emitted_at": 1699627823579} +{"stream": "call_activities", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_country_iso": "US", "date_created": "2022-11-09T12:55:26.751000+00:00", "has_recording": false, "voicemail_url": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "duration": 0, "parent_meeting_id": null, "recording_duration": null, "activity_at": "2022-11-09T12:55:26.751000+00:00", "dialer_saved_search_id": null, "dialer_id": null, "sequence_name": null, "voicemail_duration": 0, "direction": "outbound", "updated_by_name": "Airbyte Team", "recording_url": null, "call_method": "regular", "date_updated": "2022-11-09T12:57:27.648000+00:00", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "transferred_to_user_id": null, "recording_expires_at": null, "note": "", "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "status": "completed", "user_name": "Airbyte Team", "local_country_iso": "US", "transferred_from": null, "cost": null, "transferred_to": null, "sequence_id": null, "source": "Close.io", "local_phone": "+14156251293", "disposition": "answered", "coach_legs": [], "phone": "+14156236785", "transferred_from_user_id": null, "created_by_name": "Airbyte Team", "sequence_subscription_id": null, "is_forwarded": false, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "note_html": "

", "is_to_group_number": false, "forwarded_to": null, "remote_phone_formatted": "+1 415-623-6785", "local_phone_formatted": "+1 415-625-1293", "id": "acti_694PvtEQICHUY5umTjEM8nl9bLU0MmWWV6HhMEcqAEF", "remote_phone": "+14156236785", "date_answered": null, "is_joinable": false, "users": [], "_type": "Call"}, "emitted_at": 1699627823583} +{"stream": "call_activities", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_country_iso": "US", "date_created": "2022-11-09T12:55:03.314000+00:00", "has_recording": false, "voicemail_url": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "duration": 10, "parent_meeting_id": null, "recording_duration": null, "activity_at": "2022-11-09T12:55:03.314000+00:00", "dialer_saved_search_id": null, "dialer_id": null, "sequence_name": null, "voicemail_duration": 0, "direction": "outbound", "updated_by_name": "Airbyte Team", "recording_url": null, "call_method": "regular", "date_updated": "2022-11-09T12:55:23.356000+00:00", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "transferred_to_user_id": null, "recording_expires_at": null, "note": "", "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "status": "completed", "user_name": "Airbyte Team", "local_country_iso": "US", "transferred_from": null, "cost": "2", "transferred_to": null, "sequence_id": null, "source": "Close.io", "local_phone": "+14156251293", "disposition": "answered", "coach_legs": [], "phone": "+14156236785", "transferred_from_user_id": null, "created_by_name": "Airbyte Team", "sequence_subscription_id": null, "is_forwarded": false, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "note_html": "

", "is_to_group_number": false, "forwarded_to": null, "remote_phone_formatted": "+1 415-623-6785", "local_phone_formatted": "+1 415-625-1293", "id": "acti_xzjEYrejim9F9IWz4mrs16ehPCcnUcit43wJTZkwuhv", "remote_phone": "+14156236785", "date_answered": "2022-11-09T12:55:11.703000+00:00", "is_joinable": false, "users": [], "_type": "Call"}, "emitted_at": 1699627823587} +{"stream": "call_activities", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_country_iso": "US", "date_created": "2022-11-09T11:20:32.524000+00:00", "has_recording": false, "voicemail_url": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "duration": 3, "parent_meeting_id": null, "recording_duration": null, "activity_at": "2022-11-09T11:20:32.524000+00:00", "dialer_saved_search_id": null, "dialer_id": null, "sequence_name": null, "voicemail_duration": 0, "direction": "outbound", "updated_by_name": "Airbyte Team", "recording_url": null, "call_method": "regular", "date_updated": "2022-11-09T11:20:42.502000+00:00", "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "transferred_to_user_id": null, "recording_expires_at": null, "note": "", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "status": "completed", "user_name": "Airbyte Team", "local_country_iso": "US", "transferred_from": null, "cost": "2", "transferred_to": null, "sequence_id": null, "source": "Close.io", "local_phone": "+14156251293", "disposition": "answered", "coach_legs": [], "phone": "+14156236785", "transferred_from_user_id": null, "created_by_name": "Airbyte Team", "sequence_subscription_id": null, "is_forwarded": false, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "note_html": "

", "is_to_group_number": false, "forwarded_to": null, "remote_phone_formatted": "+1 415-623-6785", "local_phone_formatted": "+1 415-625-1293", "id": "acti_2PDiIs2kT5NQaipTXd1lyavwVLS5a6wipeheGVWOO54", "remote_phone": "+14156236785", "date_answered": "2022-11-09T11:20:38.986000+00:00", "is_joinable": false, "users": [], "_type": "Call"}, "emitted_at": 1699627823592} +{"stream": "call_activities", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_country_iso": "UA", "date_created": "2022-11-08T15:55:39.770000+00:00", "has_recording": false, "voicemail_url": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "duration": 0, "parent_meeting_id": null, "recording_duration": null, "activity_at": "2022-11-08T15:55:39.770000+00:00", "dialer_saved_search_id": null, "dialer_id": null, "sequence_name": null, "voicemail_duration": 0, "direction": "outbound", "updated_by_name": "Airbyte Team", "recording_url": null, "call_method": "regular", "date_updated": "2022-11-09T12:56:13.631000+00:00", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "transferred_to_user_id": null, "recording_expires_at": null, "note": "Test test", "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "status": "failed", "user_name": "Airbyte Team", "local_country_iso": "", "transferred_from": null, "cost": null, "transferred_to": null, "sequence_id": null, "source": "Close.io", "local_phone": null, "disposition": "blocked", "coach_legs": [], "phone": "+380636306253", "transferred_from_user_id": null, "created_by_name": "Airbyte Team", "sequence_subscription_id": null, "is_forwarded": false, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "note_html": "

Test test

", "is_to_group_number": false, "forwarded_to": null, "remote_phone_formatted": "+380 63 630 6253", "local_phone_formatted": null, "id": "acti_cBMvILyTlk57YSm7c8xfb9qNjgTX1z2OyjnPPsTXpRG", "remote_phone": "+380636306253", "date_answered": null, "is_joinable": false, "users": [], "_type": "Call"}, "emitted_at": 1699627823596} +{"stream": "call_activities", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_country_iso": "UA", "date_created": "2022-11-08T15:53:39.388000+00:00", "has_recording": false, "voicemail_url": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "duration": 0, "parent_meeting_id": null, "recording_duration": null, "activity_at": "2022-11-08T15:53:39.388000+00:00", "dialer_saved_search_id": null, "dialer_id": null, "sequence_name": null, "voicemail_duration": 0, "direction": "outbound", "updated_by_name": "Airbyte Team", "recording_url": null, "call_method": "regular", "date_updated": "2022-11-08T15:54:58.677000+00:00", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "transferred_to_user_id": null, "recording_expires_at": null, "note": "", "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "status": "failed", "user_name": "Airbyte Team", "local_country_iso": "", "transferred_from": null, "cost": null, "transferred_to": null, "sequence_id": null, "source": "Close.io", "local_phone": null, "disposition": "blocked", "coach_legs": [], "phone": "+380636306253", "transferred_from_user_id": null, "created_by_name": "Airbyte Team", "sequence_subscription_id": null, "is_forwarded": false, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "note_html": "

", "is_to_group_number": false, "forwarded_to": null, "remote_phone_formatted": "+380 63 630 6253", "local_phone_formatted": null, "id": "acti_HpPgUhPNiZeEkjdpfW6jmK11uXJyTZHHiubYmU0uoFZ", "remote_phone": "+380636306253", "date_answered": null, "is_joinable": false, "users": [], "_type": "Call"}, "emitted_at": 1699627823601} +{"stream": "meeting_activities", "data": {"calendar_event_link": "https://www.google.com/calendar/event?eid=MWRlcDU1cDZtamY0MGxnYnJ2OXI5ajlocG8gaXJ5bmEuZ3JhbmtvdmFAYWlyYnl0ZS5pbw", "notetaker_id": null, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "integrations": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "summary": null, "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "created_by_name": "Airbyte Team", "note": "", "user_note": null, "duration": 3600, "location": null, "date_updated": "2022-11-12T19:00:03.639000+00:00", "provider_calendar_ids": ["iryna.grankova@airbyte.io"], "users": ["user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"], "title": "Test meeting 2", "is_recurring": false, "id": "acti_AlRbqNk15jdt7Eq2phHXuV49qSpkLBB7af3mwvJkk1z", "activity_at": "2022-11-12T18:00:00+00:00", "provider_calendar_type": "google", "status": "completed", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "conference_links": [], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_note_date_updated": null, "date_created": "2022-11-12T18:00:00+00:00", "user_note_html": null, "contact_id": null, "source": "calendar", "calendar_event_uids": ["1dep55p6mjf40lgbrv9r9j9hpo"], "attached_call_ids": [], "provider_calendar_event_id": "1dep55p6mjf40lgbrv9r9j9hpo", "_type": "Meeting", "starts_at": "2022-11-12T18:00:00+00:00", "attendees": [{"email": "irina.grankova@gmail.com", "name": null, "status": "yes", "user_id": null, "contact_id": null, "is_organizer": false}, {"email": "iryna.grankova@airbyte.io", "name": null, "status": "yes", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "contact_id": null, "is_organizer": true}], "ends_at": "2022-11-12T19:00:00+00:00", "connected_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "user_name": "Airbyte Team", "updated_by_name": "Airbyte Team"}, "emitted_at": 1699628913615} {"stream": "lead_status_change_activities", "data": {"new_status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "_type": "LeadStatusChange", "old_status_label": "Potential", "users": [], "activity_at": "2021-08-25T21:15:35.163000+00:00", "date_created": "2021-08-25T21:15:35.163000+00:00", "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "user_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "acti_wSO4ltT4XHGI6wkN2oCZpRdCtw1ALhsRttt8osqVjll", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "date_updated": "2021-08-25T21:15:35.163000+00:00", "old_status_id": "stat_HrZ1aYkkxRORQSxdBcNPT31HkqxkK2w2uWGiK6yjkmK", "new_status_label": "Interested", "contact_id": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417099274} {"stream": "lead_status_change_activities", "data": {"new_status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "_type": "LeadStatusChange", "old_status_label": "Qualified", "users": [], "activity_at": "2021-08-25T21:15:34.607000+00:00", "date_created": "2021-08-25T21:15:34.607000+00:00", "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "user_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "acti_T4XA2LBQYvrqAimflFjU2KXY0I2g1BPbOUn2jHZ59Dj", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "date_updated": "2021-08-25T21:15:34.607000+00:00", "old_status_id": "stat_y6v7svdpj3v1ZHd1GoiJFcKrUGrA0jl2Af53jfGbkN9", "new_status_label": "Interested", "contact_id": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417099279} {"stream": "lead_status_change_activities", "data": {"new_status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "_type": "LeadStatusChange", "old_status_label": "Qualified", "users": [], "activity_at": "2021-08-25T21:15:34.044000+00:00", "date_created": "2021-08-25T21:15:34.044000+00:00", "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "user_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "acti_udWvk5RJ6SFuO9Ra6d1pAJg200Q0Zyq43kEq0FpceIx", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "date_updated": "2021-08-25T21:15:34.044000+00:00", "old_status_id": "stat_y6v7svdpj3v1ZHd1GoiJFcKrUGrA0jl2Af53jfGbkN9", "new_status_label": "Interested", "contact_id": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417099283} @@ -32,7 +37,7 @@ {"stream": "activity_custom_fields", "data": {"choices": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "editable_with_roles": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "cf_UftGNTS2rq9XMG9hOHkALp5cgn4Xl8nZqN7gReax3lc", "referenced_custom_type_id": null, "date_created": "2021-07-13T11:39:02.212000+00:00", "name": "Contact", "back_reference_is_visible": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "accepts_multiple_values": false, "type": "contact", "date_updated": "2021-07-13T11:39:02.212000+00:00", "is_shared": false, "description": null, "custom_activity_type_id": "actitype_0J9YvrOw4opjiYI4aDY6wj", "required": false}, "emitted_at": 1691417109700} {"stream": "activity_custom_fields", "data": {"choices": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "editable_with_roles": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "cf_3W4n175LyZ3QMfr665jUS19nt2QATHyM7QaxXQC4QqA", "referenced_custom_type_id": null, "date_created": "2021-07-13T11:39:02.744000+00:00", "name": "Current Vendor: Other (if applicable)", "back_reference_is_visible": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "accepts_multiple_values": false, "type": "text", "date_updated": "2021-07-13T11:39:02.744000+00:00", "is_shared": false, "description": null, "custom_activity_type_id": "actitype_0J9YvrOw4opjiYI4aDY6wj", "required": false}, "emitted_at": 1691417109703} {"stream": "activity_custom_fields", "data": {"choices": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "editable_with_roles": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "cf_bp93vNo2vxbmM2QhQ7JvdytgcozCCGmOklTG9E8rDYa", "referenced_custom_type_id": null, "date_created": "2021-07-13T11:39:02.478000+00:00", "name": "Industry: Other (if applicable)", "back_reference_is_visible": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "accepts_multiple_values": false, "type": "text", "date_updated": "2021-07-13T11:39:02.478000+00:00", "is_shared": false, "description": null, "custom_activity_type_id": "actitype_0J9YvrOw4opjiYI4aDY6wj", "required": false}, "emitted_at": 1691417109705} -{"stream": "users", "data": {"last_used_timezone": "America/New_York", "date_created": "2021-07-13T11:36:04.905000+00:00", "date_updated": "2023-06-29T19:39:02.749000+00:00", "first_name": "Airbyte", "last_name": "Team", "google_profile_image_url": null, "image": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef", "organizations": ["orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi"], "email": "integration-test@airbyte.io", "email_verified_at": "2021-07-13T11:37:23.175000+00:00", "id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417111009} +{"stream": "users", "data": {"google_profile_image_url": null, "date_created": "2021-07-13T11:36:04.905000+00:00", "date_updated": "2023-10-21T04:56:24.365000+00:00", "id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organizations": ["orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi"], "last_name": "Team", "email_verified_at": "2021-07-13T11:37:23.175000+00:00", "last_used_timezone": "America/New_York", "image": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef", "first_name": "Airbyte", "email": "integration-test@airbyte.io"}, "emitted_at": 1699629347222} {"stream": "contacts", "data": {"id": "cont_b4h4BcmWn7rKbnsHQ0JfADwXGgndbpc5JlQEdHHyv78", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=User1"}], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "display_name": "User1", "date_created": "2022-11-11T09:00:52.289000+00:00", "emails": [], "lead_id": "lead_AUtZm7EBlaSbYqOrDjIZEuC4tfhLxTDtaK9jlEPMb3y", "date_updated": "2023-01-30T16:02:41.174000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "phones": [{"phone_formatted": "+1 600-000-0001", "country": null, "type": "office", "phone": "+16000000001"}], "urls": [], "title": "Product Manager", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "User1"}, "emitted_at": 1691417111908} {"stream": "contacts", "data": {"id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=User2"}], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "display_name": "User2", "date_created": "2022-11-08T15:54:42.381000+00:00", "emails": [{"type": "office", "email": "user2.sample@gmail.com"}], "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "date_updated": "2023-01-30T16:03:28.787000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "phones": [{"phone_formatted": "+1 415-623-6785", "country": "US", "type": "office", "phone": "+14156236785"}], "urls": [], "title": "Test Lead", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "User2", "custom.cf_Qj3b4cWxvmvqTtZ0U5TaprRDah8g7jRsavfVh8NCPcu": "2022-12-01"}, "emitted_at": 1691417111912} {"stream": "contacts", "data": {"id": "cont_Dsi7AGMRelIZ2I6DIKicaGJU7mwxPZElLIHy33xbjCM", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Cooper"}], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "display_name": "Cooper", "date_created": "2022-07-05T21:01:25.612000+00:00", "emails": [], "lead_id": "lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY", "date_updated": "2022-07-05T21:01:25.612000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "phones": [], "urls": [], "title": "", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Cooper"}, "emitted_at": 1691417111916} @@ -48,10 +53,38 @@ {"stream": "pipelines", "data": {"created_by": null, "date_created": "2021-07-13T11:36:04.983404", "date_updated": "2021-07-13T11:36:04.983404", "id": "pipe_0IAl41rGk9OPls9CdxFpHy", "name": "Sales", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "statuses": [{"id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "label": "Demo Completed", "type": "active"}, {"id": "stat_AWXzFJkvkHVyJQPFulY0wM7LrQRiMEtQChGumG035bH", "label": "Proposal Sent", "type": "active"}, {"id": "stat_ObYTUqjVZW0nTZXjvHhzMGyK999e42WZdIhkaNq12En", "label": "Contract Sent", "type": "active"}, {"id": "stat_gaqEGSVHIFzrofTfzzg5UfjyBZ1B6KERccIy2MOp8FG", "label": "Won", "type": "won"}, {"id": "stat_CZr5826cyG8wqIg4tD6bbjaqePP4HAYOMSLgOhi1Xbf", "label": "Lost", "type": "lost"}], "updated_by": null}, "emitted_at": 1691417119534} {"stream": "email_templates", "data": {"organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "attachments": [], "body": "Hi {{ contact.first_name }},

I'm {{ user.first_name }} with {{ organization.name }}. We help companies in the {{ lead.custom.[\"Industry\"] }} space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at {{ lead.display_name }} and show you what we're working on.

Are you available for a quick call tomorrow afternoon?
", "name": "Email 1 - Intro", "id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "is_shared": true, "is_archived": false, "date_created": "2021-07-13T11:39:00.497000+00:00", "subject": "{{ lead.display_name }} + {{ organization.name }}", "date_updated": "2022-08-11T18:11:08.966000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417120444} {"stream": "email_templates", "data": {"organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "attachments": [], "body": "Hi {{ contact.first_name }},

Friendly follow-up.

I wanted to show you how {{ organization.name }} can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

- Wed @ 11AM
- Thur @ 2PM
- Fri @ 3PM
", "name": "Email 2 - Follow-up #1", "id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "is_shared": true, "is_archived": false, "date_created": "2021-07-13T11:39:00.503000+00:00", "subject": "{{ organization.name }} Follow-up", "date_updated": "2022-08-11T18:11:08.972000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417120447} -{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "next_billing_on"], "data": {"address_id": null, "bundle_id": null, "carrier": "twilio", "country": "US", "date_created": "2022-11-08T12:35:29.464000+00:00", "date_updated": "2023-08-07T10:04:12.414000+00:00", "forward_to": null, "forward_to_enabled": false, "forward_to_formatted": null, "id": "phon_jhMWlB6anhT8vcsGNEFukaVl806zfxCgbSAAkvtNBpN", "is_group_number": false, "is_verified": false, "label": null, "last_billed_price": "1.15", "mms_enabled": true, "next_billing_on": "2023-09-07", "number": "+14156251293", "number_formatted": "+1 415-625-1293", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "press_1_to_accept": true, "sms_enabled": true, "supports_mms_to_countries": ["CA", "US"], "supports_sms_to_countries": ["CA", "PR", "US"], "type": "internal", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "voicemail_greeting_url": "https://closeio-voicemail-greetings.s3.amazonaws.com/14bzVblrIaekmXSGQEKM11/undefined.mp3"}, "date_created": "2023-08-07T10:04:12.416000", "date_updated": "2023-08-07T10:04:12.416000", "id": "ev_3o6F0cl3A3CkYyoc9vrVjM", "lead_id": null, "meta": {}, "oauth_client_id": null, "oauth_scope": null, "object_id": "phon_jhMWlB6anhT8vcsGNEFukaVl806zfxCgbSAAkvtNBpN", "object_type": "phone_number", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-07-07T10:03:30.945000+00:00", "next_billing_on": "2023-08-07"}, "request_id": null, "user_id": null}, "emitted_at": 1691417122379} -{"stream": "leads", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_label": "Potential", "contacts": [{"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Cooper", "phones": [], "urls": [], "display_name": "Cooper", "id": "cont_Dsi7AGMRelIZ2I6DIKicaGJU7mwxPZElLIHy33xbjCM", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2022-07-05T21:01:25.612000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "emails": [], "title": "", "date_updated": "2022-07-05T21:01:25.612000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Cooper"}], "lead_id": "lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY"}], "status_id": "stat_HrZ1aYkkxRORQSxdBcNPT31HkqxkK2w2uWGiK6yjkmK", "id": "lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY", "addresses": [], "date_created": "2022-07-05T21:01:25.608000+00:00", "custom": {}, "html_url": "https://app.close.com/lead/lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY/", "date_updated": "2022-07-05T21:01:25.658000+00:00", "tasks": [], "name": "Alex", "created_by_name": "Airbyte Team", "display_name": "Alex", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Alex"}], "updated_by_name": "Airbyte Team", "description": "", "url": null, "opportunities": []}, "emitted_at": 1691417125238} -{"stream": "leads", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_label": "Interested", "contacts": [{"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Gob Bluth", "phones": [{"phone": "+12025550186", "phone_formatted": "+1 202-555-0186", "country": "US", "type": "office"}], "urls": [], "display_name": "Gob Bluth", "id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2021-07-13T11:39:04.430000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "emails": [{"email": "bluth@close.com", "type": "office"}], "title": "Magician", "date_updated": "2021-07-13T11:39:04.430000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Gob%20Bluth"}], "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Gatekeeper"]}, {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Tobias F\u00fcnke", "phones": [], "urls": [], "display_name": "Tobias F\u00fcnke", "id": "cont_at5uglNbyasFp2KsoWQpjQmLp4lmmqX2p1nhvmPYytq", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2021-07-13T11:39:04.441000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "emails": [{"email": "tobiasfunke@close.com", "type": "office"}], "title": "Blue Man Group (Understudy)", "date_updated": "2021-07-13T11:39:04.441000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Tobias%20F%C3%BCnke"}], "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Point of Contact"]}], "status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "addresses": [{"address_1": "100 Bluth Drive", "label": "business", "country": "US", "city": "Los Angeles", "zipcode": "90210", "state": "CA", "address_2": null}], "date_created": "2021-07-05T11:39:00.850000+00:00", "custom": {"Current Vendor/Software": "BiffCo", "Industry": "Real estate", "Lead Owner": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "Referral Source": "Google Search"}, "html_url": "https://app.close.com/lead/lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2/", "date_updated": "2022-11-08T16:28:54.174000+00:00", "tasks": [], "name": "Bluth Company (Example\u00a0Lead)", "created_by_name": "Airbyte Team", "display_name": "Bluth Company (Example\u00a0Lead)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Bluth%20Company%20%28Example%C2%A0Lead%29"}], "updated_by_name": "Airbyte Team", "description": null, "url": null, "opportunities": [{"annualized_expected_value": 225000, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_display_name": "Demo Completed", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "status_type": "active", "status_id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "id": "oppo_NkKuhUWfDoErNArh44hw7jkkx7sCl5bhlamtezmX7BE", "date_created": "2021-07-13T11:39:04.817000+00:00", "expected_value": 225000, "lead_name": "Bluth Company (Example\u00a0Lead)", "date_lost": null, "date_updated": "2021-07-13T11:39:04.817000+00:00", "note": "Gob's ready to buy a $3,000 suit.", "value_formatted": "$3,000", "status_label": "Demo Completed", "contact_name": null, "created_by_name": "Airbyte Team", "confidence": 75, "contact_id": null, "value_period": "one_time", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "value": 300000, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_won": "2021-07-16", "value_currency": "USD", "updated_by_name": "Airbyte Team", "annualized_value": 300000, "integration_links": [], "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2"}], "custom.cf_1exVDDcGOEiIdhBhBv2VEGqnpIcJZZqiWkk4O7hbU3D": "BiffCo", "custom.cf_ZuP9X9UjiQzjptNHlT7DxzRATaFil2Ysoz0aGMq0Kim": "Real estate", "custom.cf_mhBoQeiuwFRlz7zqyi4kJgzUreEoUp0hUsLwrnUTEgh": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_05o22yQYHMrFh4cCYCMSQJdSpODdabCbGQ8il5Do7X4": "Google Search"}, "emitted_at": 1691417125245} -{"stream": "leads", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_label": "Interested", "contacts": [{"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Steli Efti", "phones": [{"phone": "+16505176539", "phone_formatted": "+1 650-517-6539", "country": "US", "type": "office"}], "urls": [], "display_name": "Steli Efti", "id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2021-07-13T11:39:03.354000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "emails": [{"email": "sales@close.com", "type": "office"}], "title": "CEO & Co-Founder", "date_updated": "2021-07-13T11:39:03.354000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Steli%20Efti"}], "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Decision Maker"]}, {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Nick Persico", "phones": [{"phone": "+18334625673", "phone_formatted": "+1 833-462-5673", "country": "US", "type": "office"}], "urls": [], "display_name": "Nick Persico", "id": "cont_FY5ws8upMQQyD9vKg4jzwRb6V3MLctQeNTc2NaUmAyo", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2021-07-13T11:39:03.366000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "emails": [{"email": "nick@close.com", "type": "office"}], "title": "Director of Revenue", "date_updated": "2021-07-13T11:39:03.366000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Nick%20Persico"}], "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Gatekeeper", "Point of Contact"]}, {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Customer Success Team", "phones": [{"phone": "+18334625673", "phone_formatted": "+1 833-462-5673", "country": "US", "type": "office"}], "urls": [], "display_name": "Customer Success Team", "id": "cont_4cmimyQMTMi61kc72mLAV9XdHdddw6LR1LqzvoNdSuV", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2021-07-13T11:39:03.374000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "emails": [{"email": "success@close.com", "type": "office"}], "title": null, "date_updated": "2021-07-13T11:39:03.374000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Customer%20Success%20Team"}], "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz"}, {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Support", "phones": [], "urls": [], "display_name": "Support", "id": "cont_CI5c6Ekew0cyhSgoFIXeaz2PJVmmtigF2XNIGUDXKnu", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2021-07-13T11:39:03.380000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "emails": [{"email": "support@close.com", "type": "office"}], "title": null, "date_updated": "2021-07-13T11:39:03.380000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Support"}], "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz"}], "status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "addresses": [{"address_1": "PO Box 7775 #69574", "label": "mailing", "country": "US", "city": "San Francisco", "zipcode": "94120", "state": "CA", "address_2": null}], "date_created": "2021-07-13T11:39:03.315000+00:00", "custom": {"Current Vendor/Software": "Stark Industries", "Industry": "Software", "Lead Owner": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "Referral Source": "Website"}, "html_url": "https://app.close.com/lead/lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz/", "date_updated": "2022-11-09T02:40:26.489000+00:00", "tasks": [{"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "view": "archive", "id": "task_bboTdYSlGqTBSXF0FEwhBOywDCV6iKDyMWiEfNFi7sW", "date_created": "2021-07-13T11:39:03.520000+00:00", "lead_name": "Close (Example\u00a0Lead)", "is_dateless": null, "date_updated": "2022-11-08T12:21:15.730000+00:00", "text": "Call Steli", "is_complete": true, "contact_name": null, "created_by_name": "Airbyte Team", "date": "2021-07-18", "contact_id": null, "due_date": "2021-07-18", "assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "object_id": "acti_POPD3uA7lSfdf4xLO7PgsxKeoJS1dCRthzQRb13Xbyw", "_type": "lead", "updated_by_name": "Airbyte Team", "object_type": "taskcompleted", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz"}, {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "view": "archive", "id": "task_7kpMfXIPms858l9GKZTo3BCtVflvcIsOYPXL8mZgzyp", "date_created": "2021-07-13T11:39:03.463000+00:00", "lead_name": "Close (Example\u00a0Lead)", "is_dateless": null, "date_updated": "2021-08-18T10:31:52.081000+00:00", "text": "Send Steli an email", "is_complete": true, "contact_name": null, "created_by_name": "Airbyte Team", "date": "2021-07-16", "contact_id": null, "due_date": "2021-07-16", "assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "object_id": "acti_gdh0Iw30XYfKhKctrqPuxAbNghsxlDBDFaN8k35ZAxf", "_type": "lead", "updated_by_name": "Airbyte Team", "object_type": "taskcompleted", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz"}], "name": "Close (Example\u00a0Lead)", "created_by_name": "Airbyte Team", "display_name": "Close (Example\u00a0Lead)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Close%20%28Example%C2%A0Lead%29"}], "updated_by_name": "Airbyte Team", "description": "Visit our blog for high quality sales content, blog.close.com!", "url": "https://close.com", "opportunities": [{"annualized_expected_value": 37500, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_display_name": "Proposal Sent", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "status_type": "active", "status_id": "stat_AWXzFJkvkHVyJQPFulY0wM7LrQRiMEtQChGumG035bH", "id": "oppo_1TfmaSLuECcdSQIBnjUOBj1gAXdyrp4SFaogvcEmtbk", "date_created": "2021-07-13T11:39:04.284000+00:00", "expected_value": 37500, "lead_name": "Close (Example\u00a0Lead)", "date_lost": null, "date_updated": "2021-08-18T10:26:44.306000+00:00", "note": "Use opportunities to track which stage of the pipeline your deals are in and the revenue associated with them.", "value_formatted": "$500", "status_label": "Proposal Sent", "contact_name": "Steli Efti", "created_by_name": "Airbyte Team", "confidence": 75, "contact_id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "value_period": "one_time", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "value": 50000, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_won": "2021-07-15", "value_currency": "USD", "updated_by_name": "Airbyte Team", "annualized_value": 50000, "integration_links": [], "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz"}], "custom.cf_1exVDDcGOEiIdhBhBv2VEGqnpIcJZZqiWkk4O7hbU3D": "Stark Industries", "custom.cf_ZuP9X9UjiQzjptNHlT7DxzRATaFil2Ysoz0aGMq0Kim": "Software", "custom.cf_mhBoQeiuwFRlz7zqyi4kJgzUreEoUp0hUsLwrnUTEgh": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_05o22yQYHMrFh4cCYCMSQJdSpODdabCbGQ8il5Do7X4": "Website"}, "emitted_at": 1691417125251} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T12:58:44.679000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Gob,

Friendly follow-up.

I wanted to show you how Airbyte can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

- Wed @ 11AM
- Thur @ 2PM
- Fri @ 3PM
", "body_preview": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur ", "body_text": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur @ 2PM \n\\- Fri @ 3PM", "bulk_email_action_id": "bulkemail_bCJdPbYpJ2xfUuj8Dif3sBMHlxtbOKuEZSCdNcTgsWB", "cc": [], "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T12:58:44.579000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T12:58:44.679000+00:00", "date_updated": "2023-11-10T15:30:43.534000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 12:58:44 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166791232468.5302.9789676641910298968@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Airbyte Follow-up", "to": [{"email": "bluth@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166791232468.5302.9789676641910298968@smtpgw.close.com>", "<166791232468.5302.12366928756676498962@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "66.249.92.10", "opened_at": "2022-11-08T12:58:47.565000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.087000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.539000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.838000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.336000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:30:43.534000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by bluth@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "Airbyte Follow-up", "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "template_name": "Email 2 - Follow-up #1", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["bluth@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:30:43.537000", "date_updated": "2023-11-10T15:30:43.537000", "id": "ev_0RUlfL6sHJ5SEBgJn9CHf1", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x/M0vtraSSWF.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T15:14:45.336000+00:00", "opens": [{"ip_address": "66.249.92.10", "opened_at": "2022-11-08T12:58:47.565000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.087000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.539000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.838000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.336000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_7O5Zj8MPaQeNzSpDaq1m5N", "user_id": null}, "emitted_at": 1699630289703} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T15:21:48.934000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Gob,

I'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth Company (Example\u00a0Lead) and show you what we're working on.

Are you available for a quick call tomorrow afternoon?
", "body_preview": "Hi Gob, \n \nI'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth\nCompany (Example Lead) and show y", "body_text": "Hi Gob, \n \nI'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth\nCompany (Example Lead) and show you what we're working on. \n \nAre you available for a quick call tomorrow afternoon?", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "created_by": null, "created_by_name": null, "date_created": "2022-11-08T15:21:49.243000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T15:21:48.934000+00:00", "date_updated": "2023-11-10T15:30:43.185000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 15:21:50 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792090941.10645.3407832652098417673@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Bluth Company (Example\u00a0Lead) + Airbyte", "to": [{"email": "bluth@close.com", "name": "Gob Bluth"}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166792090941.10645.3407832652098417673@smtpgw.close.com>", "<166792090941.10645.8722656406065573871@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "66.249.92.23", "opened_at": "2022-11-08T15:21:52.108000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.461000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.208000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.120000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.534000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:20.249000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.338000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:30:43.185000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Gob Bluth (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": "seq_17gkOeJEV1QPdPaOMJ9mTX", "sequence_name": "Sequence 1", "sequence_subscription_id": "sub_77gWyXwSd44zuQaciQYW0O", "status": "sent", "subject": "Bluth Company (Example\u00a0Lead) + Airbyte", "template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "template_name": "Email 1 - Intro", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["Gob Bluth "], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:30:43.187000", "date_updated": "2023-11-10T15:30:43.187000", "id": "ev_2KbeIQIHqBOfD0z0btKUxB", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp/DIwaIsCbZM.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T15:14:45.338000+00:00", "opens": [{"ip_address": "66.249.92.23", "opened_at": "2022-11-08T15:21:52.108000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.461000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.208000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.120000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.534000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:20.249000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.338000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_1WktI0e4spbk65BgvBZPf4", "user_id": null}, "emitted_at": 1699630289709} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T12:56:52.538000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Bruce,

I'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at Wayne Enterprises (Example Lead) and show you what we're working on.

Are you available for a quick call tomorrow afternoon?", "body_preview": "Hi Bruce, \n \nI'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT\nYOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at\nWayne Enterprises (Example Lead) an", "body_text": "Hi Bruce, \n \nI'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT\nYOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at\nWayne Enterprises (Example Lead) and show you what we're working on. \n \nAre you available for a quick call tomorrow afternoon?", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T12:54:51.103000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T12:56:52.538000+00:00", "date_updated": "2023-11-10T15:30:43.106000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 12:56:52 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166791221256.1235.11589873218010129427@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Wayne Enterprises (Example\u00a0Lead) + Airbyte", "to": [{"email": "thedarkknight@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe", "in_reply_to_id": null, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "message_ids": ["<166791221256.1235.11589873218010129427@smtpgw.close.com>", "<166791221256.1235.14063741572606059919@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:25:56.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:48:01.421000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:00.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:10.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:28.676000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:24.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:34.029000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:07:41.553000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:10:30.843000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:13:05.125000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:15:47.492000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "85.209.47.207", "opened_at": "2022-11-08T15:43:38.471000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.479000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.770000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.340000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.086000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.538000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.276000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:58.767000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.335000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:30:43.106000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "Wayne Enterprises (Example\u00a0Lead) + Airbyte", "template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "template_name": "Email 1 - Intro", "thread_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "to": ["thedarkknight@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:30:43.109000", "date_updated": "2023-11-10T15:30:43.109000", "id": "ev_3tACWCm4grjX9YAlnhQ5xA", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "meta": {"request_method": "GET", "request_path": "/t/ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe/LKMjsg1Cwf.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T15:14:45.335000+00:00", "opens": [{"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:25:56.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:48:01.421000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:00.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:10.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:28.676000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:24.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:34.029000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:07:41.553000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:10:30.843000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:13:05.125000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:15:47.492000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "85.209.47.207", "opened_at": "2022-11-08T15:43:38.471000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.479000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.770000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.340000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.086000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.538000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.276000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:58.767000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.335000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_4wMd9PSmUsCUSfgGimvrqL", "user_id": null}, "emitted_at": 1699630289715} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T15:44:17.803000+00:00", "attachments": [], "bcc": [], "body_html": "Test", "body_preview": "Test", "body_text": "Test", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T15:43:48.819000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T15:44:17.803000+00:00", "date_updated": "2023-11-10T15:30:43.104000+00:00", "direction": "outgoing", "email_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 15:44:18 +0000", "from": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792225784.2629.13990873364570888607@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "subject": "(no subject)", "to": [{"email": "thedarkknight@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "in_reply_to_id": null, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "message_ids": ["<166792225784.2629.13990873364570888607@smtpgw.close.com>", "<166792225784.2629.12825104744076060180@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "85.209.47.207", "opened_at": "2022-11-09T11:12:59.685000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.460000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.207000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.084000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.536000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.339000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:30:43.104000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "(no subject)", "template_id": null, "template_name": null, "thread_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "to": ["thedarkknight@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:30:43.106000", "date_updated": "2023-11-10T15:30:43.106000", "id": "ev_5BDijLS75lw6znax3SZyAa", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "meta": {"request_method": "GET", "request_path": "/t/tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs/LKMjsg1Cwf.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T15:14:45.339000+00:00", "opens": [{"ip_address": "85.209.47.207", "opened_at": "2022-11-09T11:12:59.685000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.460000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.207000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.084000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.536000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.339000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_2Nx6LPRvRgu3mdrpqIp5xd", "user_id": null}, "emitted_at": 1699630289720} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T16:09:03.809000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Tobias,

Friendly follow-up.

I wanted to show you how Airbyte can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

- Wed @ 11AM
- Thur @ 2PM
- Fri @ 3PM
", "body_preview": "Hi Tobias, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Th", "body_text": "Hi Tobias, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur @ 2PM \n\\- Fri @ 3PM", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_at5uglNbyasFp2KsoWQpjQmLp4lmmqX2p1nhvmPYytq", "created_by": null, "created_by_name": null, "date_created": "2022-11-08T16:09:04.170000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T16:09:03.809000+00:00", "date_updated": "2023-11-10T15:30:43.104000+00:00", "direction": "outgoing", "email_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 16:09:04 +0000", "from": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792374433.22448.16558415745556798661@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "subject": "Airbyte Follow-up", "to": [{"email": "tobiasfunke@close.com", "name": "Tobias F\u00fcnke"}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166792374433.22448.16558415745556798661@smtpgw.close.com>", "<166792374433.22448.17907165931371313011@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.922000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.205000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.242000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.540000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.274000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.339000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:30:43.104000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Tobias F\u00fcnke (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": "seq_7gEZ4ByvvLv1rI6szKolhR", "sequence_name": "Sequence 2", "sequence_subscription_id": "sub_1rgSspOohLocAaucazbFbB", "status": "sent", "subject": "Airbyte Follow-up", "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "template_name": "Email 2 - Follow-up #1", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["Tobias F\u00fcnke "], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:30:43.105000", "date_updated": "2023-11-10T15:30:43.105000", "id": "ev_69tSzwIVfGiBvN8k4y6XlM", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp/lnZoSPgicp.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T15:14:45.339000+00:00", "opens": [{"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.922000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.205000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.242000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.540000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.274000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.339000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_7Ox9Zm8ChxH4nU9ctwgvKj", "user_id": null}, "emitted_at": 1699630289725} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T16:09:03.809000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Tobias,

Friendly follow-up.

I wanted to show you how Airbyte can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

- Wed @ 11AM
- Thur @ 2PM
- Fri @ 3PM
", "body_preview": "Hi Tobias, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Th", "body_text": "Hi Tobias, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur @ 2PM \n\\- Fri @ 3PM", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_at5uglNbyasFp2KsoWQpjQmLp4lmmqX2p1nhvmPYytq", "created_by": null, "created_by_name": null, "date_created": "2022-11-08T16:09:04.170000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T16:09:03.809000+00:00", "date_updated": "2023-11-10T15:14:45.339000+00:00", "direction": "outgoing", "email_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 16:09:04 +0000", "from": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792374433.22448.16558415745556798661@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "subject": "Airbyte Follow-up", "to": [{"email": "tobiasfunke@close.com", "name": "Tobias F\u00fcnke"}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166792374433.22448.16558415745556798661@smtpgw.close.com>", "<166792374433.22448.17907165931371313011@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.922000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.205000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.242000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.540000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.274000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.339000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Tobias F\u00fcnke (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": "seq_7gEZ4ByvvLv1rI6szKolhR", "sequence_name": "Sequence 2", "sequence_subscription_id": "sub_1rgSspOohLocAaucazbFbB", "status": "sent", "subject": "Airbyte Follow-up", "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "template_name": "Email 2 - Follow-up #1", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["Tobias F\u00fcnke "], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:14:45.341000", "date_updated": "2023-11-10T15:14:45.341000", "id": "ev_2b2ycyLs8ry3UeIcfbwyhO", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp/lnZoSPgicp.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T15:06:57.840000+00:00", "opens": [{"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.922000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.205000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.242000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.540000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.274000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_60PxPjrZKKbbBHR0Ys3X7s", "user_id": null}, "emitted_at": 1699630289731} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T15:44:17.803000+00:00", "attachments": [], "bcc": [], "body_html": "Test", "body_preview": "Test", "body_text": "Test", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T15:43:48.819000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T15:44:17.803000+00:00", "date_updated": "2023-11-10T15:14:45.339000+00:00", "direction": "outgoing", "email_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 15:44:18 +0000", "from": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792225784.2629.13990873364570888607@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "subject": "(no subject)", "to": [{"email": "thedarkknight@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "in_reply_to_id": null, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "message_ids": ["<166792225784.2629.13990873364570888607@smtpgw.close.com>", "<166792225784.2629.12825104744076060180@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "85.209.47.207", "opened_at": "2022-11-09T11:12:59.685000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.460000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.207000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.084000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.536000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.339000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "(no subject)", "template_id": null, "template_name": null, "thread_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "to": ["thedarkknight@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:14:45.340000", "date_updated": "2023-11-10T15:14:45.340000", "id": "ev_2qHLh4rzCWZj4hSW4BYIJD", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "meta": {"request_method": "GET", "request_path": "/t/tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs/LKMjsg1Cwf.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T15:06:57.840000+00:00", "opens": [{"ip_address": "85.209.47.207", "opened_at": "2022-11-09T11:12:59.685000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.460000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.207000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.084000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.536000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_1lvmCiZBTALUDLAXw0LaM8", "user_id": null}, "emitted_at": 1699630289737} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T15:21:48.934000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Gob,

I'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth Company (Example\u00a0Lead) and show you what we're working on.

Are you available for a quick call tomorrow afternoon?
", "body_preview": "Hi Gob, \n \nI'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth\nCompany (Example Lead) and show y", "body_text": "Hi Gob, \n \nI'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth\nCompany (Example Lead) and show you what we're working on. \n \nAre you available for a quick call tomorrow afternoon?", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "created_by": null, "created_by_name": null, "date_created": "2022-11-08T15:21:49.243000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T15:21:48.934000+00:00", "date_updated": "2023-11-10T15:14:45.338000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 15:21:50 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792090941.10645.3407832652098417673@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Bluth Company (Example\u00a0Lead) + Airbyte", "to": [{"email": "bluth@close.com", "name": "Gob Bluth"}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166792090941.10645.3407832652098417673@smtpgw.close.com>", "<166792090941.10645.8722656406065573871@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "66.249.92.23", "opened_at": "2022-11-08T15:21:52.108000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.461000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.208000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.120000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.534000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:20.249000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.338000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Gob Bluth (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": "seq_17gkOeJEV1QPdPaOMJ9mTX", "sequence_name": "Sequence 1", "sequence_subscription_id": "sub_77gWyXwSd44zuQaciQYW0O", "status": "sent", "subject": "Bluth Company (Example\u00a0Lead) + Airbyte", "template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "template_name": "Email 1 - Intro", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["Gob Bluth "], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:14:45.340000", "date_updated": "2023-11-10T15:14:45.340000", "id": "ev_29E1EPZK1xaOxSmT3BBU6Y", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp/DIwaIsCbZM.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T15:06:57.840000+00:00", "opens": [{"ip_address": "66.249.92.23", "opened_at": "2022-11-08T15:21:52.108000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.461000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.208000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.120000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.534000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:20.249000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_79TlUClmIFBrdx1OaUbMz9", "user_id": null}, "emitted_at": 1699630289742} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T12:58:44.679000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Gob,

Friendly follow-up.

I wanted to show you how Airbyte can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

- Wed @ 11AM
- Thur @ 2PM
- Fri @ 3PM
", "body_preview": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur ", "body_text": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur @ 2PM \n\\- Fri @ 3PM", "bulk_email_action_id": "bulkemail_bCJdPbYpJ2xfUuj8Dif3sBMHlxtbOKuEZSCdNcTgsWB", "cc": [], "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T12:58:44.579000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T12:58:44.679000+00:00", "date_updated": "2023-11-10T15:14:45.336000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 12:58:44 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166791232468.5302.9789676641910298968@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Airbyte Follow-up", "to": [{"email": "bluth@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166791232468.5302.9789676641910298968@smtpgw.close.com>", "<166791232468.5302.12366928756676498962@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "66.249.92.10", "opened_at": "2022-11-08T12:58:47.565000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.087000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.539000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.838000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.336000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by bluth@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "Airbyte Follow-up", "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "template_name": "Email 2 - Follow-up #1", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["bluth@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:14:45.338000", "date_updated": "2023-11-10T15:14:45.338000", "id": "ev_6PyTh6KVyKMePoEEm0jwdZ", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x/M0vtraSSWF.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T15:06:57.838000+00:00", "opens": [{"ip_address": "66.249.92.10", "opened_at": "2022-11-08T12:58:47.565000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.087000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.539000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.838000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_2JxdqbgZNqNO4mxjnS9yTp", "user_id": null}, "emitted_at": 1699630289747} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T12:56:52.538000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Bruce,

I'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at Wayne Enterprises (Example Lead) and show you what we're working on.

Are you available for a quick call tomorrow afternoon?", "body_preview": "Hi Bruce, \n \nI'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT\nYOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at\nWayne Enterprises (Example Lead) an", "body_text": "Hi Bruce, \n \nI'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT\nYOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at\nWayne Enterprises (Example Lead) and show you what we're working on. \n \nAre you available for a quick call tomorrow afternoon?", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T12:54:51.103000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T12:56:52.538000+00:00", "date_updated": "2023-11-10T15:14:45.335000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 12:56:52 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166791221256.1235.11589873218010129427@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Wayne Enterprises (Example\u00a0Lead) + Airbyte", "to": [{"email": "thedarkknight@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe", "in_reply_to_id": null, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "message_ids": ["<166791221256.1235.11589873218010129427@smtpgw.close.com>", "<166791221256.1235.14063741572606059919@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:25:56.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:48:01.421000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:00.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:10.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:28.676000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:24.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:34.029000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:07:41.553000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:10:30.843000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:13:05.125000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:15:47.492000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "85.209.47.207", "opened_at": "2022-11-08T15:43:38.471000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.479000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.770000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.340000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.086000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.538000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.276000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:58.767000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.335000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "Wayne Enterprises (Example\u00a0Lead) + Airbyte", "template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "template_name": "Email 1 - Intro", "thread_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "to": ["thedarkknight@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:14:45.336000", "date_updated": "2023-11-10T15:14:45.336000", "id": "ev_0d8wShZMOSQeoUTw6w9nyX", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "meta": {"request_method": "GET", "request_path": "/t/ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe/LKMjsg1Cwf.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T15:06:58.767000+00:00", "opens": [{"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:25:56.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:48:01.421000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:00.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:10.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:28.676000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:24.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:34.029000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:07:41.553000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:10:30.843000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:13:05.125000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:15:47.492000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "85.209.47.207", "opened_at": "2022-11-08T15:43:38.471000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.479000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.770000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.340000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.086000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.538000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.276000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:58.767000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_0iKLh9PrBVh2nUq62pYqQL", "user_id": null}, "emitted_at": 1699630289752} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T12:56:52.538000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Bruce,

I'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at Wayne Enterprises (Example Lead) and show you what we're working on.

Are you available for a quick call tomorrow afternoon?", "body_preview": "Hi Bruce, \n \nI'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT\nYOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at\nWayne Enterprises (Example Lead) an", "body_text": "Hi Bruce, \n \nI'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT\nYOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at\nWayne Enterprises (Example Lead) and show you what we're working on. \n \nAre you available for a quick call tomorrow afternoon?", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T12:54:51.103000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T12:56:52.538000+00:00", "date_updated": "2023-11-10T15:06:58.767000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 12:56:52 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166791221256.1235.11589873218010129427@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Wayne Enterprises (Example\u00a0Lead) + Airbyte", "to": [{"email": "thedarkknight@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe", "in_reply_to_id": null, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "message_ids": ["<166791221256.1235.11589873218010129427@smtpgw.close.com>", "<166791221256.1235.14063741572606059919@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:25:56.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:48:01.421000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:00.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:10.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:28.676000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:24.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:34.029000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:07:41.553000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:10:30.843000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:13:05.125000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:15:47.492000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "85.209.47.207", "opened_at": "2022-11-08T15:43:38.471000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.479000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.770000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.340000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.086000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.538000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.276000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:58.767000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "Wayne Enterprises (Example\u00a0Lead) + Airbyte", "template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "template_name": "Email 1 - Intro", "thread_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "to": ["thedarkknight@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:06:58.770000", "date_updated": "2023-11-10T15:06:58.770000", "id": "ev_55owS3VVOSN5ym6nGjJYnp", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "meta": {"request_method": "GET", "request_path": "/t/ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe/LKMjsg1Cwf.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T14:41:19.276000+00:00", "opens": [{"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:25:56.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:48:01.421000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:00.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:10.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:28.676000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:24.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:34.029000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:07:41.553000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:10:30.843000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:13:05.125000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:15:47.492000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "85.209.47.207", "opened_at": "2022-11-08T15:43:38.471000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.479000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.770000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.340000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.086000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.538000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.276000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_5s2fFhSgvg58QClPZQdkxe", "user_id": null}, "emitted_at": 1699630289758} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T15:21:48.934000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Gob,

I'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth Company (Example\u00a0Lead) and show you what we're working on.

Are you available for a quick call tomorrow afternoon?
", "body_preview": "Hi Gob, \n \nI'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth\nCompany (Example Lead) and show y", "body_text": "Hi Gob, \n \nI'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth\nCompany (Example Lead) and show you what we're working on. \n \nAre you available for a quick call tomorrow afternoon?", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "created_by": null, "created_by_name": null, "date_created": "2022-11-08T15:21:49.243000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T15:21:48.934000+00:00", "date_updated": "2023-11-10T15:06:57.840000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 15:21:50 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792090941.10645.3407832652098417673@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Bluth Company (Example\u00a0Lead) + Airbyte", "to": [{"email": "bluth@close.com", "name": "Gob Bluth"}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166792090941.10645.3407832652098417673@smtpgw.close.com>", "<166792090941.10645.8722656406065573871@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "66.249.92.23", "opened_at": "2022-11-08T15:21:52.108000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.461000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.208000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.120000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.534000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:20.249000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Gob Bluth (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": "seq_17gkOeJEV1QPdPaOMJ9mTX", "sequence_name": "Sequence 1", "sequence_subscription_id": "sub_77gWyXwSd44zuQaciQYW0O", "status": "sent", "subject": "Bluth Company (Example\u00a0Lead) + Airbyte", "template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "template_name": "Email 1 - Intro", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["Gob Bluth "], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:06:57.842000", "date_updated": "2023-11-10T15:06:57.842000", "id": "ev_4d4oWnQeD4olrMOlNcH30C", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp/DIwaIsCbZM.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T14:41:20.249000+00:00", "opens": [{"ip_address": "66.249.92.23", "opened_at": "2022-11-08T15:21:52.108000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.461000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.208000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.120000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.534000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:20.249000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_6mJCuOuq0mvtvmWdyDCcwk", "user_id": null}, "emitted_at": 1699630289763} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T15:44:17.803000+00:00", "attachments": [], "bcc": [], "body_html": "Test", "body_preview": "Test", "body_text": "Test", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T15:43:48.819000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T15:44:17.803000+00:00", "date_updated": "2023-11-10T15:06:57.840000+00:00", "direction": "outgoing", "email_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 15:44:18 +0000", "from": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792225784.2629.13990873364570888607@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "subject": "(no subject)", "to": [{"email": "thedarkknight@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "in_reply_to_id": null, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "message_ids": ["<166792225784.2629.13990873364570888607@smtpgw.close.com>", "<166792225784.2629.12825104744076060180@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "85.209.47.207", "opened_at": "2022-11-09T11:12:59.685000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.460000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.207000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.084000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.536000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "(no subject)", "template_id": null, "template_name": null, "thread_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "to": ["thedarkknight@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:06:57.842000", "date_updated": "2023-11-10T15:06:57.842000", "id": "ev_3NPqRa3FAbQINcjzP9OvnO", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "meta": {"request_method": "GET", "request_path": "/t/tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs/LKMjsg1Cwf.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T14:41:19.278000+00:00", "opens": [{"ip_address": "85.209.47.207", "opened_at": "2022-11-09T11:12:59.685000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.460000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.207000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.084000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.536000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_4ZNBg8yzqoRKPsqTdPquZS", "user_id": null}, "emitted_at": 1699630289768} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T16:09:03.809000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Tobias,

Friendly follow-up.

I wanted to show you how Airbyte can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

- Wed @ 11AM
- Thur @ 2PM
- Fri @ 3PM
", "body_preview": "Hi Tobias, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Th", "body_text": "Hi Tobias, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur @ 2PM \n\\- Fri @ 3PM", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_at5uglNbyasFp2KsoWQpjQmLp4lmmqX2p1nhvmPYytq", "created_by": null, "created_by_name": null, "date_created": "2022-11-08T16:09:04.170000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T16:09:03.809000+00:00", "date_updated": "2023-11-10T15:06:57.840000+00:00", "direction": "outgoing", "email_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 16:09:04 +0000", "from": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792374433.22448.16558415745556798661@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "subject": "Airbyte Follow-up", "to": [{"email": "tobiasfunke@close.com", "name": "Tobias F\u00fcnke"}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166792374433.22448.16558415745556798661@smtpgw.close.com>", "<166792374433.22448.17907165931371313011@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.922000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.205000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.242000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.540000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.274000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Tobias F\u00fcnke (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": "seq_7gEZ4ByvvLv1rI6szKolhR", "sequence_name": "Sequence 2", "sequence_subscription_id": "sub_1rgSspOohLocAaucazbFbB", "status": "sent", "subject": "Airbyte Follow-up", "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "template_name": "Email 2 - Follow-up #1", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["Tobias F\u00fcnke "], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:06:57.841000", "date_updated": "2023-11-10T15:06:57.841000", "id": "ev_5eJEEFysXUZesjkFWky3h4", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp/lnZoSPgicp.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T14:41:19.274000+00:00", "opens": [{"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.922000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.205000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.242000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.540000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.274000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_5AGdFTB2Wtd3QunmnoPpCM", "user_id": null}, "emitted_at": 1699630289773} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T12:58:44.679000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Gob,

Friendly follow-up.

I wanted to show you how Airbyte can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

- Wed @ 11AM
- Thur @ 2PM
- Fri @ 3PM
", "body_preview": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur ", "body_text": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur @ 2PM \n\\- Fri @ 3PM", "bulk_email_action_id": "bulkemail_bCJdPbYpJ2xfUuj8Dif3sBMHlxtbOKuEZSCdNcTgsWB", "cc": [], "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T12:58:44.579000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T12:58:44.679000+00:00", "date_updated": "2023-11-10T15:06:57.838000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 12:58:44 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166791232468.5302.9789676641910298968@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Airbyte Follow-up", "to": [{"email": "bluth@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166791232468.5302.9789676641910298968@smtpgw.close.com>", "<166791232468.5302.12366928756676498962@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "66.249.92.10", "opened_at": "2022-11-08T12:58:47.565000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.087000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.539000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.838000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by bluth@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "Airbyte Follow-up", "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "template_name": "Email 2 - Follow-up #1", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["bluth@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:06:57.840000", "date_updated": "2023-11-10T15:06:57.840000", "id": "ev_7Pcuc4cBZ0sJpICFbiXdM2", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x/M0vtraSSWF.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T14:41:19.279000+00:00", "opens": [{"ip_address": "66.249.92.10", "opened_at": "2022-11-08T12:58:47.565000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.087000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.539000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_2f6VqlA9IdvNnjzNibe5WW", "user_id": null}, "emitted_at": 1699630289777} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens", "opens_summary"], "data": {"_type": "Email", "activity_at": "2022-11-08T15:21:48.934000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Gob,

I'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth Company (Example\u00a0Lead) and show you what we're working on.

Are you available for a quick call tomorrow afternoon?
", "body_preview": "Hi Gob, \n \nI'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth\nCompany (Example Lead) and show y", "body_text": "Hi Gob, \n \nI'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth\nCompany (Example Lead) and show you what we're working on. \n \nAre you available for a quick call tomorrow afternoon?", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "created_by": null, "created_by_name": null, "date_created": "2022-11-08T15:21:49.243000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T15:21:48.934000+00:00", "date_updated": "2023-11-10T14:41:20.249000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 15:21:50 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792090941.10645.3407832652098417673@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Bluth Company (Example\u00a0Lead) + Airbyte", "to": [{"email": "bluth@close.com", "name": "Gob Bluth"}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166792090941.10645.3407832652098417673@smtpgw.close.com>", "<166792090941.10645.8722656406065573871@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "66.249.92.23", "opened_at": "2022-11-08T15:21:52.108000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.461000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.208000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.120000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.534000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:20.249000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Gob Bluth (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": "seq_17gkOeJEV1QPdPaOMJ9mTX", "sequence_name": "Sequence 1", "sequence_subscription_id": "sub_77gWyXwSd44zuQaciQYW0O", "status": "sent", "subject": "Bluth Company (Example\u00a0Lead) + Airbyte", "template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "template_name": "Email 1 - Intro", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["Gob Bluth "], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T14:41:20.253000", "date_updated": "2023-11-10T14:41:20.253000", "id": "ev_6dudv39N6yamql3fp3n1mZ", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp/DIwaIsCbZM.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-07T17:16:57.534000+00:00", "opens": [{"ip_address": "66.249.92.23", "opened_at": "2022-11-08T15:21:52.108000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.461000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.208000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.120000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.534000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Gob Bluth (5+ times, latest 2023-11-07)"}, "request_id": "req_2HPaptmDh9wUootbcVVmAU", "user_id": null}, "emitted_at": 1699630289781} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens", "opens_summary"], "data": {"_type": "Email", "activity_at": "2022-11-08T12:58:44.679000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Gob,

Friendly follow-up.

I wanted to show you how Airbyte can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

- Wed @ 11AM
- Thur @ 2PM
- Fri @ 3PM
", "body_preview": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur ", "body_text": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur @ 2PM \n\\- Fri @ 3PM", "bulk_email_action_id": "bulkemail_bCJdPbYpJ2xfUuj8Dif3sBMHlxtbOKuEZSCdNcTgsWB", "cc": [], "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T12:58:44.579000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T12:58:44.679000+00:00", "date_updated": "2023-11-10T14:41:19.279000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 12:58:44 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166791232468.5302.9789676641910298968@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Airbyte Follow-up", "to": [{"email": "bluth@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166791232468.5302.9789676641910298968@smtpgw.close.com>", "<166791232468.5302.12366928756676498962@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "66.249.92.10", "opened_at": "2022-11-08T12:58:47.565000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.087000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.539000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by bluth@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "Airbyte Follow-up", "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "template_name": "Email 2 - Follow-up #1", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["bluth@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T14:41:19.283000", "date_updated": "2023-11-10T14:41:19.283000", "id": "ev_7Ju4WMcrHkfBGyXZhpuZIu", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x/M0vtraSSWF.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-07T17:16:57.539000+00:00", "opens": [{"ip_address": "66.249.92.10", "opened_at": "2022-11-08T12:58:47.565000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.087000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.539000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by bluth@close.com (5+ times, latest 2023-11-07)"}, "request_id": "req_1j1zj4MDzVzRdrQLO9KioF", "user_id": null}, "emitted_at": 1699630289787} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens", "opens_summary"], "data": {"_type": "Email", "activity_at": "2022-11-08T15:44:17.803000+00:00", "attachments": [], "bcc": [], "body_html": "Test", "body_preview": "Test", "body_text": "Test", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T15:43:48.819000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T15:44:17.803000+00:00", "date_updated": "2023-11-10T14:41:19.278000+00:00", "direction": "outgoing", "email_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 15:44:18 +0000", "from": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792225784.2629.13990873364570888607@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "subject": "(no subject)", "to": [{"email": "thedarkknight@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "in_reply_to_id": null, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "message_ids": ["<166792225784.2629.13990873364570888607@smtpgw.close.com>", "<166792225784.2629.12825104744076060180@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "85.209.47.207", "opened_at": "2022-11-09T11:12:59.685000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.460000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.207000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.084000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.536000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "(no subject)", "template_id": null, "template_name": null, "thread_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "to": ["thedarkknight@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T14:41:19.281000", "date_updated": "2023-11-10T14:41:19.281000", "id": "ev_1srZ3pOsXl7lK7GkGZKY8h", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "meta": {"request_method": "GET", "request_path": "/t/tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs/LKMjsg1Cwf.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-07T17:16:57.536000+00:00", "opens": [{"ip_address": "85.209.47.207", "opened_at": "2022-11-09T11:12:59.685000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.460000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.207000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.084000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.536000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-07)"}, "request_id": "req_22HCK1MouFKeXP7DWW88UW", "user_id": null}, "emitted_at": 1699630289792} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens", "opens_summary"], "data": {"_type": "Email", "activity_at": "2022-11-08T12:56:52.538000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Bruce,

I'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at Wayne Enterprises (Example Lead) and show you what we're working on.

Are you available for a quick call tomorrow afternoon?", "body_preview": "Hi Bruce, \n \nI'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT\nYOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at\nWayne Enterprises (Example Lead) an", "body_text": "Hi Bruce, \n \nI'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT\nYOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at\nWayne Enterprises (Example Lead) and show you what we're working on. \n \nAre you available for a quick call tomorrow afternoon?", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T12:54:51.103000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T12:56:52.538000+00:00", "date_updated": "2023-11-10T14:41:19.276000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 12:56:52 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166791221256.1235.11589873218010129427@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Wayne Enterprises (Example\u00a0Lead) + Airbyte", "to": [{"email": "thedarkknight@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe", "in_reply_to_id": null, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "message_ids": ["<166791221256.1235.11589873218010129427@smtpgw.close.com>", "<166791221256.1235.14063741572606059919@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:25:56.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:48:01.421000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:00.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:10.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:28.676000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:24.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:34.029000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:07:41.553000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:10:30.843000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:13:05.125000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:15:47.492000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "85.209.47.207", "opened_at": "2022-11-08T15:43:38.471000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.479000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.770000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.340000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.086000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.538000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.276000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "Wayne Enterprises (Example\u00a0Lead) + Airbyte", "template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "template_name": "Email 1 - Intro", "thread_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "to": ["thedarkknight@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T14:41:19.278000", "date_updated": "2023-11-10T14:41:19.278000", "id": "ev_0j9qPXyGEdfiSdhiCn1nGw", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "meta": {"request_method": "GET", "request_path": "/t/ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe/LKMjsg1Cwf.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-07T17:16:57.538000+00:00", "opens": [{"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:25:56.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:48:01.421000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:00.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:10.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:28.676000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:24.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:34.029000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:07:41.553000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:10:30.843000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:13:05.125000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:15:47.492000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "85.209.47.207", "opened_at": "2022-11-08T15:43:38.471000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.479000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.770000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.340000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.086000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.538000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-07)"}, "request_id": "req_2qQadxDEP4DZZiDMWB9USz", "user_id": null}, "emitted_at": 1699630289796} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens", "opens_summary"], "data": {"_type": "Email", "activity_at": "2022-11-08T16:09:03.809000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Tobias,

Friendly follow-up.

I wanted to show you how Airbyte can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

- Wed @ 11AM
- Thur @ 2PM
- Fri @ 3PM
", "body_preview": "Hi Tobias, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Th", "body_text": "Hi Tobias, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur @ 2PM \n\\- Fri @ 3PM", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_at5uglNbyasFp2KsoWQpjQmLp4lmmqX2p1nhvmPYytq", "created_by": null, "created_by_name": null, "date_created": "2022-11-08T16:09:04.170000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T16:09:03.809000+00:00", "date_updated": "2023-11-10T14:41:19.274000+00:00", "direction": "outgoing", "email_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 16:09:04 +0000", "from": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792374433.22448.16558415745556798661@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "subject": "Airbyte Follow-up", "to": [{"email": "tobiasfunke@close.com", "name": "Tobias F\u00fcnke"}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166792374433.22448.16558415745556798661@smtpgw.close.com>", "<166792374433.22448.17907165931371313011@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.922000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.205000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.242000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.540000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.274000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Tobias F\u00fcnke (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": "seq_7gEZ4ByvvLv1rI6szKolhR", "sequence_name": "Sequence 2", "sequence_subscription_id": "sub_1rgSspOohLocAaucazbFbB", "status": "sent", "subject": "Airbyte Follow-up", "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "template_name": "Email 2 - Follow-up #1", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["Tobias F\u00fcnke "], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T14:41:19.277000", "date_updated": "2023-11-10T14:41:19.277000", "id": "ev_5j7Pk6YNQU9E9nZjwCLe6t", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp/lnZoSPgicp.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-07T17:16:57.540000+00:00", "opens": [{"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.922000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.205000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.242000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.540000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Tobias F\u00fcnke (5+ times, latest 2023-11-07)"}, "request_id": "req_2c5eao433qyUjb3hiqes6f", "user_id": null}, "emitted_at": 1699630289801} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens", "opens_summary"], "data": {"_type": "Email", "activity_at": "2022-11-08T16:09:03.809000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Tobias,

Friendly follow-up.

I wanted to show you how Airbyte can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

- Wed @ 11AM
- Thur @ 2PM
- Fri @ 3PM
", "body_preview": "Hi Tobias, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Th", "body_text": "Hi Tobias, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur @ 2PM \n\\- Fri @ 3PM", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_at5uglNbyasFp2KsoWQpjQmLp4lmmqX2p1nhvmPYytq", "created_by": null, "created_by_name": null, "date_created": "2022-11-08T16:09:04.170000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T16:09:03.809000+00:00", "date_updated": "2023-11-07T17:16:57.540000+00:00", "direction": "outgoing", "email_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 16:09:04 +0000", "from": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792374433.22448.16558415745556798661@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "subject": "Airbyte Follow-up", "to": [{"email": "tobiasfunke@close.com", "name": "Tobias F\u00fcnke"}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166792374433.22448.16558415745556798661@smtpgw.close.com>", "<166792374433.22448.17907165931371313011@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.922000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.205000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.242000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.540000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Tobias F\u00fcnke (5+ times, latest 2023-11-07)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": "seq_7gEZ4ByvvLv1rI6szKolhR", "sequence_name": "Sequence 2", "sequence_subscription_id": "sub_1rgSspOohLocAaucazbFbB", "status": "sent", "subject": "Airbyte Follow-up", "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "template_name": "Email 2 - Follow-up #1", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["Tobias F\u00fcnke "], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-07T17:16:57.542000", "date_updated": "2023-11-07T17:16:57.542000", "id": "ev_3yPOiNAqhOB5tzHLQlaPXq", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp/lnZoSPgicp.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-08-15T13:46:12.242000+00:00", "opens": [{"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.922000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.205000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.242000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Tobias F\u00fcnke (5 times, latest 2023-08-15)"}, "request_id": "req_59kkiOFtRVDLGEHmetdbZh", "user_id": null}, "emitted_at": 1699630289805} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens", "opens_summary"], "data": {"_type": "Email", "activity_at": "2022-11-08T12:58:44.679000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Gob,

Friendly follow-up.

I wanted to show you how Airbyte can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

- Wed @ 11AM
- Thur @ 2PM
- Fri @ 3PM
", "body_preview": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur ", "body_text": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur @ 2PM \n\\- Fri @ 3PM", "bulk_email_action_id": "bulkemail_bCJdPbYpJ2xfUuj8Dif3sBMHlxtbOKuEZSCdNcTgsWB", "cc": [], "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T12:58:44.579000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T12:58:44.679000+00:00", "date_updated": "2023-11-07T17:16:57.539000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 12:58:44 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166791232468.5302.9789676641910298968@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Airbyte Follow-up", "to": [{"email": "bluth@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166791232468.5302.9789676641910298968@smtpgw.close.com>", "<166791232468.5302.12366928756676498962@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "66.249.92.10", "opened_at": "2022-11-08T12:58:47.565000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.087000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.539000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by bluth@close.com (5+ times, latest 2023-11-07)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "Airbyte Follow-up", "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "template_name": "Email 2 - Follow-up #1", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["bluth@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-07T17:16:57.541000", "date_updated": "2023-11-07T17:16:57.541000", "id": "ev_45XKOg0O0KTHiJF0cQAH34", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x/M0vtraSSWF.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-08-15T13:46:12.087000+00:00", "opens": [{"ip_address": "66.249.92.10", "opened_at": "2022-11-08T12:58:47.565000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.087000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}], "opens_summary": "Opened by bluth@close.com (5+ times, latest 2023-08-15)"}, "request_id": "req_1l0ppWWlua2dq9abxQBEQK", "user_id": null}, "emitted_at": 1699630289810} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens", "opens_summary"], "data": {"_type": "Email", "activity_at": "2022-11-08T12:56:52.538000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Bruce,

I'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at Wayne Enterprises (Example Lead) and show you what we're working on.

Are you available for a quick call tomorrow afternoon?", "body_preview": "Hi Bruce, \n \nI'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT\nYOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at\nWayne Enterprises (Example Lead) an", "body_text": "Hi Bruce, \n \nI'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT\nYOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at\nWayne Enterprises (Example Lead) and show you what we're working on. \n \nAre you available for a quick call tomorrow afternoon?", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T12:54:51.103000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T12:56:52.538000+00:00", "date_updated": "2023-11-07T17:16:57.538000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 12:56:52 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166791221256.1235.11589873218010129427@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Wayne Enterprises (Example\u00a0Lead) + Airbyte", "to": [{"email": "thedarkknight@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe", "in_reply_to_id": null, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "message_ids": ["<166791221256.1235.11589873218010129427@smtpgw.close.com>", "<166791221256.1235.14063741572606059919@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:25:56.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:48:01.421000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:00.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:10.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:28.676000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:24.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:34.029000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:07:41.553000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:10:30.843000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:13:05.125000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:15:47.492000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "85.209.47.207", "opened_at": "2022-11-08T15:43:38.471000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.479000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.770000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.340000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.086000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.538000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-07)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "Wayne Enterprises (Example\u00a0Lead) + Airbyte", "template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "template_name": "Email 1 - Intro", "thread_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "to": ["thedarkknight@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-07T17:16:57.540000", "date_updated": "2023-11-07T17:16:57.540000", "id": "ev_3TtqVtgYKK8DFWvPVDLgy9", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "meta": {"request_method": "GET", "request_path": "/t/ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe/LKMjsg1Cwf.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-08-15T13:46:12.086000+00:00", "opens": [{"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:25:56.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:48:01.421000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:00.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:10.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:28.676000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:24.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:34.029000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:07:41.553000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:10:30.843000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:13:05.125000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:15:47.492000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "85.209.47.207", "opened_at": "2022-11-08T15:43:38.471000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.479000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.770000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.340000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.086000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-08-15)"}, "request_id": "req_6129LX7KFu95RZWKBShau3", "user_id": null}, "emitted_at": 1699630289814} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens", "opens_summary"], "data": {"_type": "Email", "activity_at": "2022-11-08T15:44:17.803000+00:00", "attachments": [], "bcc": [], "body_html": "Test", "body_preview": "Test", "body_text": "Test", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T15:43:48.819000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T15:44:17.803000+00:00", "date_updated": "2023-11-07T17:16:57.536000+00:00", "direction": "outgoing", "email_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 15:44:18 +0000", "from": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792225784.2629.13990873364570888607@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "subject": "(no subject)", "to": [{"email": "thedarkknight@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "in_reply_to_id": null, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "message_ids": ["<166792225784.2629.13990873364570888607@smtpgw.close.com>", "<166792225784.2629.12825104744076060180@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "85.209.47.207", "opened_at": "2022-11-09T11:12:59.685000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.460000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.207000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.084000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.536000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-07)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "(no subject)", "template_id": null, "template_name": null, "thread_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "to": ["thedarkknight@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-07T17:16:57.538000", "date_updated": "2023-11-07T17:16:57.538000", "id": "ev_4fYnwSM4MPvqo3VmdFLyEz", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "meta": {"request_method": "GET", "request_path": "/t/tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs/LKMjsg1Cwf.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-08-15T13:46:12.084000+00:00", "opens": [{"ip_address": "85.209.47.207", "opened_at": "2022-11-09T11:12:59.685000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.460000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.207000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.084000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-08-15)"}, "request_id": "req_2o2D7qw5scrSNtN097kCp1", "user_id": null}, "emitted_at": 1699630289819} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens", "opens_summary"], "data": {"_type": "Email", "activity_at": "2022-11-08T15:21:48.934000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Gob,

I'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth Company (Example\u00a0Lead) and show you what we're working on.

Are you available for a quick call tomorrow afternoon?
", "body_preview": "Hi Gob, \n \nI'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth\nCompany (Example Lead) and show y", "body_text": "Hi Gob, \n \nI'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth\nCompany (Example Lead) and show you what we're working on. \n \nAre you available for a quick call tomorrow afternoon?", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "created_by": null, "created_by_name": null, "date_created": "2022-11-08T15:21:49.243000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T15:21:48.934000+00:00", "date_updated": "2023-11-07T17:16:57.534000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 15:21:50 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792090941.10645.3407832652098417673@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Bluth Company (Example\u00a0Lead) + Airbyte", "to": [{"email": "bluth@close.com", "name": "Gob Bluth"}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166792090941.10645.3407832652098417673@smtpgw.close.com>", "<166792090941.10645.8722656406065573871@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "66.249.92.23", "opened_at": "2022-11-08T15:21:52.108000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.461000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.208000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.120000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.534000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Gob Bluth (5+ times, latest 2023-11-07)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": "seq_17gkOeJEV1QPdPaOMJ9mTX", "sequence_name": "Sequence 1", "sequence_subscription_id": "sub_77gWyXwSd44zuQaciQYW0O", "status": "sent", "subject": "Bluth Company (Example\u00a0Lead) + Airbyte", "template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "template_name": "Email 1 - Intro", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["Gob Bluth "], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-07T17:16:57.536000", "date_updated": "2023-11-07T17:16:57.536000", "id": "ev_2ZSra6V4WKPGYvErNty0Ms", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp/DIwaIsCbZM.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-08-15T13:46:12.120000+00:00", "opens": [{"ip_address": "66.249.92.23", "opened_at": "2022-11-08T15:21:52.108000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.461000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.208000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.120000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Gob Bluth (5+ times, latest 2023-08-15)"}, "request_id": "req_4AquNjQSZfA3wuLQ129LCR", "user_id": null}, "emitted_at": 1699630289823} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "next_billing_on"], "data": {"address_id": null, "bundle_id": null, "carrier": "twilio", "carrier_type": "local", "country": "US", "date_created": "2022-11-08T12:35:29.464000+00:00", "date_updated": "2023-11-07T10:04:24.797000+00:00", "forward_to": null, "forward_to_enabled": false, "forward_to_formatted": null, "id": "phon_jhMWlB6anhT8vcsGNEFukaVl806zfxCgbSAAkvtNBpN", "is_group_number": false, "is_verified": false, "label": null, "last_billed_price": "1.15", "mms_enabled": true, "next_billing_on": "2023-12-07", "number": "+14156251293", "number_formatted": "+1 415-625-1293", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "press_1_to_accept": true, "sms_enabled": true, "supports_mms_to_countries": ["CA", "US"], "supports_sms_to_countries": ["CA", "PR", "US"], "type": "internal", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "voicemail_greeting_url": "https://closeio-voicemail-greetings.s3.amazonaws.com/14bzVblrIaekmXSGQEKM11/undefined.mp3"}, "date_created": "2023-11-07T10:04:24.799000", "date_updated": "2023-11-07T10:04:24.799000", "id": "ev_741AbDOXqemOl5abyZTut4", "lead_id": null, "meta": {}, "oauth_client_id": null, "oauth_scope": null, "object_id": "phon_jhMWlB6anhT8vcsGNEFukaVl806zfxCgbSAAkvtNBpN", "object_type": "phone_number", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-10-07T10:03:53.255000+00:00", "next_billing_on": "2023-11-07"}, "request_id": null, "user_id": null}, "emitted_at": 1699630289827} +{"stream": "leads", "data": {"contacts": [{"created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "urls": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "display_name": "Cooper", "id": "cont_Dsi7AGMRelIZ2I6DIKicaGJU7mwxPZElLIHy33xbjCM", "title": "", "date_created": "2022-07-05T21:01:25.612000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Cooper"}], "name": "Cooper", "lead_id": "lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY", "emails": [], "phones": [], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2022-07-05T21:01:25.612000+00:00"}], "description": "", "date_created": "2022-07-05T21:01:25.608000+00:00", "opportunities": [], "date_updated": "2022-07-05T21:01:25.658000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "url": null, "custom": {}, "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "status_id": "stat_HrZ1aYkkxRORQSxdBcNPT31HkqxkK2w2uWGiK6yjkmK", "tasks": [], "display_name": "Alex", "html_url": "https://app.close.com/lead/lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY/", "addresses": [], "status_label": "Potential", "id": "lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY", "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Alex"}], "name": "Alex", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1699641295267} +{"stream": "leads", "data": {"contacts": [{"created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "urls": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "display_name": "Steli Efti", "id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "title": "CEO & Co-Founder", "date_created": "2021-07-13T11:39:03.354000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Steli%20Efti"}], "name": "Steli Efti", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "emails": [{"type": "office", "email": "sales@close.com"}], "phones": [{"phone": "+16505176539", "phone_formatted": "+1 650-517-6539", "type": "office", "country": "US"}], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2021-07-13T11:39:03.354000+00:00", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Decision Maker"]}, {"created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "urls": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "display_name": "Nick Persico", "id": "cont_FY5ws8upMQQyD9vKg4jzwRb6V3MLctQeNTc2NaUmAyo", "title": "Director of Revenue", "date_created": "2021-07-13T11:39:03.366000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Nick%20Persico"}], "name": "Nick Persico", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "emails": [{"type": "office", "email": "nick@close.com"}], "phones": [{"phone": "+18334625673", "phone_formatted": "+1 833-462-5673", "type": "office", "country": "US"}], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2021-07-13T11:39:03.366000+00:00", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Gatekeeper", "Point of Contact"]}, {"created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "urls": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "display_name": "Customer Success Team", "id": "cont_4cmimyQMTMi61kc72mLAV9XdHdddw6LR1LqzvoNdSuV", "title": null, "date_created": "2021-07-13T11:39:03.374000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Customer%20Success%20Team"}], "name": "Customer Success Team", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "emails": [{"type": "office", "email": "success@close.com"}], "phones": [{"phone": "+18334625673", "phone_formatted": "+1 833-462-5673", "type": "office", "country": "US"}], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2021-07-13T11:39:03.374000+00:00"}, {"created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "urls": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "display_name": "Support", "id": "cont_CI5c6Ekew0cyhSgoFIXeaz2PJVmmtigF2XNIGUDXKnu", "title": null, "date_created": "2021-07-13T11:39:03.380000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Support"}], "name": "Support", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "emails": [{"type": "office", "email": "support@close.com"}], "phones": [], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2021-07-13T11:39:03.380000+00:00"}], "description": "Visit our blog for high quality sales content, blog.close.com!", "date_created": "2021-07-13T11:39:03.315000+00:00", "opportunities": [{"expected_value": 37500, "confidence": 75, "date_created": "2021-07-13T11:39:04.284000+00:00", "user_name": "Airbyte Team", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "date_updated": "2021-08-18T10:26:44.306000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "contact_name": "Steli Efti", "updated_by_name": "Airbyte Team", "lead_name": "Close (Example\u00a0Lead)", "date_won": "2021-07-15", "annualized_expected_value": 37500, "value_formatted": "$500", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "status_id": "stat_AWXzFJkvkHVyJQPFulY0wM7LrQRiMEtQChGumG035bH", "contact_id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "date_lost": null, "value": 50000, "value_period": "one_time", "id": "oppo_1TfmaSLuECcdSQIBnjUOBj1gAXdyrp4SFaogvcEmtbk", "value_currency": "USD", "note": "Use opportunities to track which stage of the pipeline your deals are in and the revenue associated with them.", "status_label": "Proposal Sent", "integration_links": [], "status_display_name": "Proposal Sent", "annualized_value": 50000, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_type": "active", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}], "date_updated": "2022-11-09T02:40:26.489000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "url": "https://close.com", "custom": {"Current Vendor/Software": "Stark Industries", "Industry": "Software", "Lead Owner": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "Referral Source": "Website"}, "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "tasks": [{"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "contact_id": null, "contact_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2021-07-18", "date_created": "2021-07-13T11:39:03.520000+00:00", "date_updated": "2022-11-08T12:21:15.730000+00:00", "due_date": "2021-07-18", "_type": "lead", "view": "archive", "id": "task_bboTdYSlGqTBSXF0FEwhBOywDCV6iKDyMWiEfNFi7sW", "is_complete": true, "is_dateless": null, "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "lead_name": "Close (Example\u00a0Lead)", "object_id": "acti_POPD3uA7lSfdf4xLO7PgsxKeoJS1dCRthzQRb13Xbyw", "object_type": "taskcompleted", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "text": "Call Steli", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team"}, {"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "contact_id": null, "contact_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2021-07-16", "date_created": "2021-07-13T11:39:03.463000+00:00", "date_updated": "2021-08-18T10:31:52.081000+00:00", "due_date": "2021-07-16", "_type": "lead", "view": "archive", "id": "task_7kpMfXIPms858l9GKZTo3BCtVflvcIsOYPXL8mZgzyp", "is_complete": true, "is_dateless": null, "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "lead_name": "Close (Example\u00a0Lead)", "object_id": "acti_gdh0Iw30XYfKhKctrqPuxAbNghsxlDBDFaN8k35ZAxf", "object_type": "taskcompleted", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "text": "Send Steli an email", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team"}], "display_name": "Close (Example\u00a0Lead)", "html_url": "https://app.close.com/lead/lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz/", "addresses": [{"city": "San Francisco", "address_2": null, "label": "mailing", "zipcode": "94120", "state": "CA", "country": "US", "address_1": "PO Box 7775 #69574"}], "status_label": "Interested", "id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Close%20%28Example%C2%A0Lead%29"}], "name": "Close (Example\u00a0Lead)", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_1exVDDcGOEiIdhBhBv2VEGqnpIcJZZqiWkk4O7hbU3D": "Stark Industries", "custom.cf_ZuP9X9UjiQzjptNHlT7DxzRATaFil2Ysoz0aGMq0Kim": "Software", "custom.cf_mhBoQeiuwFRlz7zqyi4kJgzUreEoUp0hUsLwrnUTEgh": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_05o22yQYHMrFh4cCYCMSQJdSpODdabCbGQ8il5Do7X4": "Website"}, "emitted_at": 1699641295282} +{"stream": "leads", "data": {"tasks": [{"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "contact_id": null, "contact_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2022-11-30T09:00:00+00:00", "date_created": "2022-11-11T09:01:05.650000+00:00", "date_updated": "2022-11-11T09:01:24.622000+00:00", "due_date": "2022-11-30T09:00:00+00:00", "_type": "lead", "view": "inbox", "id": "task_1uqKlzwO0qLGYgBMT5DcLtklNmgtrUNVobSNEiPpoU6", "is_complete": false, "is_dateless": false, "lead_id": "lead_AUtZm7EBlaSbYqOrDjIZEuC4tfhLxTDtaK9jlEPMb3y", "lead_name": "Airbyte", "object_id": null, "object_type": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "text": "Follow up", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team"}], "url": null, "date_updated": "2023-01-30T16:02:41.186000+00:00", "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Airbyte"}], "name": "Airbyte", "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "status_id": "stat_HrZ1aYkkxRORQSxdBcNPT31HkqxkK2w2uWGiK6yjkmK", "contacts": [{"lead_id": "lead_AUtZm7EBlaSbYqOrDjIZEuC4tfhLxTDtaK9jlEPMb3y", "id": "cont_b4h4BcmWn7rKbnsHQ0JfADwXGgndbpc5JlQEdHHyv78", "date_updated": "2023-01-30T16:02:41.174000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-11T09:00:52.289000+00:00", "name": "User1", "emails": [], "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=User1"}], "urls": [], "display_name": "User1", "title": "Product Manager", "phones": [{"phone_formatted": "+1 600-000-0001", "phone": "+16000000001", "type": "office", "country": null}], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}], "addresses": [], "status_label": "Potential", "description": "", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "html_url": "https://app.close.com/lead/lead_AUtZm7EBlaSbYqOrDjIZEuC4tfhLxTDtaK9jlEPMb3y/", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-11T09:00:52.285000+00:00", "opportunities": [], "custom": {}, "display_name": "Airbyte", "id": "lead_AUtZm7EBlaSbYqOrDjIZEuC4tfhLxTDtaK9jlEPMb3y", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1699641295991} +{"stream": "leads", "data": {"tasks": [{"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "contact_id": null, "contact_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2022-11-17T08:30:00+00:00", "date_created": "2022-11-08T15:55:06.275000+00:00", "date_updated": "2022-11-08T15:55:06.275000+00:00", "due_date": "2022-11-17T08:30:00+00:00", "_type": "lead", "view": "inbox", "id": "task_Va4jRQkIhZrUwzejqF3lZ4VaJeqR0cAvzzuaN0IaE5r", "is_complete": false, "is_dateless": false, "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "lead_name": "Test Lead", "object_id": null, "object_type": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "text": "Follow up", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team"}, {"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "contact_id": null, "contact_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2022-11-23T09:00:00+00:00", "date_created": "2022-11-09T12:37:29.827000+00:00", "date_updated": "2022-11-09T12:44:26.200000+00:00", "due_date": "2022-11-23T09:00:00+00:00", "_type": "lead", "view": "archive", "id": "task_KAzmd4tqcQ1dYypqOjeZqmpKVlFZf3LWJfxCSoX29GY", "is_complete": true, "is_dateless": false, "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "lead_name": "Test Lead", "object_id": "acti_tEFyhPH4AKZ3YHaqqFEdm9Lot1oEI4yEyKzEf0ncbGE", "object_type": "taskcompleted", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "text": "Follow up", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team"}], "url": null, "date_updated": "2023-01-30T16:03:28.802000+00:00", "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Test%20Lead"}], "name": "Test Lead", "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "contacts": [{"lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "date_updated": "2023-01-30T16:03:28.787000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-08T15:54:42.381000+00:00", "name": "User2", "emails": [{"email": "user2.sample@gmail.com", "type": "office"}], "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=User2"}], "urls": [], "display_name": "User2", "title": "Test Lead", "phones": [{"phone_formatted": "+1 415-623-6785", "phone": "+14156236785", "type": "office", "country": "US"}], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_Qj3b4cWxvmvqTtZ0U5TaprRDah8g7jRsavfVh8NCPcu": "2022-12-01"}], "addresses": [], "status_label": "Interested", "description": "", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "html_url": "https://app.close.com/lead/lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j/", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-08T15:54:42.377000+00:00", "opportunities": [{"user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "date_updated": "2022-11-08T15:55:27.750000+00:00", "integration_links": [], "note": "Test", "value": 10000, "user_name": "Airbyte Team", "annualized_expected_value": 5000, "value_formatted": "$100", "annualized_value": 10000, "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "date_won": "2022-11-16", "status_id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "status_type": "active", "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "value_period": "one_time", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_display_name": "Demo Completed", "date_created": "2022-11-08T15:55:27.750000+00:00", "contact_name": "User2", "date_lost": null, "lead_name": "Test Lead", "expected_value": 5000, "value_currency": "USD", "id": "oppo_QtfwXdJFxmOLRzd3mirP7S5vWWYTEho2uWtHYNSiOUE", "confidence": 50, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_label": "Demo Completed"}], "custom": {}, "display_name": "Test Lead", "id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1699641295999} +{"stream": "leads", "data": {"tasks": [], "url": null, "date_updated": "2023-11-10T17:59:45.408000+00:00", "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Bluth%20Company%20%28Example%C2%A0Lead%29"}], "name": "Bluth Company (Example\u00a0Lead)", "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "contacts": [{"lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "date_updated": "2021-07-13T11:39:04.430000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-07-13T11:39:04.430000+00:00", "name": "Gob Bluth", "emails": [{"email": "bluth@close.com", "type": "office"}], "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Gob%20Bluth"}], "urls": [], "display_name": "Gob Bluth", "title": "Magician", "phones": [{"phone_formatted": "+1 202-555-0186", "phone": "+12025550186", "type": "office", "country": "US"}], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Gatekeeper"]}, {"lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "id": "cont_at5uglNbyasFp2KsoWQpjQmLp4lmmqX2p1nhvmPYytq", "date_updated": "2021-07-13T11:39:04.441000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-07-13T11:39:04.441000+00:00", "name": "Tobias F\u00fcnke", "emails": [{"email": "tobiasfunke@close.com", "type": "office"}], "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Tobias%20F%C3%BCnke"}], "urls": [], "display_name": "Tobias F\u00fcnke", "title": "Blue Man Group (Understudy)", "phones": [], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Point of Contact"]}], "addresses": [{"city": "Los Angeles", "zipcode": "90210", "address_1": "100 Bluth Drive", "address_2": null, "label": "business", "state": "CA", "country": "US"}], "status_label": "Interested", "description": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "html_url": "https://app.close.com/lead/lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2/", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-07-05T11:39:00.850000+00:00", "opportunities": [{"user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "contact_id": null, "date_updated": "2021-07-13T11:39:04.817000+00:00", "integration_links": [], "note": "Gob's ready to buy a $3,000 suit.", "value": 300000, "user_name": "Airbyte Team", "annualized_expected_value": 225000, "value_formatted": "$3,000", "annualized_value": 300000, "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "date_won": "2021-07-16", "status_id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "status_type": "active", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "value_period": "one_time", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_display_name": "Demo Completed", "date_created": "2021-07-13T11:39:04.817000+00:00", "contact_name": null, "date_lost": null, "lead_name": "Bluth Company (Example\u00a0Lead)", "expected_value": 225000, "value_currency": "USD", "id": "oppo_NkKuhUWfDoErNArh44hw7jkkx7sCl5bhlamtezmX7BE", "confidence": 75, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_label": "Demo Completed"}], "custom": {"Current Vendor/Software": "BiffCo", "Industry": "Real estate", "Lead Owner": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "Referral Source": "Google Search"}, "display_name": "Bluth Company (Example\u00a0Lead)", "id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_1exVDDcGOEiIdhBhBv2VEGqnpIcJZZqiWkk4O7hbU3D": "BiffCo", "custom.cf_ZuP9X9UjiQzjptNHlT7DxzRATaFil2Ysoz0aGMq0Kim": "Real estate", "custom.cf_mhBoQeiuwFRlz7zqyi4kJgzUreEoUp0hUsLwrnUTEgh": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_05o22yQYHMrFh4cCYCMSQJdSpODdabCbGQ8il5Do7X4": "Google Search"}, "emitted_at": 1699641296006} +{"stream": "leads", "data": {"tasks": [{"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "contact_id": null, "contact_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2022-11-12", "date_created": "2022-11-08T15:10:57.220000+00:00", "date_updated": "2022-11-09T11:06:06.145000+00:00", "due_date": "2022-11-12", "_type": "lead", "view": "inbox", "id": "task_Wgzt6t0ZGRlvZTnfhSfxHoFRWm6zXBMuuecKxRkfYmZ", "is_complete": false, "is_dateless": false, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "lead_name": "Wayne Enterprises (Example\u00a0Lead)", "object_id": null, "object_type": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "text": "Follow up", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team"}, {"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "contact_id": null, "contact_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2022-11-12", "date_created": "2022-11-09T11:05:42.844000+00:00", "date_updated": "2022-11-09T11:06:06.147000+00:00", "due_date": "2022-11-12", "_type": "lead", "view": "inbox", "id": "task_7NzjQt8zXoqAUMJsb58EGBP4rqTTbm5iR8jRuOY3MAJ", "is_complete": false, "is_dateless": false, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "lead_name": "Wayne Enterprises (Example\u00a0Lead)", "object_id": null, "object_type": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "text": "Follow up", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team"}, {"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "contact_id": null, "contact_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2022-11-19T09:00:00+00:00", "date_created": "2022-11-08T15:13:22.159000+00:00", "date_updated": "2022-11-08T15:13:22.159000+00:00", "due_date": "2022-11-19T09:00:00+00:00", "_type": "lead", "view": "inbox", "id": "task_pIYI4An2qA84qea5mCLggCL0dZ8QIk2c2YrzoqSV8fd", "is_complete": false, "is_dateless": false, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "lead_name": "Wayne Enterprises (Example\u00a0Lead)", "object_id": null, "object_type": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "text": "Follow up", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team"}, {"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "contact_id": null, "contact_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2022-11-30T08:00:00+00:00", "date_created": "2022-11-08T15:13:13.470000+00:00", "date_updated": "2022-11-09T10:57:24.756000+00:00", "due_date": "2022-11-30T08:00:00+00:00", "_type": "lead", "view": "archive", "id": "task_hMgl18LN4kAUM7XWO4fYlu3amtKadZahsb7OV9J5qWD", "is_complete": true, "is_dateless": false, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "lead_name": "Wayne Enterprises (Example\u00a0Lead)", "object_id": "acti_SFQmP02YDdCHlkwAWoEtIfZXEOvhoenTmKVdEw7BtJB", "object_type": "taskcompleted", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "text": "Follow up", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team"}, {"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "contact_id": null, "contact_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2022-11-09T07:30:00+00:00", "date_created": "2022-11-09T11:05:46.682000+00:00", "date_updated": "2022-11-09T11:06:11.243000+00:00", "due_date": "2022-11-09T07:30:00+00:00", "_type": "lead", "view": "archive", "id": "task_iHHSk7GuvunATuloc0hM2k2btiuJpasL12GkwHnIVjP", "is_complete": true, "is_dateless": false, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "lead_name": "Wayne Enterprises (Example\u00a0Lead)", "object_id": "acti_DJOFYNqKoXSmPB4yJHVRafxAqhnvk5olHmmihw4cqND", "object_type": "taskcompleted", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "text": "Follow up", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team"}], "url": null, "date_updated": "2023-11-10T17:59:45.437000+00:00", "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Wayne%20Enterprises%20%28Example%C2%A0Lead%29"}], "name": "Wayne Enterprises (Example\u00a0Lead)", "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "contacts": [{"lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "date_updated": "2022-11-08T15:01:17.915000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-07-13T11:39:04.977000+00:00", "name": "Bruce Wayne", "emails": [{"email": "thedarkknight@close.com", "type": "office"}], "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Bruce%20Wayne"}], "urls": [], "display_name": "Bruce Wayne", "title": "The Dark Knight", "phones": [{"phone_formatted": "+1 415-623-6785", "phone": "+14156236785", "type": "office", "country": "US"}], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Decision Maker", "Point of Contact"]}], "addresses": [], "status_label": "Interested", "description": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "html_url": "https://app.close.com/lead/lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN/", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-07-13T11:39:04.960000+00:00", "opportunities": [{"user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "contact_id": null, "date_updated": "2021-07-13T11:39:05.061000+00:00", "integration_links": [], "note": "Bruce needs new software for the Bat Cave.", "value": 50000, "user_name": "Airbyte Team", "annualized_expected_value": 37500, "value_formatted": "$500", "annualized_value": 50000, "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "date_won": "2021-07-15", "status_id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "status_type": "active", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "value_period": "one_time", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_display_name": "Demo Completed", "date_created": "2021-07-13T11:39:05.061000+00:00", "contact_name": null, "date_lost": null, "lead_name": "Wayne Enterprises (Example\u00a0Lead)", "expected_value": 37500, "value_currency": "USD", "id": "oppo_jXatpqaQ3HK50yBO9vMWg2wFyHoqbCJtgbRYAMuEGor", "confidence": 75, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_label": "Demo Completed"}], "custom": {"Current Vendor/Software": "Initech", "Industry": "Manufacturing", "Lead Owner": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "Referral Source": "Facebook"}, "display_name": "Wayne Enterprises (Example\u00a0Lead)", "id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_1exVDDcGOEiIdhBhBv2VEGqnpIcJZZqiWkk4O7hbU3D": "Initech", "custom.cf_ZuP9X9UjiQzjptNHlT7DxzRATaFil2Ysoz0aGMq0Kim": "Manufacturing", "custom.cf_mhBoQeiuwFRlz7zqyi4kJgzUreEoUp0hUsLwrnUTEgh": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_05o22yQYHMrFh4cCYCMSQJdSpODdabCbGQ8il5Do7X4": "Facebook"}, "emitted_at": 1699641296013} {"stream": "opportunities", "data": {"user_name": "Airbyte Team", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "value": 300000, "status_display_name": "Demo Completed", "created_by_name": "Airbyte Team", "status_type": "active", "status_label": "Demo Completed", "value_currency": "USD", "id": "oppo_NkKuhUWfDoErNArh44hw7jkkx7sCl5bhlamtezmX7BE", "lead_name": "Bluth Company (Example\u00a0Lead)", "date_updated": "2021-07-13T11:39:04.817000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "value_period": "one_time", "integration_links": [], "status_id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "expected_value": 225000, "date_created": "2021-07-13T11:39:04.817000+00:00", "date_won": "2021-07-16", "note": "Gob's ready to buy a $3,000 suit.", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "contact_id": null, "contact_name": null, "annualized_expected_value": 225000, "confidence": 75, "annualized_value": 300000, "updated_by_name": "Airbyte Team", "value_formatted": "$3,000", "date_lost": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417127685} {"stream": "opportunities", "data": {"user_name": "Airbyte Team", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "value": 50000, "status_display_name": "Demo Completed", "created_by_name": "Airbyte Team", "status_type": "active", "status_label": "Demo Completed", "value_currency": "USD", "id": "oppo_jXatpqaQ3HK50yBO9vMWg2wFyHoqbCJtgbRYAMuEGor", "lead_name": "Wayne Enterprises (Example\u00a0Lead)", "date_updated": "2021-07-13T11:39:05.061000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "value_period": "one_time", "integration_links": [], "status_id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "expected_value": 37500, "date_created": "2021-07-13T11:39:05.061000+00:00", "date_won": "2021-07-15", "note": "Bruce needs new software for the Bat Cave.", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "contact_id": null, "contact_name": null, "annualized_expected_value": 37500, "confidence": 75, "annualized_value": 50000, "updated_by_name": "Airbyte Team", "value_formatted": "$500", "date_lost": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417127689} {"stream": "opportunities", "data": {"user_name": "Airbyte Team", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "value": 50000, "status_display_name": "Proposal Sent", "created_by_name": "Airbyte Team", "status_type": "active", "status_label": "Proposal Sent", "value_currency": "USD", "id": "oppo_1TfmaSLuECcdSQIBnjUOBj1gAXdyrp4SFaogvcEmtbk", "lead_name": "Close (Example\u00a0Lead)", "date_updated": "2021-08-18T10:26:44.306000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "value_period": "one_time", "integration_links": [], "status_id": "stat_AWXzFJkvkHVyJQPFulY0wM7LrQRiMEtQChGumG035bH", "expected_value": 37500, "date_created": "2021-07-13T11:39:04.284000+00:00", "date_won": "2021-07-15", "note": "Use opportunities to track which stage of the pipeline your deals are in and the revenue associated with them.", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "contact_id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "contact_name": "Steli Efti", "annualized_expected_value": 37500, "confidence": 75, "annualized_value": 50000, "updated_by_name": "Airbyte Team", "value_formatted": "$500", "date_lost": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417127692} diff --git a/airbyte-integrations/connectors/source-close-com/metadata.yaml b/airbyte-integrations/connectors/source-close-com/metadata.yaml index 39d1f4c8f0dc..97847eafefac 100644 --- a/airbyte-integrations/connectors/source-close-com/metadata.yaml +++ b/airbyte-integrations/connectors/source-close-com/metadata.yaml @@ -8,7 +8,7 @@ data: connectorSubtype: api connectorType: source definitionId: dfffecb7-9a13-43e9-acdc-b92af7997ca9 - dockerImageTag: 0.4.2 + dockerImageTag: 0.4.3 dockerRepository: airbyte/source-close-com documentationUrl: https://docs.airbyte.com/integrations/sources/close-com githubIssueLabel: source-close-com diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/manifest.yaml b/airbyte-integrations/connectors/source-close-com/source_close_com/manifest.yaml index f7768c310240..f4f6ba4d0a12 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/manifest.yaml +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/manifest.yaml @@ -194,9 +194,10 @@ definitions: name: "email_thread_activities" path: "activity/emailthread" email_activities_stream: - $ref: "#/definitions/activities_base_stream" + $ref: "#/definitions/base_stream" + incremental_sync: + $ref: "#/definitions/incremental_sync__cursor_date_created" $parameters: - $ref: "#/definitions/activities_base_stream/$parameters" name: "email_activities" path: "activity/email" sms_activities_stream: diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/activity_custom_fields.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/activity_custom_fields.json index cafcf348dd9a..1675b0e43dee 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/activity_custom_fields.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/activity_custom_fields.json @@ -6,6 +6,18 @@ "description": { "date_updated": ["null", "string"] }, + "back_reference_is_visible": { + "date_updated": ["null", "boolean"] + }, + "choices": { + "date_updated": ["null", "array"] + }, + "is_shared": { + "date_updated": ["null", "boolean"] + }, + "referenced_custom_type_id": { + "date_updated": ["null", "string"] + }, "name": { "date_updated": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/call_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/call_activities.json index 886d0d164101..b6b4df319d36 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/call_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/call_activities.json @@ -6,6 +6,30 @@ "source": { "type": ["null", "string"] }, + "forwarded_to": { + "type": ["null", "string"] + }, + "is_forwarded": { + "type": ["null", "boolean"] + }, + "note_html": { + "type": ["null", "string"] + }, + "parent_meeting_id": { + "type": ["null", "string"] + }, + "recording_duration": { + "type": ["null", "string"] + }, + "sequence_id": { + "type": ["null", "string"] + }, + "sequence_name": { + "type": ["null", "string"] + }, + "sequence_subscription_id": { + "type": ["null", "string"] + }, "remote_phone_formatted": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/contact_custom_fields.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/contact_custom_fields.json index 97ed85cb4ca3..b3f26fa90b46 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/contact_custom_fields.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/contact_custom_fields.json @@ -9,6 +9,15 @@ "name": { "date_updated": ["null", "string"] }, + "back_reference_is_visible": { + "date_updated": ["null", "string"] + }, + "is_shared": { + "date_updated": ["null", "boolean"] + }, + "referenced_custom_type_id": { + "date_updated": ["null", "string"] + }, "choices": { "date_updated": ["null", "array"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/created_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/created_activities.json index e092624b7a2c..ddec9f66cfd9 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/created_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/created_activities.json @@ -6,6 +6,9 @@ "date_updated": { "type": ["null", "string"] }, + "activity_at": { + "type": ["null", "string"] + }, "date_created": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_followup_tasks.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_followup_tasks.json index 35ab577ebc36..c33a632c08c0 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_followup_tasks.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_followup_tasks.json @@ -9,6 +9,12 @@ "view": { "type": ["null", "string"] }, + "total_emails_count_in_thread": { + "type": ["null", "integer"] + }, + "updated_by_name": { + "type": ["null", "string"] + }, "assigned_to": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/events.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/events.json index 0deeddc49d12..290446df8def 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/events.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/events.json @@ -9,6 +9,12 @@ "api_key_id": { "type": ["null", "string"] }, + "oauth_client_id": { + "type": ["null", "string"] + }, + "oauth_scope": { + "type": ["null", "string"] + }, "changed_fields": { "type": ["null", "array"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_custom_fields.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_custom_fields.json index 19bc94eed623..776000912839 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_custom_fields.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_custom_fields.json @@ -9,6 +9,15 @@ "description": { "date_updated": ["null", "string"] }, + "back_reference_is_visible": { + "date_updated": ["null", "string"] + }, + "date_updated": { + "date_updated": ["null", "string"] + }, + "referenced_custom_type_id": { + "date_updated": ["null", "string"] + }, "name": { "date_updated": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/meeting_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/meeting_activities.json index 66cb927a51e9..05b86653ceff 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/meeting_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/meeting_activities.json @@ -6,6 +6,57 @@ "date_updated": { "type": ["null", "string"] }, + "activity_at": { + "type": ["null", "string"] + }, + "contact_id": { + "type": ["null", "string"] + }, + "notetaker_id": { + "type": ["null", "string"] + }, + "provider_calendar_event_id": { + "type": ["null", "string"] + }, + "provider_calendar_type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "user_note": { + "type": ["null", "string"] + }, + "user_note_date_updated": { + "type": ["null", "string"] + }, + "user_note_html": { + "type": ["null", "string"] + }, + "provider_calendar_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "integrations": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "calendar_event_uids": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "conference_links": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, "created_by_name": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_custom_fields.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_custom_fields.json index 97ed85cb4ca3..b3f26fa90b46 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_custom_fields.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_custom_fields.json @@ -9,6 +9,15 @@ "name": { "date_updated": ["null", "string"] }, + "back_reference_is_visible": { + "date_updated": ["null", "string"] + }, + "is_shared": { + "date_updated": ["null", "boolean"] + }, + "referenced_custom_type_id": { + "date_updated": ["null", "string"] + }, "choices": { "date_updated": ["null", "array"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_status_change_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_status_change_activities.json index 46a659113aa3..389307edcbcb 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_status_change_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_status_change_activities.json @@ -21,6 +21,27 @@ "date_created": { "type": ["null", "string"] }, + "activity_at": { + "type": ["null", "string"] + }, + "new_pipeline_name": { + "type": ["null", "string"] + }, + "users": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "opportunity_value_period": { + "type": ["null", "string"] + }, + "old_pipeline_name": { + "type": ["null", "string"] + }, + "opportunity_confidence": { + "type": ["null", "integer"] + }, "date_updated": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/pipelines.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/pipelines.json index 46ec2b161dbd..bafc4f534fa6 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/pipelines.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/pipelines.json @@ -6,6 +6,9 @@ "created_by": { "type": ["null", "string"] }, + "updated_by": { + "type": ["null", "string"] + }, "date_updated": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/sms_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/sms_activities.json index a31ed621a3df..ee7841862e38 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/sms_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/sms_activities.json @@ -6,6 +6,36 @@ "date_updated": { "type": ["null", "string"] }, + "activity_at": { + "type": ["null", "string"] + }, + "attachments": { + "type": ["null", "array"] + }, + "date_scheduled": { + "type": ["null", "string"] + }, + "error_message": { + "type": ["null", "string"] + }, + "local_phone_formatted": { + "type": ["null", "string"] + }, + "sequence_id": { + "type": ["null", "string"] + }, + "sequence_name": { + "type": ["null", "string"] + }, + "sequence_subscription_id": { + "type": ["null", "string"] + }, + "source": { + "type": ["null", "string"] + }, + "template_id": { + "type": ["null", "string"] + }, "date_created": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/task_completed_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/task_completed_activities.json index 3c8e3434563d..8f753d39e125 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/task_completed_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/task_completed_activities.json @@ -9,6 +9,21 @@ "id": { "type": ["null", "string"] }, + "activity_at": { + "type": ["null", "string"] + }, + "task_assigned_to": { + "type": ["null", "string"] + }, + "task_assigned_to_name": { + "type": ["null", "string"] + }, + "user_name": { + "type": ["null", "string"] + }, + "users": { + "type": ["null", "array"] + }, "task_id": { "type": ["null", "string"] }, diff --git a/docs/integrations/sources/close-com.md b/docs/integrations/sources/close-com.md index 9cb94dcc2de1..bedfdc116949 100644 --- a/docs/integrations/sources/close-com.md +++ b/docs/integrations/sources/close-com.md @@ -105,6 +105,7 @@ The Close.com connector is subject to rate limits. For more information on this | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------| +| 0.4.3 | 2023-10-28 | [31534](https://github.com/airbytehq/airbyte/pull/31534) | Fixed Email Activities Stream Pagination | | 0.4.2 | 2023-08-08 | [29206](https://github.com/airbytehq/airbyte/pull/29206) | Fixed the issue with `DatePicker` format for `start date` | | 0.4.1 | 2023-07-04 | [27950](https://github.com/airbytehq/airbyte/pull/27950) | Add human readable titles to API Key and Start Date fields | | 0.4.0 | 2023-06-27 | [27776](https://github.com/airbytehq/airbyte/pull/27776) | Update the `Email Followup Tasks` stream schema | From f4d2f7be1ceb747008550ca775789515e7e85c94 Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Fri, 10 Nov 2023 14:51:13 -0800 Subject: [PATCH 009/124] Show metadata on connector docs (#32334) --- docusaurus/docusaurus.config.js | 5 +- docusaurus/src/remark/docsHeaderDecoration.js | 111 ++++++++++++++++++ 2 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 docusaurus/src/remark/docsHeaderDecoration.js diff --git a/docusaurus/docusaurus.config.js b/docusaurus/docusaurus.config.js index 2621046e6aaf..396ec4bfe143 100644 --- a/docusaurus/docusaurus.config.js +++ b/docusaurus/docusaurus.config.js @@ -8,6 +8,8 @@ const path = require("node:path"); const lightCodeTheme = require("prism-react-renderer/themes/github"); const darkCodeTheme = require("prism-react-renderer/themes/dracula"); +const docsHeaderDecoration = require("./src/remark/docsHeaderDecoration"); + const redirects = yaml.load( fs.readFileSync(path.join(__dirname, "redirects.yml"), "utf-8") ); @@ -36,7 +38,7 @@ const config = { type: "module", id: "unifytag", "data-api-key": "wk_BEtrdAz2_2qgdexg5KRa6YWLWVwDdieFC7CAHkDKz", - } + }, ], plugins: [ @@ -78,6 +80,7 @@ const config = { editUrl: "https://github.com/airbytehq/airbyte/blob/master/docs", path: "../docs", exclude: ["**/*.inapp.md"], + remarkPlugins: [docsHeaderDecoration], }, blog: false, theme: { diff --git a/docusaurus/src/remark/docsHeaderDecoration.js b/docusaurus/src/remark/docsHeaderDecoration.js new file mode 100644 index 000000000000..1530d6473253 --- /dev/null +++ b/docusaurus/src/remark/docsHeaderDecoration.js @@ -0,0 +1,111 @@ +const fetch = require("node-fetch"); +const visit = require("unist-util-visit"); + +const registry_url = + "https://connectors.airbyte.com/files/generated_reports/connector_registry_report.json"; +let registry = []; + +const plugin = () => { + let registryFetcher = fetchCatalog(); + + const transformer = async (ast, vfile) => { + if (!isDocsPage(vfile)) return; + + const pathParts = vfile.path.split("/"); + const connectorName = pathParts.pop().split(".")[0]; + const connectorType = pathParts.pop(); + const dockerRepository = `airbyte/${connectorType.replace( + /s$/, + "" + )}-${connectorName}`; + + await Promise.resolve(registryFetcher); + + const registryEntry = registry.find( + (r) => r.dockerRepository_oss === dockerRepository + ); + + if (!registryEntry) return; + + let originalTitle = ""; + let originalId = ""; + + visit(ast, "heading", (node) => { + if (node.depth === 1 && node.children.length === 1) { + originalTitle = node.children[0].value; + originalId = node.data.hProperties.id; + + node.type = "html"; + node.children = undefined; + node.value = buildConnectorHTMLContent( + registryEntry, + originalTitle, + originalId + ); + } + }); + }; + return transformer; +}; + +const fetchCatalog = async () => { + console.log("Fetching connector registry..."); + const response = await fetch(registry_url); + registry = await response.json(); + console.log(`fetched ${registry.length} connectors form registry`); +}; + +const buildConnectorHTMLContent = ( + registryEntry, + originalTitle, + originalId +) => { + // note - you can't have any leading whitespace here + const htmlContent = `
+
+ connector logo +

${originalTitle}

+
+ + +
+ Availability: Airbyte Cloud: ${ + registryEntry.is_cloud ? "✅" : "❌" + }, Airbyte OSS: ${registryEntry.is_oss ? "✅" : "❌"} +
+ Support Level: ${capitalizeFirstLetter( + registryEntry.supportLevel_oss + )} +
+ Latest Version: ${registryEntry.dockerImageTag_oss} +
+ Definition Id: ${registryEntry.definitionId} +
+ +
`; + + return htmlContent; +}; + +const isDocsPage = (vfile) => { + if ( + !vfile.path.includes("integrations/sources") && + !vfile.path.includes("integrations/destinations") + ) { + return false; + } + + if (vfile.path.includes("-migrations.md")) { + return false; + } + + return true; +}; + +const capitalizeFirstLetter = (string) => { + return string.charAt(0).toUpperCase() + string.slice(1); +}; + +module.exports = plugin; From d99de79d34bdab2dc890f031252adf13ca6dcd77 Mon Sep 17 00:00:00 2001 From: "Roman Yermilov [GL]" <86300758+roman-yermilov-gl@users.noreply.github.com> Date: Sat, 11 Nov 2023 11:45:54 +0400 Subject: [PATCH 010/124] Source Facebook Marketing: fix expected records (#32442) --- .../integration_tests/expected_records.jsonl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/expected_records.jsonl index 1fbf44808013..6bed2dde636d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/expected_records.jsonl @@ -1,4 +1,4 @@ -{"stream": "ad_account", "data": {"id": "act_212551616838260", "account_id": "212551616838260", "account_status": 1, "age": 1297.3245138889, "amount_spent": "39125", "balance": "0", "business": {"id": "1506473679510495", "name": "Airbyte"}, "business_city": "", "business_country_code": "US", "business_name": "", "business_street": "", "business_street2": "", "can_create_brand_lift_study": false, "capabilities": ["CAN_CREATE_CALL_ADS", "CAN_SEE_GROWTH_OPPORTUNITY_DATA", "ENABLE_IA_RECIRC_AD_DISPLAY_FORMAT", "CAN_USE_MOBILE_EXTERNAL_PAGE_TYPE", "CAN_USE_FB_FEED_POSITION_IN_VIDEO_VIEW_15S", "ENABLE_BIZ_DISCO_ADS", "ENABLE_BRAND_OBJECTIVES_FOR_BIZ_DISCO_ADS", "ENABLE_DIRECT_REACH_FOR_BIZ_DISCO_ADS", "ENABLE_DYNAMIC_ADS_ON_IG_STORIES_ADS", "ENABLE_IG_STORIES_ADS_PPE_OBJECTIVE", "ENABLE_IG_STORIES_ADS_MESSENGER_DESTINATION", "ENABLE_PAC_FOR_BIZ_DISCO_ADS", "CAN_USE_FB_INSTREAM_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_FB_STORY_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_AN_INSTREAM_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_IG_STORY_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_FB_IA_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_FB_SUG_VIDEO_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_FB_MKT_PLACE_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_IG_FEED_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_IG_EXPLORE_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_AN_CLASSIC_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_AN_REWARD_VIDEO_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_REACH_AND_FREQUENCY", "CAN_USE_RECURRING_BUDGET", "HAS_VALID_PAYMENT_METHODS", "CAN_USE_LINK_CLICK_BILLING_EVENT", "CAN_USE_CPA_BILLING_EVENT", "CAN_SEE_NEW_CONVERSION_WINDOW_NUX", "ADS_INSTREAM_INTERFACE_INTEGRITY", "ADS_INSTREAM_LINK_CLICK", "ADS_INSTREAM_LINK_CLICK_IMAGE", "ADS_IN_OBJECTIVES_DEPRECATION", "MESSENGER_INBOX_ADS_PRODUCT_CATALOG_SALES", "CAN_SHOW_MESSENGER_DUPLICSTION_UPSELL", "ALLOW_INSTREAM_ONLY_FOR_REACH", "ADS_INSTREAM_VIDEO_PLACEMENT_CONVERSIONS", "CAN_CREATE_INSTAGRAM_EXPLORE_ADS", "ALLOW_INSTREAM_VIDEOS_PLACEMENT_ONLY", "ALLOW_INSTREAM_NON_INTERRUPTIVE_LEADGEN", "INSTREAM_VIDEO_AD_DESKTOP_CONVERSION_AD_PREVIEW", "ALLOW_INSTREAM_ONLY_FOR_BRAND_AWARENESS_AUCTION", "ALLOW_SUGGESTED_VIDEOS_PLACEMENT_ONLY", "WHATSAPP_DESTINATION_ADS", "CTM_ADS_CREATION_CLICK_TO_DIRECT", "CTW_ADS_ENABLE_IG_FEED_PLACEMENT", "CTW_ADS_FOR_NON_MESSAGES_OBJECTIVE", "CTW_ADS_TRUSTED_TIER_2_PLUS_ADVERTISER", "CTW_ADS_TRUSTED_TIER_ADVERTISER", "ADS_PLACEMENT_MARKETPLACE", "ADNW_DISABLE_INSTREAM_AND_WEB_PLACEMENT", "CAN_CHANGE_BILLING_THRESHOLD", "CAN_USE_APP_EVENT_AVERAGE_COST_BIDDING", "CAN_USE_LEAD_GEN_AVERAGE_COST_BIDDING", "ADS_VALUE_OPTIMIZATION_DYNAMIC_ADS_1D", "ADS_DELIVERY_INSIGHTS_IN_BIDDING_PRESET_EXPERIMENT", "ADS_DELIVERY_INSIGHTS_OPTIMIZATION_PRESET", "CAN_SEE_APP_AD_EVENTS", "CAN_SEE_NEW_STANDARD_EVENTS_BETA", "CAN_SEE_VCK_HOLIDAY_TEMPLATES", "ENABLE_DCO_FOR_FB_STORY_ADS", "CAN_USE_IG_EXPLORE_GRID_HOME_PLACEMENT", "CAN_USE_IG_EXPLORE_HOME_IN_REACH_AND_FREQUENCY", "CAN_USE_IG_EXPLORE_HOME_POST_ENGAGEMENT_MESSAGES", "CAN_USE_IG_SEARCH_PLACEMENT", "CAN_USE_IG_SEARCH_GRID_ADS", "CAN_USE_IG_SEARCH_RESULTS_AUTO_PLACEMENT", "CAN_USE_IG_REELS_PAC_CAROUSEL", "CAN_USE_IG_REELS_POSITION", "CAN_SEE_CONVERSION_LIFT_SUMMARY", "CAN_USE_IG_PROFILE_FEED_POSITION", "CAN_USE_IG_PROFILE_FEED_AUTO_PLACEMENT", "CAN_USE_IG_PROFILE_FEED_ADDITIONAL_OBJECTIVES", "CAN_USE_IG_REELS_REACH_AND_FREQUENCY", "CAN_USE_IG_REELS_OVERLAY_POSITION", "CAN_USE_IG_REELS_OVERLAY_AUTO_PLACEMENT", "CAN_USE_IG_REELS_OVERLAY_PAC", "CAN_USE_IG_SHOP_TAB_PAC", "CAN_SEE_LEARNING_STAGE", "ENABLE_WEBSITE_CONVERSIONS_FOR_FB_STORY_ADS", "ENABLE_MESSENGER_INBOX_VIDEO_ADS", "ENABLE_VIDEO_VIEWS_FOR_FB_STORY_ADS", "ENABLE_LINK_CLICKS_FOR_FB_STORY_ADS", "ENABLE_REACH_FOR_FB_STORY_ADS", "CAN_USE_CALL_TO_ACTION_LINK_IMPORT_EXPORT", "ADS_INSTREAM_VIDEO_ENABLE_SLIDE_SHOW", "ALLOW_INSTREAM_VIDEOS_PLACEMENT_ONLY_IN_VV_REACH_AND_FREQUENCY", "ENABLE_MOBILE_APP_INSTALLS_FOR_FB_STORY_ADS", "ENABLE_LEAD_GEN_FOR_FB_STORY_ADS", "CAN_USE_FB_MKT_PLACE_POSITION_IN_REACH", "CAN_USE_FB_MKT_PLACE_POSITION_IN_VIDEO_VIEW", "CAN_USE_FB_MKT_PLACE_POSITION_IN_STORE_VISIT", "ENABLE_MOBILE_APP_ENGAGEMENT_FOR_FB_STORY_ADS", "CAN_USE_FB_MKT_PLACE_POSITION_IN_BRAND_AWARENESS", "CAN_USE_FB_MKT_PLACE_POSITION_IN_APP_INSTALLS", "CAN_USE_FB_MKT_PLACE_POSITION_IN_LEAD_GENERATION", "CAN_USE_FB_MKT_PLACE_POSITION_IN_MESSAGE", "CAN_USE_FB_MKT_PLACE_POSITION_IN_PAGE_LIKE", "CAN_USE_FB_MKT_PLACE_POSITION_IN_POST_ENGAGEMENT", "RF_ALLOW_MARKETPLACE_ACCOUNT", "RF_ALLOW_SEARCH_ACCOUNT", "VERTICAL_VIDEO_PAC_INSTREAM_UPSELL", "IX_COLLECTION_ENABLED_FOR_BAO_AND_REACH", "ADS_BM_REQUIREMENTS_OCT_15_RELEASE", "ENABLE_POST_ENGAGEMENT_FOR_FB_STORY", "ENBABLE_CATALOG_SALES_FOR_FB_STORY", "CAN_USE_WHATSAPP_DESTINATION_ON_LINK_CLICKS_AND_CONVERSIONS", "CAN_USE_WHATSAPP_DESTINATION_ON_CONVERSIONS", "IS_NON_TAIL_AD_ACCOUNT", "IS_IN_DSA_GK", "IS_IN_IG_EXISTING_POST_CTA_DEFAULTING_EXPERIMENT", "IS_IN_SHORT_WA_LINK_CTWA_UNCONV_TRAFFIC_EXPERIMENT", "IS_IN_ODAX_EXPERIENCE", "IS_IN_REACH_BRAND_AWARENESS_WHATSAPP_L1_DESTINATION_EXPERIMENT", "IS_IN_VIDEO_VIEWS_WHATSAPP_L1_DESTINATION_EXPERIMENT", "IS_IN_WHATSAPP_DESTINATION_DEFAULTING_EXPERIMENT", "CAN_USE_MARKETPLACE_DESKTOP", "ADS_MERCHANT_OVERLAYS_DEPRECATION", "CONNECTIONS_DEPRECATION_V2", "CAN_USE_LIVE_VIDEO_FOR_THRUPLAY", "CAN_SEE_HEC_AM_FLOW", "CAN_SEE_POLITICAL_FLOW", "ADS_INSTREAM_PLACEMENT_CATALOG_SALES", "ENABLE_CONVERSIONS_FOR_FB_GROUP_TAB_ADS", "ENABLE_LINK_CLICK_FOR_FB_GROUP_TAB_ADS", "ENABLE_REACH_FOR_FB_GROUP_TAB_ADS", "CAN_USE_CONVERSATIONS_OPTIMIZATION", "ENABLE_THRUPLAY_OPTIMIZATION_MESSENGER_STORY_ADS", "CAN_USE_IG_STORY_POLLS_PAC_CREATION", "IOS14_CEO_CAMPAIGN_CREATION", "ENABLE_VIDEO_CHANNEL_PLACEMENT_FOR_RSVP_ADS", "DIGITAL_CIRCULAR_ADS", "CAN_SEE_SAFR_V3_FLOW", "CAN_USE_FB_REELS_POSITION", "CAN_USE_ADS_ON_FB_REELS_POSITION", "CAN_USE_FB_REELS_AUTO_PLACEMENT", "ENABLE_FB_REELS_CREATION_PAC_ADS", "ENABLE_FB_REELS_CREATION_DCO_ADS", "ENABLE_FB_REELS_POSTLOOP_CREATION_DCO_ADS", "ENABLE_FB_REELS_POSTLOOP_CREATION_PAC_ADS", "RF_CPA_BILLING_DEPRECATION_PHASE_2", "ENABLE_APP_INSTALL_CUSTOM_PRODUCT_PAGES", "ENABLE_ADS_ON_FB_REELS_PLACEMENT_UNIFICATION", "ENABLE_ADS_ON_FB_REELS_PLACEMENT_UNIFICATION_L2_NUX", "ENABLE_ADS_ON_IG_SHOP_TAB_DEPRECATION_L2_NUX", "ADS_RF_FB_REELS_PLACEMENT", "ENABLE_ADS_ON_FB_INSTANT_ARTICLE_DEPRECATION_L2_NUX", "REELS_DM_ADS_ENABLE_REACH_AND_FREQUENCY", "ADS_AEMV2_HAS_LAUNCHED", "ELIGIBLE_FOR_TEXT_GEN"], "created_time": "2020-04-13T18:04:59-0700", "currency": "USD", "disable_reason": 0.0, "end_advertiser": 1506473679510495.0, "end_advertiser_name": "Airbyte", "fb_entity": 85.0, "funding_source": 2825262454257003.0, "funding_source_details": {"id": "2825262454257003", "type": 1}, "has_migrated_permissions": true, "is_attribution_spec_system_default": true, "is_direct_deals_enabled": false, "is_in_3ds_authorization_enabled_market": false, "is_notifications_enabled": true, "is_personal": 0.0, "is_prepay_account": false, "is_tax_id_required": false, "min_campaign_group_spend_cap": 10000.0, "min_daily_budget": 100.0, "name": "Airbyte", "offsite_pixels_tos_accepted": true, "owner": 1506473679510495.0, "rf_spec": {"min_reach_limits": {"US": 200000, "CA": 200000, "GB": 200000, "AR": 200000, "AU": 200000, "AT": 200000, "BE": 200000, "BR": 200000, "CL": 200000, "CN": 200000, "CO": 200000, "HR": 200000, "DK": 200000, "DO": 200000, "EG": 200000, "FI": 200000, "FR": 200000, "DE": 200000, "GR": 200000, "HK": 200000, "IN": 200000, "ID": 200000, "IE": 200000, "IL": 200000, "IT": 200000, "JP": 200000, "JO": 200000, "KW": 200000, "LB": 200000, "MY": 200000, "MX": 200000, "NL": 200000, "NZ": 200000, "NG": 200000, "NO": 200000, "PK": 200000, "PA": 200000, "PE": 200000, "PH": 200000, "PL": 200000, "RU": 200000, "SA": 200000, "RS": 200000, "SG": 200000, "ZA": 200000, "KR": 200000, "ES": 200000, "SE": 200000, "CH": 200000, "TW": 200000, "TH": 200000, "TR": 200000, "AE": 200000, "VE": 200000, "PT": 200000, "LU": 200000, "BG": 200000, "CZ": 200000, "SI": 200000, "IS": 200000, "SK": 200000, "LT": 200000, "TT": 200000, "BD": 200000, "LK": 200000, "KE": 200000, "HU": 200000, "MA": 200000, "CY": 200000, "JM": 200000, "EC": 200000, "RO": 200000, "BO": 200000, "GT": 200000, "CR": 200000, "QA": 200000, "SV": 200000, "HN": 200000, "NI": 200000, "PY": 200000, "UY": 200000, "PR": 200000, "BA": 200000, "PS": 200000, "TN": 200000, "BH": 200000, "VN": 200000, "GH": 200000, "MU": 200000, "UA": 200000, "MT": 200000, "BS": 200000, "MV": 200000, "OM": 200000, "MK": 200000, "LV": 200000, "EE": 200000, "IQ": 200000, "DZ": 200000, "AL": 200000, "NP": 200000, "MO": 200000, "ME": 200000, "SN": 200000, "GE": 200000, "BN": 200000, "UG": 200000, "GP": 200000, "BB": 200000, "AZ": 200000, "TZ": 200000, "LY": 200000, "MQ": 200000, "CM": 200000, "BW": 200000, "ET": 200000, "KZ": 200000, "NA": 200000, "MG": 200000, "NC": 200000, "MD": 200000, "FJ": 200000, "BY": 200000, "JE": 200000, "GU": 200000, "YE": 200000, "ZM": 200000, "IM": 200000, "HT": 200000, "KH": 200000, "AW": 200000, "PF": 200000, "AF": 200000, "BM": 200000, "GY": 200000, "AM": 200000, "MW": 200000, "AG": 200000, "RW": 200000, "GG": 200000, "GM": 200000, "FO": 200000, "LC": 200000, "KY": 200000, "BJ": 200000, "AD": 200000, "GD": 200000, "VI": 200000, "BZ": 200000, "VC": 200000, "MN": 200000, "MZ": 200000, "ML": 200000, "AO": 200000, "GF": 200000, "UZ": 200000, "DJ": 200000, "BF": 200000, "MC": 200000, "TG": 200000, "GL": 200000, "GA": 200000, "GI": 200000, "CD": 200000, "KG": 200000, "PG": 200000, "BT": 200000, "KN": 200000, "SZ": 200000, "LS": 200000, "LA": 200000, "LI": 200000, "MP": 200000, "SR": 200000, "SC": 200000, "VG": 200000, "TC": 200000, "DM": 200000, "MR": 200000, "AX": 200000, "SM": 200000, "SL": 200000, "NE": 200000, "CG": 200000, "AI": 200000, "YT": 200000, "CV": 200000, "GN": 200000, "TM": 200000, "BI": 200000, "TJ": 200000, "VU": 200000, "SB": 200000, "ER": 200000, "WS": 200000, "AS": 200000, "FK": 200000, "GQ": 200000, "TO": 200000, "KM": 200000, "PW": 200000, "FM": 200000, "CF": 200000, "SO": 200000, "MH": 200000, "VA": 200000, "TD": 200000, "KI": 200000, "ST": 200000, "TV": 200000, "NR": 200000, "RE": 200000, "LR": 200000, "ZW": 200000, "CI": 200000, "MM": 200000, "AN": 200000, "AQ": 200000, "BQ": 200000, "BV": 200000, "IO": 200000, "CX": 200000, "CC": 200000, "CK": 200000, "CW": 200000, "TF": 200000, "GW": 200000, "HM": 200000, "XK": 200000, "MS": 200000, "NU": 200000, "NF": 200000, "PN": 200000, "BL": 200000, "SH": 200000, "MF": 200000, "PM": 200000, "SX": 200000, "GS": 200000, "SS": 200000, "SJ": 200000, "TL": 200000, "TK": 200000, "UM": 200000, "WF": 200000, "EH": 200000}, "countries": ["US", "CA", "GB", "AR", "AU", "AT", "BE", "BR", "CL", "CN", "CO", "HR", "DK", "DO", "EG", "FI", "FR", "DE", "GR", "HK", "IN", "ID", "IE", "IL", "IT", "JP", "JO", "KW", "LB", "MY", "MX", "NL", "NZ", "NG", "NO", "PK", "PA", "PE", "PH", "PL", "RU", "SA", "RS", "SG", "ZA", "KR", "ES", "SE", "CH", "TW", "TH", "TR", "AE", "VE", "PT", "LU", "BG", "CZ", "SI", "IS", "SK", "LT", "TT", "BD", "LK", "KE", "HU", "MA", "CY", "JM", "EC", "RO", "BO", "GT", "CR", "QA", "SV", "HN", "NI", "PY", "UY", "PR", "BA", "PS", "TN", "BH", "VN", "GH", "MU", "UA", "MT", "BS", "MV", "OM", "MK", "EE", "LV", "IQ", "DZ", "AL", "NP", "MO", "ME", "SN", "GE", "BN", "UG", "GP", "BB", "ZW", "CI", "AZ", "TZ", "LY", "MQ", "MM", "CM", "BW", "ET", "KZ", "NA", "MG", "NC", "MD", "FJ", "BY", "JE", "GU", "YE", "ZM", "IM", "HT", "KH", "AW", "PF", "AF", "BM", "GY", "AM", "MW", "AG", "RW", "GG", "GM", "FO", "LC", "KY", "BJ", "AD", "GD", "VI", "BZ", "VC", "MN", "MZ", "ML", "AO", "GF", "UZ", "DJ", "BF", "MC", "TG", "GL", "GA", "GI", "CD", "KG", "PG", "BT", "KN", "SZ", "LS", "LA", "LI", "MP", "SR", "SC", "VG", "TC", "DM", "MR", "AX", "SM", "SL", "NE", "CG", "AI", "YT", "LR", "CV", "GN", "TM", "BI", "TJ", "VU", "SB", "ER", "WS", "AS", "FK", "GQ", "TO", "KM", "PW", "FM", "CF", "SO", "MH", "VA", "TD", "KI", "ST", "TV", "NR", "RE", "AN", "AQ", "BQ", "BV", "IO", "CX", "CC", "CK", "CW", "TF", "GW", "HM", "XK", "MS", "NU", "NF", "PN", "BL", "SH", "MF", "PM", "SX", "GS", "SS", "SJ", "TL", "TK", "UM", "WF", "EH"], "min_campaign_duration": {"US": 1, "CA": 1, "GB": 1, "AR": 1, "AU": 1, "AT": 1, "BE": 1, "BR": 1, "CL": 1, "CN": 1, "CO": 1, "HR": 1, "DK": 1, "DO": 1, "EG": 1, "FI": 1, "FR": 1, "DE": 1, "GR": 1, "HK": 1, "IN": 1, "ID": 1, "IE": 1, "IL": 1, "IT": 1, "JP": 1, "JO": 1, "KW": 1, "LB": 1, "MY": 1, "MX": 1, "NL": 1, "NZ": 1, "NG": 1, "NO": 1, "PK": 1, "PA": 1, "PE": 1, "PH": 1, "PL": 1, "RU": 1, "SA": 1, "RS": 1, "SG": 1, "ZA": 1, "KR": 1, "ES": 1, "SE": 1, "CH": 1, "TW": 1, "TH": 1, "TR": 1, "AE": 1, "VE": 1, "PT": 1, "LU": 1, "BG": 1, "CZ": 1, "SI": 1, "IS": 1, "SK": 1, "LT": 1, "TT": 1, "BD": 1, "LK": 1, "KE": 1, "HU": 1, "MA": 1, "CY": 1, "JM": 1, "EC": 1, "RO": 1, "BO": 1, "GT": 1, "CR": 1, "QA": 1, "SV": 1, "HN": 1, "NI": 1, "PY": 1, "UY": 1, "PR": 1, "BA": 1, "PS": 1, "TN": 1, "BH": 1, "VN": 1, "GH": 1, "MU": 1, "UA": 1, "MT": 1, "BS": 1, "MV": 1, "OM": 1, "MK": 1, "LV": 1, "EE": 1, "IQ": 1, "DZ": 1, "AL": 1, "NP": 1, "MO": 1, "ME": 1, "SN": 1, "GE": 1, "BN": 1, "UG": 1, "GP": 1, "BB": 1, "AZ": 1, "TZ": 1, "LY": 1, "MQ": 1, "CM": 1, "BW": 1, "ET": 1, "KZ": 1, "NA": 1, "MG": 1, "NC": 1, "MD": 1, "FJ": 1, "BY": 1, "JE": 1, "GU": 1, "YE": 1, "ZM": 1, "IM": 1, "HT": 1, "KH": 1, "AW": 1, "PF": 1, "AF": 1, "BM": 1, "GY": 1, "AM": 1, "MW": 1, "AG": 1, "RW": 1, "GG": 1, "GM": 1, "FO": 1, "LC": 1, "KY": 1, "BJ": 1, "AD": 1, "GD": 1, "VI": 1, "BZ": 1, "VC": 1, "MN": 1, "MZ": 1, "ML": 1, "AO": 1, "GF": 1, "UZ": 1, "DJ": 1, "BF": 1, "MC": 1, "TG": 1, "GL": 1, "GA": 1, "GI": 1, "CD": 1, "KG": 1, "PG": 1, "BT": 1, "KN": 1, "SZ": 1, "LS": 1, "LA": 1, "LI": 1, "MP": 1, "SR": 1, "SC": 1, "VG": 1, "TC": 1, "DM": 1, "MR": 1, "AX": 1, "SM": 1, "SL": 1, "NE": 1, "CG": 1, "AI": 1, "YT": 1, "CV": 1, "GN": 1, "TM": 1, "BI": 1, "TJ": 1, "VU": 1, "SB": 1, "ER": 1, "WS": 1, "AS": 1, "FK": 1, "GQ": 1, "TO": 1, "KM": 1, "PW": 1, "FM": 1, "CF": 1, "SO": 1, "MH": 1, "VA": 1, "TD": 1, "KI": 1, "ST": 1, "TV": 1, "NR": 1, "RE": 1, "LR": 1, "ZW": 1, "CI": 1, "MM": 1, "AN": 1, "AQ": 1, "BQ": 1, "BV": 1, "IO": 1, "CX": 1, "CC": 1, "CK": 1, "CW": 1, "TF": 1, "GW": 1, "HM": 1, "XK": 1, "MS": 1, "NU": 1, "NF": 1, "PN": 1, "BL": 1, "SH": 1, "MF": 1, "PM": 1, "SX": 1, "GS": 1, "SS": 1, "SJ": 1, "TL": 1, "TK": 1, "UM": 1, "WF": 1, "EH": 1}, "max_campaign_duration": {"US": 90, "CA": 90, "GB": 90, "AR": 90, "AU": 90, "AT": 90, "BE": 90, "BR": 90, "CL": 90, "CN": 90, "CO": 90, "HR": 90, "DK": 90, "DO": 90, "EG": 90, "FI": 90, "FR": 90, "DE": 90, "GR": 90, "HK": 90, "IN": 90, "ID": 90, "IE": 90, "IL": 90, "IT": 90, "JP": 90, "JO": 90, "KW": 90, "LB": 90, "MY": 90, "MX": 90, "NL": 90, "NZ": 90, "NG": 90, "NO": 90, "PK": 90, "PA": 90, "PE": 90, "PH": 90, "PL": 90, "RU": 90, "SA": 90, "RS": 90, "SG": 90, "ZA": 90, "KR": 90, "ES": 90, "SE": 90, "CH": 90, "TW": 90, "TH": 90, "TR": 90, "AE": 90, "VE": 90, "PT": 90, "LU": 90, "BG": 90, "CZ": 90, "SI": 90, "IS": 90, "SK": 90, "LT": 90, "TT": 90, "BD": 90, "LK": 90, "KE": 90, "HU": 90, "MA": 90, "CY": 90, "JM": 90, "EC": 90, "RO": 90, "BO": 90, "GT": 90, "CR": 90, "QA": 90, "SV": 90, "HN": 90, "NI": 90, "PY": 90, "UY": 90, "PR": 90, "BA": 90, "PS": 90, "TN": 90, "BH": 90, "VN": 90, "GH": 90, "MU": 90, "UA": 90, "MT": 90, "BS": 90, "MV": 90, "OM": 90, "MK": 90, "LV": 90, "EE": 90, "IQ": 90, "DZ": 90, "AL": 90, "NP": 90, "MO": 90, "ME": 90, "SN": 90, "GE": 90, "BN": 90, "UG": 90, "GP": 90, "BB": 90, "AZ": 90, "TZ": 90, "LY": 90, "MQ": 90, "CM": 90, "BW": 90, "ET": 90, "KZ": 90, "NA": 90, "MG": 90, "NC": 90, "MD": 90, "FJ": 90, "BY": 90, "JE": 90, "GU": 90, "YE": 90, "ZM": 90, "IM": 90, "HT": 90, "KH": 90, "AW": 90, "PF": 90, "AF": 90, "BM": 90, "GY": 90, "AM": 90, "MW": 90, "AG": 90, "RW": 90, "GG": 90, "GM": 90, "FO": 90, "LC": 90, "KY": 90, "BJ": 90, "AD": 90, "GD": 90, "VI": 90, "BZ": 90, "VC": 90, "MN": 90, "MZ": 90, "ML": 90, "AO": 90, "GF": 90, "UZ": 90, "DJ": 90, "BF": 90, "MC": 90, "TG": 90, "GL": 90, "GA": 90, "GI": 90, "CD": 90, "KG": 90, "PG": 90, "BT": 90, "KN": 90, "SZ": 90, "LS": 90, "LA": 90, "LI": 90, "MP": 90, "SR": 90, "SC": 90, "VG": 90, "TC": 90, "DM": 90, "MR": 90, "AX": 90, "SM": 90, "SL": 90, "NE": 90, "CG": 90, "AI": 90, "YT": 90, "CV": 90, "GN": 90, "TM": 90, "BI": 90, "TJ": 90, "VU": 90, "SB": 90, "ER": 90, "WS": 90, "AS": 90, "FK": 90, "GQ": 90, "TO": 90, "KM": 90, "PW": 90, "FM": 90, "CF": 90, "SO": 90, "MH": 90, "VA": 90, "TD": 90, "KI": 90, "ST": 90, "TV": 90, "NR": 90, "RE": 90, "LR": 90, "ZW": 90, "CI": 90, "MM": 90, "AN": 90, "AQ": 90, "BQ": 90, "BV": 90, "IO": 90, "CX": 90, "CC": 90, "CK": 90, "CW": 90, "TF": 90, "GW": 90, "HM": 90, "XK": 90, "MS": 90, "NU": 90, "NF": 90, "PN": 90, "BL": 90, "SH": 90, "MF": 90, "PM": 90, "SX": 90, "GS": 90, "SS": 90, "SJ": 90, "TL": 90, "TK": 90, "UM": 90, "WF": 90, "EH": 90}, "max_days_to_finish": {"US": 180, "CA": 180, "GB": 180, "AR": 180, "AU": 180, "AT": 180, "BE": 180, "BR": 180, "CL": 180, "CN": 180, "CO": 180, "HR": 180, "DK": 180, "DO": 180, "EG": 180, "FI": 180, "FR": 180, "DE": 180, "GR": 180, "HK": 180, "IN": 180, "ID": 180, "IE": 180, "IL": 180, "IT": 180, "JP": 180, "JO": 180, "KW": 180, "LB": 180, "MY": 180, "MX": 180, "NL": 180, "NZ": 180, "NG": 180, "NO": 180, "PK": 180, "PA": 180, "PE": 180, "PH": 180, "PL": 180, "RU": 180, "SA": 180, "RS": 180, "SG": 180, "ZA": 180, "KR": 180, "ES": 180, "SE": 180, "CH": 180, "TW": 180, "TH": 180, "TR": 180, "AE": 180, "VE": 180, "PT": 180, "LU": 180, "BG": 180, "CZ": 180, "SI": 180, "IS": 180, "SK": 180, "LT": 180, "TT": 180, "BD": 180, "LK": 180, "KE": 180, "HU": 180, "MA": 180, "CY": 180, "JM": 180, "EC": 180, "RO": 180, "BO": 180, "GT": 180, "CR": 180, "QA": 180, "SV": 180, "HN": 180, "NI": 180, "PY": 180, "UY": 180, "PR": 180, "BA": 180, "PS": 180, "TN": 180, "BH": 180, "VN": 180, "GH": 180, "MU": 180, "UA": 180, "MT": 180, "BS": 180, "MV": 180, "OM": 180, "MK": 180, "LV": 180, "EE": 180, "IQ": 180, "DZ": 180, "AL": 180, "NP": 180, "MO": 180, "ME": 180, "SN": 180, "GE": 180, "BN": 180, "UG": 180, "GP": 180, "BB": 180, "AZ": 180, "TZ": 180, "LY": 180, "MQ": 180, "CM": 180, "BW": 180, "ET": 180, "KZ": 180, "NA": 180, "MG": 180, "NC": 180, "MD": 180, "FJ": 180, "BY": 180, "JE": 180, "GU": 180, "YE": 180, "ZM": 180, "IM": 180, "HT": 180, "KH": 180, "AW": 180, "PF": 180, "AF": 180, "BM": 180, "GY": 180, "AM": 180, "MW": 180, "AG": 180, "RW": 180, "GG": 180, "GM": 180, "FO": 180, "LC": 180, "KY": 180, "BJ": 180, "AD": 180, "GD": 180, "VI": 180, "BZ": 180, "VC": 180, "MN": 180, "MZ": 180, "ML": 180, "AO": 180, "GF": 180, "UZ": 180, "DJ": 180, "BF": 180, "MC": 180, "TG": 180, "GL": 180, "GA": 180, "GI": 180, "CD": 180, "KG": 180, "PG": 180, "BT": 180, "KN": 180, "SZ": 180, "LS": 180, "LA": 180, "LI": 180, "MP": 180, "SR": 180, "SC": 180, "VG": 180, "TC": 180, "DM": 180, "MR": 180, "AX": 180, "SM": 180, "SL": 180, "NE": 180, "CG": 180, "AI": 180, "YT": 180, "CV": 180, "GN": 180, "TM": 180, "BI": 180, "TJ": 180, "VU": 180, "SB": 180, "ER": 180, "WS": 180, "AS": 180, "FK": 180, "GQ": 180, "TO": 180, "KM": 180, "PW": 180, "FM": 180, "CF": 180, "SO": 180, "MH": 180, "VA": 180, "TD": 180, "KI": 180, "ST": 180, "TV": 180, "NR": 180, "RE": 180, "LR": 180, "ZW": 180, "CI": 180, "MM": 180, "AN": 180, "AQ": 180, "BQ": 180, "BV": 180, "IO": 180, "CX": 180, "CC": 180, "CK": 180, "CW": 180, "TF": 180, "GW": 180, "HM": 180, "XK": 180, "MS": 180, "NU": 180, "NF": 180, "PN": 180, "BL": 180, "SH": 180, "MF": 180, "PM": 180, "SX": 180, "GS": 180, "SS": 180, "SJ": 180, "TL": 180, "TK": 180, "UM": 180, "WF": 180, "EH": 180}, "global_io_max_campaign_duration": 100}, "spend_cap": "0", "tax_id_status": 0.0, "tax_id_type": "0", "timezone_id": 1.0, "timezone_name": "America/Los_Angeles", "timezone_offset_hours_utc": -7.0, "tos_accepted": {"web_custom_audience_tos": 1}, "user_tasks": ["DRAFT", "ANALYZE", "ADVERTISE", "MANAGE"]}, "emitted_at": 1698916157141} +{"stream": "ad_account", "data": {"id": "act_212551616838260", "account_id": "212551616838260", "account_status": 1, "age": 1305.7507638889, "amount_spent": "39125", "balance": "0", "business": {"id": "1506473679510495", "name": "Airbyte"}, "business_city": "", "business_country_code": "US", "business_name": "", "business_street": "", "business_street2": "", "can_create_brand_lift_study": false, "capabilities": ["CAN_CREATE_CALL_ADS", "CAN_SEE_GROWTH_OPPORTUNITY_DATA", "ENABLE_IA_RECIRC_AD_DISPLAY_FORMAT", "CAN_USE_MOBILE_EXTERNAL_PAGE_TYPE", "CAN_USE_FB_FEED_POSITION_IN_VIDEO_VIEW_15S", "ENABLE_BIZ_DISCO_ADS", "ENABLE_BRAND_OBJECTIVES_FOR_BIZ_DISCO_ADS", "ENABLE_DIRECT_REACH_FOR_BIZ_DISCO_ADS", "ENABLE_DYNAMIC_ADS_ON_IG_STORIES_ADS", "ENABLE_IG_STORIES_ADS_PPE_OBJECTIVE", "ENABLE_IG_STORIES_ADS_MESSENGER_DESTINATION", "ENABLE_PAC_FOR_BIZ_DISCO_ADS", "CAN_USE_FB_INSTREAM_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_FB_STORY_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_AN_INSTREAM_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_IG_STORY_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_FB_IA_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_FB_SUG_VIDEO_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_FB_MKT_PLACE_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_IG_FEED_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_IG_EXPLORE_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_AN_CLASSIC_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_AN_REWARD_VIDEO_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_REACH_AND_FREQUENCY", "CAN_USE_RECURRING_BUDGET", "HAS_VALID_PAYMENT_METHODS", "CAN_USE_LINK_CLICK_BILLING_EVENT", "CAN_USE_CPA_BILLING_EVENT", "CAN_SEE_NEW_CONVERSION_WINDOW_NUX", "ADS_INSTREAM_INTERFACE_INTEGRITY", "ADS_INSTREAM_LINK_CLICK", "ADS_INSTREAM_LINK_CLICK_IMAGE", "ADS_IN_OBJECTIVES_DEPRECATION", "MESSENGER_INBOX_ADS_PRODUCT_CATALOG_SALES", "CAN_SHOW_MESSENGER_DUPLICSTION_UPSELL", "ALLOW_INSTREAM_ONLY_FOR_REACH", "ADS_INSTREAM_VIDEO_PLACEMENT_CONVERSIONS", "CAN_CREATE_INSTAGRAM_EXPLORE_ADS", "ALLOW_INSTREAM_VIDEOS_PLACEMENT_ONLY", "ALLOW_INSTREAM_NON_INTERRUPTIVE_LEADGEN", "INSTREAM_VIDEO_AD_DESKTOP_CONVERSION_AD_PREVIEW", "ALLOW_INSTREAM_ONLY_FOR_BRAND_AWARENESS_AUCTION", "ALLOW_SUGGESTED_VIDEOS_PLACEMENT_ONLY", "WHATSAPP_DESTINATION_ADS", "CTM_ADS_CREATION_CLICK_TO_DIRECT", "CTW_ADS_ENABLE_IG_FEED_PLACEMENT", "CTW_ADS_FOR_NON_MESSAGES_OBJECTIVE", "CTW_ADS_TRUSTED_TIER_2_PLUS_ADVERTISER", "CTW_ADS_TRUSTED_TIER_ADVERTISER", "ADS_PLACEMENT_MARKETPLACE", "ADNW_DISABLE_INSTREAM_AND_WEB_PLACEMENT", "CAN_CHANGE_BILLING_THRESHOLD", "CAN_USE_APP_EVENT_AVERAGE_COST_BIDDING", "CAN_USE_LEAD_GEN_AVERAGE_COST_BIDDING", "ADS_VALUE_OPTIMIZATION_DYNAMIC_ADS_1D", "ADS_DELIVERY_INSIGHTS_IN_BIDDING_PRESET_EXPERIMENT", "ADS_DELIVERY_INSIGHTS_OPTIMIZATION_PRESET", "CAN_SEE_APP_AD_EVENTS", "CAN_SEE_NEW_STANDARD_EVENTS_BETA", "CAN_SEE_VCK_HOLIDAY_TEMPLATES", "ENABLE_DCO_FOR_FB_STORY_ADS", "CAN_USE_IG_EXPLORE_GRID_HOME_PLACEMENT", "CAN_USE_IG_EXPLORE_HOME_IN_REACH_AND_FREQUENCY", "CAN_USE_IG_EXPLORE_HOME_POST_ENGAGEMENT_MESSAGES", "CAN_USE_IG_SEARCH_PLACEMENT", "CAN_USE_IG_SEARCH_GRID_ADS", "CAN_USE_IG_SEARCH_RESULTS_AUTO_PLACEMENT", "CAN_USE_IG_REELS_PAC_CAROUSEL", "CAN_USE_IG_REELS_POSITION", "CAN_SEE_CONVERSION_LIFT_SUMMARY", "CAN_USE_IG_PROFILE_FEED_POSITION", "CAN_USE_IG_PROFILE_FEED_AUTO_PLACEMENT", "CAN_USE_IG_PROFILE_FEED_ADDITIONAL_OBJECTIVES", "CAN_USE_IG_REELS_REACH_AND_FREQUENCY", "CAN_USE_IG_REELS_OVERLAY_POSITION", "CAN_USE_IG_REELS_OVERLAY_AUTO_PLACEMENT", "CAN_USE_IG_REELS_OVERLAY_PAC", "CAN_USE_IG_SHOP_TAB_PAC", "CAN_SEE_LEARNING_STAGE", "ENABLE_WEBSITE_CONVERSIONS_FOR_FB_STORY_ADS", "ENABLE_MESSENGER_INBOX_VIDEO_ADS", "ENABLE_VIDEO_VIEWS_FOR_FB_STORY_ADS", "ENABLE_LINK_CLICKS_FOR_FB_STORY_ADS", "ENABLE_REACH_FOR_FB_STORY_ADS", "CAN_USE_CALL_TO_ACTION_LINK_IMPORT_EXPORT", "ADS_INSTREAM_VIDEO_ENABLE_SLIDE_SHOW", "ALLOW_INSTREAM_VIDEOS_PLACEMENT_ONLY_IN_VV_REACH_AND_FREQUENCY", "ENABLE_MOBILE_APP_INSTALLS_FOR_FB_STORY_ADS", "ENABLE_LEAD_GEN_FOR_FB_STORY_ADS", "CAN_USE_FB_MKT_PLACE_POSITION_IN_REACH", "CAN_USE_FB_MKT_PLACE_POSITION_IN_VIDEO_VIEW", "CAN_USE_FB_MKT_PLACE_POSITION_IN_STORE_VISIT", "ENABLE_MOBILE_APP_ENGAGEMENT_FOR_FB_STORY_ADS", "CAN_USE_FB_MKT_PLACE_POSITION_IN_BRAND_AWARENESS", "CAN_USE_FB_MKT_PLACE_POSITION_IN_APP_INSTALLS", "CAN_USE_FB_MKT_PLACE_POSITION_IN_LEAD_GENERATION", "CAN_USE_FB_MKT_PLACE_POSITION_IN_MESSAGE", "CAN_USE_FB_MKT_PLACE_POSITION_IN_PAGE_LIKE", "CAN_USE_FB_MKT_PLACE_POSITION_IN_POST_ENGAGEMENT", "RF_ALLOW_MARKETPLACE_ACCOUNT", "RF_ALLOW_SEARCH_ACCOUNT", "VERTICAL_VIDEO_PAC_INSTREAM_UPSELL", "IX_COLLECTION_ENABLED_FOR_BAO_AND_REACH", "ADS_BM_REQUIREMENTS_OCT_15_RELEASE", "ENABLE_POST_ENGAGEMENT_FOR_FB_STORY", "ENBABLE_CATALOG_SALES_FOR_FB_STORY", "CAN_USE_WHATSAPP_DESTINATION_ON_LINK_CLICKS_AND_CONVERSIONS", "CAN_USE_WHATSAPP_DESTINATION_ON_CONVERSIONS", "IS_NON_TAIL_AD_ACCOUNT", "IS_IN_DSA_GK", "IS_IN_IG_EXISTING_POST_CTA_DEFAULTING_EXPERIMENT", "IS_IN_SHORT_WA_LINK_CTWA_UNCONV_TRAFFIC_EXPERIMENT", "IS_IN_ODAX_EXPERIENCE", "IS_IN_REACH_BRAND_AWARENESS_WHATSAPP_L1_DESTINATION_EXPERIMENT", "IS_IN_VIDEO_VIEWS_WHATSAPP_L1_DESTINATION_EXPERIMENT", "IS_IN_WHATSAPP_DESTINATION_DEFAULTING_EXPERIMENT", "CAN_USE_MARKETPLACE_DESKTOP", "ADS_MERCHANT_OVERLAYS_DEPRECATION", "CONNECTIONS_DEPRECATION_V2", "CAN_USE_LIVE_VIDEO_FOR_THRUPLAY", "CAN_SEE_HEC_AM_FLOW", "CAN_SEE_POLITICAL_FLOW", "ADS_INSTREAM_PLACEMENT_CATALOG_SALES", "ENABLE_CONVERSIONS_FOR_FB_GROUP_TAB_ADS", "ENABLE_LINK_CLICK_FOR_FB_GROUP_TAB_ADS", "ENABLE_REACH_FOR_FB_GROUP_TAB_ADS", "CAN_USE_CONVERSATIONS_OPTIMIZATION", "ENABLE_THRUPLAY_OPTIMIZATION_MESSENGER_STORY_ADS", "CAN_USE_IG_STORY_POLLS_PAC_CREATION", "IOS14_CEO_CAMPAIGN_CREATION", "ENABLE_VIDEO_CHANNEL_PLACEMENT_FOR_RSVP_ADS", "DIGITAL_CIRCULAR_ADS", "CAN_SEE_SAFR_V3_FLOW", "CAN_USE_FB_REELS_POSITION", "CAN_USE_ADS_ON_FB_REELS_POSITION", "CAN_USE_FB_REELS_AUTO_PLACEMENT", "ENABLE_FB_REELS_CREATION_PAC_ADS", "ENABLE_FB_REELS_CREATION_DCO_ADS", "ENABLE_FB_REELS_POSTLOOP_CREATION_DCO_ADS", "ENABLE_FB_REELS_POSTLOOP_CREATION_PAC_ADS", "RF_CPA_BILLING_DEPRECATION_PHASE_2", "ENABLE_APP_INSTALL_CUSTOM_PRODUCT_PAGES", "ENABLE_ADS_ON_FB_REELS_PLACEMENT_UNIFICATION", "ENABLE_ADS_ON_IG_SHOP_TAB_DEPRECATION_L2_NUX", "ADS_RF_FB_REELS_PLACEMENT", "ENABLE_ADS_ON_FB_INSTANT_ARTICLE_DEPRECATION_L2_NUX", "REELS_DM_ADS_ENABLE_REACH_AND_FREQUENCY", "ADS_AEMV2_HAS_LAUNCHED", "ELIGIBLE_FOR_TEXT_GEN"], "created_time": "2020-04-13T18:04:59-0700", "currency": "USD", "disable_reason": 0.0, "end_advertiser": 1506473679510495.0, "end_advertiser_name": "Airbyte", "fb_entity": 85.0, "funding_source": 2825262454257003.0, "funding_source_details": {"id": "2825262454257003", "type": 1}, "has_migrated_permissions": true, "is_attribution_spec_system_default": true, "is_direct_deals_enabled": false, "is_in_3ds_authorization_enabled_market": false, "is_notifications_enabled": true, "is_personal": 0.0, "is_prepay_account": false, "is_tax_id_required": false, "min_campaign_group_spend_cap": 10000.0, "min_daily_budget": 100.0, "name": "Airbyte", "offsite_pixels_tos_accepted": true, "owner": 1506473679510495.0, "rf_spec": {"min_reach_limits": {"US": 200000, "CA": 200000, "GB": 200000, "AR": 200000, "AU": 200000, "AT": 200000, "BE": 200000, "BR": 200000, "CL": 200000, "CN": 200000, "CO": 200000, "HR": 200000, "DK": 200000, "DO": 200000, "EG": 200000, "FI": 200000, "FR": 200000, "DE": 200000, "GR": 200000, "HK": 200000, "IN": 200000, "ID": 200000, "IE": 200000, "IL": 200000, "IT": 200000, "JP": 200000, "JO": 200000, "KW": 200000, "LB": 200000, "MY": 200000, "MX": 200000, "NL": 200000, "NZ": 200000, "NG": 200000, "NO": 200000, "PK": 200000, "PA": 200000, "PE": 200000, "PH": 200000, "PL": 200000, "RU": 200000, "SA": 200000, "RS": 200000, "SG": 200000, "ZA": 200000, "KR": 200000, "ES": 200000, "SE": 200000, "CH": 200000, "TW": 200000, "TH": 200000, "TR": 200000, "AE": 200000, "VE": 200000, "PT": 200000, "LU": 200000, "BG": 200000, "CZ": 200000, "SI": 200000, "IS": 200000, "SK": 200000, "LT": 200000, "TT": 200000, "BD": 200000, "LK": 200000, "KE": 200000, "HU": 200000, "MA": 200000, "CY": 200000, "JM": 200000, "EC": 200000, "RO": 200000, "BO": 200000, "GT": 200000, "CR": 200000, "QA": 200000, "SV": 200000, "HN": 200000, "NI": 200000, "PY": 200000, "UY": 200000, "PR": 200000, "BA": 200000, "PS": 200000, "TN": 200000, "BH": 200000, "VN": 200000, "GH": 200000, "MU": 200000, "UA": 200000, "MT": 200000, "BS": 200000, "MV": 200000, "OM": 200000, "MK": 200000, "LV": 200000, "EE": 200000, "IQ": 200000, "DZ": 200000, "AL": 200000, "NP": 200000, "MO": 200000, "ME": 200000, "SN": 200000, "GE": 200000, "BN": 200000, "UG": 200000, "GP": 200000, "BB": 200000, "AZ": 200000, "TZ": 200000, "LY": 200000, "MQ": 200000, "CM": 200000, "BW": 200000, "ET": 200000, "KZ": 200000, "NA": 200000, "MG": 200000, "NC": 200000, "MD": 200000, "FJ": 200000, "BY": 200000, "JE": 200000, "GU": 200000, "YE": 200000, "ZM": 200000, "IM": 200000, "HT": 200000, "KH": 200000, "AW": 200000, "PF": 200000, "AF": 200000, "BM": 200000, "GY": 200000, "AM": 200000, "MW": 200000, "AG": 200000, "RW": 200000, "GG": 200000, "GM": 200000, "FO": 200000, "LC": 200000, "KY": 200000, "BJ": 200000, "AD": 200000, "GD": 200000, "VI": 200000, "BZ": 200000, "VC": 200000, "MN": 200000, "MZ": 200000, "ML": 200000, "AO": 200000, "GF": 200000, "UZ": 200000, "DJ": 200000, "BF": 200000, "MC": 200000, "TG": 200000, "GL": 200000, "GA": 200000, "GI": 200000, "CD": 200000, "KG": 200000, "PG": 200000, "BT": 200000, "KN": 200000, "SZ": 200000, "LS": 200000, "LA": 200000, "LI": 200000, "MP": 200000, "SR": 200000, "SC": 200000, "VG": 200000, "TC": 200000, "DM": 200000, "MR": 200000, "AX": 200000, "SM": 200000, "SL": 200000, "NE": 200000, "CG": 200000, "AI": 200000, "YT": 200000, "CV": 200000, "GN": 200000, "TM": 200000, "BI": 200000, "TJ": 200000, "VU": 200000, "SB": 200000, "ER": 200000, "WS": 200000, "AS": 200000, "FK": 200000, "GQ": 200000, "TO": 200000, "KM": 200000, "PW": 200000, "FM": 200000, "CF": 200000, "SO": 200000, "MH": 200000, "VA": 200000, "TD": 200000, "KI": 200000, "ST": 200000, "TV": 200000, "NR": 200000, "RE": 200000, "LR": 200000, "ZW": 200000, "CI": 200000, "MM": 200000, "AN": 200000, "AQ": 200000, "BQ": 200000, "BV": 200000, "IO": 200000, "CX": 200000, "CC": 200000, "CK": 200000, "CW": 200000, "TF": 200000, "GW": 200000, "HM": 200000, "XK": 200000, "MS": 200000, "NU": 200000, "NF": 200000, "PN": 200000, "BL": 200000, "SH": 200000, "MF": 200000, "PM": 200000, "SX": 200000, "GS": 200000, "SS": 200000, "SJ": 200000, "TL": 200000, "TK": 200000, "UM": 200000, "WF": 200000, "EH": 200000}, "countries": ["US", "CA", "GB", "AR", "AU", "AT", "BE", "BR", "CL", "CN", "CO", "HR", "DK", "DO", "EG", "FI", "FR", "DE", "GR", "HK", "IN", "ID", "IE", "IL", "IT", "JP", "JO", "KW", "LB", "MY", "MX", "NL", "NZ", "NG", "NO", "PK", "PA", "PE", "PH", "PL", "RU", "SA", "RS", "SG", "ZA", "KR", "ES", "SE", "CH", "TW", "TH", "TR", "AE", "VE", "PT", "LU", "BG", "CZ", "SI", "IS", "SK", "LT", "TT", "BD", "LK", "KE", "HU", "MA", "CY", "JM", "EC", "RO", "BO", "GT", "CR", "QA", "SV", "HN", "NI", "PY", "UY", "PR", "BA", "PS", "TN", "BH", "VN", "GH", "MU", "UA", "MT", "BS", "MV", "OM", "MK", "EE", "LV", "IQ", "DZ", "AL", "NP", "MO", "ME", "SN", "GE", "BN", "UG", "GP", "BB", "ZW", "CI", "AZ", "TZ", "LY", "MQ", "MM", "CM", "BW", "ET", "KZ", "NA", "MG", "NC", "MD", "FJ", "BY", "JE", "GU", "YE", "ZM", "IM", "HT", "KH", "AW", "PF", "AF", "BM", "GY", "AM", "MW", "AG", "RW", "GG", "GM", "FO", "LC", "KY", "BJ", "AD", "GD", "VI", "BZ", "VC", "MN", "MZ", "ML", "AO", "GF", "UZ", "DJ", "BF", "MC", "TG", "GL", "GA", "GI", "CD", "KG", "PG", "BT", "KN", "SZ", "LS", "LA", "LI", "MP", "SR", "SC", "VG", "TC", "DM", "MR", "AX", "SM", "SL", "NE", "CG", "AI", "YT", "LR", "CV", "GN", "TM", "BI", "TJ", "VU", "SB", "ER", "WS", "AS", "FK", "GQ", "TO", "KM", "PW", "FM", "CF", "SO", "MH", "VA", "TD", "KI", "ST", "TV", "NR", "RE", "AN", "AQ", "BQ", "BV", "IO", "CX", "CC", "CK", "CW", "TF", "GW", "HM", "XK", "MS", "NU", "NF", "PN", "BL", "SH", "MF", "PM", "SX", "GS", "SS", "SJ", "TL", "TK", "UM", "WF", "EH"], "min_campaign_duration": {"US": 1, "CA": 1, "GB": 1, "AR": 1, "AU": 1, "AT": 1, "BE": 1, "BR": 1, "CL": 1, "CN": 1, "CO": 1, "HR": 1, "DK": 1, "DO": 1, "EG": 1, "FI": 1, "FR": 1, "DE": 1, "GR": 1, "HK": 1, "IN": 1, "ID": 1, "IE": 1, "IL": 1, "IT": 1, "JP": 1, "JO": 1, "KW": 1, "LB": 1, "MY": 1, "MX": 1, "NL": 1, "NZ": 1, "NG": 1, "NO": 1, "PK": 1, "PA": 1, "PE": 1, "PH": 1, "PL": 1, "RU": 1, "SA": 1, "RS": 1, "SG": 1, "ZA": 1, "KR": 1, "ES": 1, "SE": 1, "CH": 1, "TW": 1, "TH": 1, "TR": 1, "AE": 1, "VE": 1, "PT": 1, "LU": 1, "BG": 1, "CZ": 1, "SI": 1, "IS": 1, "SK": 1, "LT": 1, "TT": 1, "BD": 1, "LK": 1, "KE": 1, "HU": 1, "MA": 1, "CY": 1, "JM": 1, "EC": 1, "RO": 1, "BO": 1, "GT": 1, "CR": 1, "QA": 1, "SV": 1, "HN": 1, "NI": 1, "PY": 1, "UY": 1, "PR": 1, "BA": 1, "PS": 1, "TN": 1, "BH": 1, "VN": 1, "GH": 1, "MU": 1, "UA": 1, "MT": 1, "BS": 1, "MV": 1, "OM": 1, "MK": 1, "LV": 1, "EE": 1, "IQ": 1, "DZ": 1, "AL": 1, "NP": 1, "MO": 1, "ME": 1, "SN": 1, "GE": 1, "BN": 1, "UG": 1, "GP": 1, "BB": 1, "AZ": 1, "TZ": 1, "LY": 1, "MQ": 1, "CM": 1, "BW": 1, "ET": 1, "KZ": 1, "NA": 1, "MG": 1, "NC": 1, "MD": 1, "FJ": 1, "BY": 1, "JE": 1, "GU": 1, "YE": 1, "ZM": 1, "IM": 1, "HT": 1, "KH": 1, "AW": 1, "PF": 1, "AF": 1, "BM": 1, "GY": 1, "AM": 1, "MW": 1, "AG": 1, "RW": 1, "GG": 1, "GM": 1, "FO": 1, "LC": 1, "KY": 1, "BJ": 1, "AD": 1, "GD": 1, "VI": 1, "BZ": 1, "VC": 1, "MN": 1, "MZ": 1, "ML": 1, "AO": 1, "GF": 1, "UZ": 1, "DJ": 1, "BF": 1, "MC": 1, "TG": 1, "GL": 1, "GA": 1, "GI": 1, "CD": 1, "KG": 1, "PG": 1, "BT": 1, "KN": 1, "SZ": 1, "LS": 1, "LA": 1, "LI": 1, "MP": 1, "SR": 1, "SC": 1, "VG": 1, "TC": 1, "DM": 1, "MR": 1, "AX": 1, "SM": 1, "SL": 1, "NE": 1, "CG": 1, "AI": 1, "YT": 1, "CV": 1, "GN": 1, "TM": 1, "BI": 1, "TJ": 1, "VU": 1, "SB": 1, "ER": 1, "WS": 1, "AS": 1, "FK": 1, "GQ": 1, "TO": 1, "KM": 1, "PW": 1, "FM": 1, "CF": 1, "SO": 1, "MH": 1, "VA": 1, "TD": 1, "KI": 1, "ST": 1, "TV": 1, "NR": 1, "RE": 1, "LR": 1, "ZW": 1, "CI": 1, "MM": 1, "AN": 1, "AQ": 1, "BQ": 1, "BV": 1, "IO": 1, "CX": 1, "CC": 1, "CK": 1, "CW": 1, "TF": 1, "GW": 1, "HM": 1, "XK": 1, "MS": 1, "NU": 1, "NF": 1, "PN": 1, "BL": 1, "SH": 1, "MF": 1, "PM": 1, "SX": 1, "GS": 1, "SS": 1, "SJ": 1, "TL": 1, "TK": 1, "UM": 1, "WF": 1, "EH": 1}, "max_campaign_duration": {"US": 90, "CA": 90, "GB": 90, "AR": 90, "AU": 90, "AT": 90, "BE": 90, "BR": 90, "CL": 90, "CN": 90, "CO": 90, "HR": 90, "DK": 90, "DO": 90, "EG": 90, "FI": 90, "FR": 90, "DE": 90, "GR": 90, "HK": 90, "IN": 90, "ID": 90, "IE": 90, "IL": 90, "IT": 90, "JP": 90, "JO": 90, "KW": 90, "LB": 90, "MY": 90, "MX": 90, "NL": 90, "NZ": 90, "NG": 90, "NO": 90, "PK": 90, "PA": 90, "PE": 90, "PH": 90, "PL": 90, "RU": 90, "SA": 90, "RS": 90, "SG": 90, "ZA": 90, "KR": 90, "ES": 90, "SE": 90, "CH": 90, "TW": 90, "TH": 90, "TR": 90, "AE": 90, "VE": 90, "PT": 90, "LU": 90, "BG": 90, "CZ": 90, "SI": 90, "IS": 90, "SK": 90, "LT": 90, "TT": 90, "BD": 90, "LK": 90, "KE": 90, "HU": 90, "MA": 90, "CY": 90, "JM": 90, "EC": 90, "RO": 90, "BO": 90, "GT": 90, "CR": 90, "QA": 90, "SV": 90, "HN": 90, "NI": 90, "PY": 90, "UY": 90, "PR": 90, "BA": 90, "PS": 90, "TN": 90, "BH": 90, "VN": 90, "GH": 90, "MU": 90, "UA": 90, "MT": 90, "BS": 90, "MV": 90, "OM": 90, "MK": 90, "LV": 90, "EE": 90, "IQ": 90, "DZ": 90, "AL": 90, "NP": 90, "MO": 90, "ME": 90, "SN": 90, "GE": 90, "BN": 90, "UG": 90, "GP": 90, "BB": 90, "AZ": 90, "TZ": 90, "LY": 90, "MQ": 90, "CM": 90, "BW": 90, "ET": 90, "KZ": 90, "NA": 90, "MG": 90, "NC": 90, "MD": 90, "FJ": 90, "BY": 90, "JE": 90, "GU": 90, "YE": 90, "ZM": 90, "IM": 90, "HT": 90, "KH": 90, "AW": 90, "PF": 90, "AF": 90, "BM": 90, "GY": 90, "AM": 90, "MW": 90, "AG": 90, "RW": 90, "GG": 90, "GM": 90, "FO": 90, "LC": 90, "KY": 90, "BJ": 90, "AD": 90, "GD": 90, "VI": 90, "BZ": 90, "VC": 90, "MN": 90, "MZ": 90, "ML": 90, "AO": 90, "GF": 90, "UZ": 90, "DJ": 90, "BF": 90, "MC": 90, "TG": 90, "GL": 90, "GA": 90, "GI": 90, "CD": 90, "KG": 90, "PG": 90, "BT": 90, "KN": 90, "SZ": 90, "LS": 90, "LA": 90, "LI": 90, "MP": 90, "SR": 90, "SC": 90, "VG": 90, "TC": 90, "DM": 90, "MR": 90, "AX": 90, "SM": 90, "SL": 90, "NE": 90, "CG": 90, "AI": 90, "YT": 90, "CV": 90, "GN": 90, "TM": 90, "BI": 90, "TJ": 90, "VU": 90, "SB": 90, "ER": 90, "WS": 90, "AS": 90, "FK": 90, "GQ": 90, "TO": 90, "KM": 90, "PW": 90, "FM": 90, "CF": 90, "SO": 90, "MH": 90, "VA": 90, "TD": 90, "KI": 90, "ST": 90, "TV": 90, "NR": 90, "RE": 90, "LR": 90, "ZW": 90, "CI": 90, "MM": 90, "AN": 90, "AQ": 90, "BQ": 90, "BV": 90, "IO": 90, "CX": 90, "CC": 90, "CK": 90, "CW": 90, "TF": 90, "GW": 90, "HM": 90, "XK": 90, "MS": 90, "NU": 90, "NF": 90, "PN": 90, "BL": 90, "SH": 90, "MF": 90, "PM": 90, "SX": 90, "GS": 90, "SS": 90, "SJ": 90, "TL": 90, "TK": 90, "UM": 90, "WF": 90, "EH": 90}, "max_days_to_finish": {"US": 180, "CA": 180, "GB": 180, "AR": 180, "AU": 180, "AT": 180, "BE": 180, "BR": 180, "CL": 180, "CN": 180, "CO": 180, "HR": 180, "DK": 180, "DO": 180, "EG": 180, "FI": 180, "FR": 180, "DE": 180, "GR": 180, "HK": 180, "IN": 180, "ID": 180, "IE": 180, "IL": 180, "IT": 180, "JP": 180, "JO": 180, "KW": 180, "LB": 180, "MY": 180, "MX": 180, "NL": 180, "NZ": 180, "NG": 180, "NO": 180, "PK": 180, "PA": 180, "PE": 180, "PH": 180, "PL": 180, "RU": 180, "SA": 180, "RS": 180, "SG": 180, "ZA": 180, "KR": 180, "ES": 180, "SE": 180, "CH": 180, "TW": 180, "TH": 180, "TR": 180, "AE": 180, "VE": 180, "PT": 180, "LU": 180, "BG": 180, "CZ": 180, "SI": 180, "IS": 180, "SK": 180, "LT": 180, "TT": 180, "BD": 180, "LK": 180, "KE": 180, "HU": 180, "MA": 180, "CY": 180, "JM": 180, "EC": 180, "RO": 180, "BO": 180, "GT": 180, "CR": 180, "QA": 180, "SV": 180, "HN": 180, "NI": 180, "PY": 180, "UY": 180, "PR": 180, "BA": 180, "PS": 180, "TN": 180, "BH": 180, "VN": 180, "GH": 180, "MU": 180, "UA": 180, "MT": 180, "BS": 180, "MV": 180, "OM": 180, "MK": 180, "LV": 180, "EE": 180, "IQ": 180, "DZ": 180, "AL": 180, "NP": 180, "MO": 180, "ME": 180, "SN": 180, "GE": 180, "BN": 180, "UG": 180, "GP": 180, "BB": 180, "AZ": 180, "TZ": 180, "LY": 180, "MQ": 180, "CM": 180, "BW": 180, "ET": 180, "KZ": 180, "NA": 180, "MG": 180, "NC": 180, "MD": 180, "FJ": 180, "BY": 180, "JE": 180, "GU": 180, "YE": 180, "ZM": 180, "IM": 180, "HT": 180, "KH": 180, "AW": 180, "PF": 180, "AF": 180, "BM": 180, "GY": 180, "AM": 180, "MW": 180, "AG": 180, "RW": 180, "GG": 180, "GM": 180, "FO": 180, "LC": 180, "KY": 180, "BJ": 180, "AD": 180, "GD": 180, "VI": 180, "BZ": 180, "VC": 180, "MN": 180, "MZ": 180, "ML": 180, "AO": 180, "GF": 180, "UZ": 180, "DJ": 180, "BF": 180, "MC": 180, "TG": 180, "GL": 180, "GA": 180, "GI": 180, "CD": 180, "KG": 180, "PG": 180, "BT": 180, "KN": 180, "SZ": 180, "LS": 180, "LA": 180, "LI": 180, "MP": 180, "SR": 180, "SC": 180, "VG": 180, "TC": 180, "DM": 180, "MR": 180, "AX": 180, "SM": 180, "SL": 180, "NE": 180, "CG": 180, "AI": 180, "YT": 180, "CV": 180, "GN": 180, "TM": 180, "BI": 180, "TJ": 180, "VU": 180, "SB": 180, "ER": 180, "WS": 180, "AS": 180, "FK": 180, "GQ": 180, "TO": 180, "KM": 180, "PW": 180, "FM": 180, "CF": 180, "SO": 180, "MH": 180, "VA": 180, "TD": 180, "KI": 180, "ST": 180, "TV": 180, "NR": 180, "RE": 180, "LR": 180, "ZW": 180, "CI": 180, "MM": 180, "AN": 180, "AQ": 180, "BQ": 180, "BV": 180, "IO": 180, "CX": 180, "CC": 180, "CK": 180, "CW": 180, "TF": 180, "GW": 180, "HM": 180, "XK": 180, "MS": 180, "NU": 180, "NF": 180, "PN": 180, "BL": 180, "SH": 180, "MF": 180, "PM": 180, "SX": 180, "GS": 180, "SS": 180, "SJ": 180, "TL": 180, "TK": 180, "UM": 180, "WF": 180, "EH": 180}, "global_io_max_campaign_duration": 100}, "spend_cap": "0", "tax_id_status": 0.0, "tax_id_type": "0", "timezone_id": 1.0, "timezone_name": "America/Los_Angeles", "timezone_offset_hours_utc": -8.0, "tos_accepted": {"web_custom_audience_tos": 1}, "user_tasks": ["DRAFT", "ANALYZE", "ADVERTISE", "MANAGE"]}, "emitted_at": 1699644186066} {"stream":"ads","data":{"bid_type":"ABSOLUTE_OCPM","account_id":"212551616838260","campaign_id":"23853619670350398","adset_id":"23853619670380398","status":"ACTIVE","creative":{"id":"23853666125630398"},"id":"23853620198790398","updated_time":"2023-03-21T22:33:56-0700","created_time":"2023-03-17T08:04:29-0700","name":"Don't Compromise Between Cost/Relaibility","targeting":{"age_max":60,"age_min":18,"custom_audiences":[{"id":"23853630753300398","name":"Lookalike (US, 10%) - Airbyte Cloud Users"},{"id":"23853683587660398","name":"Web Traffic [ALL] - _copy"}],"geo_locations":{"countries":["US"],"location_types":["home","recent"]},"brand_safety_content_filter_levels":["FACEBOOK_STANDARD","AN_STANDARD"],"targeting_relaxation_types":{"lookalike":1,"custom_audience":1},"publisher_platforms":["facebook","instagram","audience_network","messenger"],"facebook_positions":["feed","biz_disco_feed","facebook_reels","facebook_reels_overlay","right_hand_column","video_feeds","instant_article","instream_video","marketplace","story","search"],"instagram_positions":["stream","story","explore","reels","shop","explore_home","profile_feed"],"device_platforms":["mobile","desktop"],"messenger_positions":["story"],"audience_network_positions":["classic","instream_video","rewarded_video"]},"effective_status":"ACTIVE","last_updated_by_app_id":"119211728144504","source_ad_id":"0","tracking_specs":[{"action.type":["offsite_conversion"],"fb_pixel":["917042523049733"]},{"action.type":["post_engagement"],"page":["112704783733939"],"post":["660122622785523","662226992575086"]},{"action.type":["link_click"],"post":["660122622785523","662226992575086"],"post.wall":["112704783733939"]}],"conversion_specs":[{"action.type":["offsite_conversion"],"conversion_id":["6015304265216283"]}]},"emitted_at":1682686047377} {"stream":"ad_sets","data":{"name":"Lookalike audience_Free Connector Program","promoted_object":{"pixel_id":"917042523049733","custom_event_type":"COMPLETE_REGISTRATION"},"id":"23853619670380398","account_id":"212551616838260","updated_time":"2023-03-21T14:20:51-0700","daily_budget":2000,"budget_remaining":2000,"effective_status":"ACTIVE","campaign_id":"23853619670350398","created_time":"2023-03-17T08:04:28-0700","start_time":"2023-03-17T08:04:28-0700","lifetime_budget":0,"targeting":{"age_max":60,"age_min":18,"custom_audiences":[{"id":"23853630753300398","name":"Lookalike (US, 10%) - Airbyte Cloud Users"},{"id":"23853683587660398","name":"Web Traffic [ALL] - _copy"}],"geo_locations":{"countries":["US"],"location_types":["home","recent"]},"brand_safety_content_filter_levels":["FACEBOOK_STANDARD","AN_STANDARD"],"targeting_relaxation_types":{"lookalike":1,"custom_audience":1},"publisher_platforms":["facebook","instagram","audience_network","messenger"],"facebook_positions":["feed","biz_disco_feed","facebook_reels","facebook_reels_overlay","right_hand_column","video_feeds","instant_article","instream_video","marketplace","story","search"],"instagram_positions":["stream","story","explore","reels","shop","explore_home","profile_feed"],"device_platforms":["mobile","desktop"],"messenger_positions":["story"],"audience_network_positions":["classic","instream_video","rewarded_video"]},"bid_strategy":"LOWEST_COST_WITHOUT_CAP"},"emitted_at":1692180821847} {"stream":"campaigns","data":{"id":"23846542053890398","account_id":"212551616838260","budget_rebalance_flag":false,"budget_remaining":0.0,"buying_type":"AUCTION","created_time":"2021-01-18T21:36:42-0800","configured_status":"PAUSED","effective_status":"PAUSED","name":"Fake Campaign 0","objective":"MESSAGES","smart_promotion_type":"GUIDED_CREATION","source_campaign_id":0.0,"special_ad_category":"NONE","start_time":"1969-12-31T15:59:59-0800","status":"PAUSED","updated_time":"2021-02-18T01:00:02-0800"},"emitted_at":1694795155769} From 4561792e9960954327807701e7e877e710395b74 Mon Sep 17 00:00:00 2001 From: "Roman Yermilov [GL]" <86300758+roman-yermilov-gl@users.noreply.github.com> Date: Sat, 11 Nov 2023 11:46:04 +0400 Subject: [PATCH 011/124] Source Github: fix expected records (#32443) --- .../source-github/integration_tests/expected_records.jsonl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl index 5b7d50d93c49..7ccbbee8037c 100644 --- a/airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl @@ -32,8 +32,8 @@ {"stream":"teams", "data": {"name": "Zazmic", "id": 4432406, "node_id": "MDQ6VGVhbTQ0MzI0MDY=", "slug": "zazmic", "description": "", "privacy": "closed", "notification_setting": "notifications_enabled", "url": "https://api.github.com/organizations/59758427/team/4432406", "html_url": "https://github.com/orgs/airbytehq/teams/zazmic", "members_url": "https://api.github.com/organizations/59758427/team/4432406/members{/member}", "repositories_url": "https://api.github.com/organizations/59758427/team/4432406/repos", "permission": "pull", "parent": null, "organization": "airbytehq"}, "emitted_at": 1681307598422} {"stream":"users","data":{"login":"AirbyteEricksson","id":101604444,"node_id":"U_kgDOBg5cXA","avatar_url":"https://avatars.githubusercontent.com/u/101604444?v=4","gravatar_id":"","url":"https://api.github.com/users/AirbyteEricksson","html_url":"https://github.com/AirbyteEricksson","followers_url":"https://api.github.com/users/AirbyteEricksson/followers","following_url":"https://api.github.com/users/AirbyteEricksson/following{/other_user}","gists_url":"https://api.github.com/users/AirbyteEricksson/gists{/gist_id}","starred_url":"https://api.github.com/users/AirbyteEricksson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AirbyteEricksson/subscriptions","organizations_url":"https://api.github.com/users/AirbyteEricksson/orgs","repos_url":"https://api.github.com/users/AirbyteEricksson/repos","events_url":"https://api.github.com/users/AirbyteEricksson/events{/privacy}","received_events_url":"https://api.github.com/users/AirbyteEricksson/received_events","type":"User","site_admin":false,"organization":"airbytehq"},"emitted_at":1677668766142} {"stream":"workflows","data":{"id":22952989,"node_id":"W_kwDOF9hP9c4BXjwd","name":"Pull Request Labeler","path":".github/workflows/labeler.yml","state":"active","created_at":"2022-03-30T21:30:37.000+02:00","updated_at":"2022-03-30T21:30:37.000+02:00","url":"https://api.github.com/repos/airbytehq/integration-test/actions/workflows/22952989","html_url":"https://github.com/airbytehq/integration-test/blob/master/.github/workflows/labeler.yml","badge_url":"https://github.com/airbytehq/integration-test/workflows/Pull%20Request%20Labeler/badge.svg","repository":"airbytehq/integration-test"},"emitted_at":1677668766580} -{"stream":"workflow_runs","data":{"id":3184250176,"name":"Pull Request Labeler","node_id":"WFR_kwLOF9hP9c69y81A","head_branch":"feature/branch_5","head_sha":"f71e5f6894578148d52b487dff07e55804fd9cfd","path":".github/workflows/labeler.yml","display_title":"New PR from feature/branch_5","run_number":3,"event":"pull_request_target","status":"completed","conclusion":"success","workflow_id":22952989,"check_suite_id":8611635614,"check_suite_node_id":"CS_kwDOF9hP9c8AAAACAUshng","url":"https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176","html_url":"https://github.com/airbytehq/integration-test/actions/runs/3184250176","pull_requests":[{"url":"https://api.github.com/repos/airbytehq/integration-test/pulls/14","id":984835098,"number":14,"head":{"ref":"feature/branch_5","sha":"f71e5f6894578148d52b487dff07e55804fd9cfd","repo":{"id":400052213,"url":"https://api.github.com/repos/airbytehq/integration-test","name":"integration-test"}},"base":{"ref":"master","sha":"a12c9379604f7b32e54e5459122aa48473f806ee","repo":{"id":400052213,"url":"https://api.github.com/repos/airbytehq/integration-test","name":"integration-test"}}}],"created_at":"2022-10-04T17:41:18Z","updated_at":"2022-10-04T17:41:32Z","actor":{"login":"grubberr","id":195743,"node_id":"MDQ6VXNlcjE5NTc0Mw==","avatar_url":"https://avatars.githubusercontent.com/u/195743?v=4","gravatar_id":"","url":"https://api.github.com/users/grubberr","html_url":"https://github.com/grubberr","followers_url":"https://api.github.com/users/grubberr/followers","following_url":"https://api.github.com/users/grubberr/following{/other_user}","gists_url":"https://api.github.com/users/grubberr/gists{/gist_id}","starred_url":"https://api.github.com/users/grubberr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/grubberr/subscriptions","organizations_url":"https://api.github.com/users/grubberr/orgs","repos_url":"https://api.github.com/users/grubberr/repos","events_url":"https://api.github.com/users/grubberr/events{/privacy}","received_events_url":"https://api.github.com/users/grubberr/received_events","type":"User","site_admin":false},"run_attempt":1,"referenced_workflows":[],"run_started_at":"2022-10-04T17:41:18Z","triggering_actor":{"login":"grubberr","id":195743,"node_id":"MDQ6VXNlcjE5NTc0Mw==","avatar_url":"https://avatars.githubusercontent.com/u/195743?v=4","gravatar_id":"","url":"https://api.github.com/users/grubberr","html_url":"https://github.com/grubberr","followers_url":"https://api.github.com/users/grubberr/followers","following_url":"https://api.github.com/users/grubberr/following{/other_user}","gists_url":"https://api.github.com/users/grubberr/gists{/gist_id}","starred_url":"https://api.github.com/users/grubberr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/grubberr/subscriptions","organizations_url":"https://api.github.com/users/grubberr/orgs","repos_url":"https://api.github.com/users/grubberr/repos","events_url":"https://api.github.com/users/grubberr/events{/privacy}","received_events_url":"https://api.github.com/users/grubberr/received_events","type":"User","site_admin":false},"jobs_url":"https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176/jobs","logs_url":"https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176/logs","check_suite_url":"https://api.github.com/repos/airbytehq/integration-test/check-suites/8611635614","artifacts_url":"https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176/artifacts","cancel_url":"https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176/cancel","rerun_url":"https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176/rerun","previous_attempt_url":null,"workflow_url":"https://api.github.com/repos/airbytehq/integration-test/actions/workflows/22952989","head_commit":{"id":"f71e5f6894578148d52b487dff07e55804fd9cfd","tree_id":"bb78ec62be8c5c640010e7c897f40932ce59e725","message":"file_5.txt updated\n\nSigned-off-by: Sergey Chvalyuk ","timestamp":"2022-10-04T17:41:08Z","author":{"name":"Sergey Chvalyuk","email":"grubberr@gmail.com"},"committer":{"name":"Sergey Chvalyuk","email":"grubberr@gmail.com"}},"repository":{"id":400052213,"node_id":"MDEwOlJlcG9zaXRvcnk0MDAwNTIyMTM=","name":"integration-test","full_name":"airbytehq/integration-test","private":false,"owner":{"login":"airbytehq","id":59758427,"node_id":"MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3","avatar_url":"https://avatars.githubusercontent.com/u/59758427?v=4","gravatar_id":"","url":"https://api.github.com/users/airbytehq","html_url":"https://github.com/airbytehq","followers_url":"https://api.github.com/users/airbytehq/followers","following_url":"https://api.github.com/users/airbytehq/following{/other_user}","gists_url":"https://api.github.com/users/airbytehq/gists{/gist_id}","starred_url":"https://api.github.com/users/airbytehq/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/airbytehq/subscriptions","organizations_url":"https://api.github.com/users/airbytehq/orgs","repos_url":"https://api.github.com/users/airbytehq/repos","events_url":"https://api.github.com/users/airbytehq/events{/privacy}","received_events_url":"https://api.github.com/users/airbytehq/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/airbytehq/integration-test","description":"Used for integration testing the Github source connector","fork":false,"url":"https://api.github.com/repos/airbytehq/integration-test","forks_url":"https://api.github.com/repos/airbytehq/integration-test/forks","keys_url":"https://api.github.com/repos/airbytehq/integration-test/keys{/key_id}","collaborators_url":"https://api.github.com/repos/airbytehq/integration-test/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/airbytehq/integration-test/teams","hooks_url":"https://api.github.com/repos/airbytehq/integration-test/hooks","issue_events_url":"https://api.github.com/repos/airbytehq/integration-test/issues/events{/number}","events_url":"https://api.github.com/repos/airbytehq/integration-test/events","assignees_url":"https://api.github.com/repos/airbytehq/integration-test/assignees{/user}","branches_url":"https://api.github.com/repos/airbytehq/integration-test/branches{/branch}","tags_url":"https://api.github.com/repos/airbytehq/integration-test/tags","blobs_url":"https://api.github.com/repos/airbytehq/integration-test/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/airbytehq/integration-test/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/airbytehq/integration-test/git/refs{/sha}","trees_url":"https://api.github.com/repos/airbytehq/integration-test/git/trees{/sha}","statuses_url":"https://api.github.com/repos/airbytehq/integration-test/statuses/{sha}","languages_url":"https://api.github.com/repos/airbytehq/integration-test/languages","stargazers_url":"https://api.github.com/repos/airbytehq/integration-test/stargazers","contributors_url":"https://api.github.com/repos/airbytehq/integration-test/contributors","subscribers_url":"https://api.github.com/repos/airbytehq/integration-test/subscribers","subscription_url":"https://api.github.com/repos/airbytehq/integration-test/subscription","commits_url":"https://api.github.com/repos/airbytehq/integration-test/commits{/sha}","git_commits_url":"https://api.github.com/repos/airbytehq/integration-test/git/commits{/sha}","comments_url":"https://api.github.com/repos/airbytehq/integration-test/comments{/number}","issue_comment_url":"https://api.github.com/repos/airbytehq/integration-test/issues/comments{/number}","contents_url":"https://api.github.com/repos/airbytehq/integration-test/contents/{+path}","compare_url":"https://api.github.com/repos/airbytehq/integration-test/compare/{base}...{head}","merges_url":"https://api.github.com/repos/airbytehq/integration-test/merges","archive_url":"https://api.github.com/repos/airbytehq/integration-test/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/airbytehq/integration-test/downloads","issues_url":"https://api.github.com/repos/airbytehq/integration-test/issues{/number}","pulls_url":"https://api.github.com/repos/airbytehq/integration-test/pulls{/number}","milestones_url":"https://api.github.com/repos/airbytehq/integration-test/milestones{/number}","notifications_url":"https://api.github.com/repos/airbytehq/integration-test/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/airbytehq/integration-test/labels{/name}","releases_url":"https://api.github.com/repos/airbytehq/integration-test/releases{/id}","deployments_url":"https://api.github.com/repos/airbytehq/integration-test/deployments"},"head_repository":{"id":400052213,"node_id":"MDEwOlJlcG9zaXRvcnk0MDAwNTIyMTM=","name":"integration-test","full_name":"airbytehq/integration-test","private":false,"owner":{"login":"airbytehq","id":59758427,"node_id":"MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3","avatar_url":"https://avatars.githubusercontent.com/u/59758427?v=4","gravatar_id":"","url":"https://api.github.com/users/airbytehq","html_url":"https://github.com/airbytehq","followers_url":"https://api.github.com/users/airbytehq/followers","following_url":"https://api.github.com/users/airbytehq/following{/other_user}","gists_url":"https://api.github.com/users/airbytehq/gists{/gist_id}","starred_url":"https://api.github.com/users/airbytehq/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/airbytehq/subscriptions","organizations_url":"https://api.github.com/users/airbytehq/orgs","repos_url":"https://api.github.com/users/airbytehq/repos","events_url":"https://api.github.com/users/airbytehq/events{/privacy}","received_events_url":"https://api.github.com/users/airbytehq/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/airbytehq/integration-test","description":"Used for integration testing the Github source connector","fork":false,"url":"https://api.github.com/repos/airbytehq/integration-test","forks_url":"https://api.github.com/repos/airbytehq/integration-test/forks","keys_url":"https://api.github.com/repos/airbytehq/integration-test/keys{/key_id}","collaborators_url":"https://api.github.com/repos/airbytehq/integration-test/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/airbytehq/integration-test/teams","hooks_url":"https://api.github.com/repos/airbytehq/integration-test/hooks","issue_events_url":"https://api.github.com/repos/airbytehq/integration-test/issues/events{/number}","events_url":"https://api.github.com/repos/airbytehq/integration-test/events","assignees_url":"https://api.github.com/repos/airbytehq/integration-test/assignees{/user}","branches_url":"https://api.github.com/repos/airbytehq/integration-test/branches{/branch}","tags_url":"https://api.github.com/repos/airbytehq/integration-test/tags","blobs_url":"https://api.github.com/repos/airbytehq/integration-test/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/airbytehq/integration-test/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/airbytehq/integration-test/git/refs{/sha}","trees_url":"https://api.github.com/repos/airbytehq/integration-test/git/trees{/sha}","statuses_url":"https://api.github.com/repos/airbytehq/integration-test/statuses/{sha}","languages_url":"https://api.github.com/repos/airbytehq/integration-test/languages","stargazers_url":"https://api.github.com/repos/airbytehq/integration-test/stargazers","contributors_url":"https://api.github.com/repos/airbytehq/integration-test/contributors","subscribers_url":"https://api.github.com/repos/airbytehq/integration-test/subscribers","subscription_url":"https://api.github.com/repos/airbytehq/integration-test/subscription","commits_url":"https://api.github.com/repos/airbytehq/integration-test/commits{/sha}","git_commits_url":"https://api.github.com/repos/airbytehq/integration-test/git/commits{/sha}","comments_url":"https://api.github.com/repos/airbytehq/integration-test/comments{/number}","issue_comment_url":"https://api.github.com/repos/airbytehq/integration-test/issues/comments{/number}","contents_url":"https://api.github.com/repos/airbytehq/integration-test/contents/{+path}","compare_url":"https://api.github.com/repos/airbytehq/integration-test/compare/{base}...{head}","merges_url":"https://api.github.com/repos/airbytehq/integration-test/merges","archive_url":"https://api.github.com/repos/airbytehq/integration-test/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/airbytehq/integration-test/downloads","issues_url":"https://api.github.com/repos/airbytehq/integration-test/issues{/number}","pulls_url":"https://api.github.com/repos/airbytehq/integration-test/pulls{/number}","milestones_url":"https://api.github.com/repos/airbytehq/integration-test/milestones{/number}","notifications_url":"https://api.github.com/repos/airbytehq/integration-test/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/airbytehq/integration-test/labels{/name}","releases_url":"https://api.github.com/repos/airbytehq/integration-test/releases{/id}","deployments_url":"https://api.github.com/repos/airbytehq/integration-test/deployments"}},"emitted_at":1677668766993} -{"stream":"workflow_jobs","data":{"id": 8705992587, "run_id": 3184250176, "workflow_name": "Pull Request Labeler", "head_branch": "feature/branch_5", "run_url": "https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176", "run_attempt": 1, "node_id": "CR_kwDOF9hP9c8AAAACBurniw", "head_sha": "f71e5f6894578148d52b487dff07e55804fd9cfd", "url": "https://api.github.com/repos/airbytehq/integration-test/actions/jobs/8705992587", "html_url": "https://github.com/airbytehq/integration-test/actions/runs/3184250176/job/8705992587", "status": "completed", "conclusion": "success", "created_at": "2022-10-04T17:41:20Z", "started_at": "2022-10-04T17:41:27Z", "completed_at": "2022-10-04T17:41:30Z", "name": "triage", "steps": [{"name": "Set up job", "status": "completed", "conclusion": "success", "number": 1, "started_at": "2022-10-04T20:41:26.000+03:00", "completed_at": "2022-10-04T20:41:27.000+03:00"}, {"name": "Run actions/labeler@v3", "status": "completed", "conclusion": "success", "number": 2, "started_at": "2022-10-04T20:41:27.000+03:00", "completed_at": "2022-10-04T20:41:29.000+03:00"}, {"name": "Complete job", "status": "completed", "conclusion": "success", "number": 3, "started_at": "2022-10-04T20:41:29.000+03:00", "completed_at": "2022-10-04T20:41:29.000+03:00"}], "check_run_url": "https://api.github.com/repos/airbytehq/integration-test/check-runs/8705992587", "labels": ["ubuntu-latest"], "runner_id": 1, "runner_name": "Hosted Agent", "runner_group_id": 2, "runner_group_name": "GitHub Actions", "repository": "airbytehq/integration-test"},"emitted_at":1677668767830} +{"stream": "workflow_runs", "data": {"id": 3184250176, "name": "Pull Request Labeler", "node_id": "WFR_kwLOF9hP9c69y81A", "head_branch": "feature/branch_5", "head_sha": "f71e5f6894578148d52b487dff07e55804fd9cfd", "path": ".github/workflows/labeler.yml", "display_title": "New PR from feature/branch_5", "run_number": 3, "event": "pull_request_target", "status": "completed", "conclusion": "success", "workflow_id": 22952989, "check_suite_id": 8611635614, "check_suite_node_id": "CS_kwDOF9hP9c8AAAACAUshng", "url": "https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176", "html_url": "https://github.com/airbytehq/integration-test/actions/runs/3184250176", "pull_requests": [{"url": "https://api.github.com/repos/airbytehq/integration-test/pulls/14", "id": 984835098, "number": 14, "head": {"ref": "feature/branch_5", "sha": "f71e5f6894578148d52b487dff07e55804fd9cfd", "repo": {"id": 400052213, "url": "https://api.github.com/repos/airbytehq/integration-test", "name": "integration-test"}}, "base": {"ref": "master", "sha": "a12c9379604f7b32e54e5459122aa48473f806ee", "repo": {"id": 400052213, "url": "https://api.github.com/repos/airbytehq/integration-test", "name": "integration-test"}}}], "created_at": "2022-10-04T17:41:18Z", "updated_at": "2023-11-08T19:58:29Z", "actor": {"login": "grubberr", "id": 195743, "node_id": "MDQ6VXNlcjE5NTc0Mw==", "avatar_url": "https://avatars.githubusercontent.com/u/195743?v=4", "gravatar_id": "", "url": "https://api.github.com/users/grubberr", "html_url": "https://github.com/grubberr", "followers_url": "https://api.github.com/users/grubberr/followers", "following_url": "https://api.github.com/users/grubberr/following{/other_user}", "gists_url": "https://api.github.com/users/grubberr/gists{/gist_id}", "starred_url": "https://api.github.com/users/grubberr/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/grubberr/subscriptions", "organizations_url": "https://api.github.com/users/grubberr/orgs", "repos_url": "https://api.github.com/users/grubberr/repos", "events_url": "https://api.github.com/users/grubberr/events{/privacy}", "received_events_url": "https://api.github.com/users/grubberr/received_events", "type": "User", "site_admin": false}, "run_attempt": 1, "referenced_workflows": [], "run_started_at": "2022-10-04T17:41:18Z", "triggering_actor": {"login": "grubberr", "id": 195743, "node_id": "MDQ6VXNlcjE5NTc0Mw==", "avatar_url": "https://avatars.githubusercontent.com/u/195743?v=4", "gravatar_id": "", "url": "https://api.github.com/users/grubberr", "html_url": "https://github.com/grubberr", "followers_url": "https://api.github.com/users/grubberr/followers", "following_url": "https://api.github.com/users/grubberr/following{/other_user}", "gists_url": "https://api.github.com/users/grubberr/gists{/gist_id}", "starred_url": "https://api.github.com/users/grubberr/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/grubberr/subscriptions", "organizations_url": "https://api.github.com/users/grubberr/orgs", "repos_url": "https://api.github.com/users/grubberr/repos", "events_url": "https://api.github.com/users/grubberr/events{/privacy}", "received_events_url": "https://api.github.com/users/grubberr/received_events", "type": "User", "site_admin": false}, "jobs_url": "https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176/jobs", "logs_url": "https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176/logs", "check_suite_url": "https://api.github.com/repos/airbytehq/integration-test/check-suites/8611635614", "artifacts_url": "https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176/artifacts", "cancel_url": "https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176/cancel", "rerun_url": "https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176/rerun", "previous_attempt_url": null, "workflow_url": "https://api.github.com/repos/airbytehq/integration-test/actions/workflows/22952989", "head_commit": {"id": "f71e5f6894578148d52b487dff07e55804fd9cfd", "tree_id": "bb78ec62be8c5c640010e7c897f40932ce59e725", "message": "file_5.txt updated\n\nSigned-off-by: Sergey Chvalyuk ", "timestamp": "2022-10-04T17:41:08Z", "author": {"name": "Sergey Chvalyuk", "email": "grubberr@gmail.com"}, "committer": {"name": "Sergey Chvalyuk", "email": "grubberr@gmail.com"}}, "repository": {"id": 400052213, "node_id": "MDEwOlJlcG9zaXRvcnk0MDAwNTIyMTM=", "name": "integration-test", "full_name": "airbytehq/integration-test", "private": false, "owner": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "html_url": "https://github.com/airbytehq/integration-test", "description": "Used for integration testing the Github source connector", "fork": false, "url": "https://api.github.com/repos/airbytehq/integration-test", "forks_url": "https://api.github.com/repos/airbytehq/integration-test/forks", "keys_url": "https://api.github.com/repos/airbytehq/integration-test/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/airbytehq/integration-test/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/airbytehq/integration-test/teams", "hooks_url": "https://api.github.com/repos/airbytehq/integration-test/hooks", "issue_events_url": "https://api.github.com/repos/airbytehq/integration-test/issues/events{/number}", "events_url": "https://api.github.com/repos/airbytehq/integration-test/events", "assignees_url": "https://api.github.com/repos/airbytehq/integration-test/assignees{/user}", "branches_url": "https://api.github.com/repos/airbytehq/integration-test/branches{/branch}", "tags_url": "https://api.github.com/repos/airbytehq/integration-test/tags", "blobs_url": "https://api.github.com/repos/airbytehq/integration-test/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/airbytehq/integration-test/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/airbytehq/integration-test/git/refs{/sha}", "trees_url": "https://api.github.com/repos/airbytehq/integration-test/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/airbytehq/integration-test/statuses/{sha}", "languages_url": "https://api.github.com/repos/airbytehq/integration-test/languages", "stargazers_url": "https://api.github.com/repos/airbytehq/integration-test/stargazers", "contributors_url": "https://api.github.com/repos/airbytehq/integration-test/contributors", "subscribers_url": "https://api.github.com/repos/airbytehq/integration-test/subscribers", "subscription_url": "https://api.github.com/repos/airbytehq/integration-test/subscription", "commits_url": "https://api.github.com/repos/airbytehq/integration-test/commits{/sha}", "git_commits_url": "https://api.github.com/repos/airbytehq/integration-test/git/commits{/sha}", "comments_url": "https://api.github.com/repos/airbytehq/integration-test/comments{/number}", "issue_comment_url": "https://api.github.com/repos/airbytehq/integration-test/issues/comments{/number}", "contents_url": "https://api.github.com/repos/airbytehq/integration-test/contents/{+path}", "compare_url": "https://api.github.com/repos/airbytehq/integration-test/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/airbytehq/integration-test/merges", "archive_url": "https://api.github.com/repos/airbytehq/integration-test/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/airbytehq/integration-test/downloads", "issues_url": "https://api.github.com/repos/airbytehq/integration-test/issues{/number}", "pulls_url": "https://api.github.com/repos/airbytehq/integration-test/pulls{/number}", "milestones_url": "https://api.github.com/repos/airbytehq/integration-test/milestones{/number}", "notifications_url": "https://api.github.com/repos/airbytehq/integration-test/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/airbytehq/integration-test/labels{/name}", "releases_url": "https://api.github.com/repos/airbytehq/integration-test/releases{/id}", "deployments_url": "https://api.github.com/repos/airbytehq/integration-test/deployments"}, "head_repository": {"id": 400052213, "node_id": "MDEwOlJlcG9zaXRvcnk0MDAwNTIyMTM=", "name": "integration-test", "full_name": "airbytehq/integration-test", "private": false, "owner": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "html_url": "https://github.com/airbytehq/integration-test", "description": "Used for integration testing the Github source connector", "fork": false, "url": "https://api.github.com/repos/airbytehq/integration-test", "forks_url": "https://api.github.com/repos/airbytehq/integration-test/forks", "keys_url": "https://api.github.com/repos/airbytehq/integration-test/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/airbytehq/integration-test/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/airbytehq/integration-test/teams", "hooks_url": "https://api.github.com/repos/airbytehq/integration-test/hooks", "issue_events_url": "https://api.github.com/repos/airbytehq/integration-test/issues/events{/number}", "events_url": "https://api.github.com/repos/airbytehq/integration-test/events", "assignees_url": "https://api.github.com/repos/airbytehq/integration-test/assignees{/user}", "branches_url": "https://api.github.com/repos/airbytehq/integration-test/branches{/branch}", "tags_url": "https://api.github.com/repos/airbytehq/integration-test/tags", "blobs_url": "https://api.github.com/repos/airbytehq/integration-test/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/airbytehq/integration-test/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/airbytehq/integration-test/git/refs{/sha}", "trees_url": "https://api.github.com/repos/airbytehq/integration-test/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/airbytehq/integration-test/statuses/{sha}", "languages_url": "https://api.github.com/repos/airbytehq/integration-test/languages", "stargazers_url": "https://api.github.com/repos/airbytehq/integration-test/stargazers", "contributors_url": "https://api.github.com/repos/airbytehq/integration-test/contributors", "subscribers_url": "https://api.github.com/repos/airbytehq/integration-test/subscribers", "subscription_url": "https://api.github.com/repos/airbytehq/integration-test/subscription", "commits_url": "https://api.github.com/repos/airbytehq/integration-test/commits{/sha}", "git_commits_url": "https://api.github.com/repos/airbytehq/integration-test/git/commits{/sha}", "comments_url": "https://api.github.com/repos/airbytehq/integration-test/comments{/number}", "issue_comment_url": "https://api.github.com/repos/airbytehq/integration-test/issues/comments{/number}", "contents_url": "https://api.github.com/repos/airbytehq/integration-test/contents/{+path}", "compare_url": "https://api.github.com/repos/airbytehq/integration-test/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/airbytehq/integration-test/merges", "archive_url": "https://api.github.com/repos/airbytehq/integration-test/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/airbytehq/integration-test/downloads", "issues_url": "https://api.github.com/repos/airbytehq/integration-test/issues{/number}", "pulls_url": "https://api.github.com/repos/airbytehq/integration-test/pulls{/number}", "milestones_url": "https://api.github.com/repos/airbytehq/integration-test/milestones{/number}", "notifications_url": "https://api.github.com/repos/airbytehq/integration-test/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/airbytehq/integration-test/labels{/name}", "releases_url": "https://api.github.com/repos/airbytehq/integration-test/releases{/id}", "deployments_url": "https://api.github.com/repos/airbytehq/integration-test/deployments"}}, "emitted_at": 1699644824401} +{"stream": "workflow_jobs", "data": {"id": 8705992587, "run_id": 3184250176, "workflow_name": "Pull Request Labeler", "head_branch": "feature/branch_5", "run_url": "https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176", "run_attempt": 1, "node_id": "CR_kwDOF9hP9c8AAAACBurniw", "head_sha": "f71e5f6894578148d52b487dff07e55804fd9cfd", "url": "https://api.github.com/repos/airbytehq/integration-test/actions/jobs/8705992587", "html_url": "https://github.com/airbytehq/integration-test/actions/runs/3184250176/job/8705992587", "status": "completed", "conclusion": "success", "created_at": "2022-10-04T17:41:20Z", "started_at": "2022-10-04T17:41:27Z", "completed_at": "2022-10-04T17:41:30Z", "name": "triage", "steps": [], "check_run_url": "https://api.github.com/repos/airbytehq/integration-test/check-runs/8705992587", "labels": ["ubuntu-latest"], "runner_id": 1, "runner_name": "Hosted Agent", "runner_group_id": 2, "runner_group_name": "GitHub Actions", "repository": "airbytehq/integration-test"}, "emitted_at": 1699646006344} {"stream": "team_members", "data": {"login": "johnlafleur", "id": 68561602, "node_id": "MDQ6VXNlcjY4NTYxNjAy", "avatar_url": "https://avatars.githubusercontent.com/u/68561602?v=4", "gravatar_id": "", "url": "https://api.github.com/users/johnlafleur", "html_url": "https://github.com/johnlafleur", "followers_url": "https://api.github.com/users/johnlafleur/followers", "following_url": "https://api.github.com/users/johnlafleur/following{/other_user}", "gists_url": "https://api.github.com/users/johnlafleur/gists{/gist_id}", "starred_url": "https://api.github.com/users/johnlafleur/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/johnlafleur/subscriptions", "organizations_url": "https://api.github.com/users/johnlafleur/orgs", "repos_url": "https://api.github.com/users/johnlafleur/repos", "events_url": "https://api.github.com/users/johnlafleur/events{/privacy}", "received_events_url": "https://api.github.com/users/johnlafleur/received_events", "type": "User", "site_admin": false, "organization": "airbytehq", "team_slug": "airbyte-eng"}, "emitted_at": 1698750584444} {"stream": "team_memberships", "data": {"state": "active", "role": "member", "url": "https://api.github.com/organizations/59758427/team/4559297/memberships/johnlafleur", "organization": "airbytehq", "team_slug": "airbyte-core", "username": "johnlafleur"}, "emitted_at": 1698757985640} {"stream": "issue_timeline_events", "data": {"repository": "airbytehq/integration-test", "issue_number": 6, "labeled": {"id": 5219398390, "node_id": "MDEyOkxhYmVsZWRFdmVudDUyMTkzOTgzOTA=", "url": "https://api.github.com/repos/airbytehq/integration-test/issues/events/5219398390", "actor": {"login": "gaart", "id": 743901, "node_id": "MDQ6VXNlcjc0MzkwMQ==", "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gaart", "html_url": "https://github.com/gaart", "followers_url": "https://api.github.com/users/gaart/followers", "following_url": "https://api.github.com/users/gaart/following{/other_user}", "gists_url": "https://api.github.com/users/gaart/gists{/gist_id}", "starred_url": "https://api.github.com/users/gaart/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/gaart/subscriptions", "organizations_url": "https://api.github.com/users/gaart/orgs", "repos_url": "https://api.github.com/users/gaart/repos", "events_url": "https://api.github.com/users/gaart/events{/privacy}", "received_events_url": "https://api.github.com/users/gaart/received_events", "type": "User", "site_admin": false}, "event": "labeled", "commit_id": null, "commit_url": null, "created_at": "2021-08-27T15:43:58Z", "label": {"name": "critical", "color": "ededed"}, "performed_via_github_app": null}, "milestoned": {"id": 5219398392, "node_id": "MDE1Ok1pbGVzdG9uZWRFdmVudDUyMTkzOTgzOTI=", "url": "https://api.github.com/repos/airbytehq/integration-test/issues/events/5219398392", "actor": {"login": "gaart", "id": 743901, "node_id": "MDQ6VXNlcjc0MzkwMQ==", "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gaart", "html_url": "https://github.com/gaart", "followers_url": "https://api.github.com/users/gaart/followers", "following_url": "https://api.github.com/users/gaart/following{/other_user}", "gists_url": "https://api.github.com/users/gaart/gists{/gist_id}", "starred_url": "https://api.github.com/users/gaart/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/gaart/subscriptions", "organizations_url": "https://api.github.com/users/gaart/orgs", "repos_url": "https://api.github.com/users/gaart/repos", "events_url": "https://api.github.com/users/gaart/events{/privacy}", "received_events_url": "https://api.github.com/users/gaart/received_events", "type": "User", "site_admin": false}, "event": "milestoned", "commit_id": null, "commit_url": null, "created_at": "2021-08-27T15:43:58Z", "milestone": {"title": "main"}, "performed_via_github_app": null}, "commented": {"url": "https://api.github.com/repos/airbytehq/integration-test/issues/comments/907296167", "html_url": "https://github.com/airbytehq/integration-test/issues/6#issuecomment-907296167", "issue_url": "https://api.github.com/repos/airbytehq/integration-test/issues/6", "id": 907296167, "node_id": "IC_kwDOF9hP9c42FD2n", "user": {"login": "gaart", "id": 743901, "node_id": "MDQ6VXNlcjc0MzkwMQ==", "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gaart", "html_url": "https://github.com/gaart", "followers_url": "https://api.github.com/users/gaart/followers", "following_url": "https://api.github.com/users/gaart/following{/other_user}", "gists_url": "https://api.github.com/users/gaart/gists{/gist_id}", "starred_url": "https://api.github.com/users/gaart/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/gaart/subscriptions", "organizations_url": "https://api.github.com/users/gaart/orgs", "repos_url": "https://api.github.com/users/gaart/repos", "events_url": "https://api.github.com/users/gaart/events{/privacy}", "received_events_url": "https://api.github.com/users/gaart/received_events", "type": "User", "site_admin": false}, "created_at": "2021-08-27T15:43:59Z", "updated_at": "2021-08-27T15:43:59Z", "author_association": "CONTRIBUTOR", "body": "comment for issues https://api.github.com/repos/airbytehq/integration-test/issues/6/comments", "reactions": {"url": "https://api.github.com/repos/airbytehq/integration-test/issues/comments/907296167/reactions", "total_count": 0, "+1": 0, "-1": 0, "laugh": 0, "hooray": 0, "confused": 0, "heart": 0, "rocket": 0, "eyes": 0}, "performed_via_github_app": null, "event": "commented", "actor": {"login": "gaart", "id": 743901, "node_id": "MDQ6VXNlcjc0MzkwMQ==", "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gaart", "html_url": "https://github.com/gaart", "followers_url": "https://api.github.com/users/gaart/followers", "following_url": "https://api.github.com/users/gaart/following{/other_user}", "gists_url": "https://api.github.com/users/gaart/gists{/gist_id}", "starred_url": "https://api.github.com/users/gaart/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/gaart/subscriptions", "organizations_url": "https://api.github.com/users/gaart/orgs", "repos_url": "https://api.github.com/users/gaart/repos", "events_url": "https://api.github.com/users/gaart/events{/privacy}", "received_events_url": "https://api.github.com/users/gaart/received_events", "type": "User", "site_admin": false}}}, "emitted_at": 1695815681406} From 4df80965f2cd9eba7ab4967a49665efecd08a8a6 Mon Sep 17 00:00:00 2001 From: "Roman Yermilov [GL]" <86300758+roman-yermilov-gl@users.noreply.github.com> Date: Sat, 11 Nov 2023 11:46:11 +0400 Subject: [PATCH 012/124] Source Zendesk Support: increase test coverage (#32440) Co-authored-by: roman-yermilov-gl --- .../integration_tests/expected_records.jsonl | 6 +- .../source-zendesk-support/metadata.yaml | 2 +- .../source_zendesk_support/streams.py | 67 ++----------------- .../unit_tests/test_backoff_on_rate_limit.py | 9 ++- .../unit_tests/unit_test.py | 44 +++++++++++- docs/integrations/sources/zendesk-support.md | 1 + 6 files changed, 57 insertions(+), 72 deletions(-) diff --git a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl index c195fc5166bb..615d50bb3ebc 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl @@ -41,9 +41,9 @@ {"stream": "ticket_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_fields/360002833076.json", "id": 360002833076, "type": "subject", "title": "Subject", "raw_title": "Subject", "description": "", "raw_description": "", "position": 1, "active": true, "required": false, "collapsed_for_agents": false, "regexp_for_validation": null, "title_in_portal": "Subject", "raw_title_in_portal": "Subject", "visible_in_portal": true, "editable_in_portal": true, "required_in_portal": true, "tag": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z", "removable": false, "key": null, "agent_description": null}, "emitted_at": 1697714860081} {"stream": "ticket_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_fields/360002833096.json", "id": 360002833096, "type": "description", "title": "Description", "raw_title": "Description", "description": "Please enter the details of your request. A member of our support staff will respond as soon as possible.", "raw_description": "Please enter the details of your request. A member of our support staff will respond as soon as possible.", "position": 2, "active": true, "required": false, "collapsed_for_agents": false, "regexp_for_validation": null, "title_in_portal": "Description", "raw_title_in_portal": "Description", "visible_in_portal": true, "editable_in_portal": true, "required_in_portal": true, "tag": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z", "removable": false, "key": null, "agent_description": null}, "emitted_at": 1697714860083} {"stream": "ticket_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_fields/360002833116.json", "id": 360002833116, "type": "status", "title": "Status", "raw_title": "Status", "description": "Request status", "raw_description": "Request status", "position": 3, "active": true, "required": false, "collapsed_for_agents": false, "regexp_for_validation": null, "title_in_portal": "Status", "raw_title_in_portal": "Status", "visible_in_portal": false, "editable_in_portal": false, "required_in_portal": false, "tag": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z", "removable": false, "key": null, "agent_description": null, "system_field_options": [{"name": "Open", "value": "open"}, {"name": "Pending", "value": "pending"}, {"name": "Solved", "value": "solved"}], "sub_type_id": 0}, "emitted_at": 1697714860085} -{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/8154457562767.json", "id": 8154457562767, "ticket_id": 154, "created_at": "2023-10-17T14:24:53Z", "updated_at": "2023-10-17T14:28:25Z", "group_stations": 2, "assignee_stations": 1, "reopens": 0, "replies": 1, "assignee_updated_at": "2023-10-17T14:27:52Z", "requester_updated_at": "2023-10-17T14:26:45Z", "status_updated_at": "2023-10-17T14:27:52Z", "initially_assigned_at": "2023-10-17T14:26:33Z", "assigned_at": "2023-10-17T14:26:33Z", "solved_at": "2023-10-17T14:27:52Z", "latest_comment_added_at": "2023-10-17T14:28:25Z", "reply_time_in_minutes": {"calendar": 4, "business": 0}, "first_resolution_time_in_minutes": {"calendar": 3, "business": 0}, "full_resolution_time_in_minutes": {"calendar": 3, "business": 0}, "agent_wait_time_in_minutes": {"calendar": 0, "business": 0}, "requester_wait_time_in_minutes": {"calendar": 3, "business": 0}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "reply_time_in_seconds": {"calendar": 212}, "custom_status_updated_at": "2023-10-17T14:27:52Z"}, "emitted_at": 1697714861785} -{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7283000498191.json", "id": 7283000498191, "ticket_id": 153, "created_at": "2023-06-26T11:31:48Z", "updated_at": "2023-06-26T12:13:42Z", "group_stations": 2, "assignee_stations": 2, "reopens": 0, "replies": 0, "assignee_updated_at": "2023-06-26T11:31:48Z", "requester_updated_at": "2023-06-26T11:31:48Z", "status_updated_at": "2023-06-26T11:31:48Z", "initially_assigned_at": "2023-06-26T11:31:48Z", "assigned_at": "2023-06-26T12:13:42Z", "solved_at": null, "latest_comment_added_at": "2023-06-26T11:31:48Z", "reply_time_in_minutes": {"calendar": null, "business": null}, "first_resolution_time_in_minutes": {"calendar": null, "business": null}, "full_resolution_time_in_minutes": {"calendar": null, "business": null}, "agent_wait_time_in_minutes": {"calendar": null, "business": null}, "requester_wait_time_in_minutes": {"calendar": null, "business": null}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "custom_status_updated_at": "2023-06-26T11:31:48Z"}, "emitted_at": 1697714861787} -{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7282909551759.json", "id": 7282909551759, "ticket_id": 152, "created_at": "2023-06-26T11:10:33Z", "updated_at": "2023-06-26T11:25:43Z", "group_stations": 1, "assignee_stations": 1, "reopens": 0, "replies": 1, "assignee_updated_at": "2023-06-26T11:25:43Z", "requester_updated_at": "2023-06-26T11:10:33Z", "status_updated_at": "2023-07-16T12:01:39Z", "initially_assigned_at": "2023-06-26T11:10:33Z", "assigned_at": "2023-06-26T11:10:33Z", "solved_at": "2023-06-26T11:25:43Z", "latest_comment_added_at": "2023-06-26T11:21:06Z", "reply_time_in_minutes": {"calendar": 11, "business": 0}, "first_resolution_time_in_minutes": {"calendar": 15, "business": 0}, "full_resolution_time_in_minutes": {"calendar": 15, "business": 0}, "agent_wait_time_in_minutes": {"calendar": 15, "business": 0}, "requester_wait_time_in_minutes": {"calendar": 0, "business": 0}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "custom_status_updated_at": "2023-06-26T11:25:43Z"}, "emitted_at": 1697714861788} +{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/8154457562767.json", "id": 8154457562767, "ticket_id": 154, "created_at": "2023-10-17T14:24:53Z", "updated_at": "2023-10-17T14:28:25Z", "group_stations": 2, "assignee_stations": 1, "reopens": 0, "replies": 1, "assignee_updated_at": "2023-10-17T14:27:52Z", "requester_updated_at": "2023-10-17T14:26:45Z", "status_updated_at": "2023-11-06T15:01:40Z", "initially_assigned_at": "2023-10-17T14:26:33Z", "assigned_at": "2023-10-17T14:26:33Z", "solved_at": "2023-10-17T14:27:52Z", "latest_comment_added_at": "2023-10-17T14:28:25Z", "reply_time_in_minutes": {"calendar": 4, "business": 0}, "first_resolution_time_in_minutes": {"calendar": 3, "business": 0}, "full_resolution_time_in_minutes": {"calendar": 3, "business": 0}, "agent_wait_time_in_minutes": {"calendar": 0, "business": 0}, "requester_wait_time_in_minutes": {"calendar": 3, "business": 0}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "reply_time_in_seconds": {"calendar": 212}, "custom_status_updated_at": "2023-10-17T14:27:52Z"}, "emitted_at": 1699646404810} +{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7283000498191.json", "id": 7283000498191, "ticket_id": 153, "created_at": "2023-06-26T11:31:48Z", "updated_at": "2023-06-26T12:13:42Z", "group_stations": 2, "assignee_stations": 2, "reopens": 0, "replies": 0, "assignee_updated_at": "2023-06-26T11:31:48Z", "requester_updated_at": "2023-06-26T11:31:48Z", "status_updated_at": "2023-06-26T11:31:48Z", "initially_assigned_at": "2023-06-26T11:31:48Z", "assigned_at": "2023-06-26T12:13:42Z", "solved_at": null, "latest_comment_added_at": "2023-06-26T11:31:48Z", "reply_time_in_minutes": {"calendar": null, "business": null}, "first_resolution_time_in_minutes": {"calendar": null, "business": null}, "full_resolution_time_in_minutes": {"calendar": null, "business": null}, "agent_wait_time_in_minutes": {"calendar": null, "business": null}, "requester_wait_time_in_minutes": {"calendar": null, "business": null}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "custom_status_updated_at": "2023-06-26T11:31:48Z"}, "emitted_at": 1699646404810} +{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7282909551759.json", "id": 7282909551759, "ticket_id": 152, "created_at": "2023-06-26T11:10:33Z", "updated_at": "2023-06-26T11:25:43Z", "group_stations": 1, "assignee_stations": 1, "reopens": 0, "replies": 1, "assignee_updated_at": "2023-06-26T11:25:43Z", "requester_updated_at": "2023-06-26T11:10:33Z", "status_updated_at": "2023-07-16T12:01:39Z", "initially_assigned_at": "2023-06-26T11:10:33Z", "assigned_at": "2023-06-26T11:10:33Z", "solved_at": "2023-06-26T11:25:43Z", "latest_comment_added_at": "2023-06-26T11:21:06Z", "reply_time_in_minutes": {"calendar": 11, "business": 0}, "first_resolution_time_in_minutes": {"calendar": 15, "business": 0}, "full_resolution_time_in_minutes": {"calendar": 15, "business": 0}, "agent_wait_time_in_minutes": {"calendar": 15, "business": 0}, "requester_wait_time_in_minutes": {"calendar": 0, "business": 0}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "custom_status_updated_at": "2023-06-26T11:25:43Z"}, "emitted_at": 1699646404810} {"stream": "ticket_metric_events", "data": {"id": 4992797383183, "ticket_id": 121, "metric": "agent_work_time", "instance_id": 0, "type": "measure", "time": "2022-06-17T14:49:20Z"}, "emitted_at": 1697714863384} {"stream": "ticket_metric_events", "data": {"id": 4992797383311, "ticket_id": 121, "metric": "pausable_update_time", "instance_id": 0, "type": "measure", "time": "2022-06-17T14:49:20Z"}, "emitted_at": 1697714863386} {"stream": "ticket_metric_events", "data": {"id": 4992797383439, "ticket_id": 121, "metric": "reply_time", "instance_id": 0, "type": "measure", "time": "2022-06-17T14:49:20Z"}, "emitted_at": 1697714863386} diff --git a/airbyte-integrations/connectors/source-zendesk-support/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-support/metadata.yaml index a13d1d5e9029..425d8ee26e57 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-support/metadata.yaml @@ -11,7 +11,7 @@ data: connectorSubtype: api connectorType: source definitionId: 79c1aa37-dae3-42ae-b333-d1c105477715 - dockerImageTag: 2.2.0 + dockerImageTag: 2.2.1 dockerRepository: airbyte/source-zendesk-support documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-support githubIssueLabel: source-zendesk-support diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py index b667830c98a6..bff5b5fdfed1 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py @@ -351,13 +351,14 @@ def request_params( stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + next_page_token = next_page_token or {} + parsed_state = self.check_stream_state(stream_state) + params = {"start_time": next_page_token.get(self.cursor_field, parsed_state)} # check "start_time" is not in the future params["start_time"] = self.check_start_time_param(params["start_time"]) if self.sideload_param: params["include"] = self.sideload_param if next_page_token: - params.pop("start_time", None) params.update(next_page_token) return params @@ -398,23 +399,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, response_json = response.json() return None if response_json.get(END_OF_STREAM_KEY, True) else {"start_time": response_json.get("end_time")} - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - next_page_token = next_page_token or {} - parsed_state = self.check_stream_state(stream_state) - params = {"start_time": next_page_token.get(self.cursor_field, parsed_state)} - # check "start_time" is not in the future - params["start_time"] = self.check_start_time_param(params["start_time"]) - if self.sideload_param: - params["include"] = self.sideload_param - if next_page_token: - params.update(next_page_token) - return params - @property def update_event_from_record(self) -> bool: """Returns True/False based on list_entities_from_event property""" @@ -455,46 +439,12 @@ class Users(SourceZendeskIncrementalExportStream): def path(self, **kwargs) -> str: return "incremental/users/cursor.json" - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - next_page_token = next_page_token or {} - parsed_state = self.check_stream_state(stream_state) - params = {"start_time": next_page_token.get(self.cursor_field, parsed_state)} - # check "start_time" is not in the future - params["start_time"] = self.check_start_time_param(params["start_time"]) - if self.sideload_param: - params["include"] = self.sideload_param - if next_page_token: - params.update(next_page_token) - return params - class Organizations(SourceZendeskIncrementalExportStream): """Organizations stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/""" response_list_name: str = "organizations" - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - next_page_token = next_page_token or {} - parsed_state = self.check_stream_state(stream_state) - params = {"start_time": next_page_token.get(self.cursor_field, parsed_state)} - # check "start_time" is not in the future - params["start_time"] = self.check_start_time_param(params["start_time"]) - if self.sideload_param: - params["include"] = self.sideload_param - if next_page_token: - params.update(next_page_token) - return params - class Posts(CursorPaginationZendeskSupportStream): """Posts stream: https://developer.zendesk.com/api-reference/help_center/help-center-api/posts/#list-posts""" @@ -894,16 +844,7 @@ def request_params( stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - next_page_token = next_page_token or {} - parsed_state = self.check_stream_state(stream_state) - params = {"sort_by": "updated_at", "sort_order": "asc", "start_time": next_page_token.get(self.cursor_field, parsed_state)} - # check "start_time" is not in the future - params["start_time"] = self.check_start_time_param(params["start_time"]) - if self.sideload_param: - params["include"] = self.sideload_param - if next_page_token: - params.update(next_page_token) - return params + return {"sort_by": "updated_at", "sort_order": "asc", **super().request_params(stream_state, stream_slice, next_page_token)} class ArticleVotes(AbstractVotes, HttpSubStream): diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_backoff_on_rate_limit.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_backoff_on_rate_limit.py index 89349107b60b..0c17e1bcc722 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_backoff_on_rate_limit.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_backoff_on_rate_limit.py @@ -25,10 +25,13 @@ def prepare_config(config: Dict): return SourceZendeskSupport().convert_config2stream_args(config) -@pytest.mark.parametrize("retry_after, expected", [({}, None), ({"Retry-After": "5"}, 5), ({"Retry-After": "5, 4"}, 5)]) -def test_backoff(requests_mock, config, retry_after, expected): +@pytest.mark.parametrize( + "x_rate_limit, retry_after, expected", + [("60", {}, 1), ("0", {}, None), ("0", {"Retry-After": "5"}, 5), ("0", {"Retry-After": "5, 4"}, 5)], +) +def test_backoff(requests_mock, config, x_rate_limit, retry_after, expected): """ """ - test_response_header = {"X-Rate-Limit": "0"} | retry_after + test_response_header = {"X-Rate-Limit": x_rate_limit} | retry_after test_response_json = {"count": {"value": 1, "refreshed_at": "2022-03-29T10:10:51+00:00"}} # create client diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py index 223515426aad..cdb9a4d84d53 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py @@ -22,6 +22,10 @@ END_OF_STREAM_KEY, LAST_END_TIME_KEY, AccountAttributes, + ArticleComments, + ArticleCommentVotes, + Articles, + ArticleVotes, AttributeDefinitions, AuditLogs, BaseZendeskSupportStream, @@ -70,6 +74,13 @@ "credentials": {"credentials": "api_token", "email": "integration-test@airbyte.io", "api_token": "api_token"}, } +# raw old config +TEST_OLD_CONFIG = { + "auth_method": {"auth_method": "api_token", "email": "integration-test@airbyte.io", "api_token": "api_token"}, + "subdomain": "sandbox", + "start_date": "2021-06-01T00:00:00Z", +} + TEST_CONFIG_WITHOUT_START_DATE = { "subdomain": "sandbox", "credentials": {"credentials": "api_token", "email": "integration-test@airbyte.io", "api_token": "api_token"}, @@ -131,8 +142,12 @@ def test_default_start_date(): @pytest.mark.parametrize( "config, expected", - [(TEST_CONFIG, "aW50ZWdyYXRpb24tdGVzdEBhaXJieXRlLmlvL3Rva2VuOmFwaV90b2tlbg=="), (TEST_CONFIG_OAUTH, "test_access_token")], - ids=["api_token", "oauth"], + [ + (TEST_CONFIG, "aW50ZWdyYXRpb24tdGVzdEBhaXJieXRlLmlvL3Rva2VuOmFwaV90b2tlbg=="), + (TEST_CONFIG_OAUTH, "test_access_token"), + (TEST_OLD_CONFIG, "aW50ZWdyYXRpb24tdGVzdEBhaXJieXRlLmlvL3Rva2VuOmFwaV90b2tlbg=="), + ], + ids=["api_token", "oauth", "old_config"], ) def test_get_authenticator(config, expected): # we expect base64 from creds input @@ -320,6 +335,12 @@ def test_streams(self, expected_stream_cls): if expected_stream_cls in streams: assert isinstance(stream, expected_stream_cls) + def test_ticket_forms_exception_stream(self): + with patch.object(TicketForms, "read_records", return_value=[{}]) as mocked_records: + mocked_records.side_effect = Exception("The error") + streams = SourceZendeskSupport().streams(TEST_CONFIG) + assert not any([isinstance(stream, TicketForms) for stream in streams]) + @pytest.mark.parametrize( "stream_cls, expected", [ @@ -756,10 +777,12 @@ def test_next_page_token(self, requests_mock, stream_cls): [ (Users, {"start_time": 1622505600}), (Tickets, {"start_time": 1622505600}), + (Articles, {"sort_by": "updated_at", "sort_order": "asc", "start_time": 1622505600}), ], ids=[ "Users", "Tickets", + "Articles", ], ) def test_request_params(self, stream_cls, expected): @@ -787,6 +810,23 @@ def test_parse_response(self, requests_mock, stream_cls): output = list(stream.parse_response(test_response)) assert expected == output + @pytest.mark.parametrize( + "stream_cls, stream_slice, expected_path", + [ + (ArticleVotes, {"parent": {"id": 1}}, "help_center/articles/1/votes"), + (ArticleComments, {"parent": {"id": 1}}, "help_center/articles/1/comments"), + (ArticleCommentVotes, {"parent": {"id": 1, "source_id": 1}}, "help_center/articles/1/comments/1/votes"), + ], + ids=[ + "ArticleVotes_path", + "ArticleComments_path", + "ArticleCommentVotes_path", + ], + ) + def test_path(self, stream_cls, stream_slice, expected_path): + stream = stream_cls(**STREAM_ARGS) + assert stream.path(stream_slice=stream_slice) == expected_path + class TestSourceZendeskSupportTicketEventsExportStream: @pytest.mark.parametrize( diff --git a/docs/integrations/sources/zendesk-support.md b/docs/integrations/sources/zendesk-support.md index 053f9c1a3252..5f194f8322e5 100644 --- a/docs/integrations/sources/zendesk-support.md +++ b/docs/integrations/sources/zendesk-support.md @@ -134,6 +134,7 @@ The Zendesk connector ideally should not run into Zendesk API limitations under | Version | Date | Pull Request | Subject | |:---------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `2.2.1` | 2023-11-10 | [32440](https://github.com/airbytehq/airbyte/pull/32440) | Made refactoring to improve code maintainability | | `2.2.0` | 2023-10-31 | [31999](https://github.com/airbytehq/airbyte/pull/31999) | Extended the `CustomRoles` stream schema | | `2.1.1` | 2023-10-23 | [31702](https://github.com/airbytehq/airbyte/pull/31702) | Base image migration: remove Dockerfile and use the python-connector-base image | | `2.1.0` | 2023-10-19 | [31606](https://github.com/airbytehq/airbyte/pull/31606) | Added new field `reply_time_in_seconds` to the `Ticket Metrics` stream schema | From 868083c981aee88a65744c59649d911f4fb8ced9 Mon Sep 17 00:00:00 2001 From: "Roman Yermilov [GL]" <86300758+roman-yermilov-gl@users.noreply.github.com> Date: Sat, 11 Nov 2023 12:01:22 +0400 Subject: [PATCH 013/124] Source Slack: fix expected records (#32444) --- .../integration_tests/expected_records.jsonl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.jsonl index 09790a8b086b..9ddbf42168e5 100644 --- a/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.jsonl @@ -18,12 +18,12 @@ {"stream": "threads", "data": {"client_msg_id": "e27672c0-451e-42a6-8eff-a14d2db8ac1e", "type": "message", "text": "Test Thread 1", "user": "U04L65GPMKN", "ts": "1683104499.808709", "blocks": [{"type": "rich_text", "block_id": "0j7", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 1"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104528.084359", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104528.084359", "channel_id": "C04LTCM2Y56", "float_ts": 1683104499.808709}, "emitted_at": 1695112005658} {"stream": "threads", "data": {"client_msg_id": "e1e2d142-a0dd-4587-86e3-2dcb439ead82", "type": "message", "text": "<@U04LY6NARHU> Test test", "user": "U04L65GPMKN", "ts": "1683104515.919709", "blocks": [{"type": "rich_text", "block_id": "xVnQ", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04LY6NARHU"}, {"type": "text", "text": " Test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "parent_user_id": "U04L65GPMKN", "channel_id": "C04LTCM2Y56", "float_ts": 1683104515.919709}, "emitted_at": 1695112005659} {"stream": "threads", "data": {"client_msg_id": "ffccbb24-8dd6-476d-87bf-65e5fa033cb9", "type": "message", "text": "<@U04M23SBJGM> test test test", "user": "U04L65GPMKN", "ts": "1683104528.084359", "blocks": [{"type": "rich_text", "block_id": "Lvl", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04M23SBJGM"}, {"type": "text", "text": " test test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "parent_user_id": "U04L65GPMKN", "channel_id": "C04LTCM2Y56", "float_ts": 1683104528.084359}, "emitted_at": 1695112005659} -{"stream":"users","data":{"id":"USLACKBOT","team_id":"T04KX3KDDU6","name":"slackbot","deleted":false,"color":"757575","real_name":"Slackbot","tz":"America/Los_Angeles","tz_label":"Pacific Daylight Time","tz_offset":-25200,"profile":{"title":"","phone":"","skype":"","real_name":"Slackbot","real_name_normalized":"Slackbot","display_name":"Slackbot","display_name_normalized":"Slackbot","fields":{},"status_text":"","status_emoji":"","status_emoji_display_info":[],"status_expiration":0,"avatar_hash":"sv41d8cd98f0","always_active":true,"first_name":"slackbot","last_name":"","image_24":"https://a.slack-edge.com/80588/img/slackbot_24.png","image_32":"https://a.slack-edge.com/80588/img/slackbot_32.png","image_48":"https://a.slack-edge.com/80588/img/slackbot_48.png","image_72":"https://a.slack-edge.com/80588/img/slackbot_72.png","image_192":"https://a.slack-edge.com/80588/marketing/img/avatars/slackbot/avatar-slackbot.png","image_512":"https://a.slack-edge.com/80588/img/slackbot_512.png","status_text_canonical":"","team":"T04KX3KDDU6"},"is_admin":false,"is_owner":false,"is_primary_owner":false,"is_restricted":false,"is_ultra_restricted":false,"is_bot":false,"is_app_user":false,"updated":0,"is_email_confirmed":false,"who_can_share_contact_card":"EVERYONE"},"emitted_at":1695766576249} -{"stream":"users","data":{"id":"U04KUMXNYMV","team_id":"T04KX3KDDU6","name":"deactivateduser693438","deleted":true,"profile":{"title":"","phone":"","skype":"","real_name":"Deactivated User","real_name_normalized":"Deactivated User","display_name":"deactivateduser","display_name_normalized":"deactivateduser","fields":null,"status_text":"","status_emoji":"","status_emoji_display_info":[],"status_expiration":0,"avatar_hash":"g849cc56ed76","huddle_state":"default_unset","first_name":"Deactivated","last_name":"User","image_24":"https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-24.png","image_32":"https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-32.png","image_48":"https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-48.png","image_72":"https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-72.png","image_192":"https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-192.png","image_512":"https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-512.png","status_text_canonical":"","team":"T04KX3KDDU6"},"is_bot":false,"is_app_user":false,"updated":1675090804,"is_forgotten":true,"is_invited_user":true},"emitted_at":1695766576250} -{"stream":"users","data":{"id":"U04L2KY5CES","team_id":"T04KX3KDDU6","name":"deactivateduser686066","deleted":true,"profile":{"title":"","phone":"","skype":"","real_name":"Deactivated User","real_name_normalized":"Deactivated User","display_name":"deactivateduser","display_name_normalized":"deactivateduser","fields":null,"status_text":"","status_emoji":"","status_emoji_display_info":[],"status_expiration":0,"avatar_hash":"g849cc56ed76","huddle_state":"default_unset","first_name":"Deactivated","last_name":"User","image_24":"https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-24.png","image_32":"https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-32.png","image_48":"https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-48.png","image_72":"https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-72.png","image_192":"https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-192.png","image_512":"https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-512.png","status_text_canonical":"","team":"T04KX3KDDU6"},"is_bot":false,"is_app_user":false,"updated":1675090785,"is_forgotten":true,"is_invited_user":true},"emitted_at":1695766576250} -{"stream":"users","data":{"id":"U04L2LC770E","team_id":"T04KX3KDDU6","name":"deactivateduser521176","deleted":true,"profile":{"title":"","phone":"","skype":"","real_name":"Deactivated User","real_name_normalized":"Deactivated User","display_name":"deactivateduser","display_name_normalized":"deactivateduser","fields":null,"status_text":"","status_emoji":"","status_emoji_display_info":[],"status_expiration":0,"avatar_hash":"g849cc56ed76","huddle_state":"default_unset","first_name":"Deactivated","last_name":"User","image_24":"https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-24.png","image_32":"https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-32.png","image_48":"https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-48.png","image_72":"https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-72.png","image_192":"https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-192.png","image_512":"https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-512.png","status_text_canonical":"","team":"T04KX3KDDU6"},"is_bot":false,"is_app_user":false,"updated":1675090821,"is_forgotten":true,"is_invited_user":true},"emitted_at":1695766576250} -{"stream":"users","data":{"id":"U04L69BPZFX","team_id":"T04KX3KDDU6","name":"deactivateduser839125","deleted":true,"profile":{"title":"","phone":"","skype":"","real_name":"Deactivated User","real_name_normalized":"Deactivated User","display_name":"deactivateduser","display_name_normalized":"deactivateduser","fields":null,"status_text":"","status_emoji":"","status_emoji_display_info":[],"status_expiration":0,"avatar_hash":"g849cc56ed76","huddle_state":"default_unset","first_name":"Deactivated","last_name":"User","image_24":"https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-24.png","image_32":"https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-32.png","image_48":"https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-48.png","image_72":"https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-72.png","image_192":"https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-192.png","image_512":"https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-512.png","status_text_canonical":"","team":"T04KX3KDDU6"},"is_bot":false,"is_app_user":false,"updated":1681811889,"is_forgotten":true,"is_invited_user":true},"emitted_at":1695766576251} -{"stream":"users","data":{"id":"U04L94Y2JPM","team_id":"T04KX3KDDU6","name":"deactivateduser962255","deleted":true,"profile":{"title":"","phone":"","skype":"","real_name":"Deactivated User","real_name_normalized":"Deactivated User","display_name":"deactivateduser","display_name_normalized":"deactivateduser","fields":null,"status_text":"","status_emoji":"","status_emoji_display_info":[],"status_expiration":0,"avatar_hash":"g849cc56ed76","huddle_state":"default_unset","first_name":"Deactivated","last_name":"User","image_24":"https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-24.png","image_32":"https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-32.png","image_48":"https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-48.png","image_72":"https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-72.png","image_192":"https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-192.png","image_512":"https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-512.png","status_text_canonical":"","team":"T04KX3KDDU6"},"is_bot":false,"is_app_user":false,"updated":1675090815,"is_forgotten":true,"is_invited_user":true},"emitted_at":1695766576251} -{"stream":"users","data":{"id":"U04LMS8F7JM","team_id":"T04KX3KDDU6","name":"deactivateduser421996","deleted":true,"profile":{"title":"","phone":"","skype":"","real_name":"Deactivated User","real_name_normalized":"Deactivated User","display_name":"deactivateduser","display_name_normalized":"deactivateduser","fields":null,"status_text":"","status_emoji":"","status_emoji_display_info":[],"status_expiration":0,"avatar_hash":"g849cc56ed76","huddle_state":"default_unset","first_name":"Deactivated","last_name":"User","image_24":"https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-24.png","image_32":"https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-32.png","image_48":"https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-48.png","image_72":"https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-72.png","image_192":"https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-192.png","image_512":"https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-512.png","status_text_canonical":"","team":"T04KX3KDDU6"},"is_bot":false,"is_app_user":false,"updated":1681811683,"is_forgotten":true,"is_invited_user":true},"emitted_at":1695766576252} -{"stream":"users","data":{"id": "U04LY6NARHU", "team_id": "T04KX3KDDU6", "name": "user1.sample", "deleted": false, "color": "684b6c", "real_name": "User1 Sample", "tz": "Europe/Helsinki", "tz_label": "Eastern European Time", "tz_offset": 7200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "User1 Sample", "real_name_normalized": "User1 Sample", "display_name": "User1 Sample", "display_name_normalized": "User1 Sample", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g76d12585ef1", "first_name": "User1", "last_name": "Sample", "image_24": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-24.png", "image_32": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-32.png", "image_48": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-48.png", "image_72": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-72.png", "image_192": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-192.png", "image_512": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1675090572, "is_email_confirmed": true, "has_2fa": false, "who_can_share_contact_card": "EVERYONE"},"emitted_at":1695766576252} -{"stream":"users","data":{"id": "U04M23SBJGM", "team_id": "T04KX3KDDU6", "name": "user2.sample.airbyte", "deleted": false, "color": "5b89d5", "real_name": "User2 Sample", "tz": "Europe/Helsinki", "tz_label": "Eastern European Time", "tz_offset": 7200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "User2 Sample", "real_name_normalized": "User2 Sample", "display_name": "User2 Sample", "display_name_normalized": "User2 Sample", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "gce662542f72", "first_name": "User2", "last_name": "Sample", "image_24": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-24.png", "image_32": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-32.png", "image_48": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-48.png", "image_72": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-72.png", "image_192": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-192.png", "image_512": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1675092508, "is_email_confirmed": true, "has_2fa": false, "who_can_share_contact_card": "EVERYONE"},"emitted_at":1695766576252} +{"stream": "users", "data": {"id": "USLACKBOT", "team_id": "T04KX3KDDU6", "name": "slackbot", "deleted": false, "color": "757575", "real_name": "Slackbot", "tz": "America/Los_Angeles", "tz_label": "Pacific Standard Time", "tz_offset": -28800, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Slackbot", "real_name_normalized": "Slackbot", "display_name": "Slackbot", "display_name_normalized": "Slackbot", "fields": {}, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "sv41d8cd98f0", "always_active": true, "first_name": "slackbot", "last_name": "", "image_24": "https://a.slack-edge.com/80588/img/slackbot_24.png", "image_32": "https://a.slack-edge.com/80588/img/slackbot_32.png", "image_48": "https://a.slack-edge.com/80588/img/slackbot_48.png", "image_72": "https://a.slack-edge.com/80588/img/slackbot_72.png", "image_192": "https://a.slack-edge.com/80588/marketing/img/avatars/slackbot/avatar-slackbot.png", "image_512": "https://a.slack-edge.com/80588/img/slackbot_512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 0, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1699645354906} +{"stream": "users", "data": {"id": "U04KUMXNYMV", "team_id": "T04KX3KDDU6", "name": "deactivateduser693438", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-24.png", "image_32": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-32.png", "image_48": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-48.png", "image_72": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-72.png", "image_192": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-192.png", "image_512": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090804, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354909} +{"stream": "users", "data": {"id": "U04L2KY5CES", "team_id": "T04KX3KDDU6", "name": "deactivateduser686066", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-24.png", "image_32": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-32.png", "image_48": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-48.png", "image_72": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-72.png", "image_192": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-192.png", "image_512": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090785, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354909} +{"stream": "users", "data": {"id": "U04L2LC770E", "team_id": "T04KX3KDDU6", "name": "deactivateduser521176", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-24.png", "image_32": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-32.png", "image_48": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-48.png", "image_72": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-72.png", "image_192": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-192.png", "image_512": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090821, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354910} +{"stream": "users", "data": {"id": "U04L69BPZFX", "team_id": "T04KX3KDDU6", "name": "deactivateduser839125", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-24.png", "image_32": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-32.png", "image_48": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-48.png", "image_72": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-72.png", "image_192": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-192.png", "image_512": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1681811889, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354910} +{"stream": "users", "data": {"id": "U04L94Y2JPM", "team_id": "T04KX3KDDU6", "name": "deactivateduser962255", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-24.png", "image_32": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-32.png", "image_48": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-48.png", "image_72": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-72.png", "image_192": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-192.png", "image_512": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090815, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354911} +{"stream": "users", "data": {"id": "U04LMS8F7JM", "team_id": "T04KX3KDDU6", "name": "deactivateduser421996", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-24.png", "image_32": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-32.png", "image_48": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-48.png", "image_72": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-72.png", "image_192": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-192.png", "image_512": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1681811683, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354911} +{"stream": "users", "data": {"id": "U04LY6NARHU", "team_id": "T04KX3KDDU6", "name": "user1.sample", "deleted": false, "color": "684b6c", "real_name": "User1 Sample", "tz": "Europe/Helsinki", "tz_label": "Eastern European Time", "tz_offset": 7200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "User1 Sample", "real_name_normalized": "User1 Sample", "display_name": "User1 Sample", "display_name_normalized": "User1 Sample", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g76d12585ef1", "first_name": "User1", "last_name": "Sample", "image_24": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-24.png", "image_32": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-32.png", "image_48": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-48.png", "image_72": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-72.png", "image_192": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-192.png", "image_512": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1675090572, "is_email_confirmed": true, "has_2fa": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1699645354911} +{"stream": "users", "data": {"id": "U04M23SBJGM", "team_id": "T04KX3KDDU6", "name": "user2.sample.airbyte", "deleted": false, "color": "5b89d5", "real_name": "User2 Sample", "tz": "Europe/Helsinki", "tz_label": "Eastern European Time", "tz_offset": 7200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "User2 Sample", "real_name_normalized": "User2 Sample", "display_name": "User2 Sample", "display_name_normalized": "User2 Sample", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "gce662542f72", "first_name": "User2", "last_name": "Sample", "image_24": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-24.png", "image_32": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-32.png", "image_48": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-48.png", "image_72": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-72.png", "image_192": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-192.png", "image_512": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1675092508, "is_email_confirmed": true, "has_2fa": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1699645354912} From 7973311383717f56ea58385944c918a3d0e2edb3 Mon Sep 17 00:00:00 2001 From: Augustin Date: Mon, 13 Nov 2023 09:31:58 +0100 Subject: [PATCH 014/124] airbyte-ci: re-enable caching on source-postgres CI (#32427) --- .../workflows/tmp-source-postgres-test.yml | 80 ------------------- airbyte-ci/connectors/pipelines/README.md | 1 + .../pipelines/airbyte_ci/steps/gradle.py | 3 - .../connectors/pipelines/pipelines/hacks.py | 31 ------- .../connectors/pipelines/pyproject.toml | 2 +- 5 files changed, 2 insertions(+), 115 deletions(-) delete mode 100644 .github/workflows/tmp-source-postgres-test.yml diff --git a/.github/workflows/tmp-source-postgres-test.yml b/.github/workflows/tmp-source-postgres-test.yml deleted file mode 100644 index 4e8d2f82959f..000000000000 --- a/.github/workflows/tmp-source-postgres-test.yml +++ /dev/null @@ -1,80 +0,0 @@ -# This workflows runs airbyte-ci connectors --name=source-postgres test -# We created this in the context of our project to improve CI performances for this connector -# It's made to be triggered manually from the GitHub UI -# It will allow us to collect performance metrics outside of the context of nightly builds -# And also to use different runner types (e.g. conn-prod-xxlarge-runner) to test the connector with various resources. - -name: source-postgres ci - for testing only - -on: - schedule: - # Runs four times a day to observe variance - - cron: "0 6,12,16,20 * * *" - workflow_dispatch: - inputs: - runner: - description: "The runner to use for this job" - default: "conn-prod-xlarge-runner" - -jobs: - source_postgres_ci: - name: Source Postgres CI on ${{ inputs.runner || 'conn-prod-xlarge-runner'}} - runs-on: ${{ inputs.runner || 'conn-prod-xlarge-runner'}} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - - name: Extract branch name - shell: bash - run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT - id: extract_branch - - name: Check if PR is from a fork - if: github.event_name == 'pull_request' - shell: bash - run: | - if [ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]; then - echo "PR is from a fork. Exiting workflow..." - exit 78 - fi - - name: Docker login - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_PASSWORD }} - - name: Get start timestamp - id: get-start-timestamp - shell: bash - run: echo "name=start-timestamp=$(date +%s)" >> $GITHUB_OUTPUT - - name: Install Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: "3.10" - token: ${{ secrets.GITHUB_TOKEN }} - - name: Install ci-connector-ops package - shell: bash - run: | - pip install pipx - pipx ensurepath - pipx install airbyte-ci/connectors/pipelines/ - - name: Run airbyte-ci - shell: bash - run: | - export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock" - airbyte-ci --is-ci --show-dagger-logs --gha-workflow-run-id=${{ github.run_id }} connectors --name=source-postgres test - env: - _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k" - CI_CONTEXT: "master" - CI_GIT_BRANCH: ${{ github.head_ref }} - CI_GIT_REVISION: ${{ github.sha }} - CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }} - CI_REPORT_BUCKET_NAME: airbyte-ci-reports-multi - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - SENTRY_DSN: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} - DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} - # S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - # S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - CI: "True" diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 841b534368b4..11c642616f63 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -408,6 +408,7 @@ This command runs the Python tests for a airbyte-ci poetry package. ## Changelog | Version | PR | Description | | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| 2.5.9 | [#32427](https://github.com/airbytehq/airbyte/pull/32427) | Re-enable caching for source-postgres | | 2.5.8 | [#32402](https://github.com/airbytehq/airbyte/pull/32402) | Set Dagger Cloud token for airbyters only | | 2.5.7 | [#31628](https://github.com/airbytehq/airbyte/pull/31628) | Add ClickPipelineContext class | | 2.5.6 | [#32139](https://github.com/airbytehq/airbyte/pull/32139) | Test coverage report on Python connector UnitTest. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py index 7dedcf610959..bcdc19a2e307 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py @@ -7,7 +7,6 @@ import pipelines.dagger.actions.system.docker from dagger import CacheSharingMode, CacheVolume -from pipelines import hacks from pipelines.consts import AMAZONCORRETTO_IMAGE from pipelines.dagger.actions import secrets from pipelines.helpers.utils import sh_dash_c @@ -125,8 +124,6 @@ async def _run(self) -> StepResult: .with_env_variable("TESTCONTAINERS_RYUK_DISABLED", "true") # Set the current working directory. .with_workdir("/airbyte") - # TODO: remove this once we finish the project to boost source-postgres CI performance. - .with_env_variable("CACHEBUSTER", hacks.get_cachebuster(self.context, self.logger)) ) # Augment the base container with S3 build cache secrets when available. diff --git a/airbyte-ci/connectors/pipelines/pipelines/hacks.py b/airbyte-ci/connectors/pipelines/pipelines/hacks.py index 2a39c81c983d..4a65628eaea4 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/hacks.py +++ b/airbyte-ci/connectors/pipelines/pipelines/hacks.py @@ -6,7 +6,6 @@ from __future__ import annotations -from logging import Logger from typing import TYPE_CHECKING, Callable, List from pipelines import consts @@ -71,33 +70,3 @@ def never_fail_exec_inner(container: Container): return container.with_exec(["sh", "-c", f"{' '.join(command)}; echo $? > /exit_code"], skip_entrypoint=True) return never_fail_exec_inner - - -# We want to invalidate the persisted dagger cache and gradle cache for source-postgres. -# We do it in the context of a project to boost the CI speed for this connector. -# Invalidating the cache on every run will help us gather unbiased metrics on the CI speed. -# This should be removed once the project is over. -CONNECTORS_WITHOUT_CACHING = [ - "source-postgres", -] - - -def get_cachebuster(context: ConnectorContext, logger: Logger) -> str: - """ - This function will return a semi-static cachebuster value for connectors in CONNECTORS_WITHOUT_CACHING and a static value for all other connectors. - By semi-static I mean that the value (the pipeline start time) will change on each pipeline execution but will be the same for all the steps of the pipeline. - It ensures we do not use the remotely persisted dagger cache but we still benefit from the buildkit layer caching inside the pipeline execution. - This hack is useful to collect unbiased metrics on the CI speed for connectors in CONNECTORS_WITHOUT_CACHING. - - When the cachebuster value is static it won't invalidate the dagger cache because it's the same value as the previous run: no layer will be rebuilt. - When the cachebuster value is changed it will invalidate the dagger cache because it's a different value than the previous run: all downstream layers will be rebuilt. - - Returns: - str: The cachebuster value. - """ - if context.connector.technical_name in CONNECTORS_WITHOUT_CACHING: - logger.warning( - f"Invalidating the persisted dagger cache for {context.connector.technical_name}. Only used in the context of the CI performance improvements project for {context.connector.technical_name}." - ) - return str(context.pipeline_start_timestamp) - return "0" diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 8b0882b08d6f..3824b117b447 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "2.5.8" +version = "2.5.9" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] From 87f51fac42b29b55addc9c5afa3c60bca75c48c9 Mon Sep 17 00:00:00 2001 From: Anton Karpets Date: Mon, 13 Nov 2023 14:18:03 +0200 Subject: [PATCH 015/124] =?UTF-8?q?=F0=9F=90=9BSource=20Instagram:=20skip?= =?UTF-8?q?=20reading=20MediaInsights=20if=20error=20code=2010=20received?= =?UTF-8?q?=20(#32245)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Anatolii Yatsuk --- .../connectors/source-instagram/metadata.yaml | 2 +- .../source_instagram/streams.py | 17 +++++++++++++---- .../source-instagram/unit_tests/test_streams.py | 17 ++++++++++++++++- docs/integrations/sources/instagram.md | 3 ++- 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-instagram/metadata.yaml b/airbyte-integrations/connectors/source-instagram/metadata.yaml index 010093c987a2..c53b474362b2 100644 --- a/airbyte-integrations/connectors/source-instagram/metadata.yaml +++ b/airbyte-integrations/connectors/source-instagram/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6acf6b55-4f1e-4fca-944e-1a3caef8aba8 - dockerImageTag: 1.0.12 + dockerImageTag: 1.0.13 dockerRepository: airbyte/source-instagram githubIssueLabel: source-instagram icon: instagram.svg diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py index 72239476831f..3d81c18e9fbd 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py @@ -380,16 +380,25 @@ def _get_insights(self, item, account_id) -> Optional[MutableMapping[str, Any]]: insights = item.get_insights(params={"metric": metrics}) return {record.get("name"): record.get("values")[0]["value"] for record in insights} except FacebookRequestError as error: + error_code = error.api_error_code() + error_subcode = error.api_error_subcode() + error_message = error.api_error_message() + # An error might occur if the media was posted before the most recent time that # the user's account was converted to a business account from a personal account - if error.api_error_subcode() == 2108006: - details = error.body().get("error", {}).get("error_user_title") or error.api_error_message() + if error_subcode == 2108006: + details = error.body().get("error", {}).get("error_user_title") or error_message self.logger.error(f"Insights error for business_account_id {account_id}: {details}") # We receive all Media starting from the last one, and if on the next Media we get an Insight error, # then no reason to make inquiries for each Media further, since they were published even earlier. return None - elif error.api_error_code() == 100 and error.api_error_subcode() == 33: - self.logger.error(f"Check provided permissions for {account_id}: {error.api_error_message()}") + elif ( + error_code == 100 + and error_subcode == 33 + or error_code == 10 + and error_message == "(#10) Application does not have permission for this action" + ): + self.logger.error(f"Check provided permissions for {account_id}: {error_message}") return None raise error diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py index 74ca5fd2837d..cad75b446d38 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py @@ -76,7 +76,7 @@ def test_media_insights_read(api, user_stories_data, user_media_insights_data, r def test_media_insights_read_error(api, requests_mock): test_id = "test_id" stream = MediaInsights(api=api) - media_response = [{"id": "test_id"}, {"id": "test_id_2"}, {"id": "test_id_3"}, {"id": "test_id_4"}] + media_response = [{"id": "test_id"}, {"id": "test_id_2"}, {"id": "test_id_3"}, {"id": "test_id_4"}, {"id": "test_id_5"}] requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{test_id}/media", json={"data": media_response}) media_insights_response_test_id = { @@ -134,6 +134,21 @@ def test_media_insights_read_error(api, requests_mock): "GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/test_id_4/insights", json=media_insights_response_test_id_4 ) + error_response_wrong_permissions_code_10 = { + "error": { + "message": "(#10) Application does not have permission for this action", + "type": "OAuthException", + "code": 10, + "fbtrace_id": "fake_trace_id", + } + } + requests_mock.register_uri( + "GET", + FacebookSession.GRAPH + f"/{FB_API_VERSION}/test_id_5/insights", + json=error_response_wrong_permissions_code_10, + status_code=400, + ) + records = read_full_refresh(stream) expected_records = [ {"business_account_id": "test_id", "id": "test_id", "impressions": 264, "page_id": "act_unknown_account"}, diff --git a/docs/integrations/sources/instagram.md b/docs/integrations/sources/instagram.md index 99cf3aa15cb2..82f96d6fbb3b 100644 --- a/docs/integrations/sources/instagram.md +++ b/docs/integrations/sources/instagram.md @@ -86,7 +86,8 @@ AirbyteRecords are required to conform to the [Airbyte type](https://docs.airbyt ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------ | +|:--------|:-----------| :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------ | +| 1.0.13 | 2023-11-10 | [32245](https://github.com/airbytehq/airbyte/pull/32245) | Add skipping reading MediaInsights stream if an error code 10 is received | | 1.0.12 | 2023-11-07 | [32200](https://github.com/airbytehq/airbyte/pull/32200) | The backoff strategy has been updated to make some errors retriable | | 1.0.11 | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes | | 1.0.10 | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references | From a05a2930d69eaa97794a803b60f54cc8f4067880 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Mon, 13 Nov 2023 15:15:15 +0200 Subject: [PATCH 016/124] :bug: Source Stripe: fix multiple BankAccounts issues (#32146) Co-authored-by: davydov-d --- .../connectors/source-stripe/metadata.yaml | 2 +- .../source-stripe/source_stripe/source.py | 54 +-- .../source-stripe/source_stripe/streams.py | 87 ++-- .../source-stripe/unit_tests/conftest.py | 53 +-- .../unit_tests/test_availability_strategy.py | 12 +- .../source-stripe/unit_tests/test_streams.py | 377 +++++++++++------- docs/integrations/sources/stripe.md | 3 +- 7 files changed, 317 insertions(+), 271 deletions(-) diff --git a/airbyte-integrations/connectors/source-stripe/metadata.yaml b/airbyte-integrations/connectors/source-stripe/metadata.yaml index 8f40faed76db..bc5f90db6dea 100644 --- a/airbyte-integrations/connectors/source-stripe/metadata.yaml +++ b/airbyte-integrations/connectors/source-stripe/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: e094cb9a-26de-4645-8761-65c0c425d1de - dockerImageTag: 4.5.1 + dockerImageTag: 4.5.2 dockerRepository: airbyte/source-stripe documentationUrl: https://docs.airbyte.com/integrations/sources/stripe githubIssueLabel: source-stripe diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py index d8193c1796f9..9ebd06df8026 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import os from typing import Any, List, Mapping, MutableMapping, Optional, Tuple import pendulum @@ -21,7 +22,6 @@ CreatedCursorIncrementalStripeStream, CustomerBalanceTransactions, Events, - FilteringRecordExtractor, IncrementalStripeStream, Persons, SetupAttempts, @@ -33,6 +33,8 @@ ) _MAX_CONCURRENCY = 3 +_CACHE_DISABLED = os.environ.get("CACHE_DISABLED") +USE_CACHE = not _CACHE_DISABLED class SourceStripe(AbstractSource): @@ -97,6 +99,18 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> return False, str(e) return True, None + @staticmethod + def customers(**args): + # The Customers stream is instantiated in a dedicated method to allow parametrization and avoid duplicated code. + # It can be used with and without expanded items (as an independent stream or as a parent stream for other streams). + return IncrementalStripeStream( + name="customers", + path="customers", + use_cache=USE_CACHE, + event_types=["customer.created", "customer.updated", "customer.deleted"], + **args, + ) + def streams(self, config: Mapping[str, Any]) -> List[Stream]: config = self.validate_and_fill_with_defaults(config) authenticator = TokenAuthenticator(config["client_secret"]) @@ -110,7 +124,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: subscriptions = IncrementalStripeStream( name="subscriptions", path="subscriptions", - use_cache=True, + use_cache=USE_CACHE, extra_request_params={"status": "all"}, event_types=[ "customer.subscription.created", @@ -127,9 +141,9 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: subscription_items = StripeLazySubStream( name="subscription_items", path="subscription_items", - extra_request_params=lambda self, *args, stream_slice, **kwargs: {"subscription": stream_slice[self.parent_id]}, + extra_request_params=lambda self, stream_slice, *args, **kwargs: {"subscription": stream_slice[self.parent_id]}, parent=subscriptions, - use_cache=True, + use_cache=USE_CACHE, parent_id="subscription_id", sub_items_attr="items", **args, @@ -137,28 +151,21 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: transfers = IncrementalStripeStream( name="transfers", path="transfers", - use_cache=True, + use_cache=USE_CACHE, event_types=["transfer.created", "transfer.reversed", "transfer.updated"], **args, ) application_fees = IncrementalStripeStream( name="application_fees", path="application_fees", - use_cache=True, + use_cache=USE_CACHE, event_types=["application_fee.created", "application_fee.refunded"], **args, ) - customers = IncrementalStripeStream( - name="customers", - path="customers", - use_cache=True, - event_types=["customer.created", "customer.updated", "customer.deleted"], - **args, - ) invoices = IncrementalStripeStream( name="invoices", path="invoices", - use_cache=True, + use_cache=USE_CACHE, event_types=[ "invoice.created", "invoice.finalization_failed", @@ -185,7 +192,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: event_types=["account.external_account.created", "account.external_account.updated", "account.external_account.deleted"], legacy_cursor_field=None, extra_request_params={"object": "card"}, - record_extractor=FilteringRecordExtractor("updated", None, "card"), + response_filter=lambda record: record["object"] == "card", **args, ), UpdatedCursorIncrementalStripeStream( @@ -194,12 +201,12 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: event_types=["account.external_account.created", "account.external_account.updated", "account.external_account.deleted"], legacy_cursor_field=None, extra_request_params={"object": "bank_account"}, - record_extractor=FilteringRecordExtractor("updated", None, "bank_account"), + response_filter=lambda record: record["object"] == "bank_account", **args, ), Persons(**args), SetupAttempts(**incremental_args), - StripeStream(name="accounts", path="accounts", use_cache=True, **args), + StripeStream(name="accounts", path="accounts", use_cache=USE_CACHE, **args), CreatedCursorIncrementalStripeStream(name="shipping_rates", path="shipping_rates", **incremental_args), CreatedCursorIncrementalStripeStream(name="balance_transactions", path="balance_transactions", **incremental_args), CreatedCursorIncrementalStripeStream(name="files", path="files", **incremental_args), @@ -207,7 +214,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: UpdatedCursorIncrementalStripeStream( name="checkout_sessions", path="checkout/sessions", - use_cache=True, + use_cache=USE_CACHE, legacy_cursor_field="expires_at", event_types=[ "checkout.session.async_payment_failed", @@ -246,7 +253,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: event_types=["issuing_authorization.created", "issuing_authorization.request", "issuing_authorization.updated"], **args, ), - customers, + self.customers(**args), IncrementalStripeStream( name="cardholders", path="issuing/cardholders", @@ -334,7 +341,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: ), transfers, IncrementalStripeStream( - name="refunds", path="refunds", use_cache=True, event_types=["refund.created", "refund.updated"], **args + name="refunds", path="refunds", use_cache=USE_CACHE, event_types=["refund.created", "refund.updated"], **args ), IncrementalStripeStream( name="payment_intents", @@ -397,19 +404,18 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: UpdatedCursorIncrementalStripeLazySubStream( name="bank_accounts", path=lambda self, stream_slice, *args, **kwargs: f"customers/{stream_slice[self.parent_id]}/sources", - parent=customers, + parent=self.customers(expand_items=["data.sources"], **args), event_types=["customer.source.created", "customer.source.expiring", "customer.source.updated", "customer.source.deleted"], legacy_cursor_field=None, parent_id="customer_id", sub_items_attr="sources", - response_filter={"attr": "object", "value": "bank_account"}, extra_request_params={"object": "bank_account"}, - record_extractor=FilteringRecordExtractor("updated", None, "bank_account"), + response_filter=lambda record: record["object"] == "bank_account", **args, ), StripeLazySubStream( name="invoice_line_items", - path=lambda self, *args, stream_slice, **kwargs: f"invoices/{stream_slice[self.parent_id]}/lines", + path=lambda self, stream_slice, *args, **kwargs: f"invoices/{stream_slice[self.parent_id]}/lines", parent=invoices, parent_id="invoice_id", sub_items_attr="lines", diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py index ad7357658a34..460059716a75 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py @@ -4,6 +4,7 @@ import copy import math +import os from abc import ABC, abstractmethod from itertools import chain from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union @@ -18,42 +19,47 @@ from source_stripe.availability_strategy import StripeAvailabilityStrategy, StripeSubStreamAvailabilityStrategy STRIPE_API_VERSION = "2022-11-15" +CACHE_DISABLED = os.environ.get("CACHE_DISABLED") +USE_CACHE = not CACHE_DISABLED class IRecordExtractor(ABC): @abstractmethod - def extract_records(self, response: requests.Response) -> Iterable[Mapping]: + def extract_records(self, records: Iterable[MutableMapping]) -> Iterable[Mapping]: pass class DefaultRecordExtractor(IRecordExtractor): - def extract_records(self, response: requests.Response) -> Iterable[MutableMapping]: - response_json = response.json() - yield from response_json.get("data", []) + def __init__(self, response_filter: Optional[Callable] = None): + self._response_filter = response_filter or (lambda x: x) + + def extract_records(self, records: Iterable[MutableMapping]) -> Iterable[MutableMapping]: + yield from filter(self._response_filter, records) class EventRecordExtractor(DefaultRecordExtractor): - def __init__(self, cursor_field: str): + def __init__(self, cursor_field: str, response_filter: Optional[Callable] = None): + super().__init__(response_filter) self.cursor_field = cursor_field - def extract_records(self, response: requests.Response) -> Iterable[MutableMapping]: - records = super().extract_records(response) - # set the record updated date = date of event creation + def extract_records(self, records: Iterable[MutableMapping]) -> Iterable[MutableMapping]: for record in records: item = record["data"]["object"] item[self.cursor_field] = record["created"] if record["type"].endswith(".deleted"): item["is_deleted"] = True - yield item + if self._response_filter(item): + yield item class UpdatedCursorIncrementalRecordExtractor(DefaultRecordExtractor): - def __init__(self, cursor_field: str, legacy_cursor_field: Optional[str]): + def __init__(self, cursor_field: str, legacy_cursor_field: Optional[str], response_filter: Optional[Callable] = None): + super().__init__(response_filter) self.cursor_field = cursor_field self.legacy_cursor_field = legacy_cursor_field - def extract_records(self, response: requests.Response) -> Iterable[MutableMapping]: - records = super().extract_records(response) + def extract_records(self, records: Iterable[MutableMapping]) -> Iterable[MutableMapping]: + records = super().extract_records(records) for record in records: if self.cursor_field in record: yield record @@ -66,18 +72,6 @@ def extract_records(self, response: requests.Response) -> Iterable[MutableMappin yield record | {self.cursor_field: current_cursor_value} -class FilteringRecordExtractor(UpdatedCursorIncrementalRecordExtractor): - def __init__(self, cursor_field: str, legacy_cursor_field: Optional[str], object_type: str): - super().__init__(cursor_field, legacy_cursor_field) - self.object_type = object_type - - def extract_records(self, response: requests.Response) -> Iterable[MutableMapping]: - records = super().extract_records(response) - for record in records: - if record["object"] == self.object_type: - yield record - - class StripeStream(HttpStream, ABC): url_base = "https://api.stripe.com/v1/" DEFAULT_SLICE_RANGE = 365 @@ -131,13 +125,14 @@ def __init__( use_cache: bool = False, expand_items: Optional[List[str]] = None, extra_request_params: Optional[Union[Mapping[str, Any], Callable]] = None, + response_filter: Optional[Callable] = None, primary_key: Optional[str] = "id", **kwargs, ): self.account_id = account_id self.start_date = start_date self.slice_range = slice_range or self.DEFAULT_SLICE_RANGE - self._record_extractor = record_extractor or DefaultRecordExtractor() + self._record_extractor = record_extractor or DefaultRecordExtractor(response_filter) self._name = name self._path = path self._use_cache = use_cache @@ -159,7 +154,10 @@ def request_params( next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: # Stripe default pagination is 10, max is 100 - params = {"limit": 100, **self.extra_request_params(stream_state, stream_slice, next_page_token)} + params = { + "limit": 100, + **self.extra_request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + } if self.expand_items: params["expand[]"] = self.expand_items # Handle pagination by inserting the next page's token in the request parameters @@ -176,7 +174,7 @@ def parse_response( stream_slice: Optional[Mapping[str, Any]] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Iterable[Mapping[str, Any]]: - yield from self.record_extractor.extract_records(response) + yield from self.record_extractor.extract_records(response.json().get("data", [])) def request_headers(self, **kwargs) -> Mapping[str, Any]: headers = {"Stripe-Version": STRIPE_API_VERSION} @@ -321,12 +319,15 @@ def __init__( legacy_cursor_field: Optional[str] = "created", event_types: Optional[List[str]] = None, record_extractor: Optional[IRecordExtractor] = None, + response_filter: Optional[Callable] = None, **kwargs, ): self._event_types = event_types self._cursor_field = cursor_field self._legacy_cursor_field = legacy_cursor_field - record_extractor = record_extractor or UpdatedCursorIncrementalRecordExtractor(self.cursor_field, self.legacy_cursor_field) + record_extractor = record_extractor or UpdatedCursorIncrementalRecordExtractor( + self.cursor_field, self.legacy_cursor_field, response_filter + ) super().__init__(*args, record_extractor=record_extractor, **kwargs) # `lookback_window_days` is hardcoded as it does not make any sense to re-export events, # as each event holds the latest value of a record. @@ -340,7 +341,7 @@ def __init__( slice_range=self.slice_range, event_types=self.event_types, cursor_field=self.cursor_field, - record_extractor=EventRecordExtractor(cursor_field=self.cursor_field), + record_extractor=EventRecordExtractor(cursor_field=self.cursor_field, response_filter=response_filter), ) def update_cursor_field(self, stream_state: MutableMapping[str, Any]) -> MutableMapping[str, Any]: @@ -486,7 +487,7 @@ def checkout_session(self): return UpdatedCursorIncrementalStripeStream( name="checkout_sessions", path="checkout/sessions", - use_cache=True, + use_cache=USE_CACHE, legacy_cursor_field="expires_at", event_types=[ "checkout.session.async_payment_failed", @@ -518,7 +519,10 @@ def request_params( next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: # override to not refer to slice values - params = {"limit": 100, **self.extra_request_params(stream_state, stream_slice, next_page_token)} + params = { + "limit": 100, + **self.extra_request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + } if self.expand_items: params["expand[]"] = self.expand_items if next_page_token: @@ -574,7 +578,7 @@ def customers(self) -> IncrementalStripeStream: return IncrementalStripeStream( name="customers", path="customers", - use_cache=True, + use_cache=USE_CACHE, event_types=["customer.created", "customer.updated", "customer.deleted"], authenticator=self.authenticator, account_id=self.account_id, @@ -658,7 +662,7 @@ class Persons(UpdatedCursorIncrementalStripeStream, HttpSubStream): event_types = ["person.created", "person.updated", "person.deleted"] def __init__(self, *args, **kwargs): - parent = StripeStream(*args, name="accounts", path="accounts", use_cache=True, **kwargs) + parent = StripeStream(*args, name="accounts", path="accounts", use_cache=USE_CACHE, **kwargs) super().__init__(*args, parent=parent, **kwargs) def path(self, stream_slice: Mapping[str, Any] = None, **kwargs): @@ -717,10 +721,6 @@ class StripeLazySubStream(StripeStream, HttpSubStream): } """ - @property - def filter(self) -> Optional[Mapping[str, Any]]: - return self._filter - @property def add_parent_id(self) -> bool: return self._add_parent_id @@ -743,14 +743,12 @@ def sub_items_attr(self) -> str: def __init__( self, *args, - response_filter: Optional[Mapping[str, Any]] = None, add_parent_id: bool = False, parent_id: Optional[str] = None, sub_items_attr: Optional[str] = None, **kwargs, ): super().__init__(*args, **kwargs) - self._filter = response_filter self._add_parent_id = add_parent_id self._parent_id = parent_id self._sub_items_attr = sub_items_attr @@ -774,9 +772,7 @@ def read_records(self, sync_mode: SyncMode, stream_slice: Optional[Mapping[str, if not items_obj: return - items = items_obj.get("data", []) - if self.filter: - items = [i for i in items if i.get(self.filter["attr"]) == self.filter["value"]] + items = list(self.record_extractor.extract_records(items_obj.get("data", []))) # get next pages items_next_pages = [] @@ -811,7 +807,7 @@ def __init__( parent_id: Optional[str] = None, add_parent_id: bool = False, sub_items_attr: Optional[str] = None, - response_filter: Optional[Mapping[str, Any]] = None, + response_filter: Optional[Callable] = None, **kwargs, ): super().__init__(*args, **kwargs) @@ -821,6 +817,7 @@ def __init__( cursor_field=cursor_field, legacy_cursor_field=legacy_cursor_field, event_types=event_types, + response_filter=response_filter, **kwargs, ) self.lazy_substream = StripeLazySubStream( @@ -829,7 +826,9 @@ def __init__( parent_id=parent_id, add_parent_id=add_parent_id, sub_items_attr=sub_items_attr, - response_filter=response_filter, + record_extractor=UpdatedCursorIncrementalRecordExtractor( + cursor_field=cursor_field, legacy_cursor_field=legacy_cursor_field, response_filter=response_filter + ), **kwargs, ) self._parent_stream = None diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py b/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py index 9877fb85efc8..5eb8e515068a 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py @@ -6,9 +6,8 @@ import pytest from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -from source_stripe.streams import IncrementalStripeStream, StripeLazySubStream -os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" +os.environ["CACHE_DISABLED"] = "true" @pytest.fixture(name="config") @@ -34,46 +33,16 @@ def incremental_args_fixture(stream_args): return {"lookback_window_days": 14, **stream_args} -@pytest.fixture(name="invoices") -def invoices_fixture(stream_args): - def mocker(args=stream_args): - return IncrementalStripeStream( - name="invoices", - path="invoices", - use_cache=False, - event_types=[ - "invoice.created", - "invoice.finalization_failed", - "invoice.finalized", - "invoice.marked_uncollectible", - "invoice.paid", - "invoice.payment_action_required", - "invoice.payment_failed", - "invoice.payment_succeeded", - "invoice.sent", - "invoice.upcoming", - "invoice.updated", - "invoice.voided", - ], - **args, - ) +@pytest.fixture() +def stream_by_name(config): + # use local import in favour of global because we need to make imports after setting the env variables + from source_stripe.source import SourceStripe - return mocker - - -@pytest.fixture(name="invoice_line_items") -def invoice_line_items_fixture(invoices, stream_args): - parent_stream = invoices() - - def mocker(args=stream_args, parent_stream=parent_stream): - return StripeLazySubStream( - name="invoice_line_items", - path=lambda self, *args, stream_slice, **kwargs: f"invoices/{stream_slice[self.parent_id]}/lines", - parent=parent_stream, - parent_id="invoice_id", - sub_items_attr="lines", - add_parent_id=True, - **args, - ) + def mocker(stream_name, source_config=config): + source = SourceStripe() + streams = source.streams(source_config) + for stream in streams: + if stream.name == stream_name: + return stream return mocker diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_availability_strategy.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_availability_strategy.py index d9382de2be1f..0f747acac434 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_availability_strategy.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_availability_strategy.py @@ -87,13 +87,13 @@ def test_traverse_over_substreams_failure(mocker): assert id(check_availability_mock.call_args_list[1].args[0]) == id(child_1) -def test_substream_availability(mocker, invoice_line_items): +def test_substream_availability(mocker, stream_by_name): check_availability_mock = mocker.MagicMock() check_availability_mock.return_value = (True, None) mocker.patch( "airbyte_cdk.sources.streams.http.availability_strategy.HttpAvailabilityStrategy.check_availability", check_availability_mock ) - stream = invoice_line_items() + stream = stream_by_name("invoice_line_items") is_available, reason = stream.availability_strategy.check_availability(stream, mocker.Mock(), mocker.Mock()) assert is_available and reason is None @@ -102,13 +102,13 @@ def test_substream_availability(mocker, invoice_line_items): assert isinstance(check_availability_mock.call_args_list[1].args[0], StripeLazySubStream) -def test_substream_availability_no_parent(mocker, invoice_line_items): +def test_substream_availability_no_parent(mocker, stream_by_name): check_availability_mock = mocker.MagicMock() check_availability_mock.return_value = (True, None) mocker.patch( "airbyte_cdk.sources.streams.http.availability_strategy.HttpAvailabilityStrategy.check_availability", check_availability_mock ) - stream = invoice_line_items() + stream = stream_by_name("invoice_line_items") stream.parent = None stream.availability_strategy.check_availability(stream, mocker.Mock(), mocker.Mock()) @@ -117,8 +117,8 @@ def test_substream_availability_no_parent(mocker, invoice_line_items): assert isinstance(check_availability_mock.call_args_list[0].args[0], StripeLazySubStream) -def test_403_error_handling(invoices, requests_mock): - stream = invoices() +def test_403_error_handling(stream_by_name, requests_mock): + stream = stream_by_name("invoices") logger = logging.getLogger("airbyte") for error_code in STRIPE_ERROR_CODES: requests_mock.get(f"{stream.url_base}{stream.path()}", status_code=403, json={"error": {"code": f"{error_code}"}}) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py index 55cba5aa9aa5..143331e06992 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py @@ -2,180 +2,178 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from urllib.parse import urlencode + import freezegun import pendulum import pytest -from source_stripe.streams import ( - CheckoutSessionsLineItems, - CreatedCursorIncrementalStripeStream, - CustomerBalanceTransactions, - FilteringRecordExtractor, - IncrementalStripeStream, - Persons, - SetupAttempts, - StripeStream, - UpdatedCursorIncrementalStripeLazySubStream, - UpdatedCursorIncrementalStripeStream, -) - - -@pytest.fixture() -def accounts(stream_args): - def mocker(args=stream_args): - return StripeStream(name="accounts", path="accounts", **args) - - return mocker - - -@pytest.fixture() -def balance_transactions(incremental_stream_args): - def mocker(args=incremental_stream_args): - return CreatedCursorIncrementalStripeStream(name="balance_transactions", path="balance_transactions", **args) - - return mocker - - -@pytest.fixture() -def credit_notes(stream_args): - def mocker(args=stream_args): - return UpdatedCursorIncrementalStripeStream( - name="credit_notes", - path="credit_notes", - event_types=["credit_note.created", "credit_note.updated", "credit_note.voided"], - **args, - ) - - return mocker - - -@pytest.fixture() -def customers(stream_args): - def mocker(args=stream_args): - return IncrementalStripeStream( - name="customers", - path="customers", - use_cache=False, - event_types=["customer.created", "customer.updated"], - **args, - ) - - return mocker +from source_stripe.streams import CheckoutSessionsLineItems, CustomerBalanceTransactions, Persons, SetupAttempts -@pytest.fixture() -def bank_accounts(customers, stream_args): - def mocker(args=stream_args): - return UpdatedCursorIncrementalStripeLazySubStream( - name="bank_accounts", - path=lambda self, stream_slice, *args, **kwargs: f"customers/{stream_slice[self.parent_id]}/sources", - parent=customers(), - event_types=["customer.source.created", "customer.source.expiring", "customer.source.updated"], - legacy_cursor_field=None, - parent_id="customer_id", - sub_items_attr="sources", - response_filter={"attr": "object", "value": "bank_account"}, - extra_request_params={"object": "bank_account"}, - record_extractor=FilteringRecordExtractor("updated", None, "bank_account"), - **args, - ) - - return mocker - - -@pytest.fixture() -def external_bank_accounts(stream_args): - def mocker(args=stream_args): - return UpdatedCursorIncrementalStripeStream( - name="external_account_bank_accounts", - path=lambda self, *args, **kwargs: f"accounts/{self.account_id}/external_accounts", - event_types=["account.external_account.created", "account.external_account.updated"], - legacy_cursor_field=None, - extra_request_params={"object": "bank_account"}, - record_extractor=FilteringRecordExtractor("updated", None, "bank_account"), - **args, - ) - - return mocker +def read_from_stream(stream, sync_mode, state): + records = [] + for slice_ in stream.stream_slices(sync_mode=sync_mode, stream_state=state): + for record in stream.read_records(sync_mode=sync_mode, stream_slice=slice_, stream_state=state): + records.append(record) + return records -def test_request_headers(accounts): - stream = accounts() +def test_request_headers(stream_by_name): + stream = stream_by_name("accounts") headers = stream.request_headers() assert headers["Stripe-Version"] == "2022-11-15" -def test_lazy_sub_stream(requests_mock, invoice_line_items, invoices, stream_args): - # First initial request to parent stream - requests_mock.get( - "https://api.stripe.com/v1/invoices", - json={ +bank_accounts_full_refresh_test_case = ( + { + "https://api.stripe.com/v1/customers?expand%5B%5D=data.sources": { "has_more": False, "object": "list", - "url": "/v1/checkout/sessions", + "url": "/v1/customers", "data": [ { "created": 1641038947, - "customer": "cus_HezytZRkaQJC8W", - "id": "in_1KD6OVIEn5WyEQxn9xuASHsD", - "object": "invoice", + "id": "cus_HezytZRkaQJC8W", + "object": "customer", "total": 1, - "lines": { + "sources": { "data": [ { - "id": "il_1", - "object": "line_item", + "id": "cs_1", + "object": "card", }, { - "id": "il_2", - "object": "line_item", + "id": "cs_2", + "object": "bank_account", }, ], "has_more": True, "object": "list", - "total_count": 3, - "url": "/v1/invoices/in_1KD6OVIEn5WyEQxn9xuASHsD/lines", + "total_count": 4, + "url": "/v1/customers/cus_HezytZRkaQJC8W/sources", }, } ], }, - ) - - # Second pagination request to main stream - requests_mock.get( - "https://api.stripe.com/v1/invoices/in_1KD6OVIEn5WyEQxn9xuASHsD/lines", - json={ + "https://api.stripe.com/v1/customers/cus_HezytZRkaQJC8W/sources?object=bank_account&starting_after=cs_2": { "data": [ { - "id": "il_3", - "object": "line_item", + "id": "cs_3", + "object": "card", + }, + { + "id": "cs_4", + "object": "bank_account", }, ], "has_more": False, "object": "list", - "total_count": 3, - "url": "/v1/invoices/in_1KD6OVIEn5WyEQxn9xuASHsD/lines", + "total_count": 4, + "url": "/v1/customers/cus_HezytZRkaQJC8W/sources", }, - ) + }, + "bank_accounts", + [ + {"id": "cs_2", "object": "bank_account", "updated": 1692802815}, + {"id": "cs_4", "object": "bank_account", "updated": 1692802815}, + ], + "full_refresh", + {}, +) - # make start date a recent date so there's just one slice in a parent stream - stream_args["start_date"] = pendulum.today().subtract(days=3).int_timestamp - parent_stream = invoices(stream_args) - stream = invoice_line_items(stream_args, parent_stream=parent_stream) - records = [] - for slice_ in stream.stream_slices(sync_mode="full_refresh"): - records.extend(stream.read_records(sync_mode="full_refresh", stream_slice=slice_)) - assert list(records) == [ - {"id": "il_1", "invoice_id": "in_1KD6OVIEn5WyEQxn9xuASHsD", "object": "line_item"}, - {"id": "il_2", "invoice_id": "in_1KD6OVIEn5WyEQxn9xuASHsD", "object": "line_item"}, - {"id": "il_3", "invoice_id": "in_1KD6OVIEn5WyEQxn9xuASHsD", "object": "line_item"}, - ] +bank_accounts_incremental_test_case = ( + { + "https://api.stripe.com/v1/events?types%5B%5D=customer.source.created&types%5B%5D=customer.source.expiring&types" + "%5B%5D=customer.source.updated&types%5B%5D=customer.source.deleted": { + "data": [ + { + "id": "evt_1NdNFoEcXtiJtvvhBP5mxQmL", + "object": "event", + "api_version": "2020-08-27", + "created": 1692802016, + "data": {"object": {"object": "bank_account", "bank_account": "cs_1K9GK0EcXtiJtvvhSo2LvGqT", "created": 1653341716}}, + "type": "customer.source.created", + }, + { + "id": "evt_1NdNFoEcXtiJtvvhBP5mxQmL", + "object": "event", + "api_version": "2020-08-27", + "created": 1692802017, + "data": {"object": {"object": "card", "card": "cs_1K9GK0EcXtiJtvvhSo2LvGqT", "created": 1653341716}}, + "type": "customer.source.updated", + }, + ], + "has_more": False, + } + }, + "bank_accounts", + [{"object": "bank_account", "bank_account": "cs_1K9GK0EcXtiJtvvhSo2LvGqT", "created": 1653341716, "updated": 1692802016}], + "incremental", + {"updated": 1692802015}, +) +@pytest.mark.parametrize( + "requests_mock_map, stream_cls, expected_records, sync_mode, state", + (bank_accounts_incremental_test_case, bank_accounts_full_refresh_test_case), +) @freezegun.freeze_time("2023-08-23T15:00:15Z") -def test_created_cursor_incremental_stream(requests_mock, balance_transactions, incremental_stream_args): - incremental_stream_args["start_date"] = pendulum.now().subtract(months=23).int_timestamp - stream = balance_transactions(incremental_stream_args) +def test_lazy_substream_data_cursor_value_is_populated( + requests_mock, stream_by_name, config, requests_mock_map, stream_cls, expected_records, sync_mode, state +): + config["start_date"] = str(pendulum.today().subtract(days=3)) + stream = stream_by_name(stream_cls, config) + for url, body in requests_mock_map.items(): + requests_mock.get(url, json=body) + + records = read_from_stream(stream, sync_mode, state) + assert records == expected_records + for record in records: + assert bool(record[stream.cursor_field]) + + +@pytest.mark.parametrize("requests_mock_map, stream_cls, expected_records, sync_mode, state", (bank_accounts_full_refresh_test_case,)) +@freezegun.freeze_time("2023-08-23T15:00:15Z") +def test_lazy_substream_data_is_expanded( + requests_mock, stream_by_name, config, requests_mock_map, stream_cls, expected_records, sync_mode, state +): + + config["start_date"] = str(pendulum.today().subtract(days=3)) + stream = stream_by_name("bank_accounts", config) + for url, body in requests_mock_map.items(): + requests_mock.get(url, json=body) + + records = read_from_stream(stream, sync_mode, state) + + assert list(records) == expected_records + assert len(requests_mock.request_history) == 2 + assert urlencode({"expand[]": "data.sources"}) in requests_mock.request_history[0].url + + +@pytest.mark.parametrize( + "requests_mock_map, stream_cls, expected_records, sync_mode, state, expected_object", + ((*bank_accounts_full_refresh_test_case, "bank_account"), (*bank_accounts_incremental_test_case, "bank_account")), +) +@freezegun.freeze_time("2023-08-23T15:00:15Z") +def test_lazy_substream_data_is_filtered( + requests_mock, stream_by_name, config, requests_mock_map, stream_cls, expected_records, sync_mode, state, expected_object +): + config["start_date"] = str(pendulum.today().subtract(days=3)) + stream = stream_by_name(stream_cls, config) + for url, body in requests_mock_map.items(): + requests_mock.get(url, json=body) + + records = read_from_stream(stream, sync_mode, state) + assert records == expected_records + for record in records: + assert record["object"] == expected_object + + +@freezegun.freeze_time("2023-08-23T15:00:15Z") +def test_created_cursor_incremental_stream(requests_mock, stream_by_name, config): + config["start_date"] = str(pendulum.now().subtract(months=23)) + stream = stream_by_name("balance_transactions", {"lookback_window_days": 14, **config}) requests_mock.get( "/v1/balance_transactions", [ @@ -223,18 +221,18 @@ def test_created_cursor_incremental_stream(requests_mock, balance_transactions, ) @freezegun.freeze_time("2023-08-23T15:00:15Z") def test_get_start_timestamp( - balance_transactions, incremental_stream_args, start_date, lookback_window, max_days_from_now, stream_state, expected_start_timestamp + stream_by_name, config, start_date, lookback_window, max_days_from_now, stream_state, expected_start_timestamp ): - incremental_stream_args["start_date"] = pendulum.parse(start_date).int_timestamp - incremental_stream_args["lookback_window_days"] = lookback_window - incremental_stream_args["start_date_max_days_from_now"] = max_days_from_now - stream = balance_transactions(incremental_stream_args) + config["start_date"] = start_date + config["lookback_window_days"] = lookback_window + stream = stream_by_name("balance_transactions", config) + stream.start_date_max_days_from_now = max_days_from_now assert stream.get_start_timestamp(stream_state) == pendulum.parse(expected_start_timestamp).int_timestamp @pytest.mark.parametrize("sync_mode", ("full_refresh", "incremental")) -def test_updated_cursor_incremental_stream_slices(credit_notes, sync_mode): - stream = credit_notes() +def test_updated_cursor_incremental_stream_slices(stream_by_name, sync_mode): + stream = stream_by_name("credit_notes") assert list(stream.stream_slices(sync_mode)) == [{}] @@ -242,13 +240,13 @@ def test_updated_cursor_incremental_stream_slices(credit_notes, sync_mode): "last_record, stream_state, expected_state", (({"updated": 110}, {"updated": 111}, {"updated": 111}), ({"created": 110}, {"updated": 111}, {"updated": 111})), ) -def test_updated_cursor_incremental_stream_get_updated_state(credit_notes, last_record, stream_state, expected_state): - stream = credit_notes() +def test_updated_cursor_incremental_stream_get_updated_state(stream_by_name, last_record, stream_state, expected_state): + stream = stream_by_name("credit_notes") assert stream.get_updated_state(last_record, stream_state) == expected_state @pytest.mark.parametrize("sync_mode", ("full_refresh", "incremental")) -def test_updated_cursor_incremental_stream_read_wo_state(requests_mock, sync_mode, credit_notes): +def test_updated_cursor_incremental_stream_read_wo_state(requests_mock, sync_mode, stream_by_name): requests_mock.get( "/v1/credit_notes", [ @@ -275,7 +273,7 @@ def test_updated_cursor_incremental_stream_read_wo_state(requests_mock, sync_mod } ], ) - stream = credit_notes() + stream = stream_by_name("credit_notes") records = [record for record in stream.read_records(sync_mode)] assert records == [ { @@ -298,7 +296,7 @@ def test_updated_cursor_incremental_stream_read_wo_state(requests_mock, sync_mod @freezegun.freeze_time("2023-08-23T00:00:00") -def test_updated_cursor_incremental_stream_read_w_state(requests_mock, credit_notes): +def test_updated_cursor_incremental_stream_read_w_state(requests_mock, stream_by_name): requests_mock.get( "/v1/events", [ @@ -320,7 +318,7 @@ def test_updated_cursor_incremental_stream_read_w_state(requests_mock, credit_no ], ) - stream = credit_notes() + stream = stream_by_name("credit_notes") records = [ record for record in stream.read_records("incremental", stream_state={"updated": pendulum.parse("2023-01-01T15:00:15Z").int_timestamp}) @@ -490,11 +488,11 @@ def test_persons_w_state(requests_mock, stream_args): @pytest.mark.parametrize("sync_mode, stream_state", (("full_refresh", {}), ("incremental", {}), ("incremental", {"updated": 1693987430}))) -def test_cursorless_incremental_stream(requests_mock, external_bank_accounts, sync_mode, stream_state): +def test_cursorless_incremental_stream(requests_mock, stream_by_name, sync_mode, stream_state): # Testing streams that *only* have the cursor field value in incremental mode because of API discrepancies, # e.g. /bank_accounts does not return created/updated date, however /events?type=bank_account.updated returns the update date. # Key condition here is that the underlying stream has legacy cursor field set to None. - stream = external_bank_accounts() + stream = stream_by_name("external_account_bank_accounts") requests_mock.get( "/v1/accounts//external_accounts", json={ @@ -540,9 +538,9 @@ def test_cursorless_incremental_stream(requests_mock, external_bank_accounts, sy @pytest.mark.parametrize("sync_mode, stream_state", (("full_refresh", {}), ("incremental", {}), ("incremental", {"updated": 1693987430}))) -def test_cursorless_incremental_substream(requests_mock, bank_accounts, sync_mode, stream_state): +def test_cursorless_incremental_substream(requests_mock, stream_by_name, sync_mode, stream_state): # same for substreams - stream = bank_accounts() + stream = stream_by_name("bank_accounts") requests_mock.get( "/v1/customers", json={ @@ -581,9 +579,9 @@ def test_cursorless_incremental_substream(requests_mock, bank_accounts, sync_mod stream.get_updated_state(stream_state, record) -@pytest.mark.parametrize("stream", ("bank_accounts",)) -def test_get_updated_state(stream, request, requests_mock): - stream = request.getfixturevalue(stream)() +@pytest.mark.parametrize("stream_name", ("bank_accounts",)) +def test_get_updated_state(stream_name, stream_by_name, requests_mock): + stream = stream_by_name(stream_name) response = {"data": [{"id": 1, stream.cursor_field: 1695292083}]} requests_mock.get("/v1/credit_notes", json=response) requests_mock.get("/v1/balance_transactions", json=response) @@ -597,3 +595,76 @@ def test_get_updated_state(stream, request, requests_mock): for record in stream.read_records(sync_mode="incremental", stream_slice=slice_, stream_state=state): state = stream.get_updated_state(state, record) assert state + + +@freezegun.freeze_time("2023-08-23T15:00:15Z") +def test_subscription_items_extra_request_params(requests_mock, stream_by_name, config): + requests_mock.get( + "/v1/subscriptions", + json={ + "object": "list", + "url": "/v1/subscriptions", + "has_more": False, + "data": [ + { + "id": "sub_1OApco2eZvKYlo2CEDCzwLrE", + "object": "subscription", + "created": 1699603174, + "items": { + "object": "list", + "data": [ + { + "id": "si_OynDmET1kQPTbI", + "object": "subscription_item", + "created": 1699603175, + "quantity": 1, + "subscription": "sub_1OApco2eZvKYlo2CEDCzwLrE", + } + ], + "has_more": True, + }, + "latest_invoice": None, + "livemode": False, + } + ], + "has_more": False, + }, + ) + requests_mock.get( + "/v1/subscription_items?subscription=sub_1OApco2eZvKYlo2CEDCzwLrE", + json={ + "object": "list", + "url": "/v1/subscription_items", + "has_more": False, + "data": [ + { + "id": "si_OynPdzMZykmCWm", + "object": "subscription_item", + "created": 1699603884, + "quantity": 2, + "subscription": "sub_1OApco2eZvKYlo2CEDCzwLrE", + } + ], + }, + ) + config["start_date"] = str(pendulum.now().subtract(days=3)) + stream = stream_by_name("subscription_items", config) + records = read_from_stream(stream, "full_refresh", {}) + assert records == [ + { + "id": "si_OynDmET1kQPTbI", + "object": "subscription_item", + "created": 1699603175, + "quantity": 1, + "subscription": "sub_1OApco2eZvKYlo2CEDCzwLrE", + }, + { + "id": "si_OynPdzMZykmCWm", + "object": "subscription_item", + "created": 1699603884, + "quantity": 2, + "subscription": "sub_1OApco2eZvKYlo2CEDCzwLrE", + }, + ] + assert len(requests_mock.request_history) == 2 + assert "subscription=sub_1OApco2eZvKYlo2CEDCzwLrE" in requests_mock.request_history[-1].url diff --git a/docs/integrations/sources/stripe.md b/docs/integrations/sources/stripe.md index 0e04310595b7..30c101fac0d8 100644 --- a/docs/integrations/sources/stripe.md +++ b/docs/integrations/sources/stripe.md @@ -192,7 +192,8 @@ The Stripe connector should not run into Stripe API limitations under normal usa | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| -| 4.5.1 | 2023-11-01 | [32056](https://github.com/airbytehq/airbyte/pull/32056/) | Use CDK version 0.52.8 | +| 4.5.2 | 2023-11-03 | [32146](https://github.com/airbytehq/airbyte/pull/32146/) | Fix multiple BankAccount issues | +| 4.5.1 | 2023-11-01 | [32056](https://github.com/airbytehq/airbyte/pull/32056/) | Use CDK version 0.52.8 | | 4.5.0 | 2023-10-25 | [31327](https://github.com/airbytehq/airbyte/pull/31327/) | Use concurrent CDK when running in full-refresh | | 4.4.2 | 2023-10-24 | [31764](https://github.com/airbytehq/airbyte/pull/31764) | Base image migration: remove Dockerfile and use the python-connector-base image | | 4.4.1 | 2023-10-18 | [31553](https://github.com/airbytehq/airbyte/pull/31553) | Adjusted `Setup Attempts` and extended `Checkout Sessions` stream schemas | From d2ccfe72a875414784cc921fe151ffda093d1106 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Mon, 13 Nov 2023 10:57:03 -0300 Subject: [PATCH 017/124] =?UTF-8?q?=F0=9F=90=9B=20Source=20Asana:=20fix=20?= =?UTF-8?q?hidden=20config=20access=20(#32458)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-integrations/connectors/source-asana/Dockerfile | 2 +- airbyte-integrations/connectors/source-asana/metadata.yaml | 2 +- .../connectors/source-asana/source_asana/source.py | 2 +- .../connectors/source-asana/source_asana/streams.py | 1 - docs/integrations/sources/asana.md | 1 + 5 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-asana/Dockerfile b/airbyte-integrations/connectors/source-asana/Dockerfile index 267ed7d5a2cd..5c4f6e5908d0 100644 --- a/airbyte-integrations/connectors/source-asana/Dockerfile +++ b/airbyte-integrations/connectors/source-asana/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.6.0 +LABEL io.airbyte.version=0.6.1 LABEL io.airbyte.name=airbyte/source-asana diff --git a/airbyte-integrations/connectors/source-asana/metadata.yaml b/airbyte-integrations/connectors/source-asana/metadata.yaml index 9ce251365196..d3a4dd21bda3 100644 --- a/airbyte-integrations/connectors/source-asana/metadata.yaml +++ b/airbyte-integrations/connectors/source-asana/metadata.yaml @@ -8,7 +8,7 @@ data: connectorSubtype: api connectorType: source definitionId: d0243522-dccf-4978-8ba0-37ed47a0bdbf - dockerImageTag: 0.6.0 + dockerImageTag: 0.6.1 dockerRepository: airbyte/source-asana documentationUrl: https://docs.airbyte.com/integrations/sources/asana githubIssueLabel: source-asana diff --git a/airbyte-integrations/connectors/source-asana/source_asana/source.py b/airbyte-integrations/connectors/source-asana/source_asana/source.py index fe893b179f54..6c177f21b623 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/source.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/source.py @@ -62,7 +62,7 @@ def _get_authenticator(config: dict) -> Union[TokenAuthenticator, AsanaOauth2Aut ) def streams(self, config: Mapping[str, Any]) -> List[Stream]: - args = {"authenticator": self._get_authenticator(config), "test_mode": config["test_mode"]} + args = {"authenticator": self._get_authenticator(config), "test_mode": config.get("test_mode", False)} streams = [ AttachmentsCompact(**args), Attachments(**args), diff --git a/airbyte-integrations/connectors/source-asana/source_asana/streams.py b/airbyte-integrations/connectors/source-asana/source_asana/streams.py index 0b901b7477ad..64bbcdd46274 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/streams.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/streams.py @@ -27,7 +27,6 @@ class AsanaStream(HttpStream, ABC): # Asana pagination could be from 1 to 100. page_size = 100 raise_on_http_errors = True - test_mode = False @property def AsanaStreamType(self) -> Type: diff --git a/docs/integrations/sources/asana.md b/docs/integrations/sources/asana.md index 1fa2a55edc66..60f92bc33752 100644 --- a/docs/integrations/sources/asana.md +++ b/docs/integrations/sources/asana.md @@ -70,6 +70,7 @@ The connector is restricted by normal Asana [requests limitation](https://develo | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------- | +| 0.6.1 | 2023-11-13 | [31110](https://github.com/airbytehq/airbyte/pull/31110) | Fix hidden config access | | 0.6.0 | 2023-11-03 | [31110](https://github.com/airbytehq/airbyte/pull/31110) | Add new stream Portfolio Memberships with Parent Portfolio | | 0.5.0 | 2023-10-30 | [31114](https://github.com/airbytehq/airbyte/pull/31114) | Add Portfolios stream | | 0.4.0 | 2023-10-24 | [31084](https://github.com/airbytehq/airbyte/pull/31084) | Add StoriesCompact stream | From 9b6a1cced56ba95e4e111e4a5bb1a66521c2586f Mon Sep 17 00:00:00 2001 From: Artem Inzhyyants <36314070+artem1205@users.noreply.github.com> Date: Mon, 13 Nov 2023 16:38:29 +0100 Subject: [PATCH 018/124] =?UTF-8?q?=F0=9F=90=9B=20Source=20Amazon=20Seller?= =?UTF-8?q?=20Partner:=20fix=20report=20decompression=20(#32457)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../source-amazon-seller-partner/metadata.yaml | 2 +- .../source_amazon_seller_partner/streams.py | 14 ++++++++------ docs/integrations/sources/amazon-seller-partner.md | 3 ++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml index c48955d0bdf7..aa6a34bd1760 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: api connectorType: source definitionId: e55879a8-0ef8-4557-abcf-ab34c53ec460 - dockerImageTag: 1.6.0 + dockerImageTag: 1.6.1 dockerRepository: airbyte/source-amazon-seller-partner documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-seller-partner githubIssueLabel: source-amazon-seller-partner diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py index d347fa185d87..6fe4216f8490 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py @@ -3,9 +3,9 @@ # import csv +import gzip import json as json_lib import time -import zlib from abc import ABC, abstractmethod from io import StringIO from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Union @@ -281,21 +281,23 @@ def _retrieve_report(self, report_id: str) -> Mapping[str, Any]: return report_payload - def decompress_report_document(self, url, payload): + @default_backoff_handler(factor=5, max_tries=5) + def download_and_decompress_report_document(self, url, payload): """ Unpacks a report document """ - report = requests.get(url).content + report = requests.get(url) + report.raise_for_status() if "compressionAlgorithm" in payload: - return zlib.decompress(bytearray(report), 15 + 32).decode("iso-8859-1") - return report.decode("iso-8859-1") + return gzip.decompress(report.content).decode("iso-8859-1") + return report.content.decode("iso-8859-1") def parse_response( self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, **kwargs ) -> Iterable[Mapping]: payload = response.json() - document = self.decompress_report_document(payload.get("url"), payload) + document = self.download_and_decompress_report_document(payload.get("url"), payload) document_records = self.parse_document(document) yield from document_records diff --git a/docs/integrations/sources/amazon-seller-partner.md b/docs/integrations/sources/amazon-seller-partner.md index 599ff80667c1..de3abffd3caf 100644 --- a/docs/integrations/sources/amazon-seller-partner.md +++ b/docs/integrations/sources/amazon-seller-partner.md @@ -129,7 +129,8 @@ So, for any value that exceeds the limit, the `period_in_days` will be automatic | Version | Date | Pull Request | Subject | |:---------|:-----------|:--------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `1.6.0` | 2023-08-18 | [\#32259](https://github.com/airbytehq/airbyte/pull/32259) | mark "aws_secret_key" and "aws_access_key" as required in specification; update schema for stream `Orders` | +| `1.6.1` | 2023-11-13 | [\#32457](https://github.com/airbytehq/airbyte/pull/32457) | Fix report decompression | +| `1.6.0` | 2023-11-09 | [\#32259](https://github.com/airbytehq/airbyte/pull/32259) | mark "aws_secret_key" and "aws_access_key" as required in specification; update schema for stream `Orders` | | `1.5.1` | 2023-08-18 | [\#29255](https://github.com/airbytehq/airbyte/pull/29255) | role_arn is optional on UI but not really on the backend blocking connector set up using oauth | | `1.5.0` | 2023-08-08 | [\#29054](https://github.com/airbytehq/airbyte/pull/29054) | Add new stream `OrderItems` | | `1.4.1` | 2023-07-25 | [\#27050](https://github.com/airbytehq/airbyte/pull/27050) | Fix - non vendor accounts connector create/check issue | From b8f122720aa9b8ab77d44d0ae3dbe3d115440ad2 Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Mon, 13 Nov 2023 19:06:49 +0100 Subject: [PATCH 019/124] Cleanup connector metadata remark plugin (#32453) --- docusaurus/docusaurus.config.js | 2 +- docusaurus/src/css/custom.css | 78 +++++++++++++++--- docusaurus/src/remark/docsHeaderDecoration.js | 81 +++++++++++-------- 3 files changed, 114 insertions(+), 47 deletions(-) diff --git a/docusaurus/docusaurus.config.js b/docusaurus/docusaurus.config.js index 396ec4bfe143..f9081daf735d 100644 --- a/docusaurus/docusaurus.config.js +++ b/docusaurus/docusaurus.config.js @@ -30,7 +30,7 @@ const config = { projectName: "airbyte", // Usually your repo name. // Adds one off script tags to the head of each page - // .e.g + // e.g. scripts: [ { src: "https://cdn.unifygtm.com/tag/v1/unify-tag-script.js", diff --git a/docusaurus/src/css/custom.css b/docusaurus/src/css/custom.css index 688dd2a47810..374c3285ff4a 100644 --- a/docusaurus/src/css/custom.css +++ b/docusaurus/src/css/custom.css @@ -183,14 +183,70 @@ img { min-width: 1.25rem; } - /* These properties set the color of the active item in the navbar (background and text). - The variables for them have been added to :root at the top of this file */ - - .menu__link--active, - .menu__link--active:not(.menu__link--sublist), - .menu__list-item-collapsible--active, - .menu__list-item-collapsible--active:hover { - background: var(--color-active-nav-item-background); - color: var(--color-active-nav-item-text); - } - \ No newline at end of file +/* These properties set the color of the active item in the navbar (background and text). +The variables for them have been added to :root at the top of this file */ + +.menu__link--active, +.menu__link--active:not(.menu__link--sublist), +.menu__list-item-collapsible--active, +.menu__list-item-collapsible--active:hover { + background: var(--color-active-nav-item-background); + color: var(--color-active-nav-item-text); +} + +.connectorIcon { + max-height: 40px; + max-width: 40px; + float: left; + margin-right: 10px; +} + +.connectorMetadata { + background: var(--ifm-color-info-contrast-background); + border-radius: 2px; + width: 100%; + font-size: smaller; + margin: 0; + padding: 5px; + margin-bottom: 5px; +} + +.connectorMetadata > div { + display: flex; + align-items: center; + gap: 0.2em; +} + +.connectorMetadata dt { + display: inline; + font-weight: bold; +} + +.connectorMetadata dd { + display: inline; + margin: 0; +} + +.connectorMetadata dt::after { + content: ": "; +} + +.connectorMetadata .availability { + display: inline-flex; + gap: 8px; + margin-left: 2px; +} + +.connectorMetadata .availability > span { + display: inline-flex; + align-items: center; + gap: 2px; +} + +.connectorMetadata .availability .unavailable { + color: var(--ifm-color-emphasis-600); +} + +[data-theme="dark"] .connectorMetadata .availability .unavailable { + color: var(--ifm-color-emphasis-400); +} \ No newline at end of file diff --git a/docusaurus/src/remark/docsHeaderDecoration.js b/docusaurus/src/remark/docsHeaderDecoration.js index 1530d6473253..57a237e7cae0 100644 --- a/docusaurus/src/remark/docsHeaderDecoration.js +++ b/docusaurus/src/remark/docsHeaderDecoration.js @@ -1,13 +1,22 @@ const fetch = require("node-fetch"); const visit = require("unist-util-visit"); -const registry_url = +const CHECK_ICON = `Available`; +const CROSS_ICON = `Not available`; + +const REGISTRY_URL = "https://connectors.airbyte.com/files/generated_reports/connector_registry_report.json"; -let registry = []; -const plugin = () => { - let registryFetcher = fetchCatalog(); +const fetchCatalog = async () => { + console.log("Fetching connector registry..."); + const json = await fetch(REGISTRY_URL).then(resp => resp.json()); + console.log(`fetched ${json.length} connectors form registry`); + return json; +}; +const catalog = fetchCatalog(); + +const plugin = () => { const transformer = async (ast, vfile) => { if (!isDocsPage(vfile)) return; @@ -19,7 +28,7 @@ const plugin = () => { "" )}-${connectorName}`; - await Promise.resolve(registryFetcher); + const registry = await catalog; const registryEntry = registry.find( (r) => r.dockerRepository_oss === dockerRepository @@ -27,13 +36,10 @@ const plugin = () => { if (!registryEntry) return; - let originalTitle = ""; - let originalId = ""; - visit(ast, "heading", (node) => { if (node.depth === 1 && node.children.length === 1) { - originalTitle = node.children[0].value; - originalId = node.data.hProperties.id; + const originalTitle = node.children[0].value; + const originalId = node.data.hProperties.id; node.type = "html"; node.children = undefined; @@ -48,13 +54,6 @@ const plugin = () => { return transformer; }; -const fetchCatalog = async () => { - console.log("Fetching connector registry..."); - const response = await fetch(registry_url); - registry = await response.json(); - console.log(`fetched ${registry.length} connectors form registry`); -}; - const buildConnectorHTMLContent = ( registryEntry, originalTitle, @@ -63,27 +62,35 @@ const buildConnectorHTMLContent = ( // note - you can't have any leading whitespace here const htmlContent = `
- connector logo -

${originalTitle}

+ +

${originalTitle}

- -
- Availability: Airbyte Cloud: ${ - registryEntry.is_cloud ? "✅" : "❌" - }, Airbyte OSS: ${registryEntry.is_oss ? "✅" : "❌"} -
- Support Level: ${capitalizeFirstLetter( - registryEntry.supportLevel_oss - )} -
- Latest Version: ${registryEntry.dockerImageTag_oss} -
- Definition Id: ${registryEntry.definitionId} +
+
+
Availability
+
+ ${registryEntry.is_cloud ? CHECK_ICON : CROSS_ICON} Airbyte Cloud + ${registryEntry.is_oss ? CHECK_ICON : CROSS_ICON} Airbyte OSS +
+
+ - +
+
Latest Version
+
${escape(registryEntry.dockerImageTag_oss)}
+
+
+
Definition Id
+
${escape(registryEntry.definitionId)}
+
+
`; return htmlContent; @@ -104,6 +111,10 @@ const isDocsPage = (vfile) => { return true; }; +const escape = (string) => { + return string.replace(/&/g, '&').replace(/"/g, '"').replace(//g, '>'); +}; + const capitalizeFirstLetter = (string) => { return string.charAt(0).toUpperCase() + string.slice(1); }; From 53bbac865663996b0b5de66477a7dce9d494fe73 Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Mon, 13 Nov 2023 11:48:04 -0800 Subject: [PATCH 020/124] Destination Bigquery docs: Clarify service account usage (#18642) --- docs/integrations/destinations/bigquery.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index 637d0068c276..57121dc1d06f 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -26,7 +26,7 @@ To use a Google Cloud Storage bucket: 1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) with the Protection Tools set to `none` or `Object versioning`. Make sure the bucket does not have a [retention policy](https://cloud.google.com/storage/docs/samples/storage-set-retention-policy). 2. [Create an HMAC key and access ID](https://cloud.google.com/storage/docs/authentication/managing-hmackeys#create). -3. Grant the [`Storage Object Admin` role](https://cloud.google.com/storage/docs/access-control/iam-roles#standard-roles) to the Google Cloud [Service Account](https://cloud.google.com/iam/docs/service-accounts). +3. Grant the [`Storage Object Admin` role](https://cloud.google.com/storage/docs/access-control/iam-roles#standard-roles) to the Google Cloud [Service Account](https://cloud.google.com/iam/docs/service-accounts). This must be the same service account as the one you configure for BigQuery access in the [BigQuery connector setup step](#step-2-set-up-the-bigquery-connector). 4. Make sure your Cloud Storage bucket is accessible from the machine running Airbyte. The easiest way to verify if Airbyte is able to connect to your bucket is via the check connection tool in the UI. Your bucket must be encrypted using a Google-managed encryption key (this is the default setting when creating a new bucket). We currently do not support buckets using customer-managed encryption keys (CMEK). You can view this setting under the "Configuration" tab of your GCS bucket, in the `Encryption type` row. From 855d6a42aa176f3fc458ea6dafd9e3fe64909f94 Mon Sep 17 00:00:00 2001 From: Aries Gunawan Date: Mon, 13 Nov 2023 21:18:04 +0100 Subject: [PATCH 021/124] =?UTF-8?q?=E2=9C=A8=20Source=20Freshsales=20:=20M?= =?UTF-8?q?igrate=20Python=20CDK=20to=20Low-code=20CDK=20(#31685)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Marcos Marx Co-authored-by: marcosmarxm --- .../connectors/source-freshsales/Dockerfile | 2 +- .../connectors/source-freshsales/README.md | 1 - .../connectors/source-freshsales/__int__.py | 3 + .../acceptance-test-config.yml | 2 +- .../integration_tests/expected_records.jsonl | 78 ++--- .../source-freshsales/metadata.yaml | 9 +- .../connectors/source-freshsales/setup.py | 12 +- .../source_freshsales/manifest.yaml | 286 ++++++++++++++++++ .../source_freshsales/schemas/accounts.json | 4 +- .../schemas/completed_tasks.json | 4 +- .../source_freshsales/schemas/contacts.json | 14 +- .../source_freshsales/schemas/lost_deals.json | 6 +- .../source_freshsales/schemas/open_deals.json | 6 +- .../source_freshsales/schemas/open_tasks.json | 4 +- .../schemas/past_appointments.json | 4 +- .../schemas/upcoming_appointments.json | 4 +- .../source_freshsales/schemas/won_deals.json | 6 +- .../source_freshsales/source.py | 213 +------------ .../source_freshsales/spec.json | 24 -- .../source_freshsales/spec.yaml | 23 ++ .../source-freshsales/unit_tests/__init__.py | 3 - .../source-freshsales/unit_tests/conftest.py | 19 -- .../unit_tests/test_source.py | 83 ----- .../sources/freshsales-migrations.md | 6 + docs/integrations/sources/freshsales.md | 1 + 25 files changed, 409 insertions(+), 408 deletions(-) create mode 100644 airbyte-integrations/connectors/source-freshsales/__int__.py create mode 100644 airbyte-integrations/connectors/source-freshsales/source_freshsales/manifest.yaml delete mode 100644 airbyte-integrations/connectors/source-freshsales/source_freshsales/spec.json create mode 100644 airbyte-integrations/connectors/source-freshsales/source_freshsales/spec.yaml delete mode 100644 airbyte-integrations/connectors/source-freshsales/unit_tests/__init__.py delete mode 100644 airbyte-integrations/connectors/source-freshsales/unit_tests/conftest.py delete mode 100644 airbyte-integrations/connectors/source-freshsales/unit_tests/test_source.py create mode 100644 docs/integrations/sources/freshsales-migrations.md diff --git a/airbyte-integrations/connectors/source-freshsales/Dockerfile b/airbyte-integrations/connectors/source-freshsales/Dockerfile index 1c46733029af..07b190ca4b88 100644 --- a/airbyte-integrations/connectors/source-freshsales/Dockerfile +++ b/airbyte-integrations/connectors/source-freshsales/Dockerfile @@ -34,5 +34,5 @@ COPY source_freshsales ./source_freshsales ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=1.0.0 LABEL io.airbyte.name=airbyte/source-freshsales diff --git a/airbyte-integrations/connectors/source-freshsales/README.md b/airbyte-integrations/connectors/source-freshsales/README.md index da97f31f2011..c4f2450e08e3 100644 --- a/airbyte-integrations/connectors/source-freshsales/README.md +++ b/airbyte-integrations/connectors/source-freshsales/README.md @@ -97,4 +97,3 @@ You've checked out the repo, implemented a million dollar feature, and you're re 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - diff --git a/airbyte-integrations/connectors/source-freshsales/__int__.py b/airbyte-integrations/connectors/source-freshsales/__int__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshsales/__int__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-freshsales/acceptance-test-config.yml b/airbyte-integrations/connectors/source-freshsales/acceptance-test-config.yml index 8f382499e126..cefc9ad1fffa 100644 --- a/airbyte-integrations/connectors/source-freshsales/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-freshsales/acceptance-test-config.yml @@ -5,7 +5,7 @@ test_strictness_level: high acceptance_tests: spec: tests: - - spec_path: "source_freshsales/spec.json" + - spec_path: "source_freshsales/spec.yaml" connection: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-freshsales/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-freshsales/integration_tests/expected_records.jsonl index 552ba6288536..c91403d5fb19 100644 --- a/airbyte-integrations/connectors/source-freshsales/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-freshsales/integration_tests/expected_records.jsonl @@ -1,39 +1,39 @@ -{"stream": "contacts", "data": {"id": 17008318589, "first_name": "Mail Delivery", "last_name": "Subsystem", "display_name": "Mail Delivery Subsystem", "avatar": null, "job_title": null, "city": null, "state": null, "zipcode": null, "country": null, "email": "mailer-daemon@googlemail.com", "emails": "[{'id': 17007443529, 'value': 'mailer-daemon@googlemail.com', 'is_primary': True, 'label': None, '_destroy': False}]", "time_zone": null, "work_number": null, "mobile_number": null, "address": null, "last_seen": null, "lead_score": 14, "last_contacted": null, "open_deals_amount": 7700.0, "won_deals_amount": 0.0, "links": {"conversations": "/crm/sales/contacts/17008318589/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "timeline_feeds": "/crm/sales/contacts/17008318589/timeline_feeds", "document_associations": "/crm/sales/contacts/17008318589/document_associations", "notes": "/crm/sales/contacts/17008318589/notes?include=creater", "tasks": "/crm/sales/contacts/17008318589/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/contacts/17008318589/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note", "reminders": "/crm/sales/contacts/17008318589/reminders?include=creater,owner,updater,targetable", "duplicates": "/crm/sales/contacts/17008318589/duplicates", "connections": "/crm/sales/contacts/17008318589/connections"}, "last_contacted_sales_activity_mode": null, "custom_field": {}, "created_at": "2021-10-19T00:28:18-06:00", "updated_at": "2023-03-23T05:19:22-06:00", "keyword": null, "medium": null, "last_contacted_mode": null, "recent_note": null, "won_deals_count": 0, "last_contacted_via_sales_activity": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_ids": null, "open_deals_count": 2, "last_assigned_at": "2021-10-19T00:28:19-06:00", "facebook": null, "twitter": null, "linkedin": null, "is_deleted": false, "team_user_ids": null, "external_id": null, "work_email": null, "subscription_status": "1", "subscription_types": "1;2;3;4;5", "unsubscription_reason": null, "other_unsubscription_reason": null, "customer_fit": 1, "whatsapp_subscription_status": 2, "sms_subscription_status": "2", "last_seen_chat": null, "first_seen_chat": null, "locale": null, "total_sessions": null, "system_tags": "[]", "first_campaign": null, "first_medium": null, "first_source": null, "last_campaign": null, "last_medium": null, "last_source": null, "latest_campaign": null, "latest_medium": null, "latest_source": null, "mcr_id": "1450348061427834880", "phone_numbers": [], "tags": []}, "emitted_at": 1682419136691} -{"stream": "contacts", "data": {"id": 17008066468, "first_name": "Jane", "last_name": "Sampleton (sample)", "display_name": "Jane Sampleton (sample)", "avatar": "https://img.fullcontact.com/static/4df0efb1ea1a7650fef74f5e44d50d35_ca437b79617f8bbfc40c317b729d32693be1463f356b5be1015b39739859659f", "job_title": "Sales Manager", "city": "Glendale", "state": "Arizona", "zipcode": "100652", "country": "USA", "email": "janesampleton@gmail.com", "emails": "[{'id': 17007194356, 'value': 'janesampleton@gmail.com', 'is_primary': True, 'label': None, '_destroy': False}]", "time_zone": "Arizona", "work_number": "3684932360", "mobile_number": "19266529503", "address": "604-5854 Beckford St.", "last_seen": null, "lead_score": 36, "last_contacted": "2021-10-12T10:06:38-06:00", "open_deals_amount": 22780.0, "won_deals_amount": 11000.0, "links": {"conversations": "/crm/sales/contacts/17008066468/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "timeline_feeds": "/crm/sales/contacts/17008066468/timeline_feeds", "document_associations": "/crm/sales/contacts/17008066468/document_associations", "notes": "/crm/sales/contacts/17008066468/notes?include=creater", "tasks": "/crm/sales/contacts/17008066468/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/contacts/17008066468/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note", "reminders": "/crm/sales/contacts/17008066468/reminders?include=creater,owner,updater,targetable", "duplicates": "/crm/sales/contacts/17008066468/duplicates", "connections": "/crm/sales/contacts/17008066468/connections"}, "last_contacted_sales_activity_mode": "Task", "custom_field": {}, "created_at": "2021-10-07T10:06:37-06:00", "updated_at": "2023-03-23T05:26:56-06:00", "keyword": "B2B Success", "medium": "Blog", "last_contacted_mode": "Email Opened", "recent_note": "Sample note for contact create", "won_deals_count": 4, "last_contacted_via_sales_activity": "2021-10-19T00:27:33-06:00", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_ids": null, "open_deals_count": 5, "last_assigned_at": "2022-09-05T04:44:49-06:00", "facebook": "100010587455650", "twitter": "janesampleton", "linkedin": "jane-sampleton-0b0039109", "is_deleted": false, "team_user_ids": null, "external_id": null, "work_email": null, "subscription_status": "1", "subscription_types": "1;2;3;4;5", "unsubscription_reason": null, "other_unsubscription_reason": null, "customer_fit": 2, "whatsapp_subscription_status": 2, "sms_subscription_status": "2", "last_seen_chat": null, "first_seen_chat": null, "locale": null, "total_sessions": null, "system_tags": "[]", "first_campaign": null, "first_medium": null, "first_source": null, "last_campaign": null, "last_medium": null, "last_source": null, "latest_campaign": null, "latest_medium": null, "latest_source": null, "mcr_id": "1450049339590340608", "phone_numbers": [], "tags": []}, "emitted_at": 1682419136692} -{"stream": "accounts", "data": {"id": 17001321830, "name": "Widgetz.io (sample)", "address": "160-6802 Aliquet Rd.", "city": "New Haven", "state": "Connecticut", "zipcode": "68089", "country": "United States", "number_of_employees": null, "annual_revenue": 0.0, "website": "widgetz.io", "owner_id": null, "phone": "5036153947", "open_deals_amount": 0.0, "open_deals_count": 0, "won_deals_amount": 0.0, "won_deals_count": 0, "last_contacted": "2021-10-12T10:06:38-06:00", "last_contacted_mode": "Email Opened", "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17001321830/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17001321830/document_associations", "notes": "/crm/sales/sales_accounts/17001321830/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17001321830/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17001321830/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2021-10-18T04:41:17-06:00", "updated_at": "2022-06-23T04:25:57-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": null, "last_contacted_via_sales_activity": "2021-10-19T00:27:33-06:00", "last_contacted_sales_activity_mode": "Task", "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2021-10-18T09:56:31-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1682419139166} -{"stream": "accounts", "data": {"id": 17001391875, "name": "Airbyte", "address": "San Francisco, CA", "city": null, "state": null, "zipcode": "94121", "country": null, "number_of_employees": 1, "annual_revenue": null, "website": null, "owner_id": 17000038922, "phone": "+1234567890", "open_deals_amount": 25.0, "open_deals_count": 1, "won_deals_amount": 0.0, "won_deals_count": 0, "last_contacted": "2021-10-19T05:04:54-06:00", "last_contacted_mode": "Call Outgoing", "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17001391875/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17001391875/document_associations", "notes": "/crm/sales/sales_accounts/17001391875/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17001391875/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17001391875/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2021-10-19T04:57:42-06:00", "updated_at": "2022-06-23T04:25:57-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": "activate 10/10/2021", "last_contacted_via_sales_activity": "2021-10-19T05:04:54-06:00", "last_contacted_sales_activity_mode": "Phone", "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2021-10-19T04:57:43-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1682419139167} -{"stream": "accounts", "data": {"id": 17004983219, "name": "Test Account 2", "address": null, "city": null, "state": null, "zipcode": null, "country": null, "number_of_employees": null, "annual_revenue": null, "website": null, "owner_id": 17000038922, "phone": null, "open_deals_amount": 0.0, "open_deals_count": 0, "won_deals_amount": 5000.0, "won_deals_count": 2, "last_contacted": null, "last_contacted_mode": null, "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17004983219/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17004983219/document_associations", "notes": "/crm/sales/sales_accounts/17004983219/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17004983219/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17004983219/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2023-03-23T05:14:37-06:00", "updated_at": "2023-03-23T05:25:59-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": null, "last_contacted_via_sales_activity": null, "last_contacted_sales_activity_mode": null, "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2023-03-23T05:14:38-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1682419139168} -{"stream": "accounts", "data": {"id": 17004983220, "name": "Test Account 3", "address": null, "city": null, "state": null, "zipcode": null, "country": null, "number_of_employees": null, "annual_revenue": null, "website": null, "owner_id": 17000038922, "phone": null, "open_deals_amount": 3200.0, "open_deals_count": 1, "won_deals_amount": 0.0, "won_deals_count": 0, "last_contacted": null, "last_contacted_mode": null, "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17004983220/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17004983220/document_associations", "notes": "/crm/sales/sales_accounts/17004983220/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17004983220/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17004983220/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2023-03-23T05:19:11-06:00", "updated_at": "2023-03-23T05:26:29-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": null, "last_contacted_via_sales_activity": null, "last_contacted_sales_activity_mode": null, "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2023-03-23T05:19:12-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1682419139169} -{"stream": "accounts", "data": {"id": 17004983221, "name": "Test Account 4", "address": null, "city": null, "state": null, "zipcode": null, "country": null, "number_of_employees": null, "annual_revenue": null, "website": null, "owner_id": 17000038922, "phone": null, "open_deals_amount": 8080.0, "open_deals_count": 2, "won_deals_amount": 0.0, "won_deals_count": 0, "last_contacted": null, "last_contacted_mode": null, "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17004983221/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17004983221/document_associations", "notes": "/crm/sales/sales_accounts/17004983221/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17004983221/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17004983221/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2023-03-23T05:19:46-06:00", "updated_at": "2023-03-23T05:26:56-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": null, "last_contacted_via_sales_activity": null, "last_contacted_sales_activity_mode": null, "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2023-03-23T05:19:47-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1682419139169} -{"stream": "accounts", "data": {"id": 17004983218, "name": "Test Account 1", "address": null, "city": null, "state": null, "zipcode": null, "country": null, "number_of_employees": null, "annual_revenue": null, "website": null, "owner_id": 17000038922, "phone": null, "open_deals_amount": 4500.0, "open_deals_count": 1, "won_deals_amount": 6000.0, "won_deals_count": 2, "last_contacted": null, "last_contacted_mode": null, "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17004983218/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17004983218/document_associations", "notes": "/crm/sales/sales_accounts/17004983218/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17004983218/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17004983218/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2023-03-23T05:13:41-06:00", "updated_at": "2023-03-23T05:31:35-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": null, "last_contacted_via_sales_activity": "2023-03-07T11:00:00-06:00", "last_contacted_sales_activity_mode": "Test chat", "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2023-03-23T05:13:42-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1682419139170} -{"stream": "open_deals", "data": {"id": 17000512184, "name": "Gold plan (sample)", "amount": 7000.0, "base_currency_amount": 7000.0, "expected_close": "2021-10-21", "closed_date": null, "stage_updated_time": "2021-10-13T10:06:38-06:00", "custom_field": {}, "probability": 100, "updated_at": "2021-10-14T10:06:38-06:00", "created_at": "2021-10-09T10:06:37-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237830, "age": 563, "links": {"conversations": "/crm/sales/deals/17000512184/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17000512184/document_associations", "notes": "/crm/sales/deals/17000512184/notes?include=creater", "tasks": "/crm/sales/deals/17000512184/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17000512184/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2021-10-09T10:06:37-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 7000.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": 2, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 533, "tags": []}, "emitted_at": 1682419141504} -{"stream": "open_deals", "data": {"id": 17000521380, "name": "Discaunt", "amount": 25.0, "base_currency_amount": 25.0, "expected_close": null, "closed_date": null, "stage_updated_time": "2021-10-19T05:10:53-06:00", "custom_field": {}, "probability": 100, "updated_at": "2021-10-19T05:10:53-06:00", "created_at": "2021-10-19T05:04:09-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237831, "age": 553, "links": {"conversations": "/crm/sales/deals/17000521380/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17000521380/document_associations", "notes": "/crm/sales/deals/17000521380/notes?include=creater", "tasks": "/crm/sales/deals/17000521380/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17000521380/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2021-10-19T05:04:10-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 25.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 523, "tags": []}, "emitted_at": 1682419141505} -{"stream": "open_deals", "data": {"id": 17015628751, "name": "Test Open Deal 1", "amount": 4500.0, "base_currency_amount": 4500.0, "expected_close": null, "closed_date": null, "stage_updated_time": "2023-03-23T05:18:44-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:18:44-06:00", "created_at": "2023-03-23T05:18:44-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237830, "age": 33, "links": {"conversations": "/crm/sales/deals/17015628751/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628751/document_associations", "notes": "/crm/sales/deals/17015628751/notes?include=creater", "tasks": "/crm/sales/deals/17015628751/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628751/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:18:45-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 4500.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 3, "tags": []}, "emitted_at": 1682419141506} -{"stream": "open_deals", "data": {"id": 17015628806, "name": "Test Open Deal 2", "amount": 3200.0, "base_currency_amount": 3200.0, "expected_close": null, "closed_date": null, "stage_updated_time": "2023-03-23T05:19:22-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:19:22-06:00", "created_at": "2023-03-23T05:19:22-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237830, "age": 33, "links": {"conversations": "/crm/sales/deals/17015628806/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628806/document_associations", "notes": "/crm/sales/deals/17015628806/notes?include=creater", "tasks": "/crm/sales/deals/17015628806/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628806/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:19:23-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 3200.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 3, "tags": []}, "emitted_at": 1682419141507} -{"stream": "open_deals", "data": {"id": 17015628849, "name": "Test Open Deal 3", "amount": 1580.0, "base_currency_amount": 1580.0, "expected_close": null, "closed_date": null, "stage_updated_time": "2023-03-23T05:19:51-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:19:51-06:00", "created_at": "2023-03-23T05:19:51-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237830, "age": 33, "links": {"conversations": "/crm/sales/deals/17015628849/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628849/document_associations", "notes": "/crm/sales/deals/17015628849/notes?include=creater", "tasks": "/crm/sales/deals/17015628849/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628849/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:19:52-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 1580.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 3, "tags": []}, "emitted_at": 1682419141508} -{"stream": "open_deals", "data": {"id": 17015628886, "name": "Test Open Deal 4", "amount": 6500.0, "base_currency_amount": 6500.0, "expected_close": null, "closed_date": null, "stage_updated_time": "2023-03-23T05:20:17-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:20:17-06:00", "created_at": "2023-03-23T05:20:17-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237830, "age": 33, "links": {"conversations": "/crm/sales/deals/17015628886/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628886/document_associations", "notes": "/crm/sales/deals/17015628886/notes?include=creater", "tasks": "/crm/sales/deals/17015628886/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628886/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:20:18-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 6500.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 3, "tags": []}, "emitted_at": 1682419141509} -{"stream": "won_deals", "data": {"id": 17015628496, "name": "Test Won Deal 4", "amount": 5000.0, "base_currency_amount": 5000.0, "expected_close": null, "closed_date": "2023-03-23", "stage_updated_time": "2023-03-23T05:16:34-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:16:36-06:00", "created_at": "2023-03-23T05:15:44-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237835, "age": 0, "links": {"conversations": "/crm/sales/deals/17015628496/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628496/document_associations", "notes": "/crm/sales/deals/17015628496/notes?include=creater", "tasks": "/crm/sales/deals/17015628496/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628496/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:15:45-06:00", "last_contacted_sales_activity_mode": "Test chat", "last_contacted_via_sales_activity": "2023-03-07T11:00:00-06:00", "expected_deal_value": 5000.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 6, "deal_prediction_last_updated_at": "2023-03-23T05:16:34-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1682419143931} -{"stream": "won_deals", "data": {"id": 17015628427, "name": "Test Won Deal 2", "amount": 3000.0, "base_currency_amount": 3000.0, "expected_close": null, "closed_date": "2023-03-23", "stage_updated_time": "2023-03-23T05:17:00-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:17:00-06:00", "created_at": "2023-03-23T05:14:55-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237835, "age": 0, "links": {"conversations": "/crm/sales/deals/17015628427/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628427/document_associations", "notes": "/crm/sales/deals/17015628427/notes?include=creater", "tasks": "/crm/sales/deals/17015628427/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628427/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:14:56-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 3000.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 6, "deal_prediction_last_updated_at": "2023-03-23T05:17:00-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1682419143932} -{"stream": "won_deals", "data": {"id": 17015628468, "name": "Test Won Deal 3", "amount": 2000.0, "base_currency_amount": 2000.0, "expected_close": null, "closed_date": "2023-03-23", "stage_updated_time": "2023-03-23T05:17:21-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:17:26-06:00", "created_at": "2023-03-23T05:15:23-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237835, "age": 0, "links": {"conversations": "/crm/sales/deals/17015628468/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628468/document_associations", "notes": "/crm/sales/deals/17015628468/notes?include=creater", "tasks": "/crm/sales/deals/17015628468/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628468/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:15:24-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 2000.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 6, "deal_prediction_last_updated_at": "2023-03-23T05:17:21-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1682419143933} -{"stream": "won_deals", "data": {"id": 17015628368, "name": "Test Won Deal 1", "amount": 1000.0, "base_currency_amount": 1000.0, "expected_close": null, "closed_date": "2023-03-14", "stage_updated_time": "2023-03-23T05:17:48-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:17:50-06:00", "created_at": "2023-03-23T05:14:16-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237835, "age": -9, "links": {"conversations": "/crm/sales/deals/17015628368/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628368/document_associations", "notes": "/crm/sales/deals/17015628368/notes?include=creater", "tasks": "/crm/sales/deals/17015628368/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628368/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:14:17-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 1000.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 6, "deal_prediction_last_updated_at": "2023-03-23T05:17:48-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1682419143934} -{"stream": "lost_deals", "data": {"id": 17015629024, "name": "Test Lost Deal 1", "amount": 800.0, "base_currency_amount": 800.0, "expected_close": null, "closed_date": "2023-03-17", "stage_updated_time": "2023-03-23T05:24:18-06:00", "custom_field": {}, "probability": 0, "updated_at": "2023-03-23T05:24:21-06:00", "created_at": "2023-03-23T05:21:53-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237836, "age": -6, "links": {"conversations": "/crm/sales/deals/17015629024/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015629024/document_associations", "notes": "/crm/sales/deals/17015629024/notes?include=creater", "tasks": "/crm/sales/deals/17015629024/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015629024/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:21:54-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 0.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 7, "deal_prediction_last_updated_at": "2023-03-23T05:24:18-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1682419146221} -{"stream": "lost_deals", "data": {"id": 17015629056, "name": "Test Lost Deal 2", "amount": 2800.0, "base_currency_amount": 2800.0, "expected_close": null, "closed_date": "2023-03-22", "stage_updated_time": "2023-03-23T05:25:59-06:00", "custom_field": {}, "probability": 0, "updated_at": "2023-03-23T05:26:01-06:00", "created_at": "2023-03-23T05:22:14-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237836, "age": -1, "links": {"conversations": "/crm/sales/deals/17015629056/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015629056/document_associations", "notes": "/crm/sales/deals/17015629056/notes?include=creater", "tasks": "/crm/sales/deals/17015629056/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015629056/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:22:15-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 0.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 7, "deal_prediction_last_updated_at": "2023-03-23T05:25:59-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1682419146222} -{"stream": "lost_deals", "data": {"id": 17015629086, "name": "Test Lost Deal 3", "amount": 3200.0, "base_currency_amount": 3200.0, "expected_close": null, "closed_date": "2023-03-09", "stage_updated_time": "2023-03-23T05:26:29-06:00", "custom_field": {}, "probability": 0, "updated_at": "2023-03-23T05:26:31-06:00", "created_at": "2023-03-23T05:22:33-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237836, "age": -14, "links": {"conversations": "/crm/sales/deals/17015629086/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015629086/document_associations", "notes": "/crm/sales/deals/17015629086/notes?include=creater", "tasks": "/crm/sales/deals/17015629086/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015629086/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:22:34-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 0.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 7, "deal_prediction_last_updated_at": "2023-03-23T05:26:29-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1682419146223} -{"stream": "lost_deals", "data": {"id": 17015629190, "name": "Test Lost Deal 4", "amount": 895.0, "base_currency_amount": 895.0, "expected_close": null, "closed_date": "2023-03-05", "stage_updated_time": "2023-03-23T05:26:55-06:00", "custom_field": {}, "probability": 0, "updated_at": "2023-03-23T05:26:57-06:00", "created_at": "2023-03-23T05:23:44-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237836, "age": -18, "links": {"conversations": "/crm/sales/deals/17015629190/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015629190/document_associations", "notes": "/crm/sales/deals/17015629190/notes?include=creater", "tasks": "/crm/sales/deals/17015629190/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015629190/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:23:45-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 0.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 7, "deal_prediction_last_updated_at": "2023-03-23T05:26:55-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1682419146223} -{"stream": "open_tasks", "data": {"id": 17000410092, "status": "0", "title": "All reports for meeting", "description": "All reports for meeting", "created_at": "2021-10-19T00:31:18-06:00", "updated_at": "2021-10-19T00:31:18-06:00", "owner_id": 17000038922, "due_date": "2021-10-20T00:30:00-06:00", "completed_date": null, "creater_id": 17000038922, "updater_id": 17000038922, "outcome_id": 17001154298, "task_type_id": 17000098963, "targetables": "[]"}, "emitted_at": 1682419147728} -{"stream": "open_tasks", "data": {"id": 17000407414, "status": "0", "title": "(Sample) Send the pricing quote", "description": "Coordinate with Steve for the pricing quote and send it to James.", "created_at": "2021-10-14T04:41:18-06:00", "updated_at": "2021-10-18T04:41:18-06:00", "owner_id": 17000038922, "due_date": "2021-10-20T03:00:00-06:00", "completed_date": null, "creater_id": 17000038922, "updater_id": 17000038922, "outcome_id": null, "task_type_id": null, "targetables": "[{'id': 17008066468, 'type': 'Contact'}]"}, "emitted_at": 1682419147729} -{"stream": "open_tasks", "data": {"id": 17000411384, "status": "0", "title": "Meeting", "description": "Meeting with Zazmic", "created_at": "2021-10-19T05:05:54-06:00", "updated_at": "2021-10-19T05:05:54-06:00", "owner_id": 17000038922, "due_date": "2021-10-29T05:30:00-06:00", "completed_date": null, "creater_id": 17000038922, "updater_id": 17000038922, "outcome_id": 17001154298, "task_type_id": 17000098963, "targetables": "[{'id': 17001391875, 'type': 'SalesAccount'}]"}, "emitted_at": 1682419147729} -{"stream": "open_tasks", "data": {"id": 17000408204, "status": "0", "title": "Sample Task", "description": "This is just a sample task.", "created_at": "2021-10-18T09:53:02-06:00", "updated_at": "2021-10-18T09:53:02-06:00", "owner_id": 17000038922, "due_date": "2022-06-21T05:00:00-06:00", "completed_date": null, "creater_id": 17000038922, "updater_id": 17000038922, "outcome_id": null, "task_type_id": null, "targetables": "[{'id': 17008066468, 'type': 'Contact'}]"}, "emitted_at": 1682419147730} -{"stream": "completed_tasks", "data": {"id": 17000407413, "status": "1", "title": "(Sample) Send the proposal document", "description": "Send the proposal document and follow up with this contact after it.", "created_at": "2021-10-14T04:41:18-06:00", "updated_at": "2021-10-19T00:27:33-06:00", "owner_id": 17000038922, "due_date": "2021-10-19T02:00:00-06:00", "completed_date": "2021-10-19T00:27:33-06:00", "creater_id": 17000038922, "updater_id": 17000038922, "outcome_id": 17001154300, "task_type_id": null, "targetables": "[{'id': 17008066468, 'type': 'Contact'}]"}, "emitted_at": 1682419148806} -{"stream": "past_appointments", "data": {"id": 17000384293, "time_zone": "Chennai", "title": "Sample Appointment", "description": "This is just a sample Appointment.", "location": "Chennai, TN, India", "is_allday": "False", "outcome_id": null, "from_date": "2021-06-19T23:00:00-06:00", "end_date": "2022-06-20T00:00:00-06:00", "created_at": "2021-10-18T09:55:47-06:00", "updated_at": "2021-10-18T09:55:47-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[{'id': 17008066468, 'type': 'Contact'}]"}, "emitted_at": 1682419150369} -{"stream": "past_appointments", "data": {"id": 17000384297, "time_zone": "Chennai", "title": "Sample Appointment", "description": "This is just a sample Appointment.", "location": "Chennai, TN, India", "is_allday": "False", "outcome_id": null, "from_date": "2021-06-19T23:00:00-06:00", "end_date": "2022-06-20T00:00:00-06:00", "created_at": "2021-10-18T09:56:29-06:00", "updated_at": "2021-10-18T09:56:29-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[{'id': 17008066468, 'type': 'Contact'}]"}, "emitted_at": 1682419150370} -{"stream": "past_appointments", "data": {"id": 17000386736, "time_zone": "Central America", "title": "Daily meeting", "description": "Daily meeting", "location": "Zoom", "is_allday": "False", "outcome_id": null, "from_date": "2021-10-19T00:30:06-06:00", "end_date": "2022-01-19T08:30:00-06:00", "created_at": "2021-10-19T00:25:24-06:00", "updated_at": "2021-10-19T00:25:24-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[]"}, "emitted_at": 1682419150370} -{"stream": "past_appointments", "data": {"id": 17000386761, "time_zone": "Central America", "title": "Discount discussion", "description": null, "location": "Zoom", "is_allday": "False", "outcome_id": null, "from_date": "2021-10-21T00:45:00-06:00", "end_date": "2021-10-21T01:15:00-06:00", "created_at": "2021-10-19T00:45:49-06:00", "updated_at": "2021-10-19T00:45:49-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[]"}, "emitted_at": 1682419150371} -{"stream": "past_appointments", "data": {"id": 17000382511, "time_zone": "Arizona", "title": "(Sample) Meeting - final discussion about the deal", "description": "Meeting James to resolve any concerns and close the deal.", "location": "Hilton Hotel, Bucks Road", "is_allday": "False", "outcome_id": null, "from_date": "2021-10-20T10:00:00-06:00", "end_date": "2021-10-20T12:00:00-06:00", "created_at": "2021-10-18T04:41:18-06:00", "updated_at": "2021-10-16T04:41:18-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[{'id': 17008066468, 'type': 'Contact'}]"}, "emitted_at": 1682419150371} -{"stream": "past_appointments", "data": {"id": 17000384327, "time_zone": "Central America", "title": "New meeting", "description": "test meeting", "location": "zoom", "is_allday": "False", "outcome_id": null, "from_date": "2021-10-18T10:15:40-06:00", "end_date": "2021-10-18T10:45:40-06:00", "created_at": "2021-10-18T10:03:30-06:00", "updated_at": "2021-10-18T10:03:30-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[]"}, "emitted_at": 1682419150372} -{"stream": "upcoming_appointments", "data": {"id": 17000384327, "time_zone": "Central America", "title": "New meeting", "description": "test meeting", "location": "zoom", "is_allday": "False", "outcome_id": null, "from_date": "2021-10-18T10:15:40-06:00", "end_date": "2021-10-18T10:45:40-06:00", "created_at": "2021-10-18T10:03:30-06:00", "updated_at": "2021-10-18T10:03:30-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[]"}, "emitted_at": 1682419151716} -{"stream": "upcoming_appointments", "data": {"id": 17000382511, "time_zone": "Arizona", "title": "(Sample) Meeting - final discussion about the deal", "description": "Meeting James to resolve any concerns and close the deal.", "location": "Hilton Hotel, Bucks Road", "is_allday": "False", "outcome_id": null, "from_date": "2021-10-20T10:00:00-06:00", "end_date": "2021-10-20T12:00:00-06:00", "created_at": "2021-10-18T04:41:18-06:00", "updated_at": "2021-10-16T04:41:18-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[{'id': 17008066468, 'type': 'Contact'}]"}, "emitted_at": 1682419151717} -{"stream": "upcoming_appointments", "data": {"id": 17000386761, "time_zone": "Central America", "title": "Discount discussion", "description": null, "location": "Zoom", "is_allday": "False", "outcome_id": null, "from_date": "2021-10-21T00:45:00-06:00", "end_date": "2021-10-21T01:15:00-06:00", "created_at": "2021-10-19T00:45:49-06:00", "updated_at": "2021-10-19T00:45:49-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[]"}, "emitted_at": 1682419151718} -{"stream": "upcoming_appointments", "data": {"id": 17000386736, "time_zone": "Central America", "title": "Daily meeting", "description": "Daily meeting", "location": "Zoom", "is_allday": "False", "outcome_id": null, "from_date": "2021-10-19T00:30:06-06:00", "end_date": "2022-01-19T08:30:00-06:00", "created_at": "2021-10-19T00:25:24-06:00", "updated_at": "2021-10-19T00:25:24-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[]"}, "emitted_at": 1682419151719} -{"stream": "upcoming_appointments", "data": {"id": 17000384293, "time_zone": "Chennai", "title": "Sample Appointment", "description": "This is just a sample Appointment.", "location": "Chennai, TN, India", "is_allday": "False", "outcome_id": null, "from_date": "2021-06-19T23:00:00-06:00", "end_date": "2022-06-20T00:00:00-06:00", "created_at": "2021-10-18T09:55:47-06:00", "updated_at": "2021-10-18T09:55:47-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[{'id': 17008066468, 'type': 'Contact'}]"}, "emitted_at": 1682419151719} -{"stream": "upcoming_appointments", "data": {"id": 17000384297, "time_zone": "Chennai", "title": "Sample Appointment", "description": "This is just a sample Appointment.", "location": "Chennai, TN, India", "is_allday": "False", "outcome_id": null, "from_date": "2021-06-19T23:00:00-06:00", "end_date": "2022-06-20T00:00:00-06:00", "created_at": "2021-10-18T09:56:29-06:00", "updated_at": "2021-10-18T09:56:29-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[{'id': 17008066468, 'type': 'Contact'}]"}, "emitted_at": 1682419151720} +{"stream": "contacts", "data": {"id": 17008318589, "first_name": "Mail Delivery", "last_name": "Subsystem", "display_name": "Mail Delivery Subsystem", "avatar": null, "job_title": null, "city": null, "state": null, "zipcode": null, "country": null, "email": "mailer-daemon@googlemail.com", "emails": [{"id": 17007443529, "value": "mailer-daemon@googlemail.com", "is_primary": true, "label": null, "_destroy": false}], "time_zone": null, "work_number": null, "mobile_number": null, "address": null, "last_seen": null, "lead_score": 14, "last_contacted": null, "open_deals_amount": "7700.0", "won_deals_amount": "0.0", "links": {"conversations": "/crm/sales/contacts/17008318589/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "timeline_feeds": "/crm/sales/contacts/17008318589/timeline_feeds", "document_associations": "/crm/sales/contacts/17008318589/document_associations", "notes": "/crm/sales/contacts/17008318589/notes?include=creater", "tasks": "/crm/sales/contacts/17008318589/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/contacts/17008318589/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note", "reminders": "/crm/sales/contacts/17008318589/reminders?include=creater,owner,updater,targetable", "duplicates": "/crm/sales/contacts/17008318589/duplicates", "connections": "/crm/sales/contacts/17008318589/connections"}, "last_contacted_sales_activity_mode": null, "custom_field": {}, "created_at": "2021-10-19T00:28:18-06:00", "updated_at": "2023-03-23T05:19:22-06:00", "keyword": null, "medium": null, "last_contacted_mode": null, "recent_note": null, "won_deals_count": 0, "last_contacted_via_sales_activity": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_ids": null, "open_deals_count": 2, "last_assigned_at": "2021-10-19T00:28:19-06:00", "facebook": null, "twitter": null, "linkedin": null, "is_deleted": false, "team_user_ids": null, "external_id": null, "work_email": null, "subscription_status": 1, "subscription_types": "1;2;3;4;5", "unsubscription_reason": null, "other_unsubscription_reason": null, "customer_fit": 1, "whatsapp_subscription_status": 2, "sms_subscription_status": 2, "last_seen_chat": null, "first_seen_chat": null, "locale": null, "total_sessions": null, "system_tags": [], "first_campaign": null, "first_medium": null, "first_source": null, "last_campaign": null, "last_medium": null, "last_source": null, "latest_campaign": null, "latest_medium": null, "latest_source": null, "mcr_id": 1450348061427834880, "phone_numbers": [], "tags": []}, "emitted_at": 1699903253739} +{"stream": "contacts", "data": {"id": 17008066468, "first_name": "Jane", "last_name": "Sampleton (sample)", "display_name": "Jane Sampleton (sample)", "avatar": "https://img.fullcontact.com/static/4df0efb1ea1a7650fef74f5e44d50d35_ca437b79617f8bbfc40c317b729d32693be1463f356b5be1015b39739859659f", "job_title": "Sales Manager", "city": "Glendale", "state": "Arizona", "zipcode": "100652", "country": "USA", "email": "janesampleton@gmail.com", "emails": [{"id": 17007194356, "value": "janesampleton@gmail.com", "is_primary": true, "label": null, "_destroy": false}], "time_zone": "Arizona", "work_number": "3684932360", "mobile_number": "19266529503", "address": "604-5854 Beckford St.", "last_seen": null, "lead_score": 33, "last_contacted": "2021-10-12T10:06:38-06:00", "open_deals_amount": "22780.0", "won_deals_amount": "11000.0", "links": {"conversations": "/crm/sales/contacts/17008066468/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "timeline_feeds": "/crm/sales/contacts/17008066468/timeline_feeds", "document_associations": "/crm/sales/contacts/17008066468/document_associations", "notes": "/crm/sales/contacts/17008066468/notes?include=creater", "tasks": "/crm/sales/contacts/17008066468/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/contacts/17008066468/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note", "reminders": "/crm/sales/contacts/17008066468/reminders?include=creater,owner,updater,targetable", "duplicates": "/crm/sales/contacts/17008066468/duplicates", "connections": "/crm/sales/contacts/17008066468/connections"}, "last_contacted_sales_activity_mode": "Task", "custom_field": {}, "created_at": "2021-10-07T10:06:37-06:00", "updated_at": "2023-09-08T23:42:26-06:00", "keyword": "B2B Success", "medium": "Blog", "last_contacted_mode": "Email opened by recipient", "recent_note": "Sample note for contact create", "won_deals_count": 4, "last_contacted_via_sales_activity": "2021-10-19T00:27:33-06:00", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_ids": null, "open_deals_count": 5, "last_assigned_at": "2022-09-05T04:44:49-06:00", "facebook": "100010587455650", "twitter": "janesampleton", "linkedin": "jane-sampleton-0b0039109", "is_deleted": false, "team_user_ids": null, "external_id": null, "work_email": null, "subscription_status": 1, "subscription_types": "1;2;3;4;5", "unsubscription_reason": null, "other_unsubscription_reason": null, "customer_fit": 2, "whatsapp_subscription_status": 2, "sms_subscription_status": 2, "last_seen_chat": null, "first_seen_chat": null, "locale": null, "total_sessions": null, "system_tags": [], "first_campaign": null, "first_medium": null, "first_source": null, "last_campaign": null, "last_medium": null, "last_source": null, "latest_campaign": null, "latest_medium": null, "latest_source": null, "mcr_id": 1450049339590340608, "phone_numbers": [], "tags": []}, "emitted_at": 1699903253746} +{"stream": "accounts", "data": {"id": 17001321830, "name": "Widgetz.io (sample)", "address": "160-6802 Aliquet Rd.", "city": "New Haven", "state": "Connecticut", "zipcode": "68089", "country": "United States", "number_of_employees": null, "annual_revenue": 0, "website": "widgetz.io", "owner_id": null, "phone": "5036153947", "open_deals_amount": "0.0", "open_deals_count": 0, "won_deals_amount": "0.0", "won_deals_count": 0, "last_contacted": "2021-10-12T10:06:38-06:00", "last_contacted_mode": "Email opened by recipient", "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17001321830/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17001321830/document_associations", "notes": "/crm/sales/sales_accounts/17001321830/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17001321830/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17001321830/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2021-10-18T04:41:17-06:00", "updated_at": "2022-06-23T04:25:57-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": null, "last_contacted_via_sales_activity": "2021-10-19T00:27:33-06:00", "last_contacted_sales_activity_mode": "Task", "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2021-10-18T09:56:31-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1699903255739} +{"stream": "accounts", "data": {"id": 17001391875, "name": "Airbyte", "address": "San Francisco, CA", "city": null, "state": null, "zipcode": "94121", "country": null, "number_of_employees": 1, "annual_revenue": null, "website": null, "owner_id": 17000038922, "phone": "+1234567890", "open_deals_amount": "25.0", "open_deals_count": 1, "won_deals_amount": "0.0", "won_deals_count": 0, "last_contacted": "2021-10-19T05:04:54-06:00", "last_contacted_mode": "Outgoing call", "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17001391875/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17001391875/document_associations", "notes": "/crm/sales/sales_accounts/17001391875/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17001391875/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17001391875/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2021-10-19T04:57:42-06:00", "updated_at": "2022-06-23T04:25:57-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": "activate 10/10/2021", "last_contacted_via_sales_activity": "2021-10-19T05:04:54-06:00", "last_contacted_sales_activity_mode": "Phone", "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2021-10-19T04:57:43-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1699903255745} +{"stream": "accounts", "data": {"id": 17004983219, "name": "Test Account 2", "address": null, "city": null, "state": null, "zipcode": null, "country": null, "number_of_employees": null, "annual_revenue": null, "website": null, "owner_id": 17000038922, "phone": null, "open_deals_amount": "0.0", "open_deals_count": 0, "won_deals_amount": "5000.0", "won_deals_count": 2, "last_contacted": null, "last_contacted_mode": null, "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17004983219/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17004983219/document_associations", "notes": "/crm/sales/sales_accounts/17004983219/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17004983219/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17004983219/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2023-03-23T05:14:37-06:00", "updated_at": "2023-03-23T05:25:59-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": null, "last_contacted_via_sales_activity": null, "last_contacted_sales_activity_mode": null, "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2023-03-23T05:14:38-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1699903255751} +{"stream": "accounts", "data": {"id": 17004983220, "name": "Test Account 3", "address": null, "city": null, "state": null, "zipcode": null, "country": null, "number_of_employees": null, "annual_revenue": null, "website": null, "owner_id": 17000038922, "phone": null, "open_deals_amount": "3200.0", "open_deals_count": 1, "won_deals_amount": "0.0", "won_deals_count": 0, "last_contacted": null, "last_contacted_mode": null, "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17004983220/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17004983220/document_associations", "notes": "/crm/sales/sales_accounts/17004983220/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17004983220/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17004983220/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2023-03-23T05:19:11-06:00", "updated_at": "2023-03-23T05:26:29-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": null, "last_contacted_via_sales_activity": null, "last_contacted_sales_activity_mode": null, "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2023-03-23T05:19:12-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1699903255756} +{"stream": "accounts", "data": {"id": 17004983221, "name": "Test Account 4", "address": null, "city": null, "state": null, "zipcode": null, "country": null, "number_of_employees": null, "annual_revenue": null, "website": null, "owner_id": 17000038922, "phone": null, "open_deals_amount": "8080.0", "open_deals_count": 2, "won_deals_amount": "0.0", "won_deals_count": 0, "last_contacted": null, "last_contacted_mode": null, "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17004983221/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17004983221/document_associations", "notes": "/crm/sales/sales_accounts/17004983221/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17004983221/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17004983221/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2023-03-23T05:19:46-06:00", "updated_at": "2023-03-23T05:26:56-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": null, "last_contacted_via_sales_activity": null, "last_contacted_sales_activity_mode": null, "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2023-03-23T05:19:47-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1699903255762} +{"stream": "accounts", "data": {"id": 17004983218, "name": "Test Account 1", "address": null, "city": null, "state": null, "zipcode": null, "country": null, "number_of_employees": null, "annual_revenue": null, "website": null, "owner_id": 17000038922, "phone": null, "open_deals_amount": "4500.0", "open_deals_count": 1, "won_deals_amount": "6000.0", "won_deals_count": 2, "last_contacted": null, "last_contacted_mode": null, "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17004983218/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17004983218/document_associations", "notes": "/crm/sales/sales_accounts/17004983218/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17004983218/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17004983218/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2023-03-23T05:13:41-06:00", "updated_at": "2023-03-23T05:31:35-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": null, "last_contacted_via_sales_activity": "2023-03-07T11:00:00-06:00", "last_contacted_sales_activity_mode": "Test chat", "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2023-03-23T05:13:42-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1699903255768} +{"stream": "open_deals", "data": {"id": 17000512184, "name": "Gold plan (sample)", "amount": "7000.0", "base_currency_amount": "7000.0", "expected_close": "2021-10-21", "closed_date": null, "stage_updated_time": "2021-10-13T10:06:38-06:00", "custom_field": {}, "probability": 100, "updated_at": "2021-10-14T10:06:38-06:00", "created_at": "2021-10-09T10:06:37-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237830, "age": 765, "links": {"conversations": "/crm/sales/deals/17000512184/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17000512184/document_associations", "notes": "/crm/sales/deals/17000512184/notes?include=creater", "tasks": "/crm/sales/deals/17000512184/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17000512184/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2021-10-09T10:06:37-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "7000.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": 2, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 735, "tags": []}, "emitted_at": 1699903257648} +{"stream": "open_deals", "data": {"id": 17000521380, "name": "Discaunt", "amount": "25.0", "base_currency_amount": "25.0", "expected_close": null, "closed_date": null, "stage_updated_time": "2021-10-19T05:10:53-06:00", "custom_field": {}, "probability": 100, "updated_at": "2021-10-19T05:10:53-06:00", "created_at": "2021-10-19T05:04:09-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237831, "age": 755, "links": {"conversations": "/crm/sales/deals/17000521380/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17000521380/document_associations", "notes": "/crm/sales/deals/17000521380/notes?include=creater", "tasks": "/crm/sales/deals/17000521380/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17000521380/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2021-10-19T05:04:10-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "25.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 725, "tags": []}, "emitted_at": 1699903257653} +{"stream": "open_deals", "data": {"id": 17015628751, "name": "Test Open Deal 1", "amount": "4500.0", "base_currency_amount": "4500.0", "expected_close": null, "closed_date": null, "stage_updated_time": "2023-03-23T05:18:44-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:18:44-06:00", "created_at": "2023-03-23T05:18:44-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237830, "age": 235, "links": {"conversations": "/crm/sales/deals/17015628751/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628751/document_associations", "notes": "/crm/sales/deals/17015628751/notes?include=creater", "tasks": "/crm/sales/deals/17015628751/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628751/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:18:45-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "4500.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 205, "tags": []}, "emitted_at": 1699903257657} +{"stream": "open_deals", "data": {"id": 17015628806, "name": "Test Open Deal 2", "amount": "3200.0", "base_currency_amount": "3200.0", "expected_close": null, "closed_date": null, "stage_updated_time": "2023-03-23T05:19:22-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:19:22-06:00", "created_at": "2023-03-23T05:19:22-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237830, "age": 235, "links": {"conversations": "/crm/sales/deals/17015628806/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628806/document_associations", "notes": "/crm/sales/deals/17015628806/notes?include=creater", "tasks": "/crm/sales/deals/17015628806/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628806/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:19:23-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "3200.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 205, "tags": []}, "emitted_at": 1699903257661} +{"stream": "open_deals", "data": {"id": 17015628849, "name": "Test Open Deal 3", "amount": "1580.0", "base_currency_amount": "1580.0", "expected_close": null, "closed_date": null, "stage_updated_time": "2023-03-23T05:19:51-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:19:51-06:00", "created_at": "2023-03-23T05:19:51-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237830, "age": 235, "links": {"conversations": "/crm/sales/deals/17015628849/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628849/document_associations", "notes": "/crm/sales/deals/17015628849/notes?include=creater", "tasks": "/crm/sales/deals/17015628849/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628849/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:19:52-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "1580.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 205, "tags": []}, "emitted_at": 1699903257666} +{"stream": "open_deals", "data": {"id": 17015628886, "name": "Test Open Deal 4", "amount": "6500.0", "base_currency_amount": "6500.0", "expected_close": null, "closed_date": null, "stage_updated_time": "2023-03-23T05:20:17-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:20:17-06:00", "created_at": "2023-03-23T05:20:17-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237830, "age": 235, "links": {"conversations": "/crm/sales/deals/17015628886/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628886/document_associations", "notes": "/crm/sales/deals/17015628886/notes?include=creater", "tasks": "/crm/sales/deals/17015628886/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628886/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:20:18-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "6500.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 205, "tags": []}, "emitted_at": 1699903257670} +{"stream": "won_deals", "data": {"id": 17015628496, "name": "Test Won Deal 4", "amount": "5000.0", "base_currency_amount": "5000.0", "expected_close": null, "closed_date": "2023-03-23", "stage_updated_time": "2023-03-23T05:16:34-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:16:36-06:00", "created_at": "2023-03-23T05:15:44-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237835, "age": 0, "links": {"conversations": "/crm/sales/deals/17015628496/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628496/document_associations", "notes": "/crm/sales/deals/17015628496/notes?include=creater", "tasks": "/crm/sales/deals/17015628496/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628496/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:15:45-06:00", "last_contacted_sales_activity_mode": "Test chat", "last_contacted_via_sales_activity": "2023-03-07T11:00:00-06:00", "expected_deal_value": "5000.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 6, "deal_prediction_last_updated_at": "2023-03-23T05:16:34-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1699903259466} +{"stream": "won_deals", "data": {"id": 17015628427, "name": "Test Won Deal 2", "amount": "3000.0", "base_currency_amount": "3000.0", "expected_close": null, "closed_date": "2023-03-23", "stage_updated_time": "2023-03-23T05:17:00-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:17:00-06:00", "created_at": "2023-03-23T05:14:55-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237835, "age": 0, "links": {"conversations": "/crm/sales/deals/17015628427/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628427/document_associations", "notes": "/crm/sales/deals/17015628427/notes?include=creater", "tasks": "/crm/sales/deals/17015628427/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628427/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:14:56-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "3000.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 6, "deal_prediction_last_updated_at": "2023-03-23T05:17:00-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1699903259472} +{"stream": "won_deals", "data": {"id": 17015628468, "name": "Test Won Deal 3", "amount": "2000.0", "base_currency_amount": "2000.0", "expected_close": null, "closed_date": "2023-03-23", "stage_updated_time": "2023-03-23T05:17:21-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:17:26-06:00", "created_at": "2023-03-23T05:15:23-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237835, "age": 0, "links": {"conversations": "/crm/sales/deals/17015628468/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628468/document_associations", "notes": "/crm/sales/deals/17015628468/notes?include=creater", "tasks": "/crm/sales/deals/17015628468/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628468/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:15:24-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "2000.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 6, "deal_prediction_last_updated_at": "2023-03-23T05:17:21-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1699903259480} +{"stream": "won_deals", "data": {"id": 17015628368, "name": "Test Won Deal 1", "amount": "1000.0", "base_currency_amount": "1000.0", "expected_close": null, "closed_date": "2023-03-14", "stage_updated_time": "2023-03-23T05:17:48-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:17:50-06:00", "created_at": "2023-03-23T05:14:16-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237835, "age": -9, "links": {"conversations": "/crm/sales/deals/17015628368/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628368/document_associations", "notes": "/crm/sales/deals/17015628368/notes?include=creater", "tasks": "/crm/sales/deals/17015628368/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628368/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:14:17-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "1000.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 6, "deal_prediction_last_updated_at": "2023-03-23T05:17:48-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1699903259485} +{"stream": "lost_deals", "data": {"id": 17015629024, "name": "Test Lost Deal 1", "amount": "800.0", "base_currency_amount": "800.0", "expected_close": null, "closed_date": "2023-03-17", "stage_updated_time": "2023-03-23T05:24:18-06:00", "custom_field": {}, "probability": 0, "updated_at": "2023-03-23T05:24:21-06:00", "created_at": "2023-03-23T05:21:53-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237836, "age": -6, "links": {"conversations": "/crm/sales/deals/17015629024/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015629024/document_associations", "notes": "/crm/sales/deals/17015629024/notes?include=creater", "tasks": "/crm/sales/deals/17015629024/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015629024/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:21:54-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "0.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 7, "deal_prediction_last_updated_at": "2023-03-23T05:24:18-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1699903261406} +{"stream": "lost_deals", "data": {"id": 17015629056, "name": "Test Lost Deal 2", "amount": "2800.0", "base_currency_amount": "2800.0", "expected_close": null, "closed_date": "2023-03-22", "stage_updated_time": "2023-03-23T05:25:59-06:00", "custom_field": {}, "probability": 0, "updated_at": "2023-03-23T05:26:01-06:00", "created_at": "2023-03-23T05:22:14-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237836, "age": -1, "links": {"conversations": "/crm/sales/deals/17015629056/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015629056/document_associations", "notes": "/crm/sales/deals/17015629056/notes?include=creater", "tasks": "/crm/sales/deals/17015629056/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015629056/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:22:15-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "0.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 7, "deal_prediction_last_updated_at": "2023-03-23T05:25:59-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1699903261411} +{"stream": "lost_deals", "data": {"id": 17015629086, "name": "Test Lost Deal 3", "amount": "3200.0", "base_currency_amount": "3200.0", "expected_close": null, "closed_date": "2023-03-09", "stage_updated_time": "2023-03-23T05:26:29-06:00", "custom_field": {}, "probability": 0, "updated_at": "2023-03-23T05:26:31-06:00", "created_at": "2023-03-23T05:22:33-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237836, "age": -14, "links": {"conversations": "/crm/sales/deals/17015629086/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015629086/document_associations", "notes": "/crm/sales/deals/17015629086/notes?include=creater", "tasks": "/crm/sales/deals/17015629086/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015629086/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:22:34-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "0.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 7, "deal_prediction_last_updated_at": "2023-03-23T05:26:29-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1699903261416} +{"stream": "lost_deals", "data": {"id": 17015629190, "name": "Test Lost Deal 4", "amount": "895.0", "base_currency_amount": "895.0", "expected_close": null, "closed_date": "2023-03-05", "stage_updated_time": "2023-03-23T05:26:55-06:00", "custom_field": {}, "probability": 0, "updated_at": "2023-03-23T05:26:57-06:00", "created_at": "2023-03-23T05:23:44-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237836, "age": -18, "links": {"conversations": "/crm/sales/deals/17015629190/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015629190/document_associations", "notes": "/crm/sales/deals/17015629190/notes?include=creater", "tasks": "/crm/sales/deals/17015629190/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015629190/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:23:45-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "0.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 7, "deal_prediction_last_updated_at": "2023-03-23T05:26:55-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1699903261420} +{"stream": "open_tasks", "data": {"id": 17000410092, "status": 0, "title": "All reports for meeting", "description": "All reports for meeting", "created_at": "2021-10-19T00:31:18-06:00", "updated_at": "2021-10-19T00:31:18-06:00", "owner_id": 17000038922, "due_date": "2021-10-20T00:30:00-06:00", "completed_date": null, "creater_id": 17000038922, "updater_id": 17000038922, "outcome_id": 17001154298, "task_type_id": 17000098963, "targetables": []}, "emitted_at": 1699903262254} +{"stream": "open_tasks", "data": {"id": 17000407414, "status": 0, "title": "(Sample) Send the pricing quote", "description": "Coordinate with Steve for the pricing quote and send it to James.", "created_at": "2021-10-14T04:41:18-06:00", "updated_at": "2021-10-18T04:41:18-06:00", "owner_id": 17000038922, "due_date": "2021-10-20T03:00:00-06:00", "completed_date": null, "creater_id": 17000038922, "updater_id": 17000038922, "outcome_id": null, "task_type_id": null, "targetables": [{"id": 17008066468, "type": "Contact"}]}, "emitted_at": 1699903262258} +{"stream": "open_tasks", "data": {"id": 17000411384, "status": 0, "title": "Meeting", "description": "Meeting with Zazmic", "created_at": "2021-10-19T05:05:54-06:00", "updated_at": "2021-10-19T05:05:54-06:00", "owner_id": 17000038922, "due_date": "2021-10-29T05:30:00-06:00", "completed_date": null, "creater_id": 17000038922, "updater_id": 17000038922, "outcome_id": 17001154298, "task_type_id": 17000098963, "targetables": [{"id": 17001391875, "type": "SalesAccount"}]}, "emitted_at": 1699903262261} +{"stream": "open_tasks", "data": {"id": 17000408204, "status": 0, "title": "Sample Task", "description": "This is just a sample task.", "created_at": "2021-10-18T09:53:02-06:00", "updated_at": "2021-10-18T09:53:02-06:00", "owner_id": 17000038922, "due_date": "2022-06-21T05:00:00-06:00", "completed_date": null, "creater_id": 17000038922, "updater_id": 17000038922, "outcome_id": null, "task_type_id": null, "targetables": [{"id": 17008066468, "type": "Contact"}]}, "emitted_at": 1699903262265} +{"stream": "completed_tasks", "data": {"id": 17000407413, "status": 1, "title": "(Sample) Send the proposal document", "description": "Send the proposal document and follow up with this contact after it.", "created_at": "2021-10-14T04:41:18-06:00", "updated_at": "2021-10-19T00:27:33-06:00", "owner_id": 17000038922, "due_date": "2021-10-19T02:00:00-06:00", "completed_date": "2021-10-19T00:27:33-06:00", "creater_id": 17000038922, "updater_id": 17000038922, "outcome_id": 17001154300, "task_type_id": null, "targetables": [{"id": 17008066468, "type": "Contact"}]}, "emitted_at": 1699903263068} +{"stream": "past_appointments", "data": {"id": 17000384293, "time_zone": "Chennai", "title": "Sample Appointment", "description": "This is just a sample Appointment.", "location": "Chennai, TN, India", "is_allday": false, "outcome_id": null, "from_date": "2021-06-19T23:00:00-06:00", "end_date": "2022-06-20T00:00:00-06:00", "created_at": "2021-10-18T09:55:47-06:00", "updated_at": "2021-10-18T09:55:47-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": [{"id": 17008066468, "type": "Contact"}]}, "emitted_at": 1699903264022} +{"stream": "past_appointments", "data": {"id": 17000384297, "time_zone": "Chennai", "title": "Sample Appointment", "description": "This is just a sample Appointment.", "location": "Chennai, TN, India", "is_allday": false, "outcome_id": null, "from_date": "2021-06-19T23:00:00-06:00", "end_date": "2022-06-20T00:00:00-06:00", "created_at": "2021-10-18T09:56:29-06:00", "updated_at": "2021-10-18T09:56:29-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": [{"id": 17008066468, "type": "Contact"}]}, "emitted_at": 1699903264028} +{"stream": "past_appointments", "data": {"id": 17000386736, "time_zone": "Central America", "title": "Daily meeting", "description": "Daily meeting", "location": "Zoom", "is_allday": false, "outcome_id": null, "from_date": "2021-10-19T00:30:06-06:00", "end_date": "2022-01-19T08:30:00-06:00", "created_at": "2021-10-19T00:25:24-06:00", "updated_at": "2021-10-19T00:25:24-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": []}, "emitted_at": 1699903264032} +{"stream": "past_appointments", "data": {"id": 17000386761, "time_zone": "Central America", "title": "Discount discussion", "description": null, "location": "Zoom", "is_allday": false, "outcome_id": null, "from_date": "2021-10-21T00:45:00-06:00", "end_date": "2021-10-21T01:15:00-06:00", "created_at": "2021-10-19T00:45:49-06:00", "updated_at": "2021-10-19T00:45:49-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": []}, "emitted_at": 1699903264037} +{"stream": "past_appointments", "data": {"id": 17000382511, "time_zone": "Arizona", "title": "(Sample) Meeting - final discussion about the deal", "description": "Meeting James to resolve any concerns and close the deal.", "location": "Hilton Hotel, Bucks Road", "is_allday": false, "outcome_id": null, "from_date": "2021-10-20T10:00:00-06:00", "end_date": "2021-10-20T12:00:00-06:00", "created_at": "2021-10-18T04:41:18-06:00", "updated_at": "2021-10-16T04:41:18-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": [{"id": 17008066468, "type": "Contact"}]}, "emitted_at": 1699903264043} +{"stream": "past_appointments", "data": {"id": 17000384327, "time_zone": "Central America", "title": "New meeting", "description": "test meeting", "location": "zoom", "is_allday": false, "outcome_id": null, "from_date": "2021-10-18T10:15:40-06:00", "end_date": "2021-10-18T10:45:40-06:00", "created_at": "2021-10-18T10:03:30-06:00", "updated_at": "2021-10-18T10:03:30-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": []}, "emitted_at": 1699903264047} +{"stream": "upcoming_appointments", "data": {"id": 17000384327, "time_zone": "Central America", "title": "New meeting", "description": "test meeting", "location": "zoom", "is_allday": false, "outcome_id": null, "from_date": "2021-10-18T10:15:40-06:00", "end_date": "2021-10-18T10:45:40-06:00", "created_at": "2021-10-18T10:03:30-06:00", "updated_at": "2021-10-18T10:03:30-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": []}, "emitted_at": 1699903265232} +{"stream": "upcoming_appointments", "data": {"id": 17000382511, "time_zone": "Arizona", "title": "(Sample) Meeting - final discussion about the deal", "description": "Meeting James to resolve any concerns and close the deal.", "location": "Hilton Hotel, Bucks Road", "is_allday": false, "outcome_id": null, "from_date": "2021-10-20T10:00:00-06:00", "end_date": "2021-10-20T12:00:00-06:00", "created_at": "2021-10-18T04:41:18-06:00", "updated_at": "2021-10-16T04:41:18-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": [{"id": 17008066468, "type": "Contact"}]}, "emitted_at": 1699903265239} +{"stream": "upcoming_appointments", "data": {"id": 17000386761, "time_zone": "Central America", "title": "Discount discussion", "description": null, "location": "Zoom", "is_allday": false, "outcome_id": null, "from_date": "2021-10-21T00:45:00-06:00", "end_date": "2021-10-21T01:15:00-06:00", "created_at": "2021-10-19T00:45:49-06:00", "updated_at": "2021-10-19T00:45:49-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": []}, "emitted_at": 1699903265243} +{"stream": "upcoming_appointments", "data": {"id": 17000386736, "time_zone": "Central America", "title": "Daily meeting", "description": "Daily meeting", "location": "Zoom", "is_allday": false, "outcome_id": null, "from_date": "2021-10-19T00:30:06-06:00", "end_date": "2022-01-19T08:30:00-06:00", "created_at": "2021-10-19T00:25:24-06:00", "updated_at": "2021-10-19T00:25:24-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": []}, "emitted_at": 1699903265249} +{"stream": "upcoming_appointments", "data": {"id": 17000384293, "time_zone": "Chennai", "title": "Sample Appointment", "description": "This is just a sample Appointment.", "location": "Chennai, TN, India", "is_allday": false, "outcome_id": null, "from_date": "2021-06-19T23:00:00-06:00", "end_date": "2022-06-20T00:00:00-06:00", "created_at": "2021-10-18T09:55:47-06:00", "updated_at": "2021-10-18T09:55:47-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": [{"id": 17008066468, "type": "Contact"}]}, "emitted_at": 1699903265253} +{"stream": "upcoming_appointments", "data": {"id": 17000384297, "time_zone": "Chennai", "title": "Sample Appointment", "description": "This is just a sample Appointment.", "location": "Chennai, TN, India", "is_allday": false, "outcome_id": null, "from_date": "2021-06-19T23:00:00-06:00", "end_date": "2022-06-20T00:00:00-06:00", "created_at": "2021-10-18T09:56:29-06:00", "updated_at": "2021-10-18T09:56:29-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": [{"id": 17008066468, "type": "Contact"}]}, "emitted_at": 1699903265258} diff --git a/airbyte-integrations/connectors/source-freshsales/metadata.yaml b/airbyte-integrations/connectors/source-freshsales/metadata.yaml index a3fe07521671..03a031b0d259 100644 --- a/airbyte-integrations/connectors/source-freshsales/metadata.yaml +++ b/airbyte-integrations/connectors/source-freshsales/metadata.yaml @@ -8,7 +8,12 @@ data: connectorSubtype: api connectorType: source definitionId: eca08d79-7b92-4065-b7f3-79c14836ebe7 - dockerImageTag: 0.1.4 + dockerImageTag: 1.0.0 + releases: + breakingChanges: + 1.0.0: + message: "Large update in schema data types. Need refresh data" + upgradeDeadline: "2023-11-23" dockerRepository: airbyte/source-freshsales documentationUrl: https://docs.airbyte.com/integrations/sources/freshsales githubIssueLabel: source-freshsales @@ -23,5 +28,5 @@ data: releaseStage: beta supportLevel: community tags: - - language:python + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-freshsales/setup.py b/airbyte-integrations/connectors/source-freshsales/setup.py index 91ef829534e2..2cc1107f0c8b 100644 --- a/airbyte-integrations/connectors/source-freshsales/setup.py +++ b/airbyte-integrations/connectors/source-freshsales/setup.py @@ -6,23 +6,23 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk~=0.1", ] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", - "pytest", - "pytest-mock", + "pytest~=6.2", + "pytest-mock~=3.6.1", ] setup( name="source_freshsales", description="Source implementation for Freshsales.", - author="Tuan Nguyen", - author_email="anhtuan.nguyen@me.com", + author="Airbyte", + author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/manifest.yaml b/airbyte-integrations/connectors/source-freshsales/source_freshsales/manifest.yaml new file mode 100644 index 000000000000..bf292c6c56f0 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/manifest.yaml @@ -0,0 +1,286 @@ +version: 0.51.16 + +type: DeclarativeSource +check: + type: CheckStream + stream_names: + - contacts +definitions: + schema_loader: + type: JsonFileSchemaLoader + file_path: "./source_freshsales/schemas/{{ parameters['name'] }}.json" + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + basic_requester: + type: HttpRequester + url_base: "https://{{ config['domain_name'] }}/crm/sales/api/" + http_method: "GET" + authenticator: + type: ApiKeyAuthenticator + header: "Authorization" + api_token: "Token token={{ config['api_key'] }}" + requester: + $ref: "#/definitions/basic_requester" + request_parameters: + page: "{{ parameters.get('page', None) }}" + filter: "{{ parameters.get('filter', None) }}" + sort_type: "asc" + sort: "updated_at" + default_paginator: + type: "DefaultPaginator" + page_size_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "per_page" + pagination_strategy: + type: "PageIncrement" + page_size: 50 + start_from_page: 1 + inject_on_first_request: true + page_token_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "page" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + requester: + $ref: "#/definitions/requester" + paginator: + $ref: "#/definitions/default_paginator" + base_stream: + primary_key: "id" + retriever: + $ref: "#/definitions/retriever" + contacts_filters_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "contact_filters" + path: "contacts/filters" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/basic_requester" + record_selector: + type: RecordSelector + extractor: + field_path: + - filters + record_filter: + condition: "{{ record['name'] == parameters['filter'] }}" + contacts_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "contacts" + filter: "All Contacts" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + path: "contacts/view/{{ stream_slice.view_id }}" + record_selector: + type: RecordSelector + extractor: + field_path: + - contacts + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/contacts_filters_stream" + parent_key: "id" + partition_field: "view_id" + accounts_filters_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "sales_accounts_filter" + path: "sales_accounts/filters" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/basic_requester" + record_selector: + type: RecordSelector + extractor: + field_path: + - filters + record_filter: + condition: "{{ record['name'] == parameters['filter'] }}" + accounts_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "accounts" + filter: "All Accounts" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + path: "sales_accounts/view/{{ stream_slice.view_id }}" + record_selector: + type: RecordSelector + extractor: + field_path: + - sales_accounts + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/accounts_filters_stream" + parent_key: "id" + partition_field: "view_id" + deals_filters_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "deals_filter" + path: "deals/filters" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/basic_requester" + record_selector: + type: RecordSelector + extractor: + field_path: + - filters + record_filter: + condition: "{{ record['name'] == parameters['filter'] }}" + open_deals_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "open_deals" + filter: "Open Deals" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + path: "deals/view/{{ stream_slice.view_id }}" + record_selector: + type: RecordSelector + extractor: + field_path: + - deals + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/deals_filters_stream" + parent_key: "id" + partition_field: "view_id" + transformations: + - type: RemoveFields + field_pointers: + - ["fc_widget_collaboration"] + won_deals_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "won_deals" + filter: "Won Deals" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + path: "deals/view/{{ stream_slice.view_id }}" + record_selector: + type: RecordSelector + extractor: + field_path: + - deals + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/deals_filters_stream" + parent_key: "id" + partition_field: "view_id" + transformations: + - type: RemoveFields + field_pointers: + - ["fc_widget_collaboration"] + lost_deals_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "lost_deals" + filter: "Lost Deals" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + path: "deals/view/{{ stream_slice.view_id }}" + record_selector: + type: RecordSelector + extractor: + field_path: + - deals + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/deals_filters_stream" + parent_key: "id" + partition_field: "view_id" + transformations: + - type: RemoveFields + field_pointers: + - ["fc_widget_collaboration"] + open_tasks_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "open_tasks" + path: "tasks" + filter: "open" + retriever: + $ref: "#/definitions/retriever" + record_selector: + type: RecordSelector + extractor: + field_path: + - tasks + completed_tasks_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "completed_tasks" + path: "tasks" + filter: "completed" + retriever: + $ref: "#/definitions/retriever" + record_selector: + type: RecordSelector + extractor: + field_path: + - tasks + past_appointments_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "past_appointments" + path: "appointments" + filter: "past" + retriever: + $ref: "#/definitions/retriever" + record_selector: + type: RecordSelector + extractor: + field_path: + - appointments + upcoming_appointments_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "upcoming_appointments" + path: "appointments" + filter: "upcoming" + retriever: + $ref: "#/definitions/retriever" + record_selector: + type: RecordSelector + extractor: + field_path: + - appointments + +streams: + - "#/definitions/contacts_stream" + - "#/definitions/accounts_stream" + - "#/definitions/open_deals_stream" + - "#/definitions/won_deals_stream" + - "#/definitions/lost_deals_stream" + - "#/definitions/open_tasks_stream" + - "#/definitions/completed_tasks_stream" + - "#/definitions/past_appointments_stream" + - "#/definitions/upcoming_appointments_stream" diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/accounts.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/accounts.json index 231a18049eed..601134fea6b1 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/accounts.json +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/accounts.json @@ -38,8 +38,8 @@ "last_seen": { "type": ["null", "string"] }, "lead_score": { "type": ["null", "integer"] }, "last_contacted": { "type": ["null", "string"] }, - "open_deals_amount": { "type": ["null", "number"] }, - "won_deals_amount": { "type": ["null", "number"] }, + "open_deals_amount": { "type": ["null", "string"] }, + "won_deals_amount": { "type": ["null", "string"] }, "links": { "type": ["null", "object"] }, "last_contacted_sales_activity_mode": { "type": ["null", "string"] }, "custom_field": { "type": ["null", "object"] }, diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/completed_tasks.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/completed_tasks.json index 7c5325c9d8e6..5d44344dfe8a 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/completed_tasks.json +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/completed_tasks.json @@ -12,14 +12,14 @@ "targetable_type": { "type": ["null", "string"] }, "Possible": { "type": ["null", "string"] }, "owner_id": { "type": ["null", "integer"] }, - "status": { "type": ["null", "string"] }, + "status": { "type": ["null", "integer"] }, "creater_id": { "type": ["null", "integer"] }, "created_at": { "type": ["null", "string"] }, "updated_at": { "type": ["null", "string"] }, "outcome_id": { "type": ["null", "integer"] }, "task_type_id": { "type": ["null", "integer"] }, "updater_id": { "type": ["null", "integer"] }, - "targetables": { "type": ["null", "string"] }, + "targetables": { "type": ["null", "array"] }, "completed_date": { "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/contacts.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/contacts.json index cae7ab5ec745..db09dc756746 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/contacts.json @@ -7,10 +7,10 @@ "id": { "type": ["null", "integer"] }, "first_name": { "type": ["null", "string"] }, "last_name": { "type": ["null", "string"] }, - "subscription_status": { "type": ["null", "string"] }, + "subscription_status": { "type": ["null", "integer"] }, "job_title": { "type": ["null", "string"] }, "email": { "type": ["null", "string"] }, - "emails": { "type": ["null", "string"] }, + "emails": { "type": ["null", "array"] }, "work_number": { "type": ["null", "string"] }, "external_id": { "type": ["null", "string"] }, "mobile_number": { "type": ["null", "string"] }, @@ -41,8 +41,8 @@ "last_seen": { "type": ["null", "string"] }, "lead_score": { "type": ["null", "integer"] }, "last_contacted": { "type": ["null", "string"] }, - "open_deals_amount": { "type": ["null", "number"] }, - "won_deals_amount": { "type": ["null", "number"] }, + "open_deals_amount": { "type": ["null", "string"] }, + "won_deals_amount": { "type": ["null", "string"] }, "links": { "type": ["null", "object"] }, "last_contacted_sales_activity_mode": { "type": ["null", "string"] }, "custom_field": { "type": ["null", "object"] }, @@ -68,16 +68,16 @@ "unsubscription_reason": { "type": ["null", "string"] }, "first_campaign": { "type": ["null", "string"] }, "total_sessions": { "type": ["null", "string"] }, - "mcr_id": { "type": ["null", "string"] }, + "mcr_id": { "type": ["null", "integer"] }, "last_campaign": { "type": ["null", "string"] }, "last_medium": { "type": ["null", "string"] }, "last_seen_chat": { "type": ["null", "string"] }, "first_medium": { "type": ["null", "string"] }, "other_unsubscription_reason": { "type": ["null", "string"] }, - "system_tags": { "type": ["null", "string"] }, + "system_tags": { "type": ["null", "array"] }, "latest_medium": { "type": ["null", "string"] }, "first_source": { "type": ["null", "string"] }, - "sms_subscription_status": { "type": ["null", "string"] }, + "sms_subscription_status": { "type": ["null", "integer"] }, "locale": { "type": ["null", "string"] }, "last_source": { "type": ["null", "string"] } } diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/lost_deals.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/lost_deals.json index 27067bba75d7..6f8dae4aaead 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/lost_deals.json +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/lost_deals.json @@ -6,9 +6,9 @@ "properties": { "id": { "type": ["null", "integer"] }, "name": { "type": ["null", "string"] }, - "amount": { "type": ["null", "number"] }, + "amount": { "type": ["null", "string"] }, "currency_id": { "type": ["null", "integer"] }, - "base_currency_amount": { "type": ["null", "number"] }, + "base_currency_amount": { "type": ["null", "string"] }, "sales_account_id": { "type": ["null", "integer"] }, "deal_stage_id": { "type": ["null", "integer"] }, "deal_reason_id": { "type": ["null", "integer"] }, @@ -39,7 +39,7 @@ "tags": { "type": ["null", "array"] }, "last_contacted_sales_activity_mode": { "type": ["null", "string"] }, "last_contacted_via_sales_activity": { "type": ["null", "string"] }, - "expected_deal_value": { "type": ["null", "number"] }, + "expected_deal_value": { "type": ["null", "string"] }, "is_deleted": { "type": ["null", "boolean"] }, "team_user_ids": { "type": ["null", "string"] }, "avatar": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/open_deals.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/open_deals.json index 27067bba75d7..6f8dae4aaead 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/open_deals.json +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/open_deals.json @@ -6,9 +6,9 @@ "properties": { "id": { "type": ["null", "integer"] }, "name": { "type": ["null", "string"] }, - "amount": { "type": ["null", "number"] }, + "amount": { "type": ["null", "string"] }, "currency_id": { "type": ["null", "integer"] }, - "base_currency_amount": { "type": ["null", "number"] }, + "base_currency_amount": { "type": ["null", "string"] }, "sales_account_id": { "type": ["null", "integer"] }, "deal_stage_id": { "type": ["null", "integer"] }, "deal_reason_id": { "type": ["null", "integer"] }, @@ -39,7 +39,7 @@ "tags": { "type": ["null", "array"] }, "last_contacted_sales_activity_mode": { "type": ["null", "string"] }, "last_contacted_via_sales_activity": { "type": ["null", "string"] }, - "expected_deal_value": { "type": ["null", "number"] }, + "expected_deal_value": { "type": ["null", "string"] }, "is_deleted": { "type": ["null", "boolean"] }, "team_user_ids": { "type": ["null", "string"] }, "avatar": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/open_tasks.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/open_tasks.json index 7c5325c9d8e6..5d44344dfe8a 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/open_tasks.json +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/open_tasks.json @@ -12,14 +12,14 @@ "targetable_type": { "type": ["null", "string"] }, "Possible": { "type": ["null", "string"] }, "owner_id": { "type": ["null", "integer"] }, - "status": { "type": ["null", "string"] }, + "status": { "type": ["null", "integer"] }, "creater_id": { "type": ["null", "integer"] }, "created_at": { "type": ["null", "string"] }, "updated_at": { "type": ["null", "string"] }, "outcome_id": { "type": ["null", "integer"] }, "task_type_id": { "type": ["null", "integer"] }, "updater_id": { "type": ["null", "integer"] }, - "targetables": { "type": ["null", "string"] }, + "targetables": { "type": ["null", "array"] }, "completed_date": { "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/past_appointments.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/past_appointments.json index 018ba47b60f2..264f43f48153 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/past_appointments.json +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/past_appointments.json @@ -20,7 +20,7 @@ "location": { "type": ["null", "string"] }, "created_at": { "type": ["null", "string"] }, "updated_at": { "type": ["null", "string"] }, - "is_allday": { "type": ["null", "string"] }, + "is_allday": { "type": ["null", "boolean"] }, "appointment_attendees_attributes": { "type": ["null", "array"] }, "outcome_id": { "type": ["null", "integer"] }, "latitude": { "type": ["null", "string"] }, @@ -31,7 +31,7 @@ "checkedout_latitude": { "type": ["null", "string"] }, "checkedin_duration": { "type": ["null", "string"] }, "checkedout_longitude": { "type": ["null", "string"] }, - "targetables": { "type": ["null", "string"] }, + "targetables": { "type": ["null", "array"] }, "can_checkin": { "type": ["null", "boolean"] }, "provider": { "type": ["null", "string"] }, "checkedout_at": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/upcoming_appointments.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/upcoming_appointments.json index 018ba47b60f2..264f43f48153 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/upcoming_appointments.json +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/upcoming_appointments.json @@ -20,7 +20,7 @@ "location": { "type": ["null", "string"] }, "created_at": { "type": ["null", "string"] }, "updated_at": { "type": ["null", "string"] }, - "is_allday": { "type": ["null", "string"] }, + "is_allday": { "type": ["null", "boolean"] }, "appointment_attendees_attributes": { "type": ["null", "array"] }, "outcome_id": { "type": ["null", "integer"] }, "latitude": { "type": ["null", "string"] }, @@ -31,7 +31,7 @@ "checkedout_latitude": { "type": ["null", "string"] }, "checkedin_duration": { "type": ["null", "string"] }, "checkedout_longitude": { "type": ["null", "string"] }, - "targetables": { "type": ["null", "string"] }, + "targetables": { "type": ["null", "array"] }, "can_checkin": { "type": ["null", "boolean"] }, "provider": { "type": ["null", "string"] }, "checkedout_at": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/won_deals.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/won_deals.json index 27067bba75d7..6f8dae4aaead 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/won_deals.json +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/won_deals.json @@ -6,9 +6,9 @@ "properties": { "id": { "type": ["null", "integer"] }, "name": { "type": ["null", "string"] }, - "amount": { "type": ["null", "number"] }, + "amount": { "type": ["null", "string"] }, "currency_id": { "type": ["null", "integer"] }, - "base_currency_amount": { "type": ["null", "number"] }, + "base_currency_amount": { "type": ["null", "string"] }, "sales_account_id": { "type": ["null", "integer"] }, "deal_stage_id": { "type": ["null", "integer"] }, "deal_reason_id": { "type": ["null", "integer"] }, @@ -39,7 +39,7 @@ "tags": { "type": ["null", "array"] }, "last_contacted_sales_activity_mode": { "type": ["null", "string"] }, "last_contacted_via_sales_activity": { "type": ["null", "string"] }, - "expected_deal_value": { "type": ["null", "number"] }, + "expected_deal_value": { "type": ["null", "string"] }, "is_deleted": { "type": ["null", "boolean"] }, "team_user_ids": { "type": ["null", "string"] }, "avatar": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/source.py b/airbyte-integrations/connectors/source-freshsales/source_freshsales/source.py index 45fba4481963..a2a6cc96dbd0 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/source.py +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/source.py @@ -2,209 +2,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer - -# Basic full refresh stream -class FreshsalesStream(HttpStream, ABC): - - primary_key: str = "id" - order_field: str = "updated_at" - object_name: str = None - require_view_id: bool = False - filter_value: str = None - - transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - - def __init__(self, domain_name: str, **kwargs): - super().__init__(**kwargs) - self.domain_name = domain_name - self.page = 1 - - @property - def url_base(self) -> str: - return f"https://{self.domain_name}/crm/sales/api/" - - @property - def auth_headers(self) -> Mapping[str, Any]: - return self.authenticator.get_auth_header() - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - There is no next page token in the respond so incrementing the page param until there is no new result - """ - list_result = response.json().get(self.object_name, []) - if list_result: - self.page += 1 - return self.page - else: - return None - - def request_params(self, **kwargs) -> MutableMapping[str, Any]: - params = {"page": self.page, "sort": self.order_field, "sort_type": "asc"} - if self.filter_value: - params["filter"] = self.filter_value - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - json_response = response.json() or {} - records = json_response.get(self.object_name, []) if self.object_name else json_response - yield from records - - def _get_filters(self) -> List: - """ - Some streams require a filter_id to be passed in. This function gets all available filters. - """ - url = f"{self.url_base}{self.object_name}/filters" - try: - response = self._session.get(url=url, headers=self.auth_headers) - response.raise_for_status() - return response.json().get("filters") - except requests.exceptions.RequestException as e: - self.logger.error(f"Error occured while getting `Filters` for stream `{self.name}`, full message: {e}") - raise - - def get_view_id(self) -> int: - """ - This function finds a relevant filter_id among all available filters by its name. - """ - filters = self._get_filters() - return next(_filter["id"] for _filter in filters if _filter["name"] == self.filter_name) - - def path(self, **kwargs) -> str: - if self.require_view_id: - return f"{self.object_name}/view/{self.get_view_id()}" - else: - return self.object_name - - -class Contacts(FreshsalesStream): - """ - API docs: https://developers.freshworks.com/crm/api/#contacts - """ - - object_name = "contacts" - filter_name = "All Contacts" - require_view_id = True - - -class Accounts(FreshsalesStream): - """ - API docs: https://developers.freshworks.com/crm/api/#accounts - """ - - object_name = "sales_accounts" - filter_name = "All Accounts" - require_view_id = True - - -class Deals(FreshsalesStream): - object_name = "deals" - require_view_id = True - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - # This is to remove data form widget development. Keeping this in failed integration tests. - for record in super().parse_response(response): - record.pop("fc_widget_collaboration", None) - yield record - - -class OpenDeals(Deals): - """ - API docs: https://developers.freshworks.com/crm/api/#deals - """ - - filter_name = "Open Deals" - - -class WonDeals(Deals): - """ - API docs: https://developers.freshworks.com/crm/api/#deals - """ - - filter_name = "Won Deals" - - -class LostDeals(Deals): - """ - API docs: https://developers.freshworks.com/crm/api/#deals - """ - - filter_name = "Lost Deals" - - -class OpenTasks(FreshsalesStream): - """ - API docs: https://developers.freshworks.com/crm/api/#tasks - """ - - object_name = "tasks" - filter_value = "open" - - -class CompletedTasks(FreshsalesStream): - """ - API docs: https://developers.freshworks.com/crm/api/#tasks - """ - - object_name = "tasks" - filter_value = "completed" - - -class PastAppointments(FreshsalesStream): - """ - API docs: https://developers.freshworks.com/crm/api/#appointments - """ - - object_name = "appointments" - filter_value = "past" - - -class UpcomingAppointments(FreshsalesStream): - """ - API docs: https://developers.freshworks.com/crm/api/#appointments - """ - - object_name = "appointments" - filter_value = "upcoming" - - -# Source -class SourceFreshsales(AbstractSource): - @staticmethod - def get_input_stream_args(api_key: str, domain_name: str) -> Mapping[str, Any]: - return { - "authenticator": TokenAuthenticator(token=api_key, auth_method="Token"), - "domain_name": domain_name, - } - - def check_connection(self, logger, config) -> Tuple[bool, any]: - stream = Contacts(**self.get_input_stream_args(config["api_key"], config["domain_name"])) - try: - next(stream.read_records(sync_mode=None)) - return True, None - except requests.exceptions.RequestException as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - args = self.get_input_stream_args(config["api_key"], config["domain_name"]) - return [ - Contacts(**args), - Accounts(**args), - OpenDeals(**args), - WonDeals(**args), - LostDeals(**args), - OpenTasks(**args), - CompletedTasks(**args), - PastAppointments(**args), - UpcomingAppointments(**args), - ] +# Declarative Source +class SourceFreshsales(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/spec.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/spec.json deleted file mode 100644 index b3c15c2a28a8..000000000000 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/spec.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/freshsales", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Freshsales Spec", - "type": "object", - "required": ["domain_name", "api_key"], - "additionalProperties": true, - "properties": { - "domain_name": { - "type": "string", - "title": "Domain Name", - "description": "The Name of your Freshsales domain", - "examples": ["mydomain.myfreshworks.com"] - }, - "api_key": { - "type": "string", - "title": "API Key", - "description": "Freshsales API Key. See here. The key is case sensitive.", - "airbyte_secret": true - } - } - } -} diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/spec.yaml b/airbyte-integrations/connectors/source-freshsales/source_freshsales/spec.yaml new file mode 100644 index 000000000000..4ee33199d1f3 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/spec.yaml @@ -0,0 +1,23 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/freshsales +connectionSpecification: + type: object + title: Freshsales Spec + $schema: http://json-schema.org/draft-07/schema# + required: + - domain_name + - api_key + properties: + domain_name: + type: string + order: 0 + title: Domain Name + description: "The Name of your Freshsales domain" + examples: + - "mydomain.myfreshworks.com" + api_key: + type: string + order: 1 + title: API Key + description: 'Freshsales API Key. See here. The key is case sensitive.' + airbyte_secret: true + additionalProperties: true diff --git a/airbyte-integrations/connectors/source-freshsales/unit_tests/__init__.py b/airbyte-integrations/connectors/source-freshsales/unit_tests/__init__.py deleted file mode 100644 index 46b7376756ec..000000000000 --- a/airbyte-integrations/connectors/source-freshsales/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-freshsales/unit_tests/conftest.py b/airbyte-integrations/connectors/source-freshsales/unit_tests/conftest.py deleted file mode 100644 index 9cdc02949ebd..000000000000 --- a/airbyte-integrations/connectors/source-freshsales/unit_tests/conftest.py +++ /dev/null @@ -1,19 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import json - -import pytest -from source_freshsales.source import SourceFreshsales - - -@pytest.fixture(scope="session", name="config") -def config_fixture(): - with open("secrets/config.json", "r") as config_file: - return json.load(config_file) - - -@pytest.fixture(name="stream_args") -def stream_args(config): - return SourceFreshsales().get_input_stream_args(config["api_key"], config["domain_name"]) diff --git a/airbyte-integrations/connectors/source-freshsales/unit_tests/test_source.py b/airbyte-integrations/connectors/source-freshsales/unit_tests/test_source.py deleted file mode 100644 index b4a1d7fb3f31..000000000000 --- a/airbyte-integrations/connectors/source-freshsales/unit_tests/test_source.py +++ /dev/null @@ -1,83 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock, patch - -import pytest -import requests -from source_freshsales.source import Contacts, FreshsalesStream, OpenDeals, OpenTasks, SourceFreshsales - - -def test_get_input_stream_args(config): - source = SourceFreshsales() - expected_keys = ["authenticator", "domain_name"] - actual = source.get_input_stream_args(config["api_key"], config["domain_name"]) - for key in expected_keys: - assert key in actual.keys() - - -def test_check_connection(mocker, config): - source = SourceFreshsales() - logger_mock = MagicMock() - assert source.check_connection(logger_mock, config) == (True, None) - - -def test_count_streams(mocker): - source = SourceFreshsales() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 9 - assert len(streams) == expected_streams_number - - -def test_url_base(stream_args): - stream = FreshsalesStream(**stream_args) - expected = f"https://{stream_args.get('domain_name')}/crm/sales/api/" - actual = stream.url_base - assert actual == expected - - -def test_next_page_token(stream_args, requests_mock): - stream = Contacts(**stream_args) - stream_filters = [{"id": 1, "name": stream.filter_name}] - with patch.object(stream, "_get_filters", return_value=stream_filters) as mock_method: - url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json={stream.name: [{"id": 123}]}) - response = requests.get(url) - assert stream.next_page_token(response) == 2 - mock_method.assert_called() - - -def test_request_params(stream_args): - stream = OpenTasks(**stream_args) - actual = stream.request_params() - expected = {"filter": "open", "page": 1, "sort": "updated_at", "sort_type": "asc"} - assert actual == expected - - -@pytest.mark.parametrize( - "stream, response, expected", - [ - (Contacts, [{"id": 123}], [{"id": 123}]), - (OpenDeals, [{"id": 234, "fc_widget_collaboration": {"test": "test"}}], [{"id": 234}]), - ], - ids=["Contacts", "OpenDeals"], -) -def test_parse_response(stream, response, expected, stream_args, requests_mock): - stream = stream(**stream_args) - stream_filters = [{"id": 1, "name": stream.filter_name}] - with patch.object(stream, "_get_filters", return_value=stream_filters) as mock_method: - url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json={stream.object_name: response}) - _resp = requests.get(url) - assert list(stream.parse_response(_resp)) == expected - mock_method.assert_called() - - -def test_path(stream_args): - stream = Contacts(**stream_args) - stream_filters = [{"id": 1, "name": stream.filter_name}] - with patch.object(stream, "_get_filters", return_value=stream_filters) as mock_method: - assert stream.path() == "contacts/view/1" - mock_method.assert_called() diff --git a/docs/integrations/sources/freshsales-migrations.md b/docs/integrations/sources/freshsales-migrations.md new file mode 100644 index 000000000000..e059dc4e6d3c --- /dev/null +++ b/docs/integrations/sources/freshsales-migrations.md @@ -0,0 +1,6 @@ +# Freshsales Migration Guide + +## Upgrading to 1.0.0 + +All streams data types were updated to match the correct return from the API. +Need refresh schema to match latest updates. diff --git a/docs/integrations/sources/freshsales.md b/docs/integrations/sources/freshsales.md index 685398c4eae1..b8b100a0301c 100644 --- a/docs/integrations/sources/freshsales.md +++ b/docs/integrations/sources/freshsales.md @@ -69,6 +69,7 @@ The Freshsales connector should not run into Freshsales API limitations under no | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------| +| 1.0.0 | 2023-10-21 | [31685](https://github.com/airbytehq/airbyte/pull/31685) | Migrate to Low-Code CDK | | 0.1.4 | 2023-03-23 | [24396](https://github.com/airbytehq/airbyte/pull/24396) | Certify to Beta | | 0.1.3 | 2023-03-16 | [24155](https://github.com/airbytehq/airbyte/pull/24155) | Set `additionalProperties` to `True` in `spec` to support BC | | 0.1.2 | 2022-07-14 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Tune the `get_view_id` function | From a79fb06f8a41d6389f540991ff627f299f48a4d3 Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Mon, 13 Nov 2023 16:04:33 -0800 Subject: [PATCH 022/124] Destination bigquery error grouping: even more fixes (#32468) Co-authored-by: edgao --- .github/CODEOWNERS | 3 +++ airbyte-cdk/java/airbyte-cdk/README.md | 4 +++- .../integrations/base/AirbyteExceptionHandler.java | 7 +++++++ .../core/src/main/resources/version.properties | 2 +- .../destination/typing_deduping/CatalogParser.java | 11 +++++++++++ .../connectors/destination-bigquery/build.gradle | 2 +- .../connectors/destination-bigquery/metadata.yaml | 2 +- docs/integrations/destinations/bigquery.md | 3 ++- 8 files changed, 29 insertions(+), 5 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 639a288950b4..1c76017e418d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -38,6 +38,9 @@ /airbyte-integrations/connectors/source-tidb/ @airbytehq/dbsources # Java-based destination connectors +airbyte-cdk/java/airbyte-cdk/db-destinations/ @airbytehq/destinations +airbyte-cdk/java/airbyte-cdk/s3-destinations/ @airbytehq/destinations +airbyte-cdk/java/airbyte-cdk/typing-deduping/ @airbytehq/destinations /airbyte-integrations/bases/standard-destination-test/ @airbytehq/destinations /airbyte-integrations/bases/base-java-s3/ @airbytehq/destinations /airbyte-integrations/bases/bases-destination-jdbc/ @airbytehq/destinations diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md index d7048556de33..6408b52ab36a 100644 --- a/airbyte-cdk/java/airbyte-cdk/README.md +++ b/airbyte-cdk/java/airbyte-cdk/README.md @@ -156,7 +156,9 @@ MavenLocal debugging steps: | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.4.7 | 2023-11-08 | [\#31856](https://github.com/airbytehq/airbyte/pull/31856) | source-postgres: support for inifinty date and timestamps | +| 0.4.9 | 2023-11-13 | [\#32468](https://github.com/airbytehq/airbyte/pull/32468) | Further error grouping improvements for DV2 connectors | +| 0.4.8 | 2023-11-09 | [\#32377](https://github.com/airbytehq/airbyte/pull/32377) | source-postgres tests: skip dropping database | +| 0.4.7 | 2023-11-08 | [\#31856](https://github.com/airbytehq/airbyte/pull/31856) | source-postgres: support for inifinty date and timestamps | | 0.4.5 | 2023-11-07 | [\#32112](https://github.com/airbytehq/airbyte/pull/32112) | Async destinations framework: Allow configuring the queue flush threshold | | 0.4.4 | 2023-11-06 | [\#32119](https://github.com/airbytehq/airbyte/pull/32119) | Add STANDARD UUID codec to MongoDB debezium handler | | 0.4.2 | 2023-11-06 | [\#32190](https://github.com/airbytehq/airbyte/pull/32190) | Improve error deinterpolation | diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java index 84d271a3a856..64502fb55232 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java @@ -139,12 +139,19 @@ protected void terminate() { @VisibleForTesting static void addCommonStringsToDeinterpolate() { // Add some common strings to deinterpolate, regardless of what the connector is doing + STRINGS_TO_DEINTERPOLATE.add("airbyte"); + STRINGS_TO_DEINTERPOLATE.add("config"); + STRINGS_TO_DEINTERPOLATE.add("configuration"); STRINGS_TO_DEINTERPOLATE.add("description"); + STRINGS_TO_DEINTERPOLATE.add("email"); STRINGS_TO_DEINTERPOLATE.add("id"); STRINGS_TO_DEINTERPOLATE.add("location"); + STRINGS_TO_DEINTERPOLATE.add("message"); STRINGS_TO_DEINTERPOLATE.add("name"); + STRINGS_TO_DEINTERPOLATE.add("state"); STRINGS_TO_DEINTERPOLATE.add("status"); STRINGS_TO_DEINTERPOLATE.add("type"); + STRINGS_TO_DEINTERPOLATE.add("userEmail"); } } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties index 9ab25c390bc2..14b28050f1e2 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties @@ -1 +1 @@ -version=0.4.8 +version=0.4.9 diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParser.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParser.java index f46323cfe4cc..925d5037ea28 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParser.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParser.java @@ -78,6 +78,17 @@ public ParsedCatalog parseCatalog(final ConfiguredAirbyteCatalog catalog) { AirbyteExceptionHandler.addStringForDeinterpolation(columnId.name()); AirbyteExceptionHandler.addStringForDeinterpolation(columnId.originalName()); }); + // It's (unfortunately) possible for a cursor/PK to be declared that don't actually exist in the + // schema. + // Add their strings explicitly. + actualStreamConfig.cursor().ifPresent(cursor -> { + AirbyteExceptionHandler.addStringForDeinterpolation(cursor.name()); + AirbyteExceptionHandler.addStringForDeinterpolation(cursor.originalName()); + }); + actualStreamConfig.primaryKey().forEach(pk -> { + AirbyteExceptionHandler.addStringForDeinterpolation(pk.name()); + AirbyteExceptionHandler.addStringForDeinterpolation(pk.originalName()); + }); } return new ParsedCatalog(streamConfigs); } diff --git a/airbyte-integrations/connectors/destination-bigquery/build.gradle b/airbyte-integrations/connectors/destination-bigquery/build.gradle index 9f50b47bf821..33dab9b167e4 100644 --- a/airbyte-integrations/connectors/destination-bigquery/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.4.5' + cdkVersionRequired = '0.4.9' features = ['db-destinations', 's3-destinations'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml index 45cb1fd7f20d..2891145670f8 100644 --- a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml +++ b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 - dockerImageTag: 2.3.14 + dockerImageTag: 2.3.15 dockerRepository: airbyte/destination-bigquery githubIssueLabel: destination-bigquery icon: bigquery.svg diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index 57121dc1d06f..4ece99595e00 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -140,8 +140,9 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 2.3.15 | 2023-11-13 | [\#32468](https://github.com/airbytehq/airbyte/pull/32468) | Further error grouping enhancments | | 2.3.14 | 2023-11-06 | [\#32234](https://github.com/airbytehq/airbyte/pull/32234) | Remove unused config option. | -| 2.3.13 | 2023-11-08 | [\#32125](https://github.com/airbytehq/airbyte/pull/32125) | fix compiler warnings | +| 2.3.13 | 2023-11-08 | [\#32125](https://github.com/airbytehq/airbyte/pull/32125) | fix compiler warnings | | 2.3.12 | 2023-11-08 | [\#32309](https://github.com/airbytehq/airbyte/pull/32309) | Revert: Use Typed object for connection config | | 2.3.11 | 2023-11-07 | [\#32147](https://github.com/airbytehq/airbyte/pull/32147) | Use Typed object for connection config | | 2.3.10 | 2023-11-07 | [\#32261](https://github.com/airbytehq/airbyte/pull/32261) | Further improve error reporting | From 6afac8310ca478428db6058d16fd2965e95472ef Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Mon, 13 Nov 2023 16:07:35 -0800 Subject: [PATCH 023/124] remove source-xero from Cloud (#32476) --- airbyte-integrations/connectors/source-xero/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-xero/metadata.yaml b/airbyte-integrations/connectors/source-xero/metadata.yaml index f70a0bf641f2..fcf18468b1ae 100644 --- a/airbyte-integrations/connectors/source-xero/metadata.yaml +++ b/airbyte-integrations/connectors/source-xero/metadata.yaml @@ -13,7 +13,7 @@ data: name: Xero registries: cloud: - enabled: true + enabled: false oss: enabled: true releaseStage: beta From c642d81b8416f2a449217e928f35d517cfe5728a Mon Sep 17 00:00:00 2001 From: Stephane Geneix <147216312+stephane-airbyte@users.noreply.github.com> Date: Mon, 13 Nov 2023 16:07:58 -0800 Subject: [PATCH 024/124] Stephane/enable javac errors 1 (#32467) --- .../java/airbyte-cdk/airbyte-commons/build.gradle | 3 +++ airbyte-cdk/java/airbyte-cdk/core/build.gradle | 3 +++ .../java/airbyte-cdk/s3-destinations/build.gradle | 9 +++++++++ .../java/airbyte-cdk/typing-deduping/build.gradle | 6 ++++++ .../destination-azure-blob-storage/build.gradle | 7 +++++++ .../connectors/destination-cassandra/build.gradle | 7 +++++++ .../destination-clickhouse-strict-encrypt/build.gradle | 7 +++++++ .../connectors/destination-clickhouse/build.gradle | 7 +++++++ .../connectors/destination-csv/build.gradle | 7 +++++++ .../connectors/destination-databricks/build.gradle | 7 +++++++ .../connectors/destination-dev-null/build.gradle | 7 +++++++ .../connectors/destination-doris/build.gradle | 7 +++++++ .../connectors/destination-dynamodb/build.gradle | 7 +++++++ .../connectors/destination-e2e-test/build.gradle | 7 +++++++ .../build.gradle | 7 +++++++ .../connectors/destination-elasticsearch/build.gradle | 7 +++++++ .../connectors/destination-exasol/build.gradle | 7 +++++++ .../connectors/destination-gcs/build.gradle | 7 +++++++ .../connectors/destination-iceberg/build.gradle | 7 +++++++ .../connectors/destination-kafka/build.gradle | 7 +++++++ .../connectors/destination-keen/build.gradle | 7 +++++++ .../connectors/destination-kinesis/build.gradle | 7 +++++++ .../connectors/destination-local-json/build.gradle | 7 +++++++ .../destination-mariadb-columnstore/build.gradle | 7 +++++++ .../destination-mongodb-strict-encrypt/build.gradle | 7 +++++++ .../connectors/destination-mongodb/build.gradle | 7 +++++++ .../connectors/destination-mqtt/build.gradle | 7 +++++++ .../destination-mssql-strict-encrypt/build.gradle | 7 +++++++ .../connectors/destination-mssql/build.gradle | 7 +++++++ .../destination-mysql-strict-encrypt/build.gradle | 7 +++++++ .../connectors/destination-mysql/build.gradle | 7 +++++++ .../destination-oracle-strict-encrypt/build.gradle | 7 +++++++ .../connectors/destination-oracle/build.gradle | 7 +++++++ .../destination-postgres-strict-encrypt/build.gradle | 7 +++++++ .../connectors/destination-postgres/build.gradle | 7 +++++++ .../connectors/destination-pubsub/build.gradle | 7 +++++++ .../connectors/destination-pulsar/build.gradle | 7 +++++++ .../connectors/destination-r2/build.gradle | 7 +++++++ .../connectors/destination-redis/build.gradle | 7 +++++++ .../connectors/destination-redpanda/build.gradle | 7 +++++++ .../connectors/destination-redshift/build.gradle | 7 +++++++ .../connectors/destination-rockset/build.gradle | 7 +++++++ .../connectors/destination-s3-glue/build.gradle | 7 +++++++ .../connectors/destination-scylla/build.gradle | 7 +++++++ .../connectors/destination-selectdb/build.gradle | 7 +++++++ .../destination-starburst-galaxy/build.gradle | 7 +++++++ .../connectors/destination-teradata/build.gradle | 7 +++++++ .../connectors/destination-tidb/build.gradle | 7 +++++++ .../connectors/destination-vertica/build.gradle | 7 +++++++ .../connectors/destination-yugabytedb/build.gradle | 7 +++++++ .../connectors/source-bigquery/build.gradle | 10 ++++++++++ .../source-clickhouse-strict-encrypt/build.gradle | 7 +++++++ .../connectors/source-clickhouse/build.gradle | 7 +++++++ .../connectors/source-cockroachdb/build.gradle | 10 ++++++++++ .../connectors/source-db2/build.gradle | 10 ++++++++++ .../connectors/source-dynamodb/build.gradle | 7 +++++++ .../connectors/source-e2e-test-cloud/build.gradle | 7 +++++++ .../connectors/source-e2e-test/build.gradle | 10 ++++++++++ .../connectors/source-elasticsearch/build.gradle | 7 +++++++ .../connectors/source-kafka/build.gradle | 7 +++++++ .../source-mongodb-strict-encrypt/build.gradle | 7 +++++++ .../source-oracle-strict-encrypt/build.gradle | 7 +++++++ .../connectors/source-oracle/build.gradle | 10 ++++++++++ .../connectors/source-redshift/build.gradle | 10 ++++++++++ .../connectors/source-sftp/build.gradle | 7 +++++++ .../connectors/source-snowflake/build.gradle | 7 +++++++ .../connectors/source-teradata/build.gradle | 10 ++++++++++ .../connectors/source-tidb/build.gradle | 10 ++++++++++ build.gradle | 8 ++++++++ 69 files changed, 501 insertions(+) diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle index 3c0ced6ec053..9073e823a0fe 100644 --- a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle @@ -7,6 +7,9 @@ java { compileJava { options.compilerArgs += "-Xlint:-varargs,-try,-deprecation" } + compileTestJava { + options.compilerArgs += "-Xlint:-try" + } } dependencies { diff --git a/airbyte-cdk/java/airbyte-cdk/core/build.gradle b/airbyte-cdk/java/airbyte-cdk/core/build.gradle index 8fbc1f7f4249..5dccda8a8d05 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/core/build.gradle @@ -3,6 +3,9 @@ java { compileJava { options.compilerArgs += "-Xlint:-deprecation,-try,-rawtypes,-overloads,-cast,-unchecked" } + compileTestJava { + options.compilerArgs += "-Xlint:-try,-divzero,-cast" + } } configurations.all { diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle index 241d0eeadd1c..488d73eebf0b 100644 --- a/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle @@ -45,3 +45,12 @@ dependencies { } + +java { + compileJava { + options.compilerArgs.remove("-Werror") + } + compileTestJava { + options.compilerArgs += "-Xlint:-try" + } +} \ No newline at end of file diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle b/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle index 6571d5e3b761..7f001ddb88ac 100644 --- a/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle @@ -18,3 +18,9 @@ dependencies { testFixturesImplementation 'org.junit.jupiter:junit-jupiter-params' testFixturesImplementation 'org.mockito:mockito-core:4.6.1' } + +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/build.gradle b/airbyte-integrations/connectors/destination-azure-blob-storage/build.gradle index 772df9a2651f..68dfe0c1cad3 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/build.gradle +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-cassandra/build.gradle b/airbyte-integrations/connectors/destination-cassandra/build.gradle index af5ff89ff1bd..b9774a9b9c7f 100644 --- a/airbyte-integrations/connectors/destination-cassandra/build.gradle +++ b/airbyte-integrations/connectors/destination-cassandra/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle index cd1363bf76e5..d1a316d740a4 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-clickhouse/build.gradle b/airbyte-integrations/connectors/destination-clickhouse/build.gradle index a4130438a6b1..0386841d5f45 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-csv/build.gradle b/airbyte-integrations/connectors/destination-csv/build.gradle index dc0ddb9b826c..d4a73e5d7143 100644 --- a/airbyte-integrations/connectors/destination-csv/build.gradle +++ b/airbyte-integrations/connectors/destination-csv/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-databricks/build.gradle b/airbyte-integrations/connectors/destination-databricks/build.gradle index 9726e2592dda..a3a3c6bb8ca8 100644 --- a/airbyte-integrations/connectors/destination-databricks/build.gradle +++ b/airbyte-integrations/connectors/destination-databricks/build.gradle @@ -24,6 +24,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-dev-null/build.gradle b/airbyte-integrations/connectors/destination-dev-null/build.gradle index 58c82aa8de97..e167b803db50 100644 --- a/airbyte-integrations/connectors/destination-dev-null/build.gradle +++ b/airbyte-integrations/connectors/destination-dev-null/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-doris/build.gradle b/airbyte-integrations/connectors/destination-doris/build.gradle index 1fce3f43fb34..1fe67aaf8ff2 100644 --- a/airbyte-integrations/connectors/destination-doris/build.gradle +++ b/airbyte-integrations/connectors/destination-doris/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-dynamodb/build.gradle b/airbyte-integrations/connectors/destination-dynamodb/build.gradle index e55ede35ef47..4ae5c529215b 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/build.gradle +++ b/airbyte-integrations/connectors/destination-dynamodb/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-e2e-test/build.gradle b/airbyte-integrations/connectors/destination-e2e-test/build.gradle index e08a1eda3bd4..c8d98e1ddc2b 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/build.gradle +++ b/airbyte-integrations/connectors/destination-e2e-test/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle index 9583e882efba..6cd2f88febbe 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-elasticsearch/build.gradle b/airbyte-integrations/connectors/destination-elasticsearch/build.gradle index 89cf33888fd5..52c7536993c1 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/build.gradle +++ b/airbyte-integrations/connectors/destination-elasticsearch/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-exasol/build.gradle b/airbyte-integrations/connectors/destination-exasol/build.gradle index ce7a81b0d460..3380731e417d 100644 --- a/airbyte-integrations/connectors/destination-exasol/build.gradle +++ b/airbyte-integrations/connectors/destination-exasol/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-gcs/build.gradle b/airbyte-integrations/connectors/destination-gcs/build.gradle index 326920e6ef5a..a524445249d2 100644 --- a/airbyte-integrations/connectors/destination-gcs/build.gradle +++ b/airbyte-integrations/connectors/destination-gcs/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-iceberg/build.gradle b/airbyte-integrations/connectors/destination-iceberg/build.gradle index bccc1dc02e27..37f06943d35d 100644 --- a/airbyte-integrations/connectors/destination-iceberg/build.gradle +++ b/airbyte-integrations/connectors/destination-iceberg/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-kafka/build.gradle b/airbyte-integrations/connectors/destination-kafka/build.gradle index 18a718371e4a..69da18f35960 100644 --- a/airbyte-integrations/connectors/destination-kafka/build.gradle +++ b/airbyte-integrations/connectors/destination-kafka/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-keen/build.gradle b/airbyte-integrations/connectors/destination-keen/build.gradle index 70abafa1ef0c..777118dbb370 100644 --- a/airbyte-integrations/connectors/destination-keen/build.gradle +++ b/airbyte-integrations/connectors/destination-keen/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-kinesis/build.gradle b/airbyte-integrations/connectors/destination-kinesis/build.gradle index dde7ccf51adf..3abe284a89a8 100644 --- a/airbyte-integrations/connectors/destination-kinesis/build.gradle +++ b/airbyte-integrations/connectors/destination-kinesis/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-local-json/build.gradle b/airbyte-integrations/connectors/destination-local-json/build.gradle index e03588646b48..84a09417b178 100644 --- a/airbyte-integrations/connectors/destination-local-json/build.gradle +++ b/airbyte-integrations/connectors/destination-local-json/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/build.gradle b/airbyte-integrations/connectors/destination-mariadb-columnstore/build.gradle index 130202a52e8c..8d545e9b1813 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/build.gradle +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle index 2c5e40cd2265..106b17a3248c 100644 --- a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-mongodb/build.gradle b/airbyte-integrations/connectors/destination-mongodb/build.gradle index f442eee32d74..cad1ef429466 100644 --- a/airbyte-integrations/connectors/destination-mongodb/build.gradle +++ b/airbyte-integrations/connectors/destination-mongodb/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-mqtt/build.gradle b/airbyte-integrations/connectors/destination-mqtt/build.gradle index 13ae29dff017..599b538f4ac2 100644 --- a/airbyte-integrations/connectors/destination-mqtt/build.gradle +++ b/airbyte-integrations/connectors/destination-mqtt/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle index cb4ca4ea5028..09e3a703a2b6 100644 --- a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle @@ -12,6 +12,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-mssql/build.gradle b/airbyte-integrations/connectors/destination-mssql/build.gradle index 9606e4d90f08..ba588da10bce 100644 --- a/airbyte-integrations/connectors/destination-mssql/build.gradle +++ b/airbyte-integrations/connectors/destination-mssql/build.gradle @@ -12,6 +12,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle index d19379c282eb..ac8e77da74ea 100644 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-mysql/build.gradle b/airbyte-integrations/connectors/destination-mysql/build.gradle index 2523e89d85c3..8de8ca4e8497 100644 --- a/airbyte-integrations/connectors/destination-mysql/build.gradle +++ b/airbyte-integrations/connectors/destination-mysql/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle index 395c555e63dd..0e940345ab00 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-oracle/build.gradle b/airbyte-integrations/connectors/destination-oracle/build.gradle index 2113ccbaade8..a192ee34744a 100644 --- a/airbyte-integrations/connectors/destination-oracle/build.gradle +++ b/airbyte-integrations/connectors/destination-oracle/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle index 98d61dd374d3..ca5db84576a7 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-postgres/build.gradle b/airbyte-integrations/connectors/destination-postgres/build.gradle index d4015cc71204..a685564c1d60 100644 --- a/airbyte-integrations/connectors/destination-postgres/build.gradle +++ b/airbyte-integrations/connectors/destination-postgres/build.gradle @@ -12,6 +12,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-pubsub/build.gradle b/airbyte-integrations/connectors/destination-pubsub/build.gradle index 9b2755bd2496..f0c8f25ea7f4 100644 --- a/airbyte-integrations/connectors/destination-pubsub/build.gradle +++ b/airbyte-integrations/connectors/destination-pubsub/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-pulsar/build.gradle b/airbyte-integrations/connectors/destination-pulsar/build.gradle index a56f39c765b6..fad585d1ed6b 100644 --- a/airbyte-integrations/connectors/destination-pulsar/build.gradle +++ b/airbyte-integrations/connectors/destination-pulsar/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-r2/build.gradle b/airbyte-integrations/connectors/destination-r2/build.gradle index 27974417f756..94626b963c02 100644 --- a/airbyte-integrations/connectors/destination-r2/build.gradle +++ b/airbyte-integrations/connectors/destination-r2/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-redis/build.gradle b/airbyte-integrations/connectors/destination-redis/build.gradle index 4d1b5a51ad3b..83cf3a207e87 100644 --- a/airbyte-integrations/connectors/destination-redis/build.gradle +++ b/airbyte-integrations/connectors/destination-redis/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-redpanda/build.gradle b/airbyte-integrations/connectors/destination-redpanda/build.gradle index 5c0d05175509..a79982fe7c56 100644 --- a/airbyte-integrations/connectors/destination-redpanda/build.gradle +++ b/airbyte-integrations/connectors/destination-redpanda/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-redshift/build.gradle b/airbyte-integrations/connectors/destination-redshift/build.gradle index 0b233aeebe4f..8b3bd50ba5fd 100644 --- a/airbyte-integrations/connectors/destination-redshift/build.gradle +++ b/airbyte-integrations/connectors/destination-redshift/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-rockset/build.gradle b/airbyte-integrations/connectors/destination-rockset/build.gradle index 8757184533a3..a5e64c058290 100644 --- a/airbyte-integrations/connectors/destination-rockset/build.gradle +++ b/airbyte-integrations/connectors/destination-rockset/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-s3-glue/build.gradle b/airbyte-integrations/connectors/destination-s3-glue/build.gradle index 95db43ec7be1..23f96e515697 100644 --- a/airbyte-integrations/connectors/destination-s3-glue/build.gradle +++ b/airbyte-integrations/connectors/destination-s3-glue/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-scylla/build.gradle b/airbyte-integrations/connectors/destination-scylla/build.gradle index 10f7738a7944..512279a1345e 100644 --- a/airbyte-integrations/connectors/destination-scylla/build.gradle +++ b/airbyte-integrations/connectors/destination-scylla/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-selectdb/build.gradle b/airbyte-integrations/connectors/destination-selectdb/build.gradle index 34d2bf6f56e8..0a654ec66f67 100644 --- a/airbyte-integrations/connectors/destination-selectdb/build.gradle +++ b/airbyte-integrations/connectors/destination-selectdb/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/build.gradle b/airbyte-integrations/connectors/destination-starburst-galaxy/build.gradle index 2fb03ea6e7e8..ffe2bf71cf6b 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/build.gradle +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-teradata/build.gradle b/airbyte-integrations/connectors/destination-teradata/build.gradle index 3f075249bd8a..0f1f2ebe89d4 100644 --- a/airbyte-integrations/connectors/destination-teradata/build.gradle +++ b/airbyte-integrations/connectors/destination-teradata/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-tidb/build.gradle b/airbyte-integrations/connectors/destination-tidb/build.gradle index 2f8e67c4f02a..e0845bc0ee01 100644 --- a/airbyte-integrations/connectors/destination-tidb/build.gradle +++ b/airbyte-integrations/connectors/destination-tidb/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-vertica/build.gradle b/airbyte-integrations/connectors/destination-vertica/build.gradle index 04c3d69c09c4..d5392f6c238b 100644 --- a/airbyte-integrations/connectors/destination-vertica/build.gradle +++ b/airbyte-integrations/connectors/destination-vertica/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/destination-yugabytedb/build.gradle b/airbyte-integrations/connectors/destination-yugabytedb/build.gradle index 4be4d3367015..2186a1b5d8ee 100644 --- a/airbyte-integrations/connectors/destination-yugabytedb/build.gradle +++ b/airbyte-integrations/connectors/destination-yugabytedb/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/source-bigquery/build.gradle b/airbyte-integrations/connectors/source-bigquery/build.gradle index 9fd6b8bca041..8a79a509d68e 100644 --- a/airbyte-integrations/connectors/source-bigquery/build.gradle +++ b/airbyte-integrations/connectors/source-bigquery/build.gradle @@ -9,6 +9,16 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileTestJava { + options.compilerArgs.remove("-Werror") + } + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/build.gradle index 19a23187dfd8..a6c2dfa8b3d7 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/source-clickhouse/build.gradle b/airbyte-integrations/connectors/source-clickhouse/build.gradle index 04d3b5925ca6..ae4d2a7b12ca 100644 --- a/airbyte-integrations/connectors/source-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/source-clickhouse/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/source-cockroachdb/build.gradle b/airbyte-integrations/connectors/source-cockroachdb/build.gradle index bd124ce8a135..18e96169143e 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/build.gradle +++ b/airbyte-integrations/connectors/source-cockroachdb/build.gradle @@ -9,6 +9,16 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileTestJava { + options.compilerArgs.remove("-Werror") + } + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/source-db2/build.gradle b/airbyte-integrations/connectors/source-db2/build.gradle index 0a0bcff91cff..d6b508b8ada9 100644 --- a/airbyte-integrations/connectors/source-db2/build.gradle +++ b/airbyte-integrations/connectors/source-db2/build.gradle @@ -9,6 +9,16 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileTestJava { + options.compilerArgs.remove("-Werror") + } + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/source-dynamodb/build.gradle b/airbyte-integrations/connectors/source-dynamodb/build.gradle index 3a471ba3f6d6..f555ca183d4d 100644 --- a/airbyte-integrations/connectors/source-dynamodb/build.gradle +++ b/airbyte-integrations/connectors/source-dynamodb/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/source-e2e-test-cloud/build.gradle b/airbyte-integrations/connectors/source-e2e-test-cloud/build.gradle index 07e48c716746..1a3b3f3109eb 100644 --- a/airbyte-integrations/connectors/source-e2e-test-cloud/build.gradle +++ b/airbyte-integrations/connectors/source-e2e-test-cloud/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/source-e2e-test/build.gradle b/airbyte-integrations/connectors/source-e2e-test/build.gradle index a5545ffa2089..148e0cf1922b 100644 --- a/airbyte-integrations/connectors/source-e2e-test/build.gradle +++ b/airbyte-integrations/connectors/source-e2e-test/build.gradle @@ -9,6 +9,16 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileTestJava { + options.compilerArgs.remove("-Werror") + } + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/source-elasticsearch/build.gradle b/airbyte-integrations/connectors/source-elasticsearch/build.gradle index 9e3253a80456..31f1cfca5fcc 100644 --- a/airbyte-integrations/connectors/source-elasticsearch/build.gradle +++ b/airbyte-integrations/connectors/source-elasticsearch/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/source-kafka/build.gradle b/airbyte-integrations/connectors/source-kafka/build.gradle index a8ebfaf3c75c..4a3137bec286 100644 --- a/airbyte-integrations/connectors/source-kafka/build.gradle +++ b/airbyte-integrations/connectors/source-kafka/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/build.gradle index 785052ac8c0d..d2557caffc6b 100644 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-oracle-strict-encrypt/build.gradle index f8801c5d29a1..3878e876b4e9 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/source-oracle/build.gradle b/airbyte-integrations/connectors/source-oracle/build.gradle index 207c8d252d69..35abccdd41fa 100644 --- a/airbyte-integrations/connectors/source-oracle/build.gradle +++ b/airbyte-integrations/connectors/source-oracle/build.gradle @@ -9,6 +9,16 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileTestJava { + options.compilerArgs.remove("-Werror") + } + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/source-redshift/build.gradle b/airbyte-integrations/connectors/source-redshift/build.gradle index e2c9b3879051..2f4952d6e725 100644 --- a/airbyte-integrations/connectors/source-redshift/build.gradle +++ b/airbyte-integrations/connectors/source-redshift/build.gradle @@ -9,6 +9,16 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileTestJava { + options.compilerArgs.remove("-Werror") + } + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/source-sftp/build.gradle b/airbyte-integrations/connectors/source-sftp/build.gradle index 14fd4e5bf81e..da35e916d0e2 100644 --- a/airbyte-integrations/connectors/source-sftp/build.gradle +++ b/airbyte-integrations/connectors/source-sftp/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/source-snowflake/build.gradle b/airbyte-integrations/connectors/source-snowflake/build.gradle index 9270de70fd35..8b44e4582d2f 100644 --- a/airbyte-integrations/connectors/source-snowflake/build.gradle +++ b/airbyte-integrations/connectors/source-snowflake/build.gradle @@ -9,6 +9,13 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/source-teradata/build.gradle b/airbyte-integrations/connectors/source-teradata/build.gradle index a2863cf774c6..1e3ec17dcab5 100644 --- a/airbyte-integrations/connectors/source-teradata/build.gradle +++ b/airbyte-integrations/connectors/source-teradata/build.gradle @@ -9,6 +9,16 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileTestJava { + options.compilerArgs.remove("-Werror") + } + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/airbyte-integrations/connectors/source-tidb/build.gradle b/airbyte-integrations/connectors/source-tidb/build.gradle index fa8ea426cbda..1368609a3dea 100644 --- a/airbyte-integrations/connectors/source-tidb/build.gradle +++ b/airbyte-integrations/connectors/source-tidb/build.gradle @@ -9,6 +9,16 @@ airbyteJavaConnector { useLocalCdk = false } +//remove once upgrading the CDK version to 0.4.x or later +java { + compileTestJava { + options.compilerArgs.remove("-Werror") + } + compileJava { + options.compilerArgs.remove("-Werror") + } +} + airbyteJavaConnector.addCdkDependencies() application { diff --git a/build.gradle b/build.gradle index 2caf9a735fa4..0573f02fe661 100644 --- a/build.gradle +++ b/build.gradle @@ -439,6 +439,14 @@ subprojects { subproj -> java { sourceCompatibility = JavaVersion.VERSION_17 targetCompatibility = JavaVersion.VERSION_17 + compileJava { + options.compilerArgs += ["-Werror", "-Xlint:all,-serial,-processing"] + } + compileTestJava { + //rawtypes and unchecked are necessary for mockito + //deprecation and removal are removed from error since we should still test those constructs. + options.compilerArgs += ["-Werror", "-Xlint:all,-serial,-processing,-rawtypes,-unchecked,-deprecation,-removal"] + } } if (isConnectorProject(subproj)) { From e2e0aba489923630cc7f59d21f150a5cb77f33f3 Mon Sep 17 00:00:00 2001 From: Akash Kulkarni <113392464+akashkulk@users.noreply.github.com> Date: Mon, 13 Nov 2023 19:51:05 -0800 Subject: [PATCH 025/124] [source-mongodb-v2] : Fix support to read UUIDs (#32285) Co-authored-by: akashkulk --- airbyte-cdk/java/airbyte-cdk/README.md | 55 ++++++++++--------- .../src/main/resources/version.properties | 2 +- .../mongodb/MongoDbCdcEventUtils.java | 4 +- .../mongodb/MongoDbCdcEventUtilsTest.java | 14 ++++- .../connectors/source-mongodb-v2/build.gradle | 2 +- .../source-mongodb-v2/metadata.yaml | 2 +- docs/integrations/sources/mongodb-v2.md | 7 ++- 7 files changed, 50 insertions(+), 36 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md index 6408b52ab36a..870b06e675e3 100644 --- a/airbyte-cdk/java/airbyte-cdk/README.md +++ b/airbyte-cdk/java/airbyte-cdk/README.md @@ -154,30 +154,31 @@ MavenLocal debugging steps: ### Java CDK -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.4.9 | 2023-11-13 | [\#32468](https://github.com/airbytehq/airbyte/pull/32468) | Further error grouping improvements for DV2 connectors | -| 0.4.8 | 2023-11-09 | [\#32377](https://github.com/airbytehq/airbyte/pull/32377) | source-postgres tests: skip dropping database | -| 0.4.7 | 2023-11-08 | [\#31856](https://github.com/airbytehq/airbyte/pull/31856) | source-postgres: support for inifinty date and timestamps | -| 0.4.5 | 2023-11-07 | [\#32112](https://github.com/airbytehq/airbyte/pull/32112) | Async destinations framework: Allow configuring the queue flush threshold | -| 0.4.4 | 2023-11-06 | [\#32119](https://github.com/airbytehq/airbyte/pull/32119) | Add STANDARD UUID codec to MongoDB debezium handler | -| 0.4.2 | 2023-11-06 | [\#32190](https://github.com/airbytehq/airbyte/pull/32190) | Improve error deinterpolation | -| 0.4.1 | 2023-11-02 | [\#32192](https://github.com/airbytehq/airbyte/pull/32192) | Add 's3-destinations' CDK module. | -| 0.4.0 | 2023-11-02 | [\#32050](https://github.com/airbytehq/airbyte/pull/32050) | Fix compiler warnings. | -| 0.3.0 | 2023-11-02 | [\#31983](https://github.com/airbytehq/airbyte/pull/31983) | Add deinterpolation feature to AirbyteExceptionHandler. | -| 0.2.4 | 2023-10-31 | [\#31807](https://github.com/airbytehq/airbyte/pull/31807) | Handle case of debezium update and delete of records in mongodb. | -| 0.2.3 | 2023-10-31 | [\#32022](https://github.com/airbytehq/airbyte/pull/32022) | Update Debezium version from 2.20 -> 2.4.0. | -| 0.2.2 | 2023-10-31 | [\#31976](https://github.com/airbytehq/airbyte/pull/31976) | Debezium tweaks to make tests run faster. | -| 0.2.0 | 2023-10-30 | [\#31960](https://github.com/airbytehq/airbyte/pull/31960) | Hoist top-level gradle subprojects into CDK. | -| 0.1.12 | 2023-10-24 | [\#31674](https://github.com/airbytehq/airbyte/pull/31674) | Fail sync when Debezium does not shut down properly. | -| 0.1.11 | 2023-10-18 | [\#31486](https://github.com/airbytehq/airbyte/pull/31486) | Update constants in AdaptiveSourceRunner. | -| 0.1.9 | 2023-10-12 | [\#31309](https://github.com/airbytehq/airbyte/pull/31309) | Use toPlainString() when handling BigDecimals in PostgresConverter | -| 0.1.8 | 2023-10-11 | [\#31322](https://github.com/airbytehq/airbyte/pull/31322) | Cap log line length to 32KB to prevent loss of records | -| 0.1.7 | 2023-10-10 | [\#31194](https://github.com/airbytehq/airbyte/pull/31194) | Deallocate unused per stream buffer memory when empty | -| 0.1.6 | 2023-10-10 | [\#31083](https://github.com/airbytehq/airbyte/pull/31083) | Fix precision of numeric values in async destinations | -| 0.1.5 | 2023-10-09 | [\#31196](https://github.com/airbytehq/airbyte/pull/31196) | Update typo in CDK (CDN_LSN -> CDC_LSN) | -| 0.1.4 | 2023-10-06 | [\#31139](https://github.com/airbytehq/airbyte/pull/31139) | Reduce async buffer | -| 0.1.1 | 2023-09-28 | [\#30835](https://github.com/airbytehq/airbyte/pull/30835) | JDBC destinations now avoid staging area name collisions by using the raw table name as the stage name. (previously we used the stream name as the stage name) | -| 0.1.0 | 2023-09-27 | [\#30445](https://github.com/airbytehq/airbyte/pull/30445) | First launch, including shared classes for all connectors. | -| 0.0.2 | 2023-08-21 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Version bump only (no other changes). | -| 0.0.1 | 2023-08-08 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Initial release for testing. | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.4.10 | 2023-11-13 | [\#32285](https://github.com/airbytehq/airbyte/pull/32285) | Fix UUID codec ordering for MongoDB connector | +| 0.4.9 | 2023-11-13 | [\#32468](https://github.com/airbytehq/airbyte/pull/32468) | Further error grouping improvements for DV2 connectors | +| 0.4.8 | 2023-11-09 | [\#32377](https://github.com/airbytehq/airbyte/pull/32377) | source-postgres tests: skip dropping database | +| 0.4.7 | 2023-11-08 | [\#31856](https://github.com/airbytehq/airbyte/pull/31856) | source-postgres: support for inifinty date and timestamps | +| 0.4.5 | 2023-11-07 | [\#32112](https://github.com/airbytehq/airbyte/pull/32112) | Async destinations framework: Allow configuring the queue flush threshold | +| 0.4.4 | 2023-11-06 | [\#32119](https://github.com/airbytehq/airbyte/pull/32119) | Add STANDARD UUID codec to MongoDB debezium handler | +| 0.4.2 | 2023-11-06 | [\#32190](https://github.com/airbytehq/airbyte/pull/32190) | Improve error deinterpolation | +| 0.4.1 | 2023-11-02 | [\#32192](https://github.com/airbytehq/airbyte/pull/32192) | Add 's3-destinations' CDK module. | +| 0.4.0 | 2023-11-02 | [\#32050](https://github.com/airbytehq/airbyte/pull/32050) | Fix compiler warnings. | +| 0.3.0 | 2023-11-02 | [\#31983](https://github.com/airbytehq/airbyte/pull/31983) | Add deinterpolation feature to AirbyteExceptionHandler. | +| 0.2.4 | 2023-10-31 | [\#31807](https://github.com/airbytehq/airbyte/pull/31807) | Handle case of debezium update and delete of records in mongodb. | +| 0.2.3 | 2023-10-31 | [\#32022](https://github.com/airbytehq/airbyte/pull/32022) | Update Debezium version from 2.20 -> 2.4.0. | +| 0.2.2 | 2023-10-31 | [\#31976](https://github.com/airbytehq/airbyte/pull/31976) | Debezium tweaks to make tests run faster. | +| 0.2.0 | 2023-10-30 | [\#31960](https://github.com/airbytehq/airbyte/pull/31960) | Hoist top-level gradle subprojects into CDK. | +| 0.1.12 | 2023-10-24 | [\#31674](https://github.com/airbytehq/airbyte/pull/31674) | Fail sync when Debezium does not shut down properly. | +| 0.1.11 | 2023-10-18 | [\#31486](https://github.com/airbytehq/airbyte/pull/31486) | Update constants in AdaptiveSourceRunner. | +| 0.1.9 | 2023-10-12 | [\#31309](https://github.com/airbytehq/airbyte/pull/31309) | Use toPlainString() when handling BigDecimals in PostgresConverter | +| 0.1.8 | 2023-10-11 | [\#31322](https://github.com/airbytehq/airbyte/pull/31322) | Cap log line length to 32KB to prevent loss of records | +| 0.1.7 | 2023-10-10 | [\#31194](https://github.com/airbytehq/airbyte/pull/31194) | Deallocate unused per stream buffer memory when empty | +| 0.1.6 | 2023-10-10 | [\#31083](https://github.com/airbytehq/airbyte/pull/31083) | Fix precision of numeric values in async destinations | +| 0.1.5 | 2023-10-09 | [\#31196](https://github.com/airbytehq/airbyte/pull/31196) | Update typo in CDK (CDN_LSN -> CDC_LSN) | +| 0.1.4 | 2023-10-06 | [\#31139](https://github.com/airbytehq/airbyte/pull/31139) | Reduce async buffer | +| 0.1.1 | 2023-09-28 | [\#30835](https://github.com/airbytehq/airbyte/pull/30835) | JDBC destinations now avoid staging area name collisions by using the raw table name as the stage name. (previously we used the stream name as the stage name) | +| 0.1.0 | 2023-09-27 | [\#30445](https://github.com/airbytehq/airbyte/pull/30445) | First launch, including shared classes for all connectors. | +| 0.0.2 | 2023-08-21 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Version bump only (no other changes). | +| 0.0.1 | 2023-08-08 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Initial release for testing. | diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties index 14b28050f1e2..3cf6558a4b17 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties @@ -1 +1 @@ -version=0.4.9 +version=0.4.10 diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcEventUtils.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcEventUtils.java index 0cf0abb07f16..82d6a0fb3e53 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcEventUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcEventUtils.java @@ -220,6 +220,7 @@ private static BsonDocument toBsonDocument(final Document document) { try { final CodecRegistry customCodecRegistry = fromProviders(asList( + new UuidCodecProvider(UuidRepresentation.STANDARD), new ValueCodecProvider(), new BsonValueCodecProvider(), new DocumentCodecProvider(), @@ -228,8 +229,7 @@ private static BsonDocument toBsonDocument(final Document document) { new Jsr310CodecProvider(), new JsonObjectCodecProvider(), new BsonCodecProvider(), - new DBRefCodecProvider(), - new UuidCodecProvider(UuidRepresentation.STANDARD))); + new DBRefCodecProvider())); // Override the default codec registry return document.toBsonDocument(BsonDocument.class, customCodecRegistry); diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcEventUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcEventUtilsTest.java index 67e0f6449163..75146837ca89 100644 --- a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcEventUtilsTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcEventUtilsTest.java @@ -22,6 +22,7 @@ import java.nio.charset.Charset; import java.util.Map; import java.util.Set; +import java.util.UUID; import org.apache.commons.codec.binary.Base64; import org.bson.BsonBinary; import org.bson.BsonBoolean; @@ -40,6 +41,7 @@ import org.bson.BsonSymbol; import org.bson.BsonTimestamp; import org.bson.Document; +import org.bson.UuidRepresentation; import org.bson.types.Decimal128; import org.bson.types.ObjectId; import org.junit.jupiter.api.Test; @@ -83,6 +85,8 @@ void testNormalizeObjectId() { void testTransformDataTypes() { final BsonTimestamp bsonTimestamp = new BsonTimestamp(394, 1926745562); final String expectedTimestamp = DataTypeUtils.toISO8601StringWithMilliseconds(bsonTimestamp.getValue()); + final UUID standardUuid = UUID.randomUUID(); + final UUID legacyUuid = UUID.randomUUID(); final Document document = new Document("field1", new BsonBoolean(true)) .append("field2", new BsonInt32(1)) @@ -99,7 +103,9 @@ void testTransformDataTypes() { .append("field13", new BsonJavaScriptWithScope("code2", new BsonDocument("scope", new BsonString("scope")))) .append("field14", new BsonRegularExpression("pattern")) .append("field15", new BsonNull()) - .append("field16", new Document("key", "value")); + .append("field16", new Document("key", "value")) + .append("field17", new BsonBinary(standardUuid, UuidRepresentation.STANDARD)) + .append("field18", new BsonBinary(legacyUuid, UuidRepresentation.JAVA_LEGACY)); final String documentAsJson = document.toJson(); final ObjectNode transformed = MongoDbCdcEventUtils.transformDataTypes(documentAsJson, document.keySet()); @@ -123,6 +129,12 @@ void testTransformDataTypes() { assertEquals("pattern", transformed.get("field14").asText()); assertFalse(transformed.has("field15")); assertEquals("value", transformed.get("field16").get("key").asText()); + // Assert that UUIDs can be serialized. Currently, they will be represented as base 64 encoded + // strings. Since the original mongo source + // may have these UUIDs written by a variety of sources, each with different encodings - we cannot + // decode these back to the original UUID. + assertTrue(transformed.has("field17")); + assertTrue(transformed.has("field18")); } @Test diff --git a/airbyte-integrations/connectors/source-mongodb-v2/build.gradle b/airbyte-integrations/connectors/source-mongodb-v2/build.gradle index dd1ccde9ccd2..137d37a71d94 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/build.gradle +++ b/airbyte-integrations/connectors/source-mongodb-v2/build.gradle @@ -5,7 +5,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.4.4' + cdkVersionRequired = '0.4.10' features = ['db-sources'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml b/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml index db6a5013a191..ecd0e8d195a8 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml +++ b/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: source definitionId: b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e - dockerImageTag: 1.0.8 + dockerImageTag: 1.0.9 dockerRepository: airbyte/source-mongodb-v2 documentationUrl: https://docs.airbyte.com/integrations/sources/mongodb-v2 githubIssueLabel: source-mongodb-v2 diff --git a/docs/integrations/sources/mongodb-v2.md b/docs/integrations/sources/mongodb-v2.md index 2aef61d81487..86cc990a7ea6 100644 --- a/docs/integrations/sources/mongodb-v2.md +++ b/docs/integrations/sources/mongodb-v2.md @@ -190,9 +190,10 @@ For more information regarding configuration parameters, please see [MongoDb Doc | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------| -| 1.0.8 | 2023-11-08 | [32125](https://github.com/airbytehq/airbyte/pull/32125) | Fix compilation warnings -| 1.0.7 | 2023-11-07 | [32250](https://github.com/airbytehq/airbyte/pull/32250) | Add support to read UUIDs. | -| 1.0.6 | 2023-11-06 | [32193](https://github.com/airbytehq/airbyte/pull/32193) | Adopt java CDK version 0.4.1. | +| 1.0.9 | 2023-11-08 | [32285](https://github.com/airbytehq/airbyte/pull/32285) | Additional support to read UUIDs | +| 1.0.8 | 2023-11-08 | [32125](https://github.com/airbytehq/airbyte/pull/32125) | Fix compilation warnings | +| 1.0.7 | 2023-11-07 | [32250](https://github.com/airbytehq/airbyte/pull/32250) | Add support to read UUIDs. | +| 1.0.6 | 2023-11-06 | [32193](https://github.com/airbytehq/airbyte/pull/32193) | Adopt java CDK version 0.4.1. | | 1.0.5 | 2023-10-31 | [32028](https://github.com/airbytehq/airbyte/pull/32028) | url encode username and password.
Handle a case of document update and delete in a single sync. | | 1.0.3 | 2023-10-19 | [31629](https://github.com/airbytehq/airbyte/pull/31629) | Allow discover operation use of disk file when an operation goes over max allowed mem | | 1.0.2 | 2023-10-19 | [31596](https://github.com/airbytehq/airbyte/pull/31596) | Allow use of temp disk file when an operation goes over max allowed mem | From c75325b91da1900513c506cdf7838da0dfda10e0 Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Tue, 14 Nov 2023 00:11:09 -0800 Subject: [PATCH 026/124] Update README.md (#32464) --- docs/connector-development/testing-connectors/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/connector-development/testing-connectors/README.md b/docs/connector-development/testing-connectors/README.md index 4c0abe0e51fc..90842bccbccb 100644 --- a/docs/connector-development/testing-connectors/README.md +++ b/docs/connector-development/testing-connectors/README.md @@ -7,7 +7,7 @@ Connector specific tests declared in the connector code directory: * Integration tests Tests common to all connectors: -* [QA checks](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/connector_ops/ci_connector_ops/qa_checks.py#L1) +* [QA checks](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py) * [Connector Acceptance tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference/) ## Running tests @@ -41,4 +41,4 @@ Connector Acceptance tests require connector configuration to be provided as a ` ## Tests on pull requests Our CI infrastructure runs the connector tests with [`airbyte-ci` CLI](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md). Connectors tests are automatically and remotely triggered on your branch according to the changes made in your branch. -**Passing tests are required to merge a connector pull request.** \ No newline at end of file +**Passing tests are required to merge a connector pull request.** From ac3eb28de2d5b1b6a984f1ed1c33295608b4b117 Mon Sep 17 00:00:00 2001 From: Ella Rohm-Ensing Date: Tue, 14 Nov 2023 02:17:48 -0600 Subject: [PATCH 027/124] airbyte-ci: add format commands (#31831) Co-authored-by: Ben Church Co-authored-by: bnchrch Co-authored-by: alafanechere Co-authored-by: Augustin Co-authored-by: Marius Posta Co-authored-by: alafanechere --- .github/workflows/format.yml | 62 +-- .github/workflows/gradle.yml | 3 +- .../airbyte_cdk/utils/analytics_message.py | 2 + .../vector_db_based/config_test.py | 4 + airbyte-ci/connectors/pipelines/README.md | 30 +- .../airbyte_ci/connectors/commands.py | 2 +- .../airbyte_ci/connectors/list/commands.py | 4 +- .../airbyte_ci/connectors/test/pipeline.py | 28 +- .../test/steps/python_connectors.py | 50 +- .../pipelines/airbyte_ci/format/__init__.py | 0 .../pipelines/airbyte_ci/format/actions.py | 54 ++ .../airbyte_ci/format/check/commands.py | 105 ++++ .../pipelines/airbyte_ci/format/commands.py | 28 ++ .../pipelines/airbyte_ci/format/consts.py | 44 ++ .../pipelines/airbyte_ci/format/containers.py | 110 +++++ .../airbyte_ci/format/fix/commands.py | 134 +++++ .../pipelines/airbyte_ci/steps/gradle.py | 4 - .../pipelines/pipelines/cli/airbyte_ci.py | 3 +- .../pipelines/cli/click_decorators.py | 2 + .../connectors/pipelines/pipelines/consts.py | 10 +- .../pipelines/dagger/containers/python.py | 8 +- .../pipelines/pipelines/helpers/cli.py | 104 ++++ .../models/contexts/click_pipeline_context.py | 10 +- .../pipelines/pipelines/models/singleton.py | 2 + .../pipelines/pipelines/models/steps.py | 27 +- airbyte-ci/connectors/pipelines/poetry.lock | 465 +++++++++--------- .../connectors/pipelines/pyproject.toml | 2 +- .../tests/test_cli/test_click_decorators.py | 2 + .../bases/base-normalization/build.gradle | 8 +- .../stream_reader.py | 2 + .../unit_tests/unit_tests.py | 2 + .../source-github/unit_tests/conftest.py | 2 + .../source-intercom/unit_tests/test_source.py | 4 + .../source_klaviyo/exceptions.py | 3 + .../unit_tests/conftest.py | 2 + .../connectors/source-pokeapi/main.py | 1 - .../source-s3/source_s3/v4/zip_reader.py | 2 + .../unit_tests/v4/test_zip_reader.py | 2 + .../unit_tests/conftest.py | 2 + .../unit_tests/conftest.py | 2 + build.gradle | 220 +-------- octavia-cli/build.gradle | 22 - poetry.lock | 140 ++++++ pyproject.toml | 13 + settings.gradle | 1 - tools/gradle/codestyle/sql-dbeaver.properties | 8 - 46 files changed, 1104 insertions(+), 631 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/__init__.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py delete mode 100644 octavia-cli/build.gradle create mode 100644 poetry.lock delete mode 100644 tools/gradle/codestyle/sql-dbeaver.properties diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 8854146c2b9e..e826618861d5 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -1,7 +1,9 @@ -name: Format Code (Python + Java) +name: Check for formatting errors concurrency: group: ${{ github.workflow }}-${{ github.ref }} + # Cancel any previous runs on the same branch if they are still in progress + cancel-in-progress: true on: workflow_dispatch: @@ -10,8 +12,8 @@ on: - master pull_request: jobs: - format-and-commit: - runs-on: ubuntu-latest + format-fix: + runs-on: "conn-prod-xlarge-runner" name: "Apply All Formatting Rules" timeout-minutes: 40 steps: @@ -20,16 +22,16 @@ jobs: with: ref: ${{ github.head_ref }} # Important that this is set so that CI checks are triggered again - # Without this we would be be forever waiting on required checks to pass + # Without this we would be forever waiting on required checks to pass token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} - # IMPORTANT! This is nessesary to make sure that a status is reported on the PR - # even if the workflow is skipped. If we used github actions filters, the workflow + # IMPORTANT! This is necessary to make sure that a status is reported on the PR + # even if the workflow is skipped. If we used GitHub Actions filters, the workflow # would not be reported as skipped, but instead would be forever pending. # # I KNOW THIS SOUNDS CRAZY, BUT IT IS TRUE. # - # Also it gets worse + # Also, it gets worse # # IMPORTANT! DO NOT CHANGE THE QUOTES AROUND THE GLOBS. THEY ARE REQUIRED. # MAKE SURE TO TEST ANY SYNTAX CHANGES BEFORE MERGING. @@ -42,43 +44,23 @@ jobs: - '**/*' - '!**/*.md' - - uses: actions/setup-java@v3 - with: - distribution: "zulu" - java-version: "17" - - - uses: actions/setup-python@v4 + - name: Run airbyte-ci format fix + uses: ./.github/actions/run-dagger-pipeline with: - python-version: "3.10" - - - name: Set up CI Gradle Properties - run: | - mkdir -p ~/.gradle/ - cat > ~/.gradle/gradle.properties <`format` command subgroup + +Available commands: +* `airbyte-ci format check all` +* `airbyte-ci format fix all` + +### Examples +- Check for formatting errors in the repository: `airbyte-ci format check all` +- Fix formatting for only python files: `airbyte-ci format fix python` + +### `format check all` command + +This command runs formatting checks, but does not format the code in place. It will exit 1 as soon as a failure is encountered. To fix errors, use `airbyte-ci format fix all`. + +Running `airbyte-ci format check` will run checks on all different types of code. Run `airbyte-ci format check --help` for subcommands to check formatting for only certain types of files. + +### `format fix all` command + +This command runs formatting checks and reformats any code that would be reformatted, so it's recommended to stage changes you might have before running this command. + +Running `airbyte-ci format fix all` will format all of the different types of code. Run `airbyte-ci format fix --help` for subcommands to format only certain types of files. + + ### `metadata` command subgroup Available commands: @@ -408,7 +433,8 @@ This command runs the Python tests for a airbyte-ci poetry package. ## Changelog | Version | PR | Description | | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | -| 2.5.9 | [#32427](https://github.com/airbytehq/airbyte/pull/32427) | Re-enable caching for source-postgres | +| 2.6.0 | [#31831](https://github.com/airbytehq/airbyte/pull/31831) | Add `airbyte-ci format` commands, remove connector-specific formatting check | +| 2.5.9 | [#32427](https://github.com/airbytehq/airbyte/pull/32427) | Re-enable caching for source-postgres | | 2.5.8 | [#32402](https://github.com/airbytehq/airbyte/pull/32402) | Set Dagger Cloud token for airbyters only | | 2.5.7 | [#31628](https://github.com/airbytehq/airbyte/pull/31628) | Add ClickPipelineContext class | | 2.5.6 | [#32139](https://github.com/airbytehq/airbyte/pull/32139) | Test coverage report on Python connector UnitTest. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py index 539c839f94fb..cf25122268bc 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -145,7 +145,7 @@ def should_use_remote_secrets(use_remote_secrets: Optional[bool]) -> bool: lazy_subcommands={ "build": "pipelines.airbyte_ci.connectors.build_image.commands.build", "test": "pipelines.airbyte_ci.connectors.test.commands.test", - "list": "pipelines.airbyte_ci.connectors.list.commands.list", + "list": "pipelines.airbyte_ci.connectors.list.commands.list_connectors", "publish": "pipelines.airbyte_ci.connectors.publish.commands.publish", "bump_version": "pipelines.airbyte_ci.connectors.bump_version.commands.bump_version", "migrate_to_base_image": "pipelines.airbyte_ci.connectors.migrate_to_base_image.commands.migrate_to_base_image", diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py index 4b0f54e6c462..76c50648eb2e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py @@ -9,9 +9,9 @@ from rich.text import Text -@click.command(cls=DaggerPipelineCommand, help="List all selected connectors.") +@click.command(cls=DaggerPipelineCommand, help="List all selected connectors.", name="list") @click.pass_context -async def list( +async def list_connectors( ctx: click.Context, ): selected_connectors = sorted(ctx.obj["selected_connectors_with_modified_files"], key=lambda x: x.technical_name) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py index 930a392cb09f..df5a4d28888c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py @@ -21,12 +21,7 @@ ConnectorLanguage.PYTHON: python_connectors.run_all_tests, ConnectorLanguage.LOW_CODE: python_connectors.run_all_tests, ConnectorLanguage.JAVA: java_connectors.run_all_tests, - }, - "run_code_format_checks": { - ConnectorLanguage.PYTHON: python_connectors.run_code_format_checks, - ConnectorLanguage.LOW_CODE: python_connectors.run_code_format_checks, - # ConnectorLanguage.JAVA: java_connectors.run_code_format_checks - }, + } } @@ -65,22 +60,6 @@ async def run_qa_checks(context: ConnectorContext) -> List[StepResult]: return [await QaChecks(context).run()] -async def run_code_format_checks(context: ConnectorContext) -> List[StepResult]: - """Run the code format checks according to the connector language. - - Args: - context (ConnectorContext): The current connector context. - - Returns: - List[StepResult]: The results of the code format checks steps. - """ - if _run_code_format_checks := LANGUAGE_MAPPING["run_code_format_checks"].get(context.connector.language): - return await _run_code_format_checks(context) - else: - context.logger.warning(f"No code format checks defined for connector language {context.connector.language}!") - return [] - - async def run_all_tests(context: ConnectorContext) -> List[StepResult]: """Run all the tests steps according to the connector language. @@ -111,10 +90,7 @@ async def run_connector_test_pipeline(context: ConnectorContext, semaphore: anyi async with semaphore: async with context: async with asyncer.create_task_group() as task_group: - tasks = [ - task_group.soonify(run_all_tests)(context), - task_group.soonify(run_code_format_checks)(context), - ] + tasks = [task_group.soonify(run_all_tests)(context)] if not context.code_tests_only: tasks += [ task_group.soonify(run_metadata_validation)(context), diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py index ade547199d83..cf3ef8eee923 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py @@ -14,47 +14,11 @@ from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.test.steps.common import AcceptanceTests, CheckBaseImageIsUsed -from pipelines.consts import LOCAL_BUILD_PLATFORM, PYPROJECT_TOML_FILE_PATH +from pipelines.consts import LOCAL_BUILD_PLATFORM from pipelines.dagger.actions import secrets from pipelines.models.steps import Step, StepResult, StepStatus -class CodeFormatChecks(Step): - """A step to run the code format checks on a Python connector using Black, Isort and Flake.""" - - title = "Code format checks" - - RUN_BLACK_CMD = ["python", "-m", "black", f"--config=/{PYPROJECT_TOML_FILE_PATH}", "--check", "."] - RUN_ISORT_CMD = ["python", "-m", "isort", f"--settings-file=/{PYPROJECT_TOML_FILE_PATH}", "--check-only", "--diff", "."] - RUN_FLAKE_CMD = ["python", "-m", "pflake8", f"--config=/{PYPROJECT_TOML_FILE_PATH}", "."] - - async def _run(self) -> StepResult: - """Run a code format check on the container source code. - - We call black, isort and flake commands: - - Black formats the code: fails if the code is not formatted. - - Isort checks the import orders: fails if the import are not properly ordered. - - Flake enforces style-guides: fails if the style-guide is not followed. - - Args: - context (ConnectorContext): The current test context, providing a connector object, a dagger client and a repository directory. - step (Step): The step in which the code format checks are run. Defaults to Step.CODE_FORMAT_CHECKS - Returns: - StepResult: Failure or success of the code format checks with stdout and stderr. - """ - connector_under_test = pipelines.dagger.actions.python.common.with_python_connector_source(self.context) - - formatter = ( - connector_under_test.with_exec(["echo", "Running black"]) - .with_exec(self.RUN_BLACK_CMD) - .with_exec(["echo", "Running Isort"]) - .with_exec(self.RUN_ISORT_CMD) - .with_exec(["echo", "Running Flake"]) - .with_exec(self.RUN_FLAKE_CMD) - ) - return await self.get_step_result(formatter) - - class PytestStep(Step, ABC): """An abstract class to run pytest tests and evaluate success or failure according to pytest logs.""" @@ -261,15 +225,3 @@ async def run_all_tests(context: ConnectorContext) -> List[StepResult]: ] return step_results + [task.value for task in tasks] - - -async def run_code_format_checks(context: ConnectorContext) -> List[StepResult]: - """Run the code format check steps for Python connectors. - - Args: - context (ConnectorContext): The current connector context. - - Returns: - List[StepResult]: Results of the code format checks. - """ - return [await CodeFormatChecks(context).run()] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py new file mode 100644 index 000000000000..b7beab00e2e3 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py @@ -0,0 +1,54 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import List + +import dagger +from pipelines.helpers.utils import sh_dash_c + + +def run_check( + container: dagger.Container, + check_commands: List[str], +) -> dagger.Container: + """Checks whether the repository is formatted correctly. + Args: + container: (dagger.Container): The container to run the formatting check in + check_commands (List[str]): The list of commands to run to check the formatting + """ + return container.with_exec(sh_dash_c(check_commands), skip_entrypoint=True) + + +async def run_format( + container: dagger.Container, + format_commands: List[str], +) -> dagger.Container: + """Formats the repository. + Args: + container: (dagger.Container): The container to run the formatter in + format_commands (List[str]): The list of commands to run to format the repository + """ + format_container = container.with_exec(sh_dash_c(format_commands), skip_entrypoint=True) + return await format_container.directory("/src").export(".") + + +def mount_repo_for_formatting( + dagger_client: dagger.Client, + container: dagger.Container, + include: List[str], +) -> dagger.Container: + """Mounts the relevant parts of the repository: the code to format and the formatting config + Args: + container: (dagger.Container): The container to mount the repository in + include (List[str]): The list of files to include in the container + """ + container = container.with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=include, + ), + ).with_workdir("/src") + + return container diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py new file mode 100644 index 000000000000..4b97275cfe57 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/check/commands.py @@ -0,0 +1,105 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import logging + +import asyncclick as click +import dagger +from pipelines.airbyte_ci.format.actions import run_check +from pipelines.airbyte_ci.format.containers import ( + format_java_container, + format_js_container, + format_license_container, + format_python_container, +) +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand +from pipelines.helpers.cli import LogOptions, get_all_sibling_commands, invoke_commands_concurrently, log_command_results +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context +from pipelines.models.steps import CommandResult, StepStatus + + +# HELPERS +async def get_check_command_result(click_command: click.Command, checks_commands, container) -> CommandResult: + try: + stdout = await run_check(container, checks_commands).stdout() + return CommandResult(click_command, status=StepStatus.SUCCESS, stdout=stdout) + except dagger.ExecError as e: + return CommandResult(click_command, status=StepStatus.FAILURE, stderr=e.stderr, stdout=e.stdout, exc_info=e) + + +@click.group( + help="Run code format checks and fail if any checks fail.", + chain=True, +) +async def check(): + pass + + +@check.command(name="all") +@click.option("--list-errors", is_flag=True, default=False, help="Show detailed error messages for failed checks.") +@click.pass_context +async def all_checks(ctx: click.Context, list_errors: bool): + """ + Run all format checks and fail if any checks fail. + """ + all_commands = get_all_sibling_commands(ctx) + + command_results = await invoke_commands_concurrently(ctx, all_commands) + failure = any([r.status is StepStatus.FAILURE for r in command_results]) + parent_command = ctx.parent.command + logger = logging.getLogger(parent_command.name) + log_options = LogOptions( + list_errors=list_errors, + help_message="Run `airbyte-ci format check all --list-errors` to see detailed error messages for failed checks. Run `airbyte-ci format fix all` for a best effort fix.", + ) + log_command_results(ctx, command_results, logger, log_options) + if failure: + raise click.Abort() + + +@check.command(cls=DaggerPipelineCommand) +@pass_pipeline_context +async def python(pipeline_context: ClickPipelineContext) -> CommandResult: + """Format python code via black and isort.""" + + dagger_client = await pipeline_context.get_dagger_client(pipeline_name="Check python formatting") + container = format_python_container(dagger_client) + check_commands = [ + "poetry install --no-root", + "poetry run isort --settings-file pyproject.toml --check-only .", + "poetry run black --config pyproject.toml --check .", + ] + return await get_check_command_result(check.commands["python"], check_commands, container) + + +@check.command(cls=DaggerPipelineCommand) +@pass_pipeline_context +async def java(pipeline_context: ClickPipelineContext) -> CommandResult: + """Format java, groovy, and sql code via spotless.""" + dagger_client = await pipeline_context.get_dagger_client(pipeline_name="Check java formatting") + container = format_java_container(dagger_client) + check_commands = ["./gradlew spotlessCheck --scan"] + return await get_check_command_result(check.commands["java"], check_commands, container) + + +@check.command(cls=DaggerPipelineCommand) +@pass_pipeline_context +async def js(pipeline_context: ClickPipelineContext): + """Format yaml and json code via prettier.""" + dagger_client = await pipeline_context.get_dagger_client(pipeline_name="Check js formatting") + container = format_js_container(dagger_client) + check_commands = ["prettier --check ."] + return await get_check_command_result(check.commands["js"], check_commands, container) + + +@check.command("license") +@pass_pipeline_context +async def license_check(pipeline_context: ClickPipelineContext): + """Add license to python and java code via addlicense.""" + license_file = "LICENSE_SHORT" + dagger_client = await pipeline_context.get_dagger_client(pipeline_name="Check license header") + container = format_license_container(dagger_client, license_file) + check_commands = [f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_file} --check ."] + return await get_check_command_result(check.commands["license"], check_commands, container) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py new file mode 100644 index 000000000000..efa340d91c47 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +""" +Module exposing the format command. +""" + +import asyncclick as click +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.lazy_group import LazyGroup +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context + + +@click.group( + cls=LazyGroup, + name="format", + help="Commands related to formatting.", + lazy_subcommands={ + "check": "pipelines.airbyte_ci.format.check.commands.check", + "fix": "pipelines.airbyte_ci.format.fix.commands.fix", + }, +) +@click_merge_args_into_context_obj +@pass_pipeline_context +@click_ignore_unused_kwargs +async def format_code(pipeline_context: ClickPipelineContext): + pass diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py new file mode 100644 index 000000000000..add25c815518 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py @@ -0,0 +1,44 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +DEFAULT_FORMAT_IGNORE_LIST = [ + "**/__pycache__", + '"**/.pytest_cache', + "**/.venv", + "**/venv", + "**/.gradle", + "**/node_modules", + "**/.tox", + "**/.eggs", + "**/.mypy_cache", + "**/.venv", + "**/*.egg-info", + "**/build", + "**/dbt-project-template", + "**/dbt-project-template-mssql", + "**/dbt-project-template-mysql", + "**/dbt-project-template-oracle", + "**/dbt-project-template-clickhouse", + "**/dbt-project-template-snowflake", + "**/dbt-project-template-tidb", + "**/dbt-project-template-duckdb", + "**/dbt_test_config", + "**/normalization_test_output", + # '**/tools', + "**/secrets", + "**/charts", # Helm charts often have injected template strings that will fail general linting. Helm linting is done separately. + "**/resources/seed/*_catalog.json", # Do not remove - this is also necessary to prevent diffs in our github workflows + "**/resources/seed/*_registry.json", # Do not remove - this is also necessary to prevent diffs in our github workflows + "**/resources/seed/specs_secrets_mask.yaml", # Downloaded externally. + "**/resources/examples/airflow/superset/docker/pythonpath_dev/superset_config.py", + "**/source-amplitude/unit_tests/api_data/zipped.json", # Zipped file presents as non-UTF-8 making spotless sad + "**/airbyte-connector-builder-server/connector_builder/generated", # autogenerated code doesn't need to be formatted + "**/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/**/invalid", # These are deliberately invalid and unformattable. + "**/__init__.py", + "**/declarative_component_schema.py", + "**/source-stock-ticker-api-tutorial/source.py", + "**/tools/git_hooks/tests/test_spec_linter.py", + "**/tools/schema_generator/schema_generator/infer_schemas.py", + "**/.git", +] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py new file mode 100644 index 000000000000..f9414b22edfd --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py @@ -0,0 +1,110 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Dict, List, Optional + +import dagger +from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST +from pipelines.consts import AMAZONCORRETTO_IMAGE, GO_IMAGE, NODE_IMAGE, PYTHON_3_10_IMAGE +from pipelines.helpers.utils import sh_dash_c + + +def build_container( + dagger_client: dagger.Client, + base_image: str, + include: List[str], + install_commands: Optional[List[str]] = None, + env_vars: Optional[Dict[str, Any]] = {}, +) -> dagger.Container: + """Build a container for formatting code. + Args: + ctx (ClickPipelineContext): The context of the pipeline + base_image (str): The base image to use for the container + include (List[str]): The list of files to include in the container + install_commands (Optional[List[str]]): The list of commands to run to install dependencies for the formatter + env_vars (Optional[Dict[str, Any]]): The list of environment variables to set on the container + Returns: + dagger.Container: The container to use for formatting + """ + # Create container from base image + container = dagger_client.container().from_(base_image) + + # Add any environment variables + for key, value in env_vars.items(): + container = container.with_env_variable(key, value) + + # Install any dependencies of the formatter + if install_commands: + container = container.with_exec(sh_dash_c(install_commands), skip_entrypoint=True) + + # Mount the relevant parts of the repository: the code to format and the formatting config + # Exclude the default ignore list to keep things as small as possible + container = container.with_mounted_directory( + "/src", + dagger_client.host().directory( + ".", + include=include, + exclude=DEFAULT_FORMAT_IGNORE_LIST, + ), + ) + + # Set the working directory to the code to format + container = container.with_workdir("/src") + + return container + + +def format_java_container(dagger_client: dagger.Client) -> dagger.Container: + """Format java, groovy, and sql code via spotless.""" + return build_container( + dagger_client, + base_image=AMAZONCORRETTO_IMAGE, + include=[ + "**/*.java", + "**/*.gradle", + "gradlew", + "gradlew.bat", + "gradle", + "**/deps.toml", + "**/gradle.properties", + "**/version.properties", + "tools/gradle/codestyle/java-google-style.xml", + ], + install_commands=[ + "yum update -y", + "yum install -y findutils", # gradle requires xargs, which is shipped in findutils. + "yum clean all", + ], + ) + + +def format_js_container(dagger_client: dagger.Client) -> dagger.Container: + """Format yaml and json code via prettier.""" + return build_container( + dagger_client, + base_image=NODE_IMAGE, + include=["**/*.yaml", "**/*.yml", "**.*/json", "package.json", "package-lock.json"], + install_commands=["npm install -g npm@10.1.0 prettier@3.0.3"], + ) + + +def format_license_container(dagger_client: dagger.Client, license_file: str) -> dagger.Container: + return build_container( + dagger_client, + base_image=GO_IMAGE, + include=["**/*.java", "**/*.py", license_file], + install_commands=["go get -u github.com/google/addlicense"], + ) + + +def format_python_container(dagger_client: dagger.Client) -> dagger.Container: + """Format python code via black and isort.""" + + return build_container( + dagger_client, + base_image=PYTHON_3_10_IMAGE, + env_vars={"PIPX_BIN_DIR": "/usr/local/bin"}, + include=["**/*.py", "pyproject.toml", "poetry.lock"], + install_commands=["pip install pipx", "pipx ensurepath", "pipx install poetry"], + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py new file mode 100644 index 000000000000..3667ab8882d1 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/fix/commands.py @@ -0,0 +1,134 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from typing import List + +import asyncclick as click +import dagger +from pipelines.airbyte_ci.format.actions import run_format +from pipelines.airbyte_ci.format.containers import ( + format_java_container, + format_js_container, + format_license_container, + format_python_container, +) +from pipelines.cli.click_decorators import click_ignore_unused_kwargs +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand +from pipelines.helpers.cli import ( + LogOptions, + get_all_sibling_commands, + invoke_commands_concurrently, + invoke_commands_sequentially, + log_command_results, +) +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context +from pipelines.models.steps import CommandResult, StepStatus + +# HELPERS + + +async def get_format_command_result(click_command: click.Command, container: dagger.Container, format_commands: List[str]) -> CommandResult: + """Run a format command and return the CommandResult. + A command is considered successful if the export operation of run_format is successful. + + Args: + click_command (click.Command): The click command to run + container (dagger.Container): The container to run the format_commands in + format_commands (List[str]): The list of commands to run to format the repository + + Returns: + CommandResult: The result of running the command + """ + try: + successful_export = await run_format(container, format_commands) + status = StepStatus.SUCCESS if successful_export else StepStatus.FAILURE + return CommandResult(click_command, status=status) + except dagger.ExecError as e: + return CommandResult(click_command, status=StepStatus.FAILURE, stderr=e.stderr, stdout=e.stdout, exc_info=e) + + +@click.group( + help="Run code format checks and fix any failures.", + chain=True, +) +async def fix(): + pass + + +@fix.command(cls=DaggerPipelineCommand, name="all") +@click.pass_context +async def all_fix(ctx: click.Context): + """Run code format checks and fix any failures.""" + parent_command = ctx.parent.command + logger = logging.getLogger(parent_command.name) + + concurrent_commands = [ + fix.commands["python"], + fix.commands["java"], + fix.commands["js"], + ] + sequential_commands = [fix.commands["license"]] + + # We can run language commands concurrently because they modify different set of files. + command_results = await invoke_commands_concurrently(ctx, concurrent_commands) + + # We have to run license command sequentially because it modifies the same set of files as other commands. + # If we ran it concurrently with language commands, we face race condition issues. + command_results += await invoke_commands_sequentially(ctx, sequential_commands) + failure = any([r.status is StepStatus.FAILURE for r in command_results]) + + log_options = LogOptions(list_errors=True) + + log_command_results(ctx, command_results, logger, log_options) + + return not failure + + +@fix.command(cls=DaggerPipelineCommand) +@pass_pipeline_context +@click_ignore_unused_kwargs +async def java(ctx: ClickPipelineContext) -> CommandResult: + """Format java, groovy, and sql code via spotless.""" + dagger_client = await ctx.get_dagger_client(pipeline_name="Format java") + container = format_java_container(dagger_client) + format_commands = ["./gradlew spotlessApply --scan"] + return await get_format_command_result(fix.commands["java"], container, format_commands) + + +@fix.command(cls=DaggerPipelineCommand) +@pass_pipeline_context +@click_ignore_unused_kwargs +async def js(ctx: ClickPipelineContext) -> CommandResult: + dagger_client = await ctx.get_dagger_client(pipeline_name="Format js") + container = format_js_container(dagger_client) + format_commands = ["prettier --write ."] + return await get_format_command_result(fix.commands["js"], container, format_commands) + + +@fix.command("license") +@pass_pipeline_context +@click_ignore_unused_kwargs +async def license_fix(ctx: ClickPipelineContext) -> CommandResult: + """Add license to python and java code via addlicense.""" + license_file = "LICENSE_SHORT" + dagger_client = await ctx.get_dagger_client(pipeline_name="Add license") + container = format_license_container(dagger_client, license_file) + format_commands = [f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {license_file} ."] + return await get_format_command_result(fix.commands["license"], container, format_commands) + + +@fix.command(cls=DaggerPipelineCommand) +@pass_pipeline_context +@click_ignore_unused_kwargs +async def python(ctx: ClickPipelineContext) -> CommandResult: + """Format python code via black and isort.""" + dagger_client = await ctx.get_dagger_client(pipeline_name="Format python") + container = format_python_container(dagger_client) + format_commands = [ + "poetry install --no-root", + "poetry run isort --settings-file pyproject.toml .", + "poetry run black --config pyproject.toml .", + ] + return await get_format_command_result(fix.commands["python"], container, format_commands) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py index bcdc19a2e307..0f04429afb34 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py @@ -68,7 +68,6 @@ async def _run(self) -> StepResult: "gradle.properties", "gradle", "gradlew", - "LICENSE_SHORT", "settings.gradle", "build.gradle", "tools/gradle", @@ -76,7 +75,6 @@ async def _run(self) -> StepResult: "buildSrc", "tools/bin/build_image.sh", "tools/lib/lib.sh", - "tools/gradle/codestyle", "pyproject.toml", ] + self.build_include @@ -84,8 +82,6 @@ async def _run(self) -> StepResult: "docker", # required by :integrationTestJava. "findutils", # gradle requires xargs, which is shipped in findutils. "jq", # required by :acceptance-test-harness to inspect docker images. - "npm", # required by :format. - "python3.11-pip", # required by :format. "rsync", # required for gradle cache synchronization. ] diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index e063efd83245..3a8912b986c8 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -52,7 +52,7 @@ def display_welcome_message() -> None: │ │ │ │ ╚─────────────────────────────────────────────────────────────────────────────────────────────────╝ - """ + """ # noqa: W605 ) @@ -226,6 +226,7 @@ async def get_modified_files_str(ctx: click.Context): help="Airbyte CI top-level command group.", lazy_subcommands={ "connectors": "pipelines.airbyte_ci.connectors.commands.connectors", + "format": "pipelines.airbyte_ci.format.commands.format_code", "metadata": "pipelines.airbyte_ci.metadata.commands.metadata", "test": "pipelines.airbyte_ci.test.commands.test", }, diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py index 6aa64a618e31..c7740a0f0f65 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -1,4 +1,6 @@ +# # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# import functools import inspect diff --git a/airbyte-ci/connectors/pipelines/pipelines/consts.py b/airbyte-ci/connectors/pipelines/pipelines/consts.py index 5fdba5b6baf2..ddfbbb841298 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/consts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/consts.py @@ -13,14 +13,11 @@ CONNECTOR_TESTING_REQUIREMENTS = [ "pip==21.3.1", "mccabe==0.6.1", - "flake8==4.0.1", - "pyproject-flake8==0.0.1a2", - "black==22.3.0", - "isort==5.6.4", + # "flake8==4.0.1", + # "pyproject-flake8==0.0.1a2", "pytest==6.2.5", "coverage[toml]==6.3.1", "pytest-custom_exit_code", - "licenseheaders==0.8.8", ] BUILD_PLATFORMS = [Platform("linux/amd64"), Platform("linux/arm64")] @@ -33,6 +30,9 @@ } LOCAL_BUILD_PLATFORM = PLATFORM_MACHINE_TO_DAGGER_PLATFORM[platform.machine()] AMAZONCORRETTO_IMAGE = "amazoncorretto:17.0.8-al2023" +NODE_IMAGE = "node:18.18.0-slim" +GO_IMAGE = "golang:1.17" +PYTHON_3_10_IMAGE = "python:3.10.13-slim" DOCKER_VERSION = "24.0.2" DOCKER_DIND_IMAGE = f"docker:{DOCKER_VERSION}-dind" DOCKER_CLI_IMAGE = f"docker:{DOCKER_VERSION}-cli" diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py index c9eed1bf8e85..98227fe9c81d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py @@ -6,7 +6,6 @@ from pipelines.airbyte_ci.connectors.context import PipelineContext from pipelines.consts import ( CONNECTOR_TESTING_REQUIREMENTS, - LICENSE_SHORT_FILE_PATH, PIP_CACHE_PATH, PIP_CACHE_VOLUME_NAME, POETRY_CACHE_PATH, @@ -61,12 +60,9 @@ def with_testing_dependencies(context: PipelineContext) -> Container: """ python_environment: Container = with_python_base(context) pyproject_toml_file = context.get_repo_dir(".", include=[PYPROJECT_TOML_FILE_PATH]).file(PYPROJECT_TOML_FILE_PATH) - license_short_file = context.get_repo_dir(".", include=[LICENSE_SHORT_FILE_PATH]).file(LICENSE_SHORT_FILE_PATH) - return ( - python_environment.with_exec(["pip", "install"] + CONNECTOR_TESTING_REQUIREMENTS) - .with_file(f"/{PYPROJECT_TOML_FILE_PATH}", pyproject_toml_file) - .with_file(f"/{LICENSE_SHORT_FILE_PATH}", license_short_file) + return python_environment.with_exec(["pip", "install"] + CONNECTOR_TESTING_REQUIREMENTS).with_file( + f"/{PYPROJECT_TOML_FILE_PATH}", pyproject_toml_file ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py new file mode 100644 index 000000000000..2aaa77258a7f --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py @@ -0,0 +1,104 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from dataclasses import dataclass +from logging import Logger +from typing import Any, List + +import asyncclick as click +import asyncer +from jinja2 import Template +from pipelines.models.steps import CommandResult, StepStatus + +ALL_RESULTS_KEY = "_run_all_results" + +SUMMARY_TEMPLATE_STR = """ + +Summary of commands results +======================== +{% for command_name, success in results %} +{{ '✅' if success else '❌' }} {{ command_prefix }} {{ command_name }} +{% endfor %} +""" + +DETAILS_TEMPLATE_STR = """ + +Detailed Errors for failed commands +================================= +{% for command_name, error in failed_commands_details %} +❌ {{ command_prefix }} {{ command_name }} + +Error: {{ error }} +{% endfor %} +================================= + +""" + + +@dataclass +class LogOptions: + list_errors: bool = False + help_message: str = None + + +def log_command_results(ctx: click.Context, command_results: List[CommandResult], logger: Logger, options: LogOptions = LogOptions()): + """ + Log the output of the subcommands run by `run_all_subcommands`. + """ + command_path = ctx.command_path + + summary_template = Template(SUMMARY_TEMPLATE_STR) + results = [(r.command.name, r.status is StepStatus.SUCCESS) for r in command_results] + summary_message = summary_template.render(results=results, command_prefix=command_path) + logger.info(summary_message) + + result_contains_failures = any([r.status is StepStatus.FAILURE for r in command_results]) + + if result_contains_failures: + if options.list_errors: + failed_commands_details = [ + (command_result.command.name, command_result.stderr) + for command_result in command_results + if command_result.status is StepStatus.FAILURE + ] + if failed_commands_details: + details_template = Template(DETAILS_TEMPLATE_STR) + details_message = details_template.render(failed_commands_details=failed_commands_details, command_prefix=command_path) + logger.info(details_message) + else: + logger.info(f"Run `{command_path} --list-errors` to see detailed error messages for failed checks.") + + if options.help_message: + logger.info(options.help_message) + + +async def invoke_commands_concurrently(ctx: click.Context, commands: List[click.Command]) -> List[Any]: + """ + Run click commands concurrently and return a list of their return values. + """ + + soon_command_executions_results = [] + async with asyncer.create_task_group() as command_group: + for command in commands: + soon_command_execution_result = command_group.soonify(ctx.invoke)(command) + soon_command_executions_results.append(soon_command_execution_result) + return [r.value for r in soon_command_executions_results] + + +async def invoke_commands_sequentially(ctx: click.Context, commands: List[click.Command]) -> List[Any]: + """ + Run click commands sequentially and return a list of their return values. + """ + command_executions_results = [] + for command in commands: + command_executions_results.append(await ctx.invoke(command)) + return command_executions_results + + +def get_all_sibling_commands(ctx: click.Context) -> List[click.Command]: + """ + Get all sibling commands of the current command. + """ + return [c for c in ctx.parent.command.commands.values() if c.name != ctx.command.name] diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py index 60d64a752370..6f9878f38f5c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -1,4 +1,6 @@ +# # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# import sys from typing import Any, Callable, Optional @@ -24,6 +26,7 @@ class ClickPipelineContext(BaseModel, Singleton): dockerd_service: Optional[Container] = Field(default=None) _dagger_client: Optional[Client] = PrivateAttr(default=None) _click_context: Callable[[], Context] = PrivateAttr(default_factory=lambda: get_current_context) + _og_click_context: Callable[[], Context] = PrivateAttr(default=None) @property def params(self): @@ -66,6 +69,11 @@ def __init__(self, **data: dict[str, Any]): super().__init__(**data) Singleton._initialized[ClickPipelineContext] = True + """ + Note: Its important to hold onto the original click context object, as it is used to hold onto the Dagger client. + """ + self._og_click_context = self._click_context() + _dagger_client_lock: anyio.Lock = PrivateAttr(default_factory=anyio.Lock) async def get_dagger_client(self, pipeline_name: Optional[str] = None) -> Client: @@ -84,7 +92,7 @@ async def get_dagger_client(self, pipeline_name: Optional[str] = None) -> Client Avoid using this client across multiple thread pools, as it can lead to errors. Cross-thread pool calls are generally considered an anti-pattern. """ - self._dagger_client = await self._click_context().with_async_resource(connection) # type: ignore + self._dagger_client = await self._og_click_context.with_async_resource(connection) # type: ignore assert self._dagger_client, "Error initializing Dagger client" return self._dagger_client.pipeline(pipeline_name) if pipeline_name else self._dagger_client diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py index e9d759d401f6..349d9a46f995 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py @@ -1,4 +1,6 @@ +# # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# from typing import Any, Type diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/steps.py b/airbyte-ci/connectors/pipelines/pipelines/models/steps.py index 5cc119ed6440..9959653cdf4f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/models/steps.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/steps.py @@ -5,26 +5,25 @@ from __future__ import annotations import logging -import typing from abc import abstractmethod from dataclasses import dataclass, field from datetime import datetime, timedelta from enum import Enum from pathlib import Path -from typing import Any, Optional +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union import anyio import asyncer +import click from dagger import Container, DaggerError from pipelines import main_logger from pipelines.helpers import sentry_utils from pipelines.helpers.utils import format_duration, get_exec_result -if typing.TYPE_CHECKING: +if TYPE_CHECKING: from pipelines.models.contexts.pipeline_context import PipelineContext from abc import ABC -from typing import ClassVar from rich.style import Style @@ -62,7 +61,7 @@ def is_file(self) -> bool: class StepResult: """A dataclass to capture the result of a step.""" - step: Step + step: Union[Step, click.command] status: StepStatus created_at: datetime = field(default_factory=datetime.utcnow) stderr: Optional[str] = None @@ -88,6 +87,24 @@ def redact_secrets_from_string(self, value: str) -> str: return value +@dataclass(frozen=True) +class CommandResult: + """A dataclass to capture the result of a command.""" + + command: click.command + status: StepStatus + created_at: datetime = field(default_factory=datetime.utcnow) + stderr: Optional[str] = None + stdout: Optional[str] = None + exc_info: Optional[Exception] = None + + def __repr__(self) -> str: # noqa D105 + return f"{self.command.name}: {self.status.value}" + + def __str__(self) -> str: # noqa D105 + return f"{self.command.name}: {self.status.value}\n\nSTDOUT:\n{self.stdout}\n\nSTDERR:\n{self.stderr}" + + class StepStatus(Enum): """An Enum to characterize the success, failure or skipping of a Step.""" diff --git a/airbyte-ci/connectors/pipelines/poetry.lock b/airbyte-ci/connectors/pipelines/poetry.lock index 015469e3f429..6a938b4db1d5 100644 --- a/airbyte-ci/connectors/pipelines/poetry.lock +++ b/airbyte-ci/connectors/pipelines/poetry.lock @@ -282,101 +282,101 @@ pycparser = "*" [[package]] name = "charset-normalizer" -version = "3.3.1" +version = "3.3.2" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7.0" files = [ - {file = "charset-normalizer-3.3.1.tar.gz", hash = "sha256:d9137a876020661972ca6eec0766d81aef8a5627df628b664b234b73396e727e"}, - {file = "charset_normalizer-3.3.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8aee051c89e13565c6bd366813c386939f8e928af93c29fda4af86d25b73d8f8"}, - {file = "charset_normalizer-3.3.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:352a88c3df0d1fa886562384b86f9a9e27563d4704ee0e9d56ec6fcd270ea690"}, - {file = "charset_normalizer-3.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:223b4d54561c01048f657fa6ce41461d5ad8ff128b9678cfe8b2ecd951e3f8a2"}, - {file = "charset_normalizer-3.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f861d94c2a450b974b86093c6c027888627b8082f1299dfd5a4bae8e2292821"}, - {file = "charset_normalizer-3.3.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1171ef1fc5ab4693c5d151ae0fdad7f7349920eabbaca6271f95969fa0756c2d"}, - {file = "charset_normalizer-3.3.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28f512b9a33235545fbbdac6a330a510b63be278a50071a336afc1b78781b147"}, - {file = "charset_normalizer-3.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0e842112fe3f1a4ffcf64b06dc4c61a88441c2f02f373367f7b4c1aa9be2ad5"}, - {file = "charset_normalizer-3.3.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3f9bc2ce123637a60ebe819f9fccc614da1bcc05798bbbaf2dd4ec91f3e08846"}, - {file = "charset_normalizer-3.3.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:f194cce575e59ffe442c10a360182a986535fd90b57f7debfaa5c845c409ecc3"}, - {file = "charset_normalizer-3.3.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9a74041ba0bfa9bc9b9bb2cd3238a6ab3b7618e759b41bd15b5f6ad958d17605"}, - {file = "charset_normalizer-3.3.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b578cbe580e3b41ad17b1c428f382c814b32a6ce90f2d8e39e2e635d49e498d1"}, - {file = "charset_normalizer-3.3.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:6db3cfb9b4fcecb4390db154e75b49578c87a3b9979b40cdf90d7e4b945656e1"}, - {file = "charset_normalizer-3.3.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:debb633f3f7856f95ad957d9b9c781f8e2c6303ef21724ec94bea2ce2fcbd056"}, - {file = "charset_normalizer-3.3.1-cp310-cp310-win32.whl", hash = "sha256:87071618d3d8ec8b186d53cb6e66955ef2a0e4fa63ccd3709c0c90ac5a43520f"}, - {file = "charset_normalizer-3.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:e372d7dfd154009142631de2d316adad3cc1c36c32a38b16a4751ba78da2a397"}, - {file = "charset_normalizer-3.3.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ae4070f741f8d809075ef697877fd350ecf0b7c5837ed68738607ee0a2c572cf"}, - {file = "charset_normalizer-3.3.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:58e875eb7016fd014c0eea46c6fa92b87b62c0cb31b9feae25cbbe62c919f54d"}, - {file = "charset_normalizer-3.3.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dbd95e300367aa0827496fe75a1766d198d34385a58f97683fe6e07f89ca3e3c"}, - {file = "charset_normalizer-3.3.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de0b4caa1c8a21394e8ce971997614a17648f94e1cd0640fbd6b4d14cab13a72"}, - {file = "charset_normalizer-3.3.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:985c7965f62f6f32bf432e2681173db41336a9c2611693247069288bcb0c7f8b"}, - {file = "charset_normalizer-3.3.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a15c1fe6d26e83fd2e5972425a772cca158eae58b05d4a25a4e474c221053e2d"}, - {file = "charset_normalizer-3.3.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae55d592b02c4349525b6ed8f74c692509e5adffa842e582c0f861751701a673"}, - {file = "charset_normalizer-3.3.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:be4d9c2770044a59715eb57c1144dedea7c5d5ae80c68fb9959515037cde2008"}, - {file = "charset_normalizer-3.3.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:851cf693fb3aaef71031237cd68699dded198657ec1e76a76eb8be58c03a5d1f"}, - {file = "charset_normalizer-3.3.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:31bbaba7218904d2eabecf4feec0d07469284e952a27400f23b6628439439fa7"}, - {file = "charset_normalizer-3.3.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:871d045d6ccc181fd863a3cd66ee8e395523ebfbc57f85f91f035f50cee8e3d4"}, - {file = "charset_normalizer-3.3.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:501adc5eb6cd5f40a6f77fbd90e5ab915c8fd6e8c614af2db5561e16c600d6f3"}, - {file = "charset_normalizer-3.3.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f5fb672c396d826ca16a022ac04c9dce74e00a1c344f6ad1a0fdc1ba1f332213"}, - {file = "charset_normalizer-3.3.1-cp311-cp311-win32.whl", hash = "sha256:bb06098d019766ca16fc915ecaa455c1f1cd594204e7f840cd6258237b5079a8"}, - {file = "charset_normalizer-3.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:8af5a8917b8af42295e86b64903156b4f110a30dca5f3b5aedea123fbd638bff"}, - {file = "charset_normalizer-3.3.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:7ae8e5142dcc7a49168f4055255dbcced01dc1714a90a21f87448dc8d90617d1"}, - {file = "charset_normalizer-3.3.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5b70bab78accbc672f50e878a5b73ca692f45f5b5e25c8066d748c09405e6a55"}, - {file = "charset_normalizer-3.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5ceca5876032362ae73b83347be8b5dbd2d1faf3358deb38c9c88776779b2e2f"}, - {file = "charset_normalizer-3.3.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34d95638ff3613849f473afc33f65c401a89f3b9528d0d213c7037c398a51296"}, - {file = "charset_normalizer-3.3.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9edbe6a5bf8b56a4a84533ba2b2f489d0046e755c29616ef8830f9e7d9cf5728"}, - {file = "charset_normalizer-3.3.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f6a02a3c7950cafaadcd46a226ad9e12fc9744652cc69f9e5534f98b47f3bbcf"}, - {file = "charset_normalizer-3.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10b8dd31e10f32410751b3430996f9807fc4d1587ca69772e2aa940a82ab571a"}, - {file = "charset_normalizer-3.3.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edc0202099ea1d82844316604e17d2b175044f9bcb6b398aab781eba957224bd"}, - {file = "charset_normalizer-3.3.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b891a2f68e09c5ef989007fac11476ed33c5c9994449a4e2c3386529d703dc8b"}, - {file = "charset_normalizer-3.3.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:71ef3b9be10070360f289aea4838c784f8b851be3ba58cf796262b57775c2f14"}, - {file = "charset_normalizer-3.3.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:55602981b2dbf8184c098bc10287e8c245e351cd4fdcad050bd7199d5a8bf514"}, - {file = "charset_normalizer-3.3.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:46fb9970aa5eeca547d7aa0de5d4b124a288b42eaefac677bde805013c95725c"}, - {file = "charset_normalizer-3.3.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:520b7a142d2524f999447b3a0cf95115df81c4f33003c51a6ab637cbda9d0bf4"}, - {file = "charset_normalizer-3.3.1-cp312-cp312-win32.whl", hash = "sha256:8ec8ef42c6cd5856a7613dcd1eaf21e5573b2185263d87d27c8edcae33b62a61"}, - {file = "charset_normalizer-3.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:baec8148d6b8bd5cee1ae138ba658c71f5b03e0d69d5907703e3e1df96db5e41"}, - {file = "charset_normalizer-3.3.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63a6f59e2d01310f754c270e4a257426fe5a591dc487f1983b3bbe793cf6bac6"}, - {file = "charset_normalizer-3.3.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d6bfc32a68bc0933819cfdfe45f9abc3cae3877e1d90aac7259d57e6e0f85b1"}, - {file = "charset_normalizer-3.3.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4f3100d86dcd03c03f7e9c3fdb23d92e32abbca07e7c13ebd7ddfbcb06f5991f"}, - {file = "charset_normalizer-3.3.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39b70a6f88eebe239fa775190796d55a33cfb6d36b9ffdd37843f7c4c1b5dc67"}, - {file = "charset_normalizer-3.3.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e12f8ee80aa35e746230a2af83e81bd6b52daa92a8afaef4fea4a2ce9b9f4fa"}, - {file = "charset_normalizer-3.3.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b6cefa579e1237ce198619b76eaa148b71894fb0d6bcf9024460f9bf30fd228"}, - {file = "charset_normalizer-3.3.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:61f1e3fb621f5420523abb71f5771a204b33c21d31e7d9d86881b2cffe92c47c"}, - {file = "charset_normalizer-3.3.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4f6e2a839f83a6a76854d12dbebde50e4b1afa63e27761549d006fa53e9aa80e"}, - {file = "charset_normalizer-3.3.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:1ec937546cad86d0dce5396748bf392bb7b62a9eeb8c66efac60e947697f0e58"}, - {file = "charset_normalizer-3.3.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:82ca51ff0fc5b641a2d4e1cc8c5ff108699b7a56d7f3ad6f6da9dbb6f0145b48"}, - {file = "charset_normalizer-3.3.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:633968254f8d421e70f91c6ebe71ed0ab140220469cf87a9857e21c16687c034"}, - {file = "charset_normalizer-3.3.1-cp37-cp37m-win32.whl", hash = "sha256:c0c72d34e7de5604df0fde3644cc079feee5e55464967d10b24b1de268deceb9"}, - {file = "charset_normalizer-3.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:63accd11149c0f9a99e3bc095bbdb5a464862d77a7e309ad5938fbc8721235ae"}, - {file = "charset_normalizer-3.3.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5a3580a4fdc4ac05f9e53c57f965e3594b2f99796231380adb2baaab96e22761"}, - {file = "charset_normalizer-3.3.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2465aa50c9299d615d757c1c888bc6fef384b7c4aec81c05a0172b4400f98557"}, - {file = "charset_normalizer-3.3.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cb7cd68814308aade9d0c93c5bd2ade9f9441666f8ba5aa9c2d4b389cb5e2a45"}, - {file = "charset_normalizer-3.3.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91e43805ccafa0a91831f9cd5443aa34528c0c3f2cc48c4cb3d9a7721053874b"}, - {file = "charset_normalizer-3.3.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:854cc74367180beb327ab9d00f964f6d91da06450b0855cbbb09187bcdb02de5"}, - {file = "charset_normalizer-3.3.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c15070ebf11b8b7fd1bfff7217e9324963c82dbdf6182ff7050519e350e7ad9f"}, - {file = "charset_normalizer-3.3.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c4c99f98fc3a1835af8179dcc9013f93594d0670e2fa80c83aa36346ee763d2"}, - {file = "charset_normalizer-3.3.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3fb765362688821404ad6cf86772fc54993ec11577cd5a92ac44b4c2ba52155b"}, - {file = "charset_normalizer-3.3.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:dced27917823df984fe0c80a5c4ad75cf58df0fbfae890bc08004cd3888922a2"}, - {file = "charset_normalizer-3.3.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a66bcdf19c1a523e41b8e9d53d0cedbfbac2e93c649a2e9502cb26c014d0980c"}, - {file = "charset_normalizer-3.3.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:ecd26be9f112c4f96718290c10f4caea6cc798459a3a76636b817a0ed7874e42"}, - {file = "charset_normalizer-3.3.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:3f70fd716855cd3b855316b226a1ac8bdb3caf4f7ea96edcccc6f484217c9597"}, - {file = "charset_normalizer-3.3.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:17a866d61259c7de1bdadef418a37755050ddb4b922df8b356503234fff7932c"}, - {file = "charset_normalizer-3.3.1-cp38-cp38-win32.whl", hash = "sha256:548eefad783ed787b38cb6f9a574bd8664468cc76d1538215d510a3cd41406cb"}, - {file = "charset_normalizer-3.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:45f053a0ece92c734d874861ffe6e3cc92150e32136dd59ab1fb070575189c97"}, - {file = "charset_normalizer-3.3.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bc791ec3fd0c4309a753f95bb6c749ef0d8ea3aea91f07ee1cf06b7b02118f2f"}, - {file = "charset_normalizer-3.3.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0c8c61fb505c7dad1d251c284e712d4e0372cef3b067f7ddf82a7fa82e1e9a93"}, - {file = "charset_normalizer-3.3.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2c092be3885a1b7899cd85ce24acedc1034199d6fca1483fa2c3a35c86e43041"}, - {file = "charset_normalizer-3.3.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c2000c54c395d9e5e44c99dc7c20a64dc371f777faf8bae4919ad3e99ce5253e"}, - {file = "charset_normalizer-3.3.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4cb50a0335382aac15c31b61d8531bc9bb657cfd848b1d7158009472189f3d62"}, - {file = "charset_normalizer-3.3.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c30187840d36d0ba2893bc3271a36a517a717f9fd383a98e2697ee890a37c273"}, - {file = "charset_normalizer-3.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe81b35c33772e56f4b6cf62cf4aedc1762ef7162a31e6ac7fe5e40d0149eb67"}, - {file = "charset_normalizer-3.3.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d0bf89afcbcf4d1bb2652f6580e5e55a840fdf87384f6063c4a4f0c95e378656"}, - {file = "charset_normalizer-3.3.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:06cf46bdff72f58645434d467bf5228080801298fbba19fe268a01b4534467f5"}, - {file = "charset_normalizer-3.3.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:3c66df3f41abee950d6638adc7eac4730a306b022570f71dd0bd6ba53503ab57"}, - {file = "charset_normalizer-3.3.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:cd805513198304026bd379d1d516afbf6c3c13f4382134a2c526b8b854da1c2e"}, - {file = "charset_normalizer-3.3.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:9505dc359edb6a330efcd2be825fdb73ee3e628d9010597aa1aee5aa63442e97"}, - {file = "charset_normalizer-3.3.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:31445f38053476a0c4e6d12b047b08ced81e2c7c712e5a1ad97bc913256f91b2"}, - {file = "charset_normalizer-3.3.1-cp39-cp39-win32.whl", hash = "sha256:bd28b31730f0e982ace8663d108e01199098432a30a4c410d06fe08fdb9e93f4"}, - {file = "charset_normalizer-3.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:555fe186da0068d3354cdf4bbcbc609b0ecae4d04c921cc13e209eece7720727"}, - {file = "charset_normalizer-3.3.1-py3-none-any.whl", hash = "sha256:800561453acdecedaac137bf09cd719c7a440b6800ec182f077bb8e7025fb708"}, + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, ] [[package]] @@ -443,7 +443,7 @@ url = "../common_utils" [[package]] name = "connector-ops" -version = "0.3.1" +version = "0.3.2" description = "Packaged maintained by the connector operations team to perform CI for connectors" optional = false python-versions = "^3.10" @@ -698,13 +698,13 @@ test = ["black", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre [[package]] name = "google-api-core" -version = "2.12.0" +version = "2.14.0" description = "Google API client core library" optional = false python-versions = ">=3.7" files = [ - {file = "google-api-core-2.12.0.tar.gz", hash = "sha256:c22e01b1e3c4dcd90998494879612c38d0a3411d1f7b679eb89e2abe3ce1f553"}, - {file = "google_api_core-2.12.0-py3-none-any.whl", hash = "sha256:ec6054f7d64ad13b41e43d96f735acbd763b0f3b695dabaa2d579673f6a6e160"}, + {file = "google-api-core-2.14.0.tar.gz", hash = "sha256:5368a4502b793d9bbf812a5912e13e4e69f9bd87f6efb508460c43f5bbd1ce41"}, + {file = "google_api_core-2.14.0-py3-none-any.whl", hash = "sha256:de2fb50ed34d47ddbb2bd2dcf680ee8fead46279f4ed6b16de362aca23a18952"}, ] [package.dependencies] @@ -720,13 +720,13 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-auth" -version = "2.23.3" +version = "2.23.4" description = "Google Authentication Library" optional = false python-versions = ">=3.7" files = [ - {file = "google-auth-2.23.3.tar.gz", hash = "sha256:6864247895eea5d13b9c57c9e03abb49cb94ce2dc7c58e91cba3248c7477c9e3"}, - {file = "google_auth-2.23.3-py2.py3-none-any.whl", hash = "sha256:a8f4608e65c244ead9e0538f181a96c6e11199ec114d41f1d7b1bffa96937bda"}, + {file = "google-auth-2.23.4.tar.gz", hash = "sha256:79905d6b1652187def79d491d6e23d0cbb3a21d3c7ba0dbaa9c8a01906b13ff3"}, + {file = "google_auth-2.23.4-py2.py3-none-any.whl", hash = "sha256:d4bbc92fe4b8bfd2f3e8d88e5ba7085935da208ee38a134fc280e7ce682a05f2"}, ] [package.dependencies] @@ -761,13 +761,13 @@ grpc = ["grpcio (>=1.38.0,<2.0dev)"] [[package]] name = "google-cloud-storage" -version = "2.12.0" +version = "2.13.0" description = "Google Cloud Storage API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-storage-2.12.0.tar.gz", hash = "sha256:57c0bcda2f5e11f008a155d8636d8381d5abab46b58e0cae0e46dd5e595e6b46"}, - {file = "google_cloud_storage-2.12.0-py2.py3-none-any.whl", hash = "sha256:bc52563439d42981b6e21b071a76da2791672776eda3ba99d13a8061ebbd6e5e"}, + {file = "google-cloud-storage-2.13.0.tar.gz", hash = "sha256:f62dc4c7b6cd4360d072e3deb28035fbdad491ac3d9b0b1815a12daea10f37c7"}, + {file = "google_cloud_storage-2.13.0-py2.py3-none-any.whl", hash = "sha256:ab0bf2e1780a1b74cf17fccb13788070b729f50c252f0c94ada2aae0ca95437d"}, ] [package.dependencies] @@ -946,39 +946,40 @@ files = [ [[package]] name = "httpcore" -version = "0.18.0" +version = "1.0.1" description = "A minimal low-level HTTP client." optional = false python-versions = ">=3.8" files = [ - {file = "httpcore-0.18.0-py3-none-any.whl", hash = "sha256:adc5398ee0a476567bf87467063ee63584a8bce86078bf748e48754f60202ced"}, - {file = "httpcore-0.18.0.tar.gz", hash = "sha256:13b5e5cd1dca1a6636a6aaea212b19f4f85cd88c366a2b82304181b769aab3c9"}, + {file = "httpcore-1.0.1-py3-none-any.whl", hash = "sha256:c5e97ef177dca2023d0b9aad98e49507ef5423e9f1d94ffe2cfe250aa28e63b0"}, + {file = "httpcore-1.0.1.tar.gz", hash = "sha256:fce1ddf9b606cfb98132ab58865c3728c52c8e4c3c46e2aabb3674464a186e92"}, ] [package.dependencies] -anyio = ">=3.0,<5.0" certifi = "*" h11 = ">=0.13,<0.15" -sniffio = "==1.*" [package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<0.23.0)"] [[package]] name = "httpx" -version = "0.25.0" +version = "0.25.1" description = "The next generation HTTP client." optional = false python-versions = ">=3.8" files = [ - {file = "httpx-0.25.0-py3-none-any.whl", hash = "sha256:181ea7f8ba3a82578be86ef4171554dd45fec26a02556a744db029a0a27b7100"}, - {file = "httpx-0.25.0.tar.gz", hash = "sha256:47ecda285389cb32bb2691cc6e069e3ab0205956f681c5b2ad2325719751d875"}, + {file = "httpx-0.25.1-py3-none-any.whl", hash = "sha256:fec7d6cc5c27c578a391f7e87b9aa7d3d8fbcd034f6399f9f79b45bcc12a866a"}, + {file = "httpx-0.25.1.tar.gz", hash = "sha256:ffd96d5cf901e63863d9f1b4b6807861dbea4d301613415d9e6e57ead15fc5d0"}, ] [package.dependencies] +anyio = "*" certifi = "*" -httpcore = ">=0.18.0,<0.19.0" +httpcore = "*" idna = "*" sniffio = "*" @@ -1414,24 +1415,22 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "protobuf" -version = "4.24.4" +version = "4.25.0" description = "" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "protobuf-4.24.4-cp310-abi3-win32.whl", hash = "sha256:ec9912d5cb6714a5710e28e592ee1093d68c5ebfeda61983b3f40331da0b1ebb"}, - {file = "protobuf-4.24.4-cp310-abi3-win_amd64.whl", hash = "sha256:1badab72aa8a3a2b812eacfede5020472e16c6b2212d737cefd685884c191085"}, - {file = "protobuf-4.24.4-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8e61a27f362369c2f33248a0ff6896c20dcd47b5d48239cb9720134bef6082e4"}, - {file = "protobuf-4.24.4-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:bffa46ad9612e6779d0e51ae586fde768339b791a50610d85eb162daeb23661e"}, - {file = "protobuf-4.24.4-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:b493cb590960ff863743b9ff1452c413c2ee12b782f48beca77c8da3e2ffe9d9"}, - {file = "protobuf-4.24.4-cp37-cp37m-win32.whl", hash = "sha256:dbbed8a56e56cee8d9d522ce844a1379a72a70f453bde6243e3c86c30c2a3d46"}, - {file = "protobuf-4.24.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6b7d2e1c753715dcfe9d284a25a52d67818dd43c4932574307daf836f0071e37"}, - {file = "protobuf-4.24.4-cp38-cp38-win32.whl", hash = "sha256:02212557a76cd99574775a81fefeba8738d0f668d6abd0c6b1d3adcc75503dbe"}, - {file = "protobuf-4.24.4-cp38-cp38-win_amd64.whl", hash = "sha256:2fa3886dfaae6b4c5ed2730d3bf47c7a38a72b3a1f0acb4d4caf68e6874b947b"}, - {file = "protobuf-4.24.4-cp39-cp39-win32.whl", hash = "sha256:b77272f3e28bb416e2071186cb39efd4abbf696d682cbb5dc731308ad37fa6dd"}, - {file = "protobuf-4.24.4-cp39-cp39-win_amd64.whl", hash = "sha256:9fee5e8aa20ef1b84123bb9232b3f4a5114d9897ed89b4b8142d81924e05d79b"}, - {file = "protobuf-4.24.4-py3-none-any.whl", hash = "sha256:80797ce7424f8c8d2f2547e2d42bfbb6c08230ce5832d6c099a37335c9c90a92"}, - {file = "protobuf-4.24.4.tar.gz", hash = "sha256:5a70731910cd9104762161719c3d883c960151eea077134458503723b60e3667"}, + {file = "protobuf-4.25.0-cp310-abi3-win32.whl", hash = "sha256:5c1203ac9f50e4853b0a0bfffd32c67118ef552a33942982eeab543f5c634395"}, + {file = "protobuf-4.25.0-cp310-abi3-win_amd64.whl", hash = "sha256:c40ff8f00aa737938c5378d461637d15c442a12275a81019cc2fef06d81c9419"}, + {file = "protobuf-4.25.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:cf21faba64cd2c9a3ed92b7a67f226296b10159dbb8fbc5e854fc90657d908e4"}, + {file = "protobuf-4.25.0-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:32ac2100b0e23412413d948c03060184d34a7c50b3e5d7524ee96ac2b10acf51"}, + {file = "protobuf-4.25.0-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:683dc44c61f2620b32ce4927de2108f3ebe8ccf2fd716e1e684e5a50da154054"}, + {file = "protobuf-4.25.0-cp38-cp38-win32.whl", hash = "sha256:1a3ba712877e6d37013cdc3476040ea1e313a6c2e1580836a94f76b3c176d575"}, + {file = "protobuf-4.25.0-cp38-cp38-win_amd64.whl", hash = "sha256:b2cf8b5d381f9378afe84618288b239e75665fe58d0f3fd5db400959274296e9"}, + {file = "protobuf-4.25.0-cp39-cp39-win32.whl", hash = "sha256:63714e79b761a37048c9701a37438aa29945cd2417a97076048232c1df07b701"}, + {file = "protobuf-4.25.0-cp39-cp39-win_amd64.whl", hash = "sha256:d94a33db8b7ddbd0af7c467475fb9fde0c705fb315a8433c0e2020942b863a1f"}, + {file = "protobuf-4.25.0-py3-none-any.whl", hash = "sha256:1a53d6f64b00eecf53b65ff4a8c23dc95df1fa1e97bb06b8122e5a64f49fc90a"}, + {file = "protobuf-4.25.0.tar.gz", hash = "sha256:68f7caf0d4f012fd194a301420cf6aa258366144d814f358c5b32558228afa7c"}, ] [[package]] @@ -1551,36 +1550,41 @@ dev = ["Sphinx", "black", "build", "coverage", "docformatter", "flake8", "flake8 [[package]] name = "pygit2" -version = "1.13.1" +version = "1.13.2" description = "Python bindings for libgit2." optional = false python-versions = ">=3.8" files = [ - {file = "pygit2-1.13.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:30db67f73ef28b07864f2509978b7396d1ed3b72f6f252d301db12a4c9f90f5b"}, - {file = "pygit2-1.13.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:825f22a1bbf73c7a11c69e53a29485d10b4df6a635ccd120cf2966e6535a5b52"}, - {file = "pygit2-1.13.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7356d4e41f122a066fa1cce3f5dbedf73c03781692f5eab3687bc355a083575"}, - {file = "pygit2-1.13.1-cp310-cp310-win32.whl", hash = "sha256:cf47de2e21cdeb5d8c35f0d1a381b56fdb365dac3dcd8ea7fa057b390ce83d40"}, - {file = "pygit2-1.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:6ede9700fdbf78a5a1513549f37884233f29d3343412272c0800cda40c4c2c56"}, - {file = "pygit2-1.13.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9c84eff2223e5fd442b746785b9cd21f98c1f53a0f3fe8d4ed06aee60a09ea35"}, - {file = "pygit2-1.13.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5bd649b9ef17564b642f59e1a2751e30fdd07d3707b0642d8012062615651039"}, - {file = "pygit2-1.13.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:787ea717bb9fadb3ee2836ed32a9ed2110ef861862bfe6b693becda75a2eaa5c"}, - {file = "pygit2-1.13.1-cp311-cp311-win32.whl", hash = "sha256:d2dbf3d6976b0626fafd7d1c7363ae92dcacaa63789e8c432bc8caea86132235"}, - {file = "pygit2-1.13.1-cp311-cp311-win_amd64.whl", hash = "sha256:ce8618e5876b4c54942587d72a0d84f6e6a5b0e69db5f8d06dc5f567abd07ed1"}, - {file = "pygit2-1.13.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:8d2d97cfe2bf2abbb0ef5984771578d1b05053942bfe1b46d4ac48d19c5eda56"}, - {file = "pygit2-1.13.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6a162db1afdc5bae608d739395a248a373165176f83c7fe57a1073e9168b459"}, - {file = "pygit2-1.13.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357c11b30d6c63ff58401a897df39b27903beffebb24842c8ce9ce77b90fe0b1"}, - {file = "pygit2-1.13.1-cp38-cp38-win32.whl", hash = "sha256:ed7fc70bc8f6db227c9919958d064cb49eaa68cc97f51c1f9de920a4500c6766"}, - {file = "pygit2-1.13.1-cp38-cp38-win_amd64.whl", hash = "sha256:949ad31e0fab408449721cc5b582350f6c5c56ab068bfa10cd6d10c2830deaa9"}, - {file = "pygit2-1.13.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ea2b870675ef1a2bef3300dda725aae9f8c68265e633ed683fce85588cfb4d37"}, - {file = "pygit2-1.13.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74af678b98f6a08ef4315f5b64889011e05ad702e340cc6cde59926906650039"}, - {file = "pygit2-1.13.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33bbe4d7501a600be320147c63a8d1a966fb7424595e6eb53fdc30259b8921dc"}, - {file = "pygit2-1.13.1-cp39-cp39-win32.whl", hash = "sha256:697044df77c8b3849fec8d7dd454acd347b180212c6bc5526eeb9309eff63a65"}, - {file = "pygit2-1.13.1-cp39-cp39-win_amd64.whl", hash = "sha256:8004244da8183fcefcf7c3d4d119806e9c705543bcf24045b97e3eddaa869aef"}, - {file = "pygit2-1.13.1.tar.gz", hash = "sha256:d8e6d540aad9ded1cf2c6bda31ba48b1e20c18525807dbd837317bef4dccb994"}, + {file = "pygit2-1.13.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:781aefab7efc464852e688965bf3b4acc7af951cebea174d69f86b213aa5d5fb"}, + {file = "pygit2-1.13.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3038b5ecef43e2c853e7cf405676241e0395bb37b37ae477ef3b73a91f12378"}, + {file = "pygit2-1.13.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c00927a2626325b64ebc9f860f024a3ae0b4c036663f6ada8d5de0e2393560ca"}, + {file = "pygit2-1.13.2-cp310-cp310-win32.whl", hash = "sha256:6988fc6cf99a3dbc03bd64060888c3b194ee27c810cb61624519ee3813f2da3d"}, + {file = "pygit2-1.13.2-cp310-cp310-win_amd64.whl", hash = "sha256:aec3df351b722ec7cdf7a7e642e421e3a15f3f2e3a51e57380d62d4992acf36d"}, + {file = "pygit2-1.13.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0eb53cc5212fad90e36693c0cd2ffd0d470efaea2506ce1c0d04f8d7fcf6767c"}, + {file = "pygit2-1.13.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32803ec881cd8f7dba91e03927e1fb13857e795bbe85cd3ec156b4798b933294"}, + {file = "pygit2-1.13.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba7297204e72c5cfdcd7a0c0d318af9d654a1d79b1cfe6cc8330570c749bec1f"}, + {file = "pygit2-1.13.2-cp311-cp311-win32.whl", hash = "sha256:2291707e648f5bba5b5c5e7ed652bc4563bd520718eb31e19525ccaceba5503c"}, + {file = "pygit2-1.13.2-cp311-cp311-win_amd64.whl", hash = "sha256:96e534e92e485c4c1d4c3e151ce960655fed38ab9a1d65e2b16650cf24b3e088"}, + {file = "pygit2-1.13.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:75f3b6d754d91dbe47b27b53d5a4440d861906b2f476284e6fb7c46cafe244d7"}, + {file = "pygit2-1.13.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30e145730dc65a9b902a889efdca0126d6b274c0b14427ebb085e090b50f6470"}, + {file = "pygit2-1.13.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2311ca16e1d0b77bc3694407c367391c7f2f78e2f725dc858721a5d4e3635fdd"}, + {file = "pygit2-1.13.2-cp312-cp312-win32.whl", hash = "sha256:a027e06c44f987a217c6197970bb29de9fbc78524c81b1f37888711978a64ce2"}, + {file = "pygit2-1.13.2-cp312-cp312-win_amd64.whl", hash = "sha256:9844fb5a38119a34b31012dddc9b439f81bb0411cbf4a4f8e92a044f6f3e7462"}, + {file = "pygit2-1.13.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2f3a5362c702a42e28c3bc84ff324b57676c8bfdbfab445c96f5e776873630a6"}, + {file = "pygit2-1.13.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7d5d1c3508b66e5e13883ff472b616d2d60feb7a4afea52d3b501e9f5ee5d08"}, + {file = "pygit2-1.13.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2507d99584c7e3976342566adf6bc48aca825c170b86f999fe7bd32f8aa1858e"}, + {file = "pygit2-1.13.2-cp38-cp38-win32.whl", hash = "sha256:acda61b726c33ada3639cac5ddc5898678f7bb7b8415e84e3ff07a2af94b1ac3"}, + {file = "pygit2-1.13.2-cp38-cp38-win_amd64.whl", hash = "sha256:4a86c4cae2e717acdd9d7ff00d196395fafe1abfc5efab5ada63650b49d5d47f"}, + {file = "pygit2-1.13.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ae9a77be5c5df5f4c9e586fbd53f1095bced2bba86ec669ead92c4c1e02f8373"}, + {file = "pygit2-1.13.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25ef1dcfb59e73f6a59491343393b6e843739cbc92e8088a551c73cd367a54d0"}, + {file = "pygit2-1.13.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b88d21ed961fe422275c9c20d2613e6ecff2fa8127ac7620a29aba1d001fc41"}, + {file = "pygit2-1.13.2-cp39-cp39-win32.whl", hash = "sha256:14b458af1e8c6b634d55110edeab055e3bd9075543792cb75d2fdb8b434c202a"}, + {file = "pygit2-1.13.2-cp39-cp39-win_amd64.whl", hash = "sha256:565b311c783a07768b91382620ad2b474fe40778411cb18c576f667be43d1299"}, + {file = "pygit2-1.13.2.tar.gz", hash = "sha256:75c7eb86b47c70f6f1434bcf3b5eb41f4e8006a15cee6bef606651b97d23788c"}, ] [package.dependencies] -cffi = ">=1.9.1" +cffi = ">=1.16.0" [[package]] name = "pygithub" @@ -1937,13 +1941,13 @@ files = [ [[package]] name = "sentry-sdk" -version = "1.32.0" +version = "1.34.0" description = "Python client for Sentry (https://sentry.io)" optional = false python-versions = "*" files = [ - {file = "sentry-sdk-1.32.0.tar.gz", hash = "sha256:935e8fbd7787a3702457393b74b13d89a5afb67185bc0af85c00cb27cbd42e7c"}, - {file = "sentry_sdk-1.32.0-py2.py3-none-any.whl", hash = "sha256:eeb0b3550536f3bbc05bb1c7e0feb3a78d74acb43b607159a606ed2ec0a33a4d"}, + {file = "sentry-sdk-1.34.0.tar.gz", hash = "sha256:e5d0d2b25931d88fa10986da59d941ac6037f742ab6ff2fce4143a27981d60c3"}, + {file = "sentry_sdk-1.34.0-py2.py3-none-any.whl", hash = "sha256:76dd087f38062ac6c1e30ed6feb533ee0037ff9e709974802db7b5dbf2e5db21"}, ] [package.dependencies] @@ -2102,13 +2106,13 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "wcwidth" -version = "0.2.8" +version = "0.2.9" description = "Measures the displayed width of unicode strings in a terminal" optional = false python-versions = "*" files = [ - {file = "wcwidth-0.2.8-py2.py3-none-any.whl", hash = "sha256:77f719e01648ed600dfa5402c347481c0992263b81a027344f3e1ba25493a704"}, - {file = "wcwidth-0.2.8.tar.gz", hash = "sha256:8705c569999ffbb4f6a87c6d1b80f324bd6db952f5eb0b95bc07517f4c1813d4"}, + {file = "wcwidth-0.2.9-py2.py3-none-any.whl", hash = "sha256:9a929bd8380f6cd9571a968a9c8f4353ca58d7cd812a4822bba831f8d685b223"}, + {file = "wcwidth-0.2.9.tar.gz", hash = "sha256:a675d1a4a2d24ef67096a04b85b02deeecd8e226f57b5e3a72dbb9ed99d27da8"}, ] [[package]] @@ -2129,86 +2133,81 @@ test = ["websockets"] [[package]] name = "wrapt" -version = "1.15.0" +version = "1.16.0" description = "Module for decorators, wrappers and monkey patching." optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -files = [ - {file = "wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975"}, - {file = "wrapt-1.15.0-cp310-cp310-win32.whl", hash = "sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1"}, - {file = "wrapt-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98"}, - {file = "wrapt-1.15.0-cp311-cp311-win32.whl", hash = "sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416"}, - {file = "wrapt-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248"}, - {file = "wrapt-1.15.0-cp35-cp35m-win32.whl", hash = "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559"}, - {file = "wrapt-1.15.0-cp35-cp35m-win_amd64.whl", hash = "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639"}, - {file = "wrapt-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2"}, - {file = "wrapt-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1"}, - {file = "wrapt-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420"}, - {file = "wrapt-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653"}, - {file = "wrapt-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0"}, - {file = "wrapt-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7"}, - {file = "wrapt-1.15.0-cp38-cp38-win32.whl", hash = "sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b"}, - {file = "wrapt-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9"}, - {file = "wrapt-1.15.0-cp39-cp39-win32.whl", hash = "sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff"}, - {file = "wrapt-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6"}, - {file = "wrapt-1.15.0-py3-none-any.whl", hash = "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640"}, - {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, ] [[package]] @@ -2301,4 +2300,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "~3.10" -content-hash = "cf81e8fa302ee7329d6fc89906bc14fb6ef11d3bd4a90abe2a09cc907b73b3d0" +content-hash = "9e99c86b95e7aefd35f67c0da45d99201a5f3043fd63d66b33ec05511f9dc322" diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 3824b117b447..afb023d33e14 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "2.5.9" +version = "2.6.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] diff --git a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py index 28ff2f9da8fc..cf503c9d6588 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py +++ b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py @@ -1,4 +1,6 @@ +# # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# import asyncclick as click import pytest diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle index 5b4d5de4355b..961ff66d0097 100644 --- a/airbyte-integrations/bases/base-normalization/build.gradle +++ b/airbyte-integrations/bases/base-normalization/build.gradle @@ -51,13 +51,7 @@ tasks.named('check').configure { 'tidb', 'duckdb', ].each {destinationName -> - def integrationTestPython = tasks.named('integrationTestPython') - integrationTestPython.configure { - dependsOn project(":airbyte-integrations:connectors:destination-$destinationName").tasks.named('assemble') - } - // Not really sure what this task does differently from customIntegrationTestPython, - // but it seems to also run integration tests and as such it depends on the docker images. - integrationTestPython.configure { + tasks.matching { it.name == 'integrationTestPython' }.configureEach { dependsOn project(":airbyte-integrations:connectors:destination-$destinationName").tasks.named('assemble') } } diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py index 0f6ff2081174..47a235c00f14 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import logging from contextlib import contextmanager from io import IOBase diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/unit_tests.py b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/unit_tests.py index 17b5f7fc5913..88d003f81475 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/unit_tests.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/unit_tests.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + from source_azure_blob_storage.legacy_config_transformer import LegacyConfigTransformer diff --git a/airbyte-integrations/connectors/source-github/unit_tests/conftest.py b/airbyte-integrations/connectors/source-github/unit_tests/conftest.py index 3604b32db597..c3d9c1c98188 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/conftest.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import os os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" diff --git a/airbyte-integrations/connectors/source-intercom/unit_tests/test_source.py b/airbyte-integrations/connectors/source-intercom/unit_tests/test_source.py index 0cf50ccbc171..fe7a765a2ee0 100644 --- a/airbyte-integrations/connectors/source-intercom/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-intercom/unit_tests/test_source.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + from source_intercom import SourceIntercom diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/exceptions.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/exceptions.py index f68eecb35695..63df81365345 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/exceptions.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/exceptions.py @@ -1,2 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + + class KlaviyoBackoffError(Exception): """An exception which is raised when 'retry-after' time is longer than 'max_time' specified""" diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py index 3604b32db597..c3d9c1c98188 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import os os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" diff --git a/airbyte-integrations/connectors/source-pokeapi/main.py b/airbyte-integrations/connectors/source-pokeapi/main.py index 9ec4c483ecf9..38a510a3f2d7 100644 --- a/airbyte-integrations/connectors/source-pokeapi/main.py +++ b/airbyte-integrations/connectors/source-pokeapi/main.py @@ -2,7 +2,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - import sys from airbyte_cdk.entrypoint import launch diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py index 8168c53508c6..4f475b80c797 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import io import struct import zipfile diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_zip_reader.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_zip_reader.py index 8120fe52e434..c97e468cba6d 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_zip_reader.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_zip_reader.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import datetime import io import struct diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py index a2c4975260d2..969410ee24e1 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import os import pytest diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py index 3604b32db597..c3d9c1c98188 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import os os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" diff --git a/build.gradle b/build.gradle index 0573f02fe661..99ea87e6f74c 100644 --- a/build.gradle +++ b/build.gradle @@ -1,7 +1,4 @@ import com.github.spotbugs.snom.SpotBugsTask -import ru.vyarus.gradle.plugin.python.task.PythonTask -import com.hierynomus.gradle.license.tasks.LicenseCheck -import com.hierynomus.gradle.license.tasks.LicenseFormat // The buildscript block defines dependencies in order for .gradle file evaluation. // This is separate from application dependencies. @@ -9,7 +6,6 @@ import com.hierynomus.gradle.license.tasks.LicenseFormat buildscript { dependencies { classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.20.0' - classpath 'gradle.plugin.com.hierynomus.gradle.plugins:license-gradle-plugin:0.16.1' } } @@ -67,57 +63,7 @@ allprojects { version = rootProject.ext.version } -// License generation logic. -def createLicenseWith = { File license, String startComment, String endComment, String lineComment, boolean isPython -> - /* - In java, we don't have a second linter/styling tool other than spotless so it doesn't really - matter if we write a newline or not for startComment/endComment. - - However, in python, we are using black that double-checks and reformats the code. - Thus, writing an extra empty newline (not removed by trimTrailingWhitespace() is actually a - big deal and would be reformatted (removed) because of black's specs. - */ - def tmp = File.createTempFile('tmp', '.tmp') - tmp.withWriter { - def w = it - if (startComment.length() > 0 || !isPython) { - w.writeLine(startComment) - } - license.eachLine { - w << lineComment - w.writeLine(it) - } - if (endComment.length() > 0 || !isPython) { - w.writeLine(endComment) - } - w.writeLine("") - if (isPython) { - w.writeLine("") - } - } - return tmp -} -def createPythonLicenseWith = { license -> - return createLicenseWith(license, '', '', '', true) -} -def createJavaLicenseWith = { license -> - return createLicenseWith(license, '/*', ' */', ' * ', false) -} - -// node is required by the root project to apply the prettier formatter repo-wide. -node { - download = true - version = '18.18.0' - npmVersion = '10.1.0' - // When setting both these directories, npm and node will be in separate directories. - workDir = file("${buildDir}/nodejs") - npmWorkDir = file("${buildDir}/npm") - // Do not declare the repository. - distBaseUrl = null - -} - -// python is required by the root project to apply python formatters like isort or black. +// python is required by the root project to run CAT tests for connectors python { envPath = '.venv' minPythonVersion = '3.10' // should be 3.10 for local development @@ -149,12 +95,10 @@ python { scope = 'VIRTUALENV' installVirtualenv = true - // black and isort are required for formatting - pip 'black:22.3.0' - pip 'isort:5.6.4' // poetry is required for installing and running airbyte-ci pip 'poetry:1.5.1' } + def cleanPythonVenv = rootProject.tasks.register('cleanPythonVenv', Exec) { commandLine 'rm' args '-rf', "${rootProject.projectDir.absolutePath}/.venv" @@ -167,56 +111,10 @@ rootProject.tasks.named('clean').configure { allprojects { def createFormatTarget = { pattern -> - // We are the spotless exclusions rules using file tree. It seems the excludeTarget option is super finicky in a - // monorepo setup and it doesn't actually exclude directories reliably. - // This code makes the behavior predictable. ArrayList excludes = [ - '.gradle', - 'node_modules', - '.eggs', - '.mypy_cache', - '.venv', - '*.egg-info', - 'build', - 'dbt-project-template', - 'dbt-project-template-mssql', - 'dbt-project-template-mysql', - 'dbt-project-template-oracle', - 'dbt-project-template-clickhouse', - 'dbt-project-template-snowflake', - 'dbt-project-template-tidb', - 'dbt-project-template-duckdb', - 'dbt_test_config', - 'normalization_test_output', - 'tools', - 'secrets', - 'charts', // Helm charts often have injected template strings that will fail general linting. Helm linting is done separately. - 'resources/seed/*_catalog.json', // Do not remove - this is also necessary to prevent diffs in our github workflows - 'resources/seed/*_registry.json', // Do not remove - this is also necessary to prevent diffs in our github workflows - 'resources/seed/specs_secrets_mask.yaml', // Downloaded externally. - 'resources/examples/airflow/superset/docker/pythonpath_dev/superset_config.py', - 'source-amplitude/unit_tests/api_data/zipped.json', // Zipped file presents as non-UTF-8 making spotless sad - 'airbyte-connector-builder-server/connector_builder/generated', // autogenerated code doesn't need to be formatted - 'airbyte-ci/connectors/metadata_service/lib/tests/fixtures/**/invalid', // These are deliberately invalid and unformattable. - '__init__.py', - 'declarative_component_schema.py', - 'source-stock-ticker-api-tutorial/source.py', - 'tools/git_hooks/tests/test_spec_linter.py', - 'tools/schema_generator/schema_generator/infer_schemas.py', + '**/build', // Ignore new build files as well as the ones ignored when running this through airbyte-ci + '.gradle' // Ignore the gradle version that is downloaded in the container ] - // Ensure that the excludes work when we're already partly in their path. - excludes.addAll excludes.collectMany { - it.startsWith("${project.name}/") ? [it, it.substring("${project.name}/".length())] : [it] - } - // Prefix everything with double-globs because we only care about path suffixes. - excludes = excludes.collect { "**/${it}" } - // Remove whatever's covered by the project's subprojects, so that each file is targeted by at most one task. - def currentProject = project - project.subprojects { - if (it.parent.projectDir == currentProject.projectDir) { - excludes.add "${it.projectDir}/**" - } - } // Build the FileTree. return fileTree(dir: projectDir, include: pattern, exclude: excludes) } @@ -230,7 +128,6 @@ allprojects { target javaTarget importOrder() eclipse('4.21').configFile(rootProject.file('tools/gradle/codestyle/java-google-style.xml')) - licenseHeaderFile createJavaLicenseWith(rootProject.file('LICENSE_SHORT')) removeUnusedImports() trimTrailingWhitespace() } @@ -241,118 +138,17 @@ allprojects { target groovyGradleTarget } } - def sqlTarget = createFormatTarget('**/*.sql') - if (!sqlTarget.isEmpty()) { - sql { - target sqlTarget - dbeaver().configFile(rootProject.file('tools/gradle/codestyle/sql-dbeaver.properties')) - } - } - def stylingTarget = createFormatTarget(['**/*.yaml', '**/*.yml', '**/*.json']) - if (!stylingTarget.isEmpty()) { - format 'styling', { - target stylingTarget - def npmPath = "${rootProject.tasks.named('npmSetup').get().npmDir.get()}/bin/npm" - def nodePath = "${rootProject.tasks.named('nodeSetup').get().nodeDir.get()}/bin/node" - prettier().npmExecutable(npmPath).nodeExecutable(nodePath) - } - } - } - tasks.matching { it.name =~ /spotless.*/ }.configureEach { - dependsOn tasks.matching { it.name == 'generate' } } - tasks.matching { it.name =~ /spotlessStyling.*/ }.configureEach { - dependsOn rootProject.tasks.named('nodeSetup') - dependsOn rootProject.tasks.named('npmSetup') - } - // TODO: remove this once the CI routinely enforces formatting checks. - tasks.matching { it.name =~ /spotless.*Check/ }.configureEach { - enabled = false - } - - // python license header generation is part of 'format' - def pythonTarget = createFormatTarget('**/*.py') - if (!project.name.endsWith('source-stock-ticker-api-tutorial') && !pythonTarget.isEmpty()) { - apply plugin: 'com.github.hierynomus.license' - - license { - header rootProject.file("LICENSE_SHORT") - } - - // Disable auto-generated java tasks, we rely on spotless there. - tasks.matching { it.name =~ /license.*/ }.configureEach { + if (rootProject.ext.skipSlowTests) { + // Format checks should be run via airbyte-ci. + tasks.matching { it.name =~ /spotless.*Check/ }.configureEach { enabled = false } - - def licensePythonCheck = tasks.register('licensePythonCheck', LicenseCheck) { - header = createPythonLicenseWith(rootProject.file('LICENSE_SHORT')) - source = pythonTarget - strictCheck = true - } - licensePythonCheck.configure { - dependsOn tasks.matching { it.name == 'generate' } - } - def licensePythonFormat = tasks.register('licensePythonFormat', LicenseFormat) { - header = createPythonLicenseWith(rootProject.file('LICENSE_SHORT')) - source = pythonTarget - strictCheck = true - } - licensePythonFormat.configure { - dependsOn tasks.matching { it.name == 'generate' } - } - - def registerPythonFormatTask = { - String taskName, String moduleName, String settingsFileFlagName, String... extraArgs -> - def task = tasks.register(taskName, Exec) { - workingDir projectDir - commandLine rootProject.file('.venv/bin/python') - args "-m", moduleName, settingsFileFlagName, rootProject.file('pyproject.toml').absolutePath - args extraArgs - pythonTarget.getFiles().forEach { - args projectDir.relativePath(it) - } - } - task.configure { - dependsOn rootProject.tasks.named('pipInstall') - dependsOn tasks.matching { it.name == 'generate' } - } - return task - } - - def isortPythonCheck = registerPythonFormatTask('isortPythonCheck', 'isort', '--settings-file', '--check-only') - isortPythonCheck.configure { - dependsOn licensePythonCheck - } - def isortPythonFormat = registerPythonFormatTask('isortPythonFormat', 'isort', '--settings-file') - isortPythonFormat.configure { - dependsOn licensePythonFormat - } - - def blackPythonCheck = registerPythonFormatTask('blackPythonCheck', 'black', '--config', '--check') - blackPythonCheck.configure { - dependsOn licensePythonCheck - dependsOn isortPythonCheck - } - def blackPythonFormat = registerPythonFormatTask('blackPythonFormat', 'black', '--config') - blackPythonFormat.configure { - dependsOn licensePythonFormat - dependsOn isortPythonFormat - } - } - - tasks.register('format') { - dependsOn tasks.matching { it.name == 'generate' } - dependsOn tasks.matching { it.name =~ /spotless.*Apply/ } - dependsOn tasks.matching { it.name =~ /.*PythonFormat/ } - } - tasks.named('check').configure { - dependsOn tasks.matching { it.name == 'licensePythonCheck' } - dependsOn tasks.matching { it.name == 'isortPythonCheck' } - dependsOn tasks.matching { it.name == 'blackPythonCheck' } } } + def getCDKTargetVersion() { def props = new Properties() file("airbyte-cdk/java/airbyte-cdk/src/main/resources/version.properties").withInputStream { props.load(it) } diff --git a/octavia-cli/build.gradle b/octavia-cli/build.gradle deleted file mode 100644 index 0f332639c632..000000000000 --- a/octavia-cli/build.gradle +++ /dev/null @@ -1,22 +0,0 @@ -import org.openapitools.generator.gradle.plugin.tasks.GenerateTask - -plugins { - id "org.openapi.generator" version "5.3.1" - id 'airbyte-python' - id 'airbyte-docker-legacy' -} - -def generateApiClient = tasks.register('generateApiClient', GenerateTask) { - inputSpec = "$rootDir.absolutePath/airbyte-cdk/java/airbyte-cdk/airbyte-api/src/main/openapi/config.yaml" - outputDir = "$buildDir/airbyte_api_client" - - generatorName = "python" - packageName = "airbyte_api_client" -} -tasks.register('generate').configure { - dependsOn generateApiClient -} - -tasks.named('installReqs').configure { - dependsOn generateApiClient -} diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 000000000000..7e3ee153c35a --- /dev/null +++ b/poetry.lock @@ -0,0 +1,140 @@ +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. + +[[package]] +name = "black" +version = "22.3.0" +description = "The uncompromising code formatter." +optional = false +python-versions = ">=3.6.2" +files = [ + {file = "black-22.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09"}, + {file = "black-22.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5795a0375eb87bfe902e80e0c8cfaedf8af4d49694d69161e5bd3206c18618bb"}, + {file = "black-22.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e3556168e2e5c49629f7b0f377070240bd5511e45e25a4497bb0073d9dda776a"}, + {file = "black-22.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67c8301ec94e3bcc8906740fe071391bce40a862b7be0b86fb5382beefecd968"}, + {file = "black-22.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:fd57160949179ec517d32ac2ac898b5f20d68ed1a9c977346efbac9c2f1e779d"}, + {file = "black-22.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cc1e1de68c8e5444e8f94c3670bb48a2beef0e91dddfd4fcc29595ebd90bb9ce"}, + {file = "black-22.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2fc92002d44746d3e7db7cf9313cf4452f43e9ea77a2c939defce3b10b5c82"}, + {file = "black-22.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:a6342964b43a99dbc72f72812bf88cad8f0217ae9acb47c0d4f141a6416d2d7b"}, + {file = "black-22.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:328efc0cc70ccb23429d6be184a15ce613f676bdfc85e5fe8ea2a9354b4e9015"}, + {file = "black-22.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06f9d8846f2340dfac80ceb20200ea5d1b3f181dd0556b47af4e8e0b24fa0a6b"}, + {file = "black-22.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4efa5fad66b903b4a5f96d91461d90b9507a812b3c5de657d544215bb7877a"}, + {file = "black-22.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8477ec6bbfe0312c128e74644ac8a02ca06bcdb8982d4ee06f209be28cdf163"}, + {file = "black-22.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:637a4014c63fbf42a692d22b55d8ad6968a946b4a6ebc385c5505d9625b6a464"}, + {file = "black-22.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:863714200ada56cbc366dc9ae5291ceb936573155f8bf8e9de92aef51f3ad0f0"}, + {file = "black-22.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10dbe6e6d2988049b4655b2b739f98785a884d4d6b85bc35133a8fb9a2233176"}, + {file = "black-22.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:cee3e11161dde1b2a33a904b850b0899e0424cc331b7295f2a9698e79f9a69a0"}, + {file = "black-22.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5891ef8abc06576985de8fa88e95ab70641de6c1fca97e2a15820a9b69e51b20"}, + {file = "black-22.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:30d78ba6bf080eeaf0b7b875d924b15cd46fec5fd044ddfbad38c8ea9171043a"}, + {file = "black-22.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ee8f1f7228cce7dffc2b464f07ce769f478968bfb3dd1254a4c2eeed84928aad"}, + {file = "black-22.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ee227b696ca60dd1c507be80a6bc849a5a6ab57ac7352aad1ffec9e8b805f21"}, + {file = "black-22.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:9b542ced1ec0ceeff5b37d69838106a6348e60db7b8fdd245294dc1d26136265"}, + {file = "black-22.3.0-py3-none-any.whl", hash = "sha256:bc58025940a896d7e5356952228b68f793cf5fcb342be703c3a2669a1488cb72"}, + {file = "black-22.3.0.tar.gz", hash = "sha256:35020b8886c022ced9282b51b5a875b6d1ab0c387b31a065b84db7c33085ca79"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +pathspec = ">=0.9.0" +platformdirs = ">=2" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + +[[package]] +name = "click" +version = "8.1.7" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "isort" +version = "5.6.4" +description = "A Python utility / library to sort Python imports." +optional = false +python-versions = ">=3.6,<4.0" +files = [ + {file = "isort-5.6.4-py3-none-any.whl", hash = "sha256:dcab1d98b469a12a1a624ead220584391648790275560e1a43e54c5dceae65e7"}, + {file = "isort-5.6.4.tar.gz", hash = "sha256:dcaeec1b5f0eca77faea2a35ab790b4f3680ff75590bfcb7145986905aab2f58"}, +] + +[package.extras] +colors = ["colorama (>=0.4.3,<0.5.0)"] +pipfile-deprecated-finder = ["pipreqs", "requirementslib"] +requirements-deprecated-finder = ["pip-api", "pipreqs"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "pathspec" +version = "0.11.2" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pathspec-0.11.2-py3-none-any.whl", hash = "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20"}, + {file = "pathspec-0.11.2.tar.gz", hash = "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"}, +] + +[[package]] +name = "platformdirs" +version = "3.11.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-3.11.0-py3-none-any.whl", hash = "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e"}, + {file = "platformdirs-3.11.0.tar.gz", hash = "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3"}, +] + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "~3.10" +content-hash = "352be223e781ec8ab7dc7326b50ba69733b74792f65832f00f185a102785caf4" diff --git a/pyproject.toml b/pyproject.toml index 80a70a5bd99e..fb3ac8175446 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,16 @@ +[tool.poetry] +name = "airbyte" +version = "0.1.0" +description = "Airbyte open source connector code" +authors = ["Airbyte "] + +[tool.poetry.dependencies] +python = "~3.10" + +[tool.poetry.group.dev.dependencies] +isort = "5.6.4" +black = "~22.3.0" + [tool.black] line-length = 140 target-version = ["py37"] diff --git a/settings.gradle b/settings.gradle index c7e237404a43..91cb8e6871f1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -160,7 +160,6 @@ if (isCiServer || isAirbyteCI) { rootProject.name = 'airbyte' include ':tools:code-generator' -include ':octavia-cli' include ':airbyte-cdk:python' include ':airbyte-cdk:java:airbyte-cdk' diff --git a/tools/gradle/codestyle/sql-dbeaver.properties b/tools/gradle/codestyle/sql-dbeaver.properties deleted file mode 100644 index cecd2e3813c6..000000000000 --- a/tools/gradle/codestyle/sql-dbeaver.properties +++ /dev/null @@ -1,8 +0,0 @@ -# case of the keywords (UPPER, LOWER or ORIGINAL) -sql.formatter.keyword.case=UPPER -# Statement delimiter -sql.formatter.statement.delimiter=; -# Indentation style (space or tab) -sql.formatter.indent.type=space -# Number of identation characters -sql.formatter.indent.size=4 From 0e47bda725e30f867e299019eb29a173efc60292 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Tue, 14 Nov 2023 00:45:26 -0800 Subject: [PATCH 028/124] Airbyte-ci: Generate binary files for airbyte-ci (#31930) Co-authored-by: bnchrch Co-authored-by: alafanechere --- .../actions/run-dagger-pipeline/action.yml | 2 +- .../airbyte-ci-release-experiment.yml | 119 +++++++- airbyte-ci/connectors/pipelines/README.md | 23 +- .../connectors/bump_version/pipeline.py | 4 +- .../migrate_to_base_image/pipeline.py | 6 +- .../pipelines/pipelines/cli/airbyte_ci.py | 56 +++- .../pipelines/pipelines/cli/dagger_run.py | 28 +- .../connectors/pipelines/pipelines/consts.py | 8 +- .../pipelines/pipelines/helpers/git.py | 17 +- airbyte-ci/connectors/pipelines/poetry.lock | 277 +++++++++++++----- .../connectors/pipelines/pyproject.toml | 11 +- 11 files changed, 420 insertions(+), 131 deletions(-) diff --git a/.github/actions/run-dagger-pipeline/action.yml b/.github/actions/run-dagger-pipeline/action.yml index afb0d8e69c1a..31af4caf9ada 100644 --- a/.github/actions/run-dagger-pipeline/action.yml +++ b/.github/actions/run-dagger-pipeline/action.yml @@ -104,7 +104,7 @@ runs: shell: bash run: | export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock" - airbyte-ci-internal --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }} + airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }} env: _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k" CI_CONTEXT: "${{ inputs.context }}" diff --git a/.github/workflows/airbyte-ci-release-experiment.yml b/.github/workflows/airbyte-ci-release-experiment.yml index cdd572066739..b2a50045340e 100644 --- a/.github/workflows/airbyte-ci-release-experiment.yml +++ b/.github/workflows/airbyte-ci-release-experiment.yml @@ -1,32 +1,125 @@ -name: Connector Ops CI - Experimental Airbyte CI Release - -# Note this is a workflow simply to test out if github can build using macos +# TODO remove -experiment from the workflow file name after merging +name: Connector Ops CI - Airbyte CI Release concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true on: + push: + branches: + - master + paths: + - "airbyte-ci/connectors/pipelines/**" workflow_dispatch: + +env: + DEV_GCS_BUCKET_NAME: dev-airbyte-cloud-connector-metadata-service + PROD_GCS_BUCKET_NAME: prod-airbyte-cloud-connector-metadata-service + BINARY_FILE_NAME: airbyte-ci + jobs: build: - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os }}-latest strategy: fail-fast: false matrix: - os: ["ubuntu-latest", "macos-latest"] + os: ["ubuntu", "macos"] steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - name: Checkout Airbyte + id: checkout_airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.sha }} # This is required to make sure that the same commit is checked out on all runners + + - name: Get short SHA + id: get_short_sha + uses: benjlevesque/short-sha@v2.2 + + - name: Install Python + id: install_python + uses: actions/setup-python@v4 with: - python-version: 3.10 + python-version: "3.10" - - run: curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - + - name: Install Poetry + id: install_poetry + uses: snok/install-poetry@v1 + + - name: Install Dependencies + id: install_dependencies + working-directory: airbyte-ci/connectors/pipelines/ + run: poetry install --with dev + + - name: Build release + id: build_release + working-directory: airbyte-ci/connectors/pipelines/ + run: poetry run poe build-release-binary ${{ env.BINARY_FILE_NAME }} - - run: cd airbyte-ci/pipelines/airbyte_ci - - run: poetry install --with dev - - run: poetry run pyinstaller --collect-all pipelines --collect-all beartype --collect-all dagger --hidden-import strawberry --name airbyte-ci-${{ matrix.os }} --onefile pipelines/cli/airbyte_ci.py - uses: actions/upload-artifact@v2 with: - path: dist/* + name: airbyte-ci-${{ matrix.os }}-${{ steps.get_short_sha.outputs.sha }} + path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }} + + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v1 + with: + credentials_json: "${{ secrets.METADATA_SERVICE_DEV_GCS_CREDENTIALS }}" + + - name: Upload pre-release to GCS dev bucket + id: upload_pre_release_to_gcs + if: github.ref != 'refs/heads/master' + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }} + destination: ${{ env.DEV_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/${{ steps.get_short_sha.outputs.sha }} + headers: |- + cache-control:public, max-age=10 + + - name: Print pre-release public url + id: print_pre_release_public_url + run: | + echo "https://storage.googleapis.com/${{ env.DEV_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/${{ steps.get_short_sha.outputs.sha }}/${{ env.BINARY_FILE_NAME }}" + + # if master, upload per version and latest to prod bucket + + - name: Set version from poetry version --short + id: set_version + if: github.ref == 'refs/heads/master' + working-directory: airbyte-ci/connectors/pipelines/ + run: | + echo "::set-output name=version::$(poetry version --short)" + + - name: Upload version release to GCS prod bucket + id: upload_version_release_to_gcs + if: github.ref == 'refs/heads/master' + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }} + destination: ${{ env.PROD_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/${{ steps.set_version.outputs.version }} + headers: |- + cache-control:public, max-age=10 + + - name: Print release version public url + id: print_version_release_public_url + if: github.ref == 'refs/heads/master' + run: | + echo "https://storage.googleapis.com/${{ env.PROD_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/${{ steps.set_version.outputs.version }}/${{ env.BINARY_FILE_NAME }}" + + - name: Upload latest release to GCS prod bucket + id: upload_latest_release_to_gcs + if: github.ref == 'refs/heads/master' + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }} + destination: ${{ env.PROD_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/latest + headers: |- + cache-control:public, max-age=10 + + - name: Print latest release public url + id: print_latest_release_public_url + if: github.ref == 'refs/heads/master' + run: | + echo "https://storage.googleapis.com/${{ env.PROD_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/latest/${{ env.BINARY_FILE_NAME }}" diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index 69ae79ccf4a7..bc9ff2271bca 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -124,17 +124,17 @@ At this point you can run `airbyte-ci` commands. #### Options -| Option | Default value | Mapped environment variable | Description | -| --------------------------------------- | ------------------------------- | ----------------------------- | ------------------------------------------------------------------------------------------- | -| `--no-tui` | | | Disables the Dagger terminal UI. | -| `--is-local/--is-ci` | `--is-local` | | Determines the environment in which the CLI runs: local environment or CI environment. | -| `--git-branch` | The checked out git branch name | `CI_GIT_BRANCH` | The git branch on which the pipelines will run. | -| `--git-revision` | The current branch head | `CI_GIT_REVISION` | The commit hash on which the pipelines will run. | -| `--diffed-branch` | `origin/master` | | Branch to which the git diff will happen to detect new or modified files. | -| `--gha-workflow-run-id` | | | GHA CI only - The run id of the GitHub action workflow | -| `--ci-context` | `manual` | | The current CI context: `manual` for manual run, `pull_request`, `nightly_builds`, `master` | -| `--pipeline-start-timestamp` | Current epoch time | `CI_PIPELINE_START_TIMESTAMP` | Start time of the pipeline as epoch time. Used for pipeline run duration computation. | -| `--show-dagger-logs/--hide-dagger-logs` | `--hide-dagger-logs` | | Flag to show or hide the dagger logs. | +| Option | Default value | Mapped environment variable | Description | +| ------------------------------------------ | ---------------------------------------------------------------------------------------------- | ----------------------------- | ------------------------------------------------------------------------------------------- | +| `--enable-dagger-run/--disable-dagger-run` | `--enable-dagger-run`` | | Disables the Dagger terminal UI. | | | +| `--is-local/--is-ci` | `--is-local` | | Determines the environment in which the CLI runs: local environment or CI environment. | +| `--git-branch` | The checked out git branch name | `CI_GIT_BRANCH` | The git branch on which the pipelines will run. | +| `--git-revision` | The current branch head | `CI_GIT_REVISION` | The commit hash on which the pipelines will run. | +| `--diffed-branch` | `origin/master` | | Branch to which the git diff will happen to detect new or modified files. | +| `--gha-workflow-run-id` | | | GHA CI only - The run id of the GitHub action workflow | +| `--ci-context` | `manual` | | The current CI context: `manual` for manual run, `pull_request`, `nightly_builds`, `master` | +| `--pipeline-start-timestamp` | Current epoch time | `CI_PIPELINE_START_TIMESTAMP` | Start time of the pipeline as epoch time. Used for pipeline run duration computation. | +| `--show-dagger-logs/--hide-dagger-logs` | `--hide-dagger-logs` | | Flag to show or hide the dagger logs. | ### `connectors` command subgroup @@ -433,6 +433,7 @@ This command runs the Python tests for a airbyte-ci poetry package. ## Changelog | Version | PR | Description | | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| 2.7.0 | [#31930](https://github.com/airbytehq/airbyte/pull/31930) | Merge airbyte-ci-internal into airbyte-ci | | 2.6.0 | [#31831](https://github.com/airbytehq/airbyte/pull/31831) | Add `airbyte-ci format` commands, remove connector-specific formatting check | | 2.5.9 | [#32427](https://github.com/airbytehq/airbyte/pull/32427) | Re-enable caching for source-postgres | | 2.5.8 | [#32402](https://github.com/airbytehq/airbyte/pull/32402) | Set Dagger Cloud token for airbyters only | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/pipeline.py index f2b7f266788a..f387e5a98284 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/pipeline.py @@ -7,9 +7,9 @@ import semver from dagger import Container -from pipelines import consts from pipelines.airbyte_ci.connectors.context import ConnectorContext from pipelines.airbyte_ci.connectors.reports import ConnectorReport +from pipelines.helpers import git from pipelines.helpers.connectors import metadata_change_helpers from pipelines.models.steps import Step, StepResult, StepStatus @@ -171,6 +171,6 @@ async def run_connector_version_bump_pipeline( add_changelog_entry_result = await add_changelog_entry.run() steps_results.append(add_changelog_entry_result) final_repo_dir = add_changelog_entry_result.output_artifact - await og_repo_dir.diff(final_repo_dir).export(str(consts.REPO_PATH)) + await og_repo_dir.diff(final_repo_dir).export(str(git.get_git_repo_path())) context.report = ConnectorReport(context, steps_results, name="CONNECTOR VERSION BUMP RESULTS") return context.report diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py index 22ad8424980a..291d6a52a281 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py @@ -10,10 +10,10 @@ from connector_ops.utils import ConnectorLanguage from dagger import Directory from jinja2 import Template -from pipelines import consts from pipelines.airbyte_ci.connectors.bump_version.pipeline import AddChangelogEntry, BumpDockerImageTagInMetadata, get_bumped_version from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext from pipelines.airbyte_ci.connectors.reports import ConnectorReport +from pipelines.helpers import git from pipelines.helpers.connectors import metadata_change_helpers from pipelines.models.steps import Step, StepResult, StepStatus @@ -266,7 +266,7 @@ async def run_connector_base_image_upgrade_pipeline(context: ConnectorContext, s update_base_image_in_metadata_result = await update_base_image_in_metadata.run() steps_results.append(update_base_image_in_metadata_result) final_repo_dir = update_base_image_in_metadata_result.output_artifact - await og_repo_dir.diff(final_repo_dir).export(str(consts.REPO_PATH)) + await og_repo_dir.diff(final_repo_dir).export(str(git.get_git_repo_path())) context.report = ConnectorReport(context, steps_results, name="BASE IMAGE UPGRADE RESULTS") return context.report @@ -337,7 +337,7 @@ async def run_connector_migration_to_base_image_pipeline(context: ConnectorConte # EXPORT MODIFIED FILES BACK TO HOST final_repo_dir = add_build_instructions_to_doc_results.output_artifact - await og_repo_dir.diff(final_repo_dir).export(str(consts.REPO_PATH)) + await og_repo_dir.diff(final_repo_dir).export(str(git.get_git_repo_path())) context.report = ConnectorReport(context, steps_results, name="MIGRATE TO BASE IMAGE RESULTS") return context.report diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 3a8912b986c8..82ca893e67cb 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -8,6 +8,7 @@ import logging import multiprocessing import os +import sys from pathlib import Path from typing import List, Optional @@ -19,7 +20,7 @@ from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.lazy_group import LazyGroup from pipelines.cli.telemetry import click_track_command -from pipelines.consts import LOCAL_PIPELINE_PACKAGE_PATH, CIContext +from pipelines.consts import DAGGER_WRAP_ENV_VAR_NAME, LOCAL_PIPELINE_PACKAGE_PATH, CIContext from pipelines.helpers import github from pipelines.helpers.git import ( get_current_git_branch, @@ -56,7 +57,7 @@ def display_welcome_message() -> None: ) -def check_up_to_date() -> bool: +def check_up_to_date(throw_as_error=False) -> bool: """Check if the installed version of pipelines is up to date.""" latest_version = get_latest_version() if latest_version != __installed_version__: @@ -68,7 +69,12 @@ def check_up_to_date() -> bool: 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 """ - raise Exception(upgrade_error_message) + + if throw_as_error: + raise Exception(upgrade_error_message) + else: + logging.warning(upgrade_error_message) + return False main_logger.info(f"pipelines is up to date. Installed version: {__installed_version__}. Latest version: {latest_version}") return True @@ -206,6 +212,38 @@ def check_local_docker_configuration(): ) +def is_dagger_run_enabled_by_default() -> bool: + dagger_run_by_default = [ + ["connectors", "test"], + ["connectors", "build"], + ["test"], + ["metadata_service"], + ] + + for command_tokens in dagger_run_by_default: + if all(token in sys.argv for token in command_tokens): + return True + + return False + + +def check_dagger_wrap(): + """ + Check if the command is already wrapped by dagger run. + This is useful to avoid infinite recursion when calling dagger run from dagger run. + """ + return os.getenv(DAGGER_WRAP_ENV_VAR_NAME) == "true" + + +def is_current_process_wrapped_by_dagger_run() -> bool: + """ + Check if the current process is wrapped by dagger run. + """ + called_with_dagger_run = check_dagger_wrap() + main_logger.info(f"Called with dagger run: {called_with_dagger_run}") + return called_with_dagger_run + + async def get_modified_files_str(ctx: click.Context): modified_files = await get_modified_files( ctx.obj["git_branch"], @@ -232,6 +270,7 @@ async def get_modified_files_str(ctx: click.Context): }, ) @click.version_option(__installed_version__) +@click.option("--enable-dagger-run/--disable-dagger-run", default=is_dagger_run_enabled_by_default) @click.option("--is-local/--is-ci", default=True) @click.option("--git-branch", default=get_current_git_branch, envvar="CI_GIT_BRANCH") @click.option("--git-revision", default=get_current_git_revision, envvar="CI_GIT_REVISION") @@ -248,6 +287,7 @@ async def get_modified_files_str(ctx: click.Context): @click.option("--ci-git-user", default="octavia-squidington-iii", envvar="CI_GIT_USER", type=str) @click.option("--ci-github-access-token", envvar="CI_GITHUB_ACCESS_TOKEN", type=str) @click.option("--ci-report-bucket-name", envvar="CI_REPORT_BUCKET_NAME", type=str) +@click.option("--ci-artifact-bucket-name", envvar="CI_ARTIFACT_BUCKET_NAME", type=str) @click.option( "--ci-gcs-credentials", help="The service account to use during CI.", @@ -269,12 +309,20 @@ async def get_modified_files_str(ctx: click.Context): @click_ignore_unused_kwargs async def airbyte_ci(ctx: click.Context): # noqa D103 display_welcome_message() + + if ctx.obj["enable_dagger_run"] and not is_current_process_wrapped_by_dagger_run(): + main_logger.debug("Re-Running airbyte-ci with dagger run.") + from pipelines.cli.dagger_run import call_current_command_with_dagger_run + + call_current_command_with_dagger_run() + return + if ctx.obj["is_local"]: # This check is meaningful only when running locally # In our CI the docker host used by the Dagger Engine is different from the one used by the runner. check_local_docker_configuration() - check_up_to_date() + check_up_to_date(throw_as_error=ctx.obj["is_local"]) if not ctx.obj["is_local"]: log_git_info(ctx) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py b/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py index 74e891720d20..155332f8d0ab 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py @@ -14,7 +14,7 @@ import pkg_resources import requests -from pipelines.cli.airbyte_ci import set_working_directory_to_root +from pipelines.consts import DAGGER_WRAP_ENV_VAR_NAME LOGGER = logging.getLogger(__name__) BIN_DIR = Path.home() / "bin" @@ -72,6 +72,10 @@ def get_dagger_cli_version(dagger_path: Optional[str]) -> Optional[str]: def check_dagger_cli_install() -> str: + """ + If the dagger CLI is not installed, install it. + """ + expected_dagger_cli_version = get_current_dagger_sdk_version() dagger_path = get_dagger_path() if dagger_path is None: @@ -89,17 +93,21 @@ def check_dagger_cli_install() -> str: return dagger_path -def main(): - set_working_directory_to_root() +def mark_dagger_wrap(): + """ + Mark that the dagger wrap has been applied. + """ + os.environ[DAGGER_WRAP_ENV_VAR_NAME] = "true" + + +def call_current_command_with_dagger_run(): + mark_dagger_wrap() if (os.environ.get("AIRBYTE_ROLE") == "airbyter") or (os.environ.get("CI") == "True"): os.environ[DAGGER_CLOUD_TOKEN_ENV_VAR_NAME_VALUE[0]] = DAGGER_CLOUD_TOKEN_ENV_VAR_NAME_VALUE[1] exit_code = 0 - if len(sys.argv) > 1 and any([arg in ARGS_DISABLING_TUI for arg in sys.argv]): - command = ["airbyte-ci-internal"] + [arg for arg in sys.argv[1:] if arg != "--no-tui"] - else: - dagger_path = check_dagger_cli_install() - command = [dagger_path, "run", "airbyte-ci-internal"] + sys.argv[1:] + dagger_path = check_dagger_cli_install() + command = [dagger_path, "run"] + sys.argv try: try: LOGGER.info(f"Running command: {command}") @@ -110,7 +118,3 @@ def main(): except subprocess.CalledProcessError as e: exit_code = e.returncode sys.exit(exit_code) - - -if __name__ == "__main__": - main() diff --git a/airbyte-ci/connectors/pipelines/pipelines/consts.py b/airbyte-ci/connectors/pipelines/pipelines/consts.py index ddfbbb841298..5f5f4a278cd7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/consts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/consts.py @@ -5,7 +5,6 @@ import platform from enum import Enum -import git from dagger import Platform PYPROJECT_TOML_FILE_PATH = "pyproject.toml" @@ -24,8 +23,8 @@ PLATFORM_MACHINE_TO_DAGGER_PLATFORM = { "x86_64": Platform("linux/amd64"), - "aarch64": Platform("linux/amd64"), "arm64": Platform("linux/arm64"), + "aarch64": Platform("linux/amd64"), "amd64": Platform("linux/amd64"), } LOCAL_BUILD_PLATFORM = PLATFORM_MACHINE_TO_DAGGER_PLATFORM[platform.machine()] @@ -47,8 +46,6 @@ DOCKER_HOST_PORT = 2375 DOCKER_TMP_VOLUME_NAME = "shared-tmp" DOCKER_VAR_LIB_VOLUME_NAME = "docker-cache" -REPO = git.Repo(search_parent_directories=True) -REPO_PATH = REPO.working_tree_dir STATIC_REPORT_PREFIX = "airbyte-ci" PIP_CACHE_VOLUME_NAME = "pip_cache" PIP_CACHE_PATH = "/root/.cache/pip" @@ -82,3 +79,6 @@ class INTERNAL_TOOL_PATHS(str, Enum): CI_CREDENTIALS = "airbyte-ci/connectors/ci_credentials" CONNECTOR_OPS = "airbyte-ci/connectors/connector_ops" METADATA_SERVICE = "airbyte-ci/connectors/metadata_service/lib" + + +DAGGER_WRAP_ENV_VAR_NAME = "_DAGGER_WRAP_APPLIED" diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py index ce822343b393..c27d3f138db4 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import functools from typing import List, Set import git @@ -11,11 +12,11 @@ def get_current_git_revision() -> str: # noqa D103 - return git.Repo().head.object.hexsha + return git.Repo(search_parent_directories=True).head.object.hexsha def get_current_git_branch() -> str: # noqa D103 - return git.Repo().active_branch.name + return git.Repo(search_parent_directories=True).active_branch.name async def get_modified_files_in_branch_remote( @@ -115,3 +116,15 @@ async def get_modified_files_in_commit(current_git_branch: str, current_git_revi def get_modified_files_in_pull_request(pull_request: PullRequest) -> List[str]: """Retrieve the list of modified files in a pull request.""" return [f.filename for f in pull_request.get_files()] + + +@functools.cache +def get_git_repo() -> git.Repo: + """Retrieve the git repo.""" + return git.Repo(search_parent_directories=True) + + +@functools.cache +def get_git_repo_path() -> str: + """Retrieve the git repo path.""" + return get_git_repo().working_tree_dir diff --git a/airbyte-ci/connectors/pipelines/poetry.lock b/airbyte-ci/connectors/pipelines/poetry.lock index 6a938b4db1d5..5b9f01101000 100644 --- a/airbyte-ci/connectors/pipelines/poetry.lock +++ b/airbyte-ci/connectors/pipelines/poetry.lock @@ -36,6 +36,17 @@ files = [ [package.dependencies] pydantic = ">=1.9.2,<1.10.0" +[[package]] +name = "altgraph" +version = "0.17.4" +description = "Python graph (network) package" +optional = false +python-versions = "*" +files = [ + {file = "altgraph-0.17.4-py2.py3-none-any.whl", hash = "sha256:642743b4750de17e655e6711601b077bc6598dbfa3ba5fa2b2a35ce12b508dff"}, + {file = "altgraph-0.17.4.tar.gz", hash = "sha256:1b5afbb98f6c4dcadb2e2ae6ab9fa994bbb8c1d75f4fa96d340f9437ae454406"}, +] + [[package]] name = "ansicon" version = "1.89.0" @@ -946,13 +957,13 @@ files = [ [[package]] name = "httpcore" -version = "1.0.1" +version = "1.0.2" description = "A minimal low-level HTTP client." optional = false python-versions = ">=3.8" files = [ - {file = "httpcore-1.0.1-py3-none-any.whl", hash = "sha256:c5e97ef177dca2023d0b9aad98e49507ef5423e9f1d94ffe2cfe250aa28e63b0"}, - {file = "httpcore-1.0.1.tar.gz", hash = "sha256:fce1ddf9b606cfb98132ab58865c3728c52c8e4c3c46e2aabb3674464a186e92"}, + {file = "httpcore-1.0.2-py3-none-any.whl", hash = "sha256:096cc05bca73b8e459a1fc3dcf585148f63e534eae4339559c9b8a8d6399acc7"}, + {file = "httpcore-1.0.2.tar.gz", hash = "sha256:9fc092e4799b26174648e54b74ed5f683132a464e95643b226e00c2ed2fa6535"}, ] [package.dependencies] @@ -1058,6 +1069,20 @@ files = [ [package.dependencies] ansicon = {version = "*", markers = "platform_system == \"Windows\""} +[[package]] +name = "macholib" +version = "1.16.3" +description = "Mach-O header analysis and editing" +optional = false +python-versions = "*" +files = [ + {file = "macholib-1.16.3-py2.py3-none-any.whl", hash = "sha256:0e315d7583d38b8c77e815b1ecbdbf504a8258d8b3e17b61165c6feb60d18f2c"}, + {file = "macholib-1.16.3.tar.gz", hash = "sha256:07ae9e15e8e4cd9a788013d81f5908b3609aa76f9b1421bae9c4d7606ec86a30"}, +] + +[package.dependencies] +altgraph = ">=0.17" + [[package]] name = "markdown-it-py" version = "3.0.0" @@ -1269,43 +1294,47 @@ files = [ [[package]] name = "numpy" -version = "1.26.1" +version = "1.26.2" description = "Fundamental package for array computing in Python" optional = false -python-versions = "<3.13,>=3.9" -files = [ - {file = "numpy-1.26.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:82e871307a6331b5f09efda3c22e03c095d957f04bf6bc1804f30048d0e5e7af"}, - {file = "numpy-1.26.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdd9ec98f0063d93baeb01aad472a1a0840dee302842a2746a7a8e92968f9575"}, - {file = "numpy-1.26.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d78f269e0c4fd365fc2992c00353e4530d274ba68f15e968d8bc3c69ce5f5244"}, - {file = "numpy-1.26.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ab9163ca8aeb7fd32fe93866490654d2f7dda4e61bc6297bf72ce07fdc02f67"}, - {file = "numpy-1.26.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:78ca54b2f9daffa5f323f34cdf21e1d9779a54073f0018a3094ab907938331a2"}, - {file = "numpy-1.26.1-cp310-cp310-win32.whl", hash = "sha256:d1cfc92db6af1fd37a7bb58e55c8383b4aa1ba23d012bdbba26b4bcca45ac297"}, - {file = "numpy-1.26.1-cp310-cp310-win_amd64.whl", hash = "sha256:d2984cb6caaf05294b8466966627e80bf6c7afd273279077679cb010acb0e5ab"}, - {file = "numpy-1.26.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cd7837b2b734ca72959a1caf3309457a318c934abef7a43a14bb984e574bbb9a"}, - {file = "numpy-1.26.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c59c046c31a43310ad0199d6299e59f57a289e22f0f36951ced1c9eac3665b9"}, - {file = "numpy-1.26.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d58e8c51a7cf43090d124d5073bc29ab2755822181fcad978b12e144e5e5a4b3"}, - {file = "numpy-1.26.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6081aed64714a18c72b168a9276095ef9155dd7888b9e74b5987808f0dd0a974"}, - {file = "numpy-1.26.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:97e5d6a9f0702c2863aaabf19f0d1b6c2628fbe476438ce0b5ce06e83085064c"}, - {file = "numpy-1.26.1-cp311-cp311-win32.whl", hash = "sha256:b9d45d1dbb9de84894cc50efece5b09939752a2d75aab3a8b0cef6f3a35ecd6b"}, - {file = "numpy-1.26.1-cp311-cp311-win_amd64.whl", hash = "sha256:3649d566e2fc067597125428db15d60eb42a4e0897fc48d28cb75dc2e0454e53"}, - {file = "numpy-1.26.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:1d1bd82d539607951cac963388534da3b7ea0e18b149a53cf883d8f699178c0f"}, - {file = "numpy-1.26.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:afd5ced4e5a96dac6725daeb5242a35494243f2239244fad10a90ce58b071d24"}, - {file = "numpy-1.26.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a03fb25610ef560a6201ff06df4f8105292ba56e7cdd196ea350d123fc32e24e"}, - {file = "numpy-1.26.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcfaf015b79d1f9f9c9fd0731a907407dc3e45769262d657d754c3a028586124"}, - {file = "numpy-1.26.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e509cbc488c735b43b5ffea175235cec24bbc57b227ef1acc691725beb230d1c"}, - {file = "numpy-1.26.1-cp312-cp312-win32.whl", hash = "sha256:af22f3d8e228d84d1c0c44c1fbdeb80f97a15a0abe4f080960393a00db733b66"}, - {file = "numpy-1.26.1-cp312-cp312-win_amd64.whl", hash = "sha256:9f42284ebf91bdf32fafac29d29d4c07e5e9d1af862ea73686581773ef9e73a7"}, - {file = "numpy-1.26.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bb894accfd16b867d8643fc2ba6c8617c78ba2828051e9a69511644ce86ce83e"}, - {file = "numpy-1.26.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e44ccb93f30c75dfc0c3aa3ce38f33486a75ec9abadabd4e59f114994a9c4617"}, - {file = "numpy-1.26.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9696aa2e35cc41e398a6d42d147cf326f8f9d81befcb399bc1ed7ffea339b64e"}, - {file = "numpy-1.26.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5b411040beead47a228bde3b2241100454a6abde9df139ed087bd73fc0a4908"}, - {file = "numpy-1.26.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1e11668d6f756ca5ef534b5be8653d16c5352cbb210a5c2a79ff288e937010d5"}, - {file = "numpy-1.26.1-cp39-cp39-win32.whl", hash = "sha256:d1d2c6b7dd618c41e202c59c1413ef9b2c8e8a15f5039e344af64195459e3104"}, - {file = "numpy-1.26.1-cp39-cp39-win_amd64.whl", hash = "sha256:59227c981d43425ca5e5c01094d59eb14e8772ce6975d4b2fc1e106a833d5ae2"}, - {file = "numpy-1.26.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:06934e1a22c54636a059215d6da99e23286424f316fddd979f5071093b648668"}, - {file = "numpy-1.26.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76ff661a867d9272cd2a99eed002470f46dbe0943a5ffd140f49be84f68ffc42"}, - {file = "numpy-1.26.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:6965888d65d2848e8768824ca8288db0a81263c1efccec881cb35a0d805fcd2f"}, - {file = "numpy-1.26.1.tar.gz", hash = "sha256:c8c6c72d4a9f831f328efb1312642a1cafafaa88981d9ab76368d50d07d93cbe"}, +python-versions = ">=3.9" +files = [ + {file = "numpy-1.26.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3703fc9258a4a122d17043e57b35e5ef1c5a5837c3db8be396c82e04c1cf9b0f"}, + {file = "numpy-1.26.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cc392fdcbd21d4be6ae1bb4475a03ce3b025cd49a9be5345d76d7585aea69440"}, + {file = "numpy-1.26.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36340109af8da8805d8851ef1d74761b3b88e81a9bd80b290bbfed61bd2b4f75"}, + {file = "numpy-1.26.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcc008217145b3d77abd3e4d5ef586e3bdfba8fe17940769f8aa09b99e856c00"}, + {file = "numpy-1.26.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3ced40d4e9e18242f70dd02d739e44698df3dcb010d31f495ff00a31ef6014fe"}, + {file = "numpy-1.26.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b272d4cecc32c9e19911891446b72e986157e6a1809b7b56518b4f3755267523"}, + {file = "numpy-1.26.2-cp310-cp310-win32.whl", hash = "sha256:22f8fc02fdbc829e7a8c578dd8d2e15a9074b630d4da29cda483337e300e3ee9"}, + {file = "numpy-1.26.2-cp310-cp310-win_amd64.whl", hash = "sha256:26c9d33f8e8b846d5a65dd068c14e04018d05533b348d9eaeef6c1bd787f9919"}, + {file = "numpy-1.26.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b96e7b9c624ef3ae2ae0e04fa9b460f6b9f17ad8b4bec6d7756510f1f6c0c841"}, + {file = "numpy-1.26.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:aa18428111fb9a591d7a9cc1b48150097ba6a7e8299fb56bdf574df650e7d1f1"}, + {file = "numpy-1.26.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:06fa1ed84aa60ea6ef9f91ba57b5ed963c3729534e6e54055fc151fad0423f0a"}, + {file = "numpy-1.26.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96ca5482c3dbdd051bcd1fce8034603d6ebfc125a7bd59f55b40d8f5d246832b"}, + {file = "numpy-1.26.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:854ab91a2906ef29dc3925a064fcd365c7b4da743f84b123002f6139bcb3f8a7"}, + {file = "numpy-1.26.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f43740ab089277d403aa07567be138fc2a89d4d9892d113b76153e0e412409f8"}, + {file = "numpy-1.26.2-cp311-cp311-win32.whl", hash = "sha256:a2bbc29fcb1771cd7b7425f98b05307776a6baf43035d3b80c4b0f29e9545186"}, + {file = "numpy-1.26.2-cp311-cp311-win_amd64.whl", hash = "sha256:2b3fca8a5b00184828d12b073af4d0fc5fdd94b1632c2477526f6bd7842d700d"}, + {file = "numpy-1.26.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a4cd6ed4a339c21f1d1b0fdf13426cb3b284555c27ac2f156dfdaaa7e16bfab0"}, + {file = "numpy-1.26.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5d5244aabd6ed7f312268b9247be47343a654ebea52a60f002dc70c769048e75"}, + {file = "numpy-1.26.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a3cdb4d9c70e6b8c0814239ead47da00934666f668426fc6e94cce869e13fd7"}, + {file = "numpy-1.26.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa317b2325f7aa0a9471663e6093c210cb2ae9c0ad824732b307d2c51983d5b6"}, + {file = "numpy-1.26.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:174a8880739c16c925799c018f3f55b8130c1f7c8e75ab0a6fa9d41cab092fd6"}, + {file = "numpy-1.26.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f79b231bf5c16b1f39c7f4875e1ded36abee1591e98742b05d8a0fb55d8a3eec"}, + {file = "numpy-1.26.2-cp312-cp312-win32.whl", hash = "sha256:4a06263321dfd3598cacb252f51e521a8cb4b6df471bb12a7ee5cbab20ea9167"}, + {file = "numpy-1.26.2-cp312-cp312-win_amd64.whl", hash = "sha256:b04f5dc6b3efdaab541f7857351aac359e6ae3c126e2edb376929bd3b7f92d7e"}, + {file = "numpy-1.26.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4eb8df4bf8d3d90d091e0146f6c28492b0be84da3e409ebef54349f71ed271ef"}, + {file = "numpy-1.26.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1a13860fdcd95de7cf58bd6f8bc5a5ef81c0b0625eb2c9a783948847abbef2c2"}, + {file = "numpy-1.26.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64308ebc366a8ed63fd0bf426b6a9468060962f1a4339ab1074c228fa6ade8e3"}, + {file = "numpy-1.26.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baf8aab04a2c0e859da118f0b38617e5ee65d75b83795055fb66c0d5e9e9b818"}, + {file = "numpy-1.26.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d73a3abcac238250091b11caef9ad12413dab01669511779bc9b29261dd50210"}, + {file = "numpy-1.26.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b361d369fc7e5e1714cf827b731ca32bff8d411212fccd29ad98ad622449cc36"}, + {file = "numpy-1.26.2-cp39-cp39-win32.whl", hash = "sha256:bd3f0091e845164a20bd5a326860c840fe2af79fa12e0469a12768a3ec578d80"}, + {file = "numpy-1.26.2-cp39-cp39-win_amd64.whl", hash = "sha256:2beef57fb031dcc0dc8fa4fe297a742027b954949cabb52a2a376c144e5e6060"}, + {file = "numpy-1.26.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1cc3d5029a30fb5f06704ad6b23b35e11309491c999838c31f124fee32107c79"}, + {file = "numpy-1.26.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94cc3c222bb9fb5a12e334d0479b97bb2df446fbe622b470928f5284ffca3f8d"}, + {file = "numpy-1.26.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fe6b44fb8fcdf7eda4ef4461b97b3f63c466b27ab151bec2366db8b197387841"}, + {file = "numpy-1.26.2.tar.gz", hash = "sha256:f65738447676ab5777f11e6bbbdb8ce11b785e105f690bc45966574816b6d3ea"}, ] [[package]] @@ -1321,36 +1350,36 @@ files = [ [[package]] name = "pandas" -version = "2.1.2" +version = "2.1.3" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.9" files = [ - {file = "pandas-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:24057459f19db9ebb02984c6fdd164a970b31a95f38e4a49cf7615b36a1b532c"}, - {file = "pandas-2.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a6cf8fcc8a63d333970b950a7331a30544cf59b1a97baf0a7409e09eafc1ac38"}, - {file = "pandas-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ae6ffbd9d614c20d028c7117ee911fc4e266b4dca2065d5c5909e401f8ff683"}, - {file = "pandas-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eff794eeb7883c5aefb1ed572e7ff533ae779f6c6277849eab9e77986e352688"}, - {file = "pandas-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:02954e285e8e2f4006b6f22be6f0df1f1c3c97adbb7ed211c6b483426f20d5c8"}, - {file = "pandas-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:5b40c9f494e1f27588c369b9e4a6ca19cd924b3a0e1ef9ef1a8e30a07a438f43"}, - {file = "pandas-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:08d287b68fd28906a94564f15118a7ca8c242e50ae7f8bd91130c362b2108a81"}, - {file = "pandas-2.1.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bbd98dcdcd32f408947afdb3f7434fade6edd408c3077bbce7bd840d654d92c6"}, - {file = "pandas-2.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e90c95abb3285d06f6e4feedafc134306a8eced93cb78e08cf50e224d5ce22e2"}, - {file = "pandas-2.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52867d69a54e71666cd184b04e839cff7dfc8ed0cd6b936995117fdae8790b69"}, - {file = "pandas-2.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8d0382645ede2fde352da2a885aac28ec37d38587864c0689b4b2361d17b1d4c"}, - {file = "pandas-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:65177d1c519b55e5b7f094c660ed357bb7d86e799686bb71653b8a4803d8ff0d"}, - {file = "pandas-2.1.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5aa6b86802e8cf7716bf4b4b5a3c99b12d34e9c6a9d06dad254447a620437931"}, - {file = "pandas-2.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d594e2ce51b8e0b4074e6644758865dc2bb13fd654450c1eae51201260a539f1"}, - {file = "pandas-2.1.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3223f997b6d2ebf9c010260cf3d889848a93f5d22bb4d14cd32638b3d8bba7ad"}, - {file = "pandas-2.1.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc4944dc004ca6cc701dfa19afb8bdb26ad36b9bed5bcec617d2a11e9cae6902"}, - {file = "pandas-2.1.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3f76280ce8ec216dde336e55b2b82e883401cf466da0fe3be317c03fb8ee7c7d"}, - {file = "pandas-2.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:7ad20d24acf3a0042512b7e8d8fdc2e827126ed519d6bd1ed8e6c14ec8a2c813"}, - {file = "pandas-2.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:021f09c15e1381e202d95d4a21ece8e7f2bf1388b6d7e9cae09dfe27bd2043d1"}, - {file = "pandas-2.1.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e7f12b2de0060b0b858cfec0016e7d980ae5bae455a1746bfcc70929100ee633"}, - {file = "pandas-2.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83c166b9bb27c1715bed94495d9598a7f02950b4749dba9349c1dd2cbf10729d"}, - {file = "pandas-2.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25c9976c17311388fcd953cb3d0697999b2205333f4e11e669d90ff8d830d429"}, - {file = "pandas-2.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:851b5afbb0d62f6129ae891b533aa508cc357d5892c240c91933d945fff15731"}, - {file = "pandas-2.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:e78507adcc730533619de07bfdd1c62b2918a68cd4419ea386e28abf7f6a1e5c"}, - {file = "pandas-2.1.2.tar.gz", hash = "sha256:52897edc2774d2779fbeb6880d2cfb305daa0b1a29c16b91f531a18918a6e0f3"}, + {file = "pandas-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:acf08a73b5022b479c1be155d4988b72f3020f308f7a87c527702c5f8966d34f"}, + {file = "pandas-2.1.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3cc4469ff0cf9aa3a005870cb49ab8969942b7156e0a46cc3f5abd6b11051dfb"}, + {file = "pandas-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35172bff95f598cc5866c047f43c7f4df2c893acd8e10e6653a4b792ed7f19bb"}, + {file = "pandas-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59dfe0e65a2f3988e940224e2a70932edc964df79f3356e5f2997c7d63e758b4"}, + {file = "pandas-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0296a66200dee556850d99b24c54c7dfa53a3264b1ca6f440e42bad424caea03"}, + {file = "pandas-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:465571472267a2d6e00657900afadbe6097c8e1dc43746917db4dfc862e8863e"}, + {file = "pandas-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:04d4c58e1f112a74689da707be31cf689db086949c71828ef5da86727cfe3f82"}, + {file = "pandas-2.1.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7fa2ad4ff196768ae63a33f8062e6838efed3a319cf938fdf8b95e956c813042"}, + {file = "pandas-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4441ac94a2a2613e3982e502ccec3bdedefe871e8cea54b8775992485c5660ef"}, + {file = "pandas-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5ded6ff28abbf0ea7689f251754d3789e1edb0c4d0d91028f0b980598418a58"}, + {file = "pandas-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fca5680368a5139d4920ae3dc993eb5106d49f814ff24018b64d8850a52c6ed2"}, + {file = "pandas-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:de21e12bf1511190fc1e9ebc067f14ca09fccfb189a813b38d63211d54832f5f"}, + {file = "pandas-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a5d53c725832e5f1645e7674989f4c106e4b7249c1d57549023ed5462d73b140"}, + {file = "pandas-2.1.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7cf4cf26042476e39394f1f86868d25b265ff787c9b2f0d367280f11afbdee6d"}, + {file = "pandas-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:72c84ec1b1d8e5efcbff5312abe92bfb9d5b558f11e0cf077f5496c4f4a3c99e"}, + {file = "pandas-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f539e113739a3e0cc15176bf1231a553db0239bfa47a2c870283fd93ba4f683"}, + {file = "pandas-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fc77309da3b55732059e484a1efc0897f6149183c522390772d3561f9bf96c00"}, + {file = "pandas-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:08637041279b8981a062899da0ef47828df52a1838204d2b3761fbd3e9fcb549"}, + {file = "pandas-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b99c4e51ef2ed98f69099c72c75ec904dd610eb41a32847c4fcbc1a975f2d2b8"}, + {file = "pandas-2.1.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f7ea8ae8004de0381a2376662c0505bb0a4f679f4c61fbfd122aa3d1b0e5f09d"}, + {file = "pandas-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fcd76d67ca2d48f56e2db45833cf9d58f548f97f61eecd3fdc74268417632b8a"}, + {file = "pandas-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1329dbe93a880a3d7893149979caa82d6ba64a25e471682637f846d9dbc10dd2"}, + {file = "pandas-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:321ecdb117bf0f16c339cc6d5c9a06063854f12d4d9bc422a84bb2ed3207380a"}, + {file = "pandas-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:11a771450f36cebf2a4c9dbd3a19dfa8c46c4b905a3ea09dc8e556626060fe71"}, + {file = "pandas-2.1.3.tar.gz", hash = "sha256:22929f84bca106921917eb73c1521317ddd0a4c71b395bcf767a106e3494209f"}, ] [package.dependencies] @@ -1360,7 +1389,7 @@ pytz = ">=2020.1" tzdata = ">=2022.1" [package.extras] -all = ["PyQt5 (>=5.15.6)", "SQLAlchemy (>=1.4.36)", "beautifulsoup4 (>=4.11.1)", "bottleneck (>=1.3.4)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=0.8.1)", "fsspec (>=2022.05.0)", "gcsfs (>=2022.05.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.8.0)", "matplotlib (>=3.6.1)", "numba (>=0.55.2)", "numexpr (>=2.8.0)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pandas-gbq (>=0.17.5)", "psycopg2 (>=2.9.3)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.5)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "pyxlsb (>=1.0.9)", "qtpy (>=2.2.0)", "s3fs (>=2022.05.0)", "scipy (>=1.8.1)", "tables (>=3.7.0)", "tabulate (>=0.8.10)", "xarray (>=2022.03.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)", "zstandard (>=0.17.0)"] +all = ["PyQt5 (>=5.15.6)", "SQLAlchemy (>=1.4.36)", "beautifulsoup4 (>=4.11.1)", "bottleneck (>=1.3.4)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=0.8.1)", "fsspec (>=2022.05.0)", "gcsfs (>=2022.05.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.8.0)", "matplotlib (>=3.6.1)", "numba (>=0.55.2)", "numexpr (>=2.8.0)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pandas-gbq (>=0.17.5)", "psycopg2 (>=2.9.3)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.5)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "pyxlsb (>=1.0.9)", "qtpy (>=2.2.0)", "s3fs (>=2022.05.0)", "scipy (>=1.8.1)", "tables (>=3.7.0)", "tabulate (>=0.8.10)", "xarray (>=2022.03.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)", "zstandard (>=0.17.0)"] aws = ["s3fs (>=2022.05.0)"] clipboard = ["PyQt5 (>=5.15.6)", "qtpy (>=2.2.0)"] compression = ["zstandard (>=0.17.0)"] @@ -1380,18 +1409,40 @@ plot = ["matplotlib (>=3.6.1)"] postgresql = ["SQLAlchemy (>=1.4.36)", "psycopg2 (>=2.9.3)"] spss = ["pyreadstat (>=1.1.5)"] sql-other = ["SQLAlchemy (>=1.4.36)"] -test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] xml = ["lxml (>=4.8.0)"] +[[package]] +name = "pastel" +version = "0.2.1" +description = "Bring colors to your terminal." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pastel-0.2.1-py2.py3-none-any.whl", hash = "sha256:4349225fcdf6c2bb34d483e523475de5bb04a5c10ef711263452cb37d7dd4364"}, + {file = "pastel-0.2.1.tar.gz", hash = "sha256:e6581ac04e973cac858828c6202c1e1e81fee1dc7de7683f3e1ffe0bfd8a573d"}, +] + +[[package]] +name = "pefile" +version = "2023.2.7" +description = "Python PE parsing module" +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "pefile-2023.2.7-py3-none-any.whl", hash = "sha256:da185cd2af68c08a6cd4481f7325ed600a88f6a813bad9dea07ab3ef73d8d8d6"}, + {file = "pefile-2023.2.7.tar.gz", hash = "sha256:82e6114004b3d6911c77c3953e3838654b04511b8b66e8583db70c65998017dc"}, +] + [[package]] name = "platformdirs" -version = "3.11.0" +version = "4.0.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." optional = false python-versions = ">=3.7" files = [ - {file = "platformdirs-3.11.0-py3-none-any.whl", hash = "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e"}, - {file = "platformdirs-3.11.0.tar.gz", hash = "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3"}, + {file = "platformdirs-4.0.0-py3-none-any.whl", hash = "sha256:118c954d7e949b35437270383a3f2531e99dd93cf7ce4dc8340d3356d30f173b"}, + {file = "platformdirs-4.0.0.tar.gz", hash = "sha256:cb633b2bcf10c51af60beb0ab06d2f1d69064b43abf4c185ca6b28865f3f9731"}, ] [package.extras] @@ -1413,6 +1464,24 @@ files = [ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] +[[package]] +name = "poethepoet" +version = "0.24.3" +description = "A task runner that works well with poetry." +optional = false +python-versions = ">=3.8" +files = [ + {file = "poethepoet-0.24.3-py3-none-any.whl", hash = "sha256:8817c6d3d8492776bbb17eb29b7b815b2905aefaa0ad887137e69e53349e2235"}, + {file = "poethepoet-0.24.3.tar.gz", hash = "sha256:73f1060200d1c8f21e303d06a879c5c0e17b96ab16740da70aee2dcc3e4350e4"}, +] + +[package.dependencies] +pastel = ">=0.2.1,<0.3.0" +tomli = ">=1.2.2" + +[package.extras] +poetry-plugin = ["poetry (>=1.0,<2.0)"] + [[package]] name = "protobuf" version = "4.25.0" @@ -1617,6 +1686,51 @@ files = [ [package.extras] plugins = ["importlib-metadata"] +[[package]] +name = "pyinstaller" +version = "6.2.0" +description = "PyInstaller bundles a Python application and all its dependencies into a single package." +optional = false +python-versions = "<3.13,>=3.8" +files = [ + {file = "pyinstaller-6.2.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:a1adbd3cf25dc90926d783eae0f444d65cdfecc7bcdf6da522c3ae3ff47b4c25"}, + {file = "pyinstaller-6.2.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:29d164394f1e949072f78a64c1e040f1c47b7f4aff08514c7666a031c8b44996"}, + {file = "pyinstaller-6.2.0-py3-none-manylinux2014_i686.whl", hash = "sha256:ba602a38d7403de89c38b8956b221ce6de0280730d269bab522492fcad82ee33"}, + {file = "pyinstaller-6.2.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:ebac06d99b80d2035594c3cc2fb5f2612d86289edd0510dbcbeb20a873f51d5a"}, + {file = "pyinstaller-6.2.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:fcfabc0ff1d38a4262c051dea3fdc1f7f106405c1f1b491b4c79cd28df19cab6"}, + {file = "pyinstaller-6.2.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:104430686149b2f1c135b2c17aa2967c85d54ef77dc92feb4e179ec846c0c467"}, + {file = "pyinstaller-6.2.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:e87fd60292b53bb9965cb5a84122875469a2bd475fd0d0db0052a3f1be351f75"}, + {file = "pyinstaller-6.2.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:8ec9d6c98972bb922cedb16a6638257aa66e5deadd79e2953f3464696237c413"}, + {file = "pyinstaller-6.2.0-py3-none-win32.whl", hash = "sha256:e5561e9a9b946d835c8dbc11ae4c16cc21e62bc77d10cc043406dc2992dfb4c6"}, + {file = "pyinstaller-6.2.0-py3-none-win_amd64.whl", hash = "sha256:3b586196277c4c54b69880650984c39c28bb6258c2b4b64200032e6ac69d53a0"}, + {file = "pyinstaller-6.2.0-py3-none-win_arm64.whl", hash = "sha256:d0c87b605bf13c3a04dfaa1d2fa7cd36765b8137000eeadccba865e1d6a19bf0"}, + {file = "pyinstaller-6.2.0.tar.gz", hash = "sha256:1ce77043929bf525be38289d78feecde0fcf15506215eda6500176a8715c5047"}, +] + +[package.dependencies] +altgraph = "*" +macholib = {version = ">=1.8", markers = "sys_platform == \"darwin\""} +packaging = ">=22.0" +pefile = {version = ">=2022.5.30", markers = "sys_platform == \"win32\""} +pyinstaller-hooks-contrib = ">=2021.4" +pywin32-ctypes = {version = ">=0.2.1", markers = "sys_platform == \"win32\""} +setuptools = ">=42.0.0" + +[package.extras] +completion = ["argcomplete"] +hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] + +[[package]] +name = "pyinstaller-hooks-contrib" +version = "2023.10" +description = "Community maintained hooks for PyInstaller" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyinstaller-hooks-contrib-2023.10.tar.gz", hash = "sha256:4b4a998036abb713774cb26534ca06b7e6e09e4c628196017a10deb11a48747f"}, + {file = "pyinstaller_hooks_contrib-2023.10-py2.py3-none-any.whl", hash = "sha256:6dc1786a8f452941245d5bb85893e2a33632ebdcbc4c23eea41f2ee08281b0c0"}, +] + [[package]] name = "pyjwt" version = "2.8.0" @@ -1782,6 +1896,17 @@ files = [ {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, ] +[[package]] +name = "pywin32-ctypes" +version = "0.2.2" +description = "A (partial) reimplementation of pywin32 using ctypes/cffi" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pywin32-ctypes-0.2.2.tar.gz", hash = "sha256:3426e063bdd5fd4df74a14fa3cf80a0b42845a87e1d1e81f6549f9daec593a60"}, + {file = "pywin32_ctypes-0.2.2-py3-none-any.whl", hash = "sha256:bf490a1a709baf35d688fe0ecf980ed4de11d2b3e37b51e5442587a75d9957e7"}, +] + [[package]] name = "pyyaml" version = "6.0.1" @@ -1941,13 +2066,13 @@ files = [ [[package]] name = "sentry-sdk" -version = "1.34.0" +version = "1.35.0" description = "Python client for Sentry (https://sentry.io)" optional = false python-versions = "*" files = [ - {file = "sentry-sdk-1.34.0.tar.gz", hash = "sha256:e5d0d2b25931d88fa10986da59d941ac6037f742ab6ff2fce4143a27981d60c3"}, - {file = "sentry_sdk-1.34.0-py2.py3-none-any.whl", hash = "sha256:76dd087f38062ac6c1e30ed6feb533ee0037ff9e709974802db7b5dbf2e5db21"}, + {file = "sentry-sdk-1.35.0.tar.gz", hash = "sha256:04e392db9a0d59bd49a51b9e3a92410ac5867556820465057c2ef89a38e953e9"}, + {file = "sentry_sdk-1.35.0-py2.py3-none-any.whl", hash = "sha256:a7865952701e46d38b41315c16c075367675c48d049b90a4cc2e41991ebc7efa"}, ] [package.dependencies] @@ -2106,13 +2231,13 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "wcwidth" -version = "0.2.9" +version = "0.2.10" description = "Measures the displayed width of unicode strings in a terminal" optional = false python-versions = "*" files = [ - {file = "wcwidth-0.2.9-py2.py3-none-any.whl", hash = "sha256:9a929bd8380f6cd9571a968a9c8f4353ca58d7cd812a4822bba831f8d685b223"}, - {file = "wcwidth-0.2.9.tar.gz", hash = "sha256:a675d1a4a2d24ef67096a04b85b02deeecd8e226f57b5e3a72dbb9ed99d27da8"}, + {file = "wcwidth-0.2.10-py2.py3-none-any.whl", hash = "sha256:aec5179002dd0f0d40c456026e74a729661c9d468e1ed64405e3a6c2176ca36f"}, + {file = "wcwidth-0.2.10.tar.gz", hash = "sha256:390c7454101092a6a5e43baad8f83de615463af459201709556b6e4b1c861f97"}, ] [[package]] @@ -2300,4 +2425,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "~3.10" -content-hash = "9e99c86b95e7aefd35f67c0da45d99201a5f3043fd63d66b33ec05511f9dc322" +content-hash = "f17190cee40ebac7d333e6f82e60c5fff15189233972e5f90a31773c41544b2b" diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index afb023d33e14..d98c22f8be33 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "2.6.0" +version = "2.7.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] @@ -35,7 +35,12 @@ pytest-mock = "^3.10.0" [tool.poetry.group.dev.dependencies] freezegun = "^1.2.2" pytest-cov = "^4.1.0" +pyinstaller = "^6.1.0" +poethepoet = "^0.24.2" [tool.poetry.scripts] -airbyte-ci-internal = "pipelines.cli.airbyte_ci:airbyte_ci" -airbyte-ci = "pipelines.cli.dagger_run:main" +airbyte-ci = "pipelines.cli.airbyte_ci:airbyte_ci" + +[tool.poe.tasks.build-release-binary] +shell = "pyinstaller --collect-all pipelines --collect-all beartype --collect-all dagger --hidden-import strawberry --name $ARTIFACT_NAME --onefile pipelines/cli/airbyte_ci.py" +args = [{name = "ARTIFACT_NAME", default="airbyte-ci", positional = true}] From 3867caedc907a67433e47905a5da57e28b98a14d Mon Sep 17 00:00:00 2001 From: Augustin Date: Tue, 14 Nov 2023 10:09:57 +0100 Subject: [PATCH 029/124] airbyte-ci: improve release workflow (#32488) --- ...te-ci-release-experiment.yml => airbyte-ci-release.yml} | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) rename .github/workflows/{airbyte-ci-release-experiment.yml => airbyte-ci-release.yml} (95%) diff --git a/.github/workflows/airbyte-ci-release-experiment.yml b/.github/workflows/airbyte-ci-release.yml similarity index 95% rename from .github/workflows/airbyte-ci-release-experiment.yml rename to .github/workflows/airbyte-ci-release.yml index b2a50045340e..a70d4dd70078 100644 --- a/.github/workflows/airbyte-ci-release-experiment.yml +++ b/.github/workflows/airbyte-ci-release.yml @@ -1,4 +1,3 @@ -# TODO remove -experiment from the workflow file name after merging name: Connector Ops CI - Airbyte CI Release concurrency: @@ -92,6 +91,12 @@ jobs: run: | echo "::set-output name=version::$(poetry version --short)" + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v1 + with: + credentials_json: "${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}" + - name: Upload version release to GCS prod bucket id: upload_version_release_to_gcs if: github.ref == 'refs/heads/master' From bf41a37f2a8ca04a6e96431589c7f91965c7fef0 Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk <35109939+tolik0@users.noreply.github.com> Date: Tue, 14 Nov 2023 11:15:16 +0200 Subject: [PATCH 030/124] =?UTF-8?q?=F0=9F=90=9B=20Source=20Instagram:=20In?= =?UTF-8?q?clude=20media=5Fproduct=5Ftype=20in=20Media=20and=20Stories=20S?= =?UTF-8?q?chemas=20(#32423)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Dhroov Makwana --- .../integration_tests/expected_records.jsonl | 14 +++--- .../connectors/source-instagram/metadata.yaml | 2 +- .../source_instagram/schemas/media.json | 3 ++ .../source_instagram/schemas/stories.json | 3 ++ .../source_instagram/streams.py | 2 +- docs/integrations/sources/instagram.md | 45 ++++++++++--------- 6 files changed, 38 insertions(+), 31 deletions(-) diff --git a/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl index 78eff4518f29..f3dc892818c0 100644 --- a/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl @@ -1,7 +1,7 @@ -{"stream": "users", "data": {"id": "17841408147298757", "website": "https://www.airbyte.io/", "biography": "Airbyte is the new open-source data integration platform that consolidates your data into your warehouses.", "profile_picture_url": "https://scontent-iev1-1.xx.fbcdn.net/v/t51.2885-15/153169696_890787328349641_8382928081987798464_n.jpg?_nc_cat=111&_nc_sid=7d201b&_nc_ohc=_R1t6pmIljcAX_4SCqZ&_nc_ht=scontent-iev1-1.xx&edm=AL-3X8kEAAAA&oh=00_AfBjHz1cywYrZ7Epl5qfAv74F0nHFgsNQ3h6yJqD9A6_5w&oe=6550EE3E", "media_count": 258, "name": "Jean Lafleur", "username": "airbytehq", "followers_count": 1253, "follows_count": 14, "ig_id": 8070063576, "page_id": "144706962067225"}, "emitted_at": 1699532816445} -{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_city", "date": "2023-11-09T08:00:00+0000", "value": {"London, England": 7, "Sydney, New South Wales": 19, "Atlanta, Georgia": 5, "Algiers, Algiers Province": 5, "S\u00e3o Paulo, S\u00e3o Paulo (state)": 13, "Rio de Janeiro, Rio de Janeiro (state)": 5, "Hong Kong, Hong Kong": 4, "Montreal, Quebec": 4, "Berlin, Berlin": 7, "Kolkata, West Bengal": 4, "Tulsa, Oklahoma": 4, "Lagos, Lagos State": 18, "Ahmedabad, Gujarat": 4, "Skopje, Municipality of Centar (Skopje)": 4, "Moscow, Moscow": 5, "Bogot\u00e1, Distrito Especial": 5, "Karachi, Sindh": 5, "Dar es Salaam, Dar es Salaam": 7, "Jakarta, Jakarta": 10, "Accra, Greater Accra Region": 4, "Buenos Aires, Ciudad Aut\u00f3noma de Buenos Aires": 9, "Melbourne, Victoria": 7, "Delhi, Delhi": 5, "Gurugram, Haryana": 6, "Kuala Lumpur, Kuala Lumpur": 4, "Los Angeles, California": 5, "Lima, Lima Region": 5, "Istanbul, Istanbul Province": 9, "Toronto, Ontario": 4, "Chennai, Tamil Nadu": 6, "Abuja, Federal Capital Territory": 6, "Bangkok, Bangkok": 5, "Mexico City, Distrito Federal": 6, "Cape Town, Western Cape": 5, "San Francisco, California": 6, "Tehran, Tehran Province": 4, "New York, New York": 13, "Cairo, Cairo Governorate": 4, "Santiago, Santiago Metropolitan Region": 6, "Dubai, Dubai": 8, "Mumbai, Maharashtra": 9, "Bangalore, Karnataka": 18, "Singapore, Singapore": 6, "Hyderabad, Telangana": 7, "San Diego, California": 6}}, "emitted_at": 1699532816665} -{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_country", "date": "2023-11-09T08:00:00+0000", "value": {"DE": 31, "BD": 4, "TW": 5, "RU": 9, "PT": 4, "TZ": 8, "FR": 9, "SA": 7, "BR": 63, "SE": 6, "MA": 6, "SG": 6, "DZ": 6, "ID": 29, "GB": 46, "CA": 26, "US": 263, "GH": 4, "EG": 10, "AE": 9, "CH": 7, "IN": 125, "ZA": 16, "IQ": 6, "CL": 9, "IR": 12, "GR": 6, "IT": 19, "MX": 23, "MY": 9, "CO": 11, "ES": 12, "VE": 9, "AR": 23, "TH": 7, "AU": 35, "PE": 5, "PH": 7, "NG": 30, "TN": 6, "PK": 10, "PL": 5, "TR": 10, "NL": 13, "LK": 4}}, "emitted_at": 1699532816666} -{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_gender_age", "date": "2023-11-09T08:00:00+0000", "value": {"F.18-24": 11, "F.25-34": 76, "F.35-44": 71, "F.45-54": 17, "F.55-64": 1, "F.65+": 2, "M.13-17": 2, "M.18-24": 49, "M.25-34": 366, "M.35-44": 226, "M.45-54": 83, "M.55-64": 20, "M.65+": 12, "U.18-24": 18, "U.25-34": 67, "U.35-44": 43, "U.45-54": 19, "U.55-64": 5}}, "emitted_at": 1699532816666} -{"stream": "media", "data": {"id": "17884386203808767", "comments_count": 2, "owner": {"id": "17841408147298757"}, "caption": "Terraform Explained Part 1\n.\n.\n.\n#airbyte #dataengineering #tech #terraform #cloud #cloudengineer #coding #reels", "ig_id": "3123724930722523505", "media_type": "VIDEO", "is_comment_enabled": true, "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/B34BFFBB0614049AD69F066D153FDD8C_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=107&vs=986202625710684_1200838240&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC9CMzRCRkZCQjA2MTQwNDlBRDY5RjA2NkQxNTNGREQ4Q192aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dDQm9HQlV3a2JxUWwtY0JBRnZGTnFBUkdQeHpicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJtDf4euHnbtAFQIoAkMzLBdAUBtDlYEGJRgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfDob4qgnW95lOhpSj4VrMWJE0DgJkqaAYcbkQPd3KrQJA&oe=654EAA9A&_nc_sid=1d576d", "permalink": "https://www.instagram.com/reel/CtZs0Y3v2lx/", "timestamp": "2023-06-12T19:20:02+0000", "like_count": 9, "username": "airbytehq", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/353022694_609901831117241_2447211336606431614_n.jpg?_nc_cat=100&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=8xgXGzm1iRIAX-Q2FuK&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfCc0YkR683nBbUGvqI89zTzwegJXwxKVIWCeBb0O4Zjkg&oe=65529A0E", "shortcode": "CtZs0Y3v2lx", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1699532772760} -{"stream": "media", "data": {"id": "17864256500936159", "comments_count": 0, "owner": {"id": "17841408147298757"}, "caption": "When and why you should be using Rust for Data Engineering! \n\n#rust #airbyte #coding #programming #tech #dataengineering #data", "ig_id": "3106359072491902976", "media_type": "VIDEO", "is_comment_enabled": true, "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/BE4F848CC97FBA35A1AE1B1150B989A7_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=110&vs=6290041361087047_1877877688&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC9CRTRGODQ4Q0M5N0ZCQTM1QTFBRTFCMTE1MEI5ODlBN192aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dEaE94aFJJdk1BWGZaWURBQXQyS0FLWWxOSlhicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJrD%2B6LaRwf1AFQIoAkMzLBdARDmZmZmZmhgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfDynJmluL-Qe4y6WoQEQnoAMIDsrp-Gwb8793zMm-K1hA&oe=654EBF26&_nc_sid=1d576d", "permalink": "https://www.instagram.com/reel/CscAR5EsRgA/", "timestamp": "2023-05-19T20:08:33+0000", "like_count": 6, "username": "airbytehq", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/347441626_604256678433845_716271787932876577_n.jpg?_nc_cat=108&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=v0GfB4fQB2wAX_epYcL&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfDXZqVwGSKkoHRF4VI4fVDWGOhSJ2V4d6yHnTX4w3ultg&oe=65515ED9", "shortcode": "CscAR5EsRgA", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1699532772760} -{"stream": "media", "data": {"id": "17964324206288599", "comments_count": 0, "owner": {"id": "17841408147298757"}, "caption": "We've all been there right? \ud83e\udd23\n\n#airbyte #data #dataengineering #datascience #dataanalytics #tech #softwareengineer", "ig_id": "3104241732634871967", "media_type": "VIDEO", "is_comment_enabled": true, "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/274503D36EA0F6E79A7CF3797A8D5985_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNTc2LmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=106&vs=1336282350269744_3931649106&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC8yNzQ1MDNEMzZFQTBGNkU3OUE3Q0YzNzk3QThENTk4NV92aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dQdzNzaFRId3VlSlBFWURBSDFmTjUzcUNhd0JicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJrDwmtqO44lAFQIoAkMzLBdAIewIMSbpeRgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfAEnsrWGqakgnSxAhFmKsqTAukx-v52gXNECxPa7OtZJQ&oe=654E4FB2&_nc_sid=1d576d", "permalink": "https://www.instagram.com/reel/CsUe2iqpQif/", "timestamp": "2023-05-16T22:01:45+0000", "like_count": 13, "username": "airbytehq", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/347429218_1848940842145573_5975413208994727174_n.jpg?_nc_cat=101&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=AGiJHqHavZIAX8DSQPd&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfBNvwjxSb-6D_Vl9QyngS1zEkIASrT2WeQ62gqNfebnCw&oe=65514B7C", "shortcode": "CsUe2iqpQif", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1699532772761} +{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_city", "date": "2023-11-13T08:00:00+0000", "value": {"London, England": 7, "Sydney, New South Wales": 19, "Algiers, Algiers Province": 4, "Caracas, Capital District": 4, "Atlanta, Georgia": 5, "S\u00e3o Paulo, S\u00e3o Paulo (state)": 14, "Munich, Bayern": 3, "Rio de Janeiro, Rio de Janeiro (state)": 5, "Hong Kong, Hong Kong": 4, "Berlin, Berlin": 7, "Kolkata, West Bengal": 4, "Tulsa, Oklahoma": 4, "Lagos, Lagos State": 18, "Ahmedabad, Gujarat": 4, "Moscow, Moscow": 5, "Karachi, Sindh": 4, "Bogot\u00e1, Distrito Especial": 5, "Dar es Salaam, Dar es Salaam": 7, "Jakarta, Jakarta": 10, "Accra, Greater Accra Region": 4, "Buenos Aires, Ciudad Aut\u00f3noma de Buenos Aires": 9, "Taipei, Taipei": 3, "Melbourne, Victoria": 7, "Delhi, Delhi": 6, "Gurugram, Haryana": 6, "Kuala Lumpur, Kuala Lumpur": 4, "Los Angeles, California": 5, "Lima, Lima Region": 5, "Istanbul, Istanbul Province": 9, "Chennai, Tamil Nadu": 6, "Abuja, Federal Capital Territory": 6, "Bangkok, Bangkok": 5, "Mexico City, Distrito Federal": 6, "Cape Town, Western Cape": 4, "San Francisco, California": 6, "Tehran, Tehran Province": 4, "New York, New York": 14, "Cairo, Cairo Governorate": 4, "Santiago, Santiago Metropolitan Region": 6, "Dubai, Dubai": 8, "Mumbai, Maharashtra": 8, "Bangalore, Karnataka": 18, "Singapore, Singapore": 6, "Hyderabad, Telangana": 7, "San Diego, California": 6}}, "emitted_at": 1699881329196} +{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_country", "date": "2023-11-13T08:00:00+0000", "value": {"DE": 31, "BD": 4, "HK": 4, "FI": 5, "TW": 5, "RU": 9, "TZ": 8, "UA": 4, "FR": 10, "SA": 8, "BR": 64, "SE": 6, "MA": 6, "SG": 6, "DZ": 6, "ID": 29, "GB": 45, "CA": 24, "US": 264, "EG": 10, "AE": 9, "CH": 7, "IN": 125, "ZA": 16, "IQ": 6, "CL": 9, "IR": 12, "GR": 6, "IT": 19, "MX": 23, "MY": 9, "CO": 11, "ES": 12, "VE": 9, "AR": 23, "TH": 7, "AU": 35, "PE": 5, "PH": 7, "NG": 30, "TN": 6, "PK": 10, "PL": 5, "TR": 10, "NL": 13}}, "emitted_at": 1699881329198} +{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_gender_age", "date": "2023-11-13T08:00:00+0000", "value": {"F.18-24": 11, "F.25-34": 75, "F.35-44": 72, "F.45-54": 17, "F.55-64": 1, "F.65+": 2, "M.13-17": 2, "M.18-24": 49, "M.25-34": 367, "M.35-44": 226, "M.45-54": 83, "M.55-64": 20, "M.65+": 12, "U.18-24": 18, "U.25-34": 66, "U.35-44": 43, "U.45-54": 19, "U.55-64": 5}}, "emitted_at": 1699881329198} +{"stream": "media", "data": {"id": "17884386203808767", "ig_id": "3123724930722523505", "is_comment_enabled": true, "owner": {"id": "17841408147298757"}, "username": "airbytehq", "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/B34BFFBB0614049AD69F066D153FDD8C_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=107&vs=986202625710684_1200838240&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC9CMzRCRkZCQjA2MTQwNDlBRDY5RjA2NkQxNTNGREQ4Q192aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dDQm9HQlV3a2JxUWwtY0JBRnZGTnFBUkdQeHpicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJtDf4euHnbtAFQIoAkMzLBdAUBtDlYEGJRgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfD22CPBcEjf5k7O8xXQHo8O0VkV7ScqVYH4tiXi1gP0cg&oe=6553F09A&_nc_sid=1d576d", "like_count": 9, "permalink": "https://www.instagram.com/reel/CtZs0Y3v2lx/", "media_product_type": "REELS", "timestamp": "2023-06-12T19:20:02+0000", "media_type": "VIDEO", "shortcode": "CtZs0Y3v2lx", "comments_count": 2, "caption": "Terraform Explained Part 1\n.\n.\n.\n#airbyte #dataengineering #tech #terraform #cloud #cloudengineer #coding #reels", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/353022694_609901831117241_2447211336606431614_n.jpg?_nc_cat=100&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=m968I3SdqJAAX9vop_P&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfBhRikvxkwg1fvXBxym8F6XHMvIYpsdij9PgRelILL2Cw&oe=65568E8E", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1699881269892} +{"stream": "media", "data": {"id": "17864256500936159", "ig_id": "3106359072491902976", "is_comment_enabled": true, "owner": {"id": "17841408147298757"}, "username": "airbytehq", "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/BE4F848CC97FBA35A1AE1B1150B989A7_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=110&vs=6290041361087047_1877877688&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC9CRTRGODQ4Q0M5N0ZCQTM1QTFBRTFCMTE1MEI5ODlBN192aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dEaE94aFJJdk1BWGZaWURBQXQyS0FLWWxOSlhicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJrD%2B6LaRwf1AFQIoAkMzLBdARDmZmZmZmhgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfCIURiEF3hTsWaXvIE6ft-jP1WxcZzXQU8cZygCLJFBxQ&oe=65540526&_nc_sid=1d576d", "like_count": 7, "permalink": "https://www.instagram.com/reel/CscAR5EsRgA/", "media_product_type": "REELS", "timestamp": "2023-05-19T20:08:33+0000", "media_type": "VIDEO", "shortcode": "CscAR5EsRgA", "comments_count": 0, "caption": "When and why you should be using Rust for Data Engineering! \n\n#rust #airbyte #coding #programming #tech #dataengineering #data", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/347441626_604256678433845_716271787932876577_n.jpg?_nc_cat=108&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=2ACJfSHiIRkAX8S0ZFU&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfDsFKul5lyAG2EzGOY-RCmm_QmmPhCkDzf4nfp2ylG2dA&oe=65574D99", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1699881269893} +{"stream": "media", "data": {"id": "17964324206288599", "ig_id": "3104241732634871967", "is_comment_enabled": true, "owner": {"id": "17841408147298757"}, "username": "airbytehq", "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/274503D36EA0F6E79A7CF3797A8D5985_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNTc2LmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=106&vs=1336282350269744_3931649106&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC8yNzQ1MDNEMzZFQTBGNkU3OUE3Q0YzNzk3QThENTk4NV92aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dQdzNzaFRId3VlSlBFWURBSDFmTjUzcUNhd0JicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJrDwmtqO44lAFQIoAkMzLBdAIewIMSbpeRgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfAdvwyUYd2Op-U2iwVZkaxB8PtLApdBwMYkp3eP7qdeeQ&oe=655395B2&_nc_sid=1d576d", "like_count": 13, "permalink": "https://www.instagram.com/reel/CsUe2iqpQif/", "media_product_type": "REELS", "timestamp": "2023-05-16T22:01:45+0000", "media_type": "VIDEO", "shortcode": "CsUe2iqpQif", "comments_count": 0, "caption": "We've all been there right? \ud83e\udd23\n\n#airbyte #data #dataengineering #datascience #dataanalytics #tech #softwareengineer", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/347429218_1848940842145573_5975413208994727174_n.jpg?_nc_cat=101&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=3He_36rMQuYAX9Pz0NM&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfAHCtC-iKyfdquGHnCLMwz_yPD5yNroFnaY8qs0etCc6A&oe=65573A3C", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1699881269893} +{"stream": "users", "data": {"id": "17841408147298757", "ig_id": 8070063576, "profile_picture_url": "https://scontent-iev1-1.xx.fbcdn.net/v/t51.2885-15/153169696_890787328349641_8382928081987798464_n.jpg?_nc_cat=111&_nc_sid=7d201b&_nc_ohc=uQq3P1OLNOYAX_JjmNN&_nc_ht=scontent-iev1-1.xx&edm=AL-3X8kEAAAA&oh=00_AfBcZD3D9MiBkz50iVXih1HCe1L52Wz3t1TKVMJv7UEDWg&oe=6556DCFE", "username": "airbytehq", "name": "Jean Lafleur", "biography": "Airbyte is the new open-source data integration platform that consolidates your data into your warehouses.", "website": "https://www.airbyte.io/", "media_count": 258, "followers_count": 1253, "follows_count": 14, "page_id": "144706962067225"}, "emitted_at": 1699881328891} diff --git a/airbyte-integrations/connectors/source-instagram/metadata.yaml b/airbyte-integrations/connectors/source-instagram/metadata.yaml index c53b474362b2..c6a4e09f6df5 100644 --- a/airbyte-integrations/connectors/source-instagram/metadata.yaml +++ b/airbyte-integrations/connectors/source-instagram/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6acf6b55-4f1e-4fca-944e-1a3caef8aba8 - dockerImageTag: 1.0.13 + dockerImageTag: 1.0.14 dockerRepository: airbyte/source-instagram githubIssueLabel: source-instagram icon: instagram.svg diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media.json b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media.json index 02eea4f43f92..4185de5f66cb 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media.json +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media.json @@ -28,6 +28,9 @@ "media_type": { "type": ["null", "string"] }, + "media_product_type": { + "type": ["null", "string"] + }, "media_url": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/stories.json b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/stories.json index 47e4af00aeb9..7fc7fa7b40a3 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/stories.json +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/stories.json @@ -22,6 +22,9 @@ "media_type": { "type": ["null", "string"] }, + "media_product_type": { + "type": ["null", "string"] + }, "media_url": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py index 3d81c18e9fbd..e83c64de6fdd 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py @@ -301,7 +301,7 @@ class Media(InstagramStream): so they are excluded when trying to get child objects to avoid the error """ - INVALID_CHILDREN_FIELDS = ["caption", "comments_count", "is_comment_enabled", "like_count", "children"] + INVALID_CHILDREN_FIELDS = ["caption", "comments_count", "is_comment_enabled", "like_count", "children", "media_product_type"] def read_records( self, diff --git a/docs/integrations/sources/instagram.md b/docs/integrations/sources/instagram.md index 82f96d6fbb3b..1270d59f4cc0 100644 --- a/docs/integrations/sources/instagram.md +++ b/docs/integrations/sources/instagram.md @@ -85,25 +85,26 @@ AirbyteRecords are required to conform to the [Airbyte type](https://docs.airbyt ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------| :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------ | -| 1.0.13 | 2023-11-10 | [32245](https://github.com/airbytehq/airbyte/pull/32245) | Add skipping reading MediaInsights stream if an error code 10 is received | -| 1.0.12 | 2023-11-07 | [32200](https://github.com/airbytehq/airbyte/pull/32200) | The backoff strategy has been updated to make some errors retriable | -| 1.0.11 | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes | -| 1.0.10 | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references | -| 1.0.9 | 2023-07-01 | [27908](https://github.com/airbytehq/airbyte/pull/27908) | Fix bug when `user_lifetime_insights` stream returns `Key Error (end_time)`, refactored `state` to use `IncrementalMixin` | -| 1.0.8 | 2023-05-26 | [26767](https://github.com/airbytehq/airbyte/pull/26767) | Handle permission error for `insights` | -| 1.0.7 | 2023-05-26 | [26656](https://github.com/airbytehq/airbyte/pull/26656) | Remove `authSpecification` from connector specification in favour of `advancedAuth` | -| 1.0.6 | 2023-03-28 | [26599](https://github.com/airbytehq/airbyte/pull/26599) | Handle error for Media posted before business account conversion | -| 1.0.5 | 2023-03-28 | [24634](https://github.com/airbytehq/airbyte/pull/24634) | Add user-friendly message for no instagram_business_accounts case | -| 1.0.4 | 2023-03-15 | [23671](https://github.com/airbytehq/airbyte/pull/23671) | Add info about main permissions in spec and doc links in error message to navigate user | -| 1.0.3 | 2023-03-14 | [24043](https://github.com/airbytehq/airbyte/pull/24043) | Do not emit incomplete records for `user_insights` stream | -| 1.0.2 | 2023-03-14 | [24042](https://github.com/airbytehq/airbyte/pull/24042) | Test publish flow | -| 1.0.1 | 2023-01-19 | [21602](https://github.com/airbytehq/airbyte/pull/21602) | Handle abnormally large state values | -| 1.0.0 | 2022-09-23 | [17110](https://github.com/airbytehq/airbyte/pull/17110) | Remove custom read function and migrate to per-stream state | -| 0.1.11 | 2022-09-08 | [16428](https://github.com/airbytehq/airbyte/pull/16428) | Fix requests metrics for Reels media product type | -| 0.1.10 | 2022-09-05 | [16340](https://github.com/airbytehq/airbyte/pull/16340) | Update to latest version of the CDK (v0.1.81) | -| 0.1.9 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification | -| 0.1.8 | 2021-08-11 | [5354](https://github.com/airbytehq/airbyte/pull/5354) | Added check for empty state and fixed tests | -| 0.1.7 | 2021-07-19 | [4805](https://github.com/airbytehq/airbyte/pull/4805) | Add support for previous `STATE` format | -| 0.1.6 | 2021-07-07 | [4210](https://github.com/airbytehq/airbyte/pull/4210) | Refactor connector to use CDK: - improve error handling - fix sync fail with HTTP status 400 - integrate SAT | +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :------------------------------------------------------- |:-------------------------------------------------------------------------------------------------------------------------------| +| 1.0.14 | 2023-11-13 | [32423](https://github.com/airbytehq/airbyte/pull/32423) | Capture media_product_type column in media and stories stream | +| 1.0.13 | 2023-11-10 | [32245](https://github.com/airbytehq/airbyte/pull/32245) | Add skipping reading MediaInsights stream if an error code 10 is received | +| 1.0.12 | 2023-11-07 | [32200](https://github.com/airbytehq/airbyte/pull/32200) | The backoff strategy has been updated to make some errors retriable | +| 1.0.11 | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes | +| 1.0.10 | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references | +| 1.0.9 | 2023-07-01 | [27908](https://github.com/airbytehq/airbyte/pull/27908) | Fix bug when `user_lifetime_insights` stream returns `Key Error (end_time)`, refactored `state` to use `IncrementalMixin` | +| 1.0.8 | 2023-05-26 | [26767](https://github.com/airbytehq/airbyte/pull/26767) | Handle permission error for `insights` | +| 1.0.7 | 2023-05-26 | [26656](https://github.com/airbytehq/airbyte/pull/26656) | Remove `authSpecification` from connector specification in favour of `advancedAuth` | +| 1.0.6 | 2023-03-28 | [26599](https://github.com/airbytehq/airbyte/pull/26599) | Handle error for Media posted before business account conversion | +| 1.0.5 | 2023-03-28 | [24634](https://github.com/airbytehq/airbyte/pull/24634) | Add user-friendly message for no instagram_business_accounts case | +| 1.0.4 | 2023-03-15 | [23671](https://github.com/airbytehq/airbyte/pull/23671) | Add info about main permissions in spec and doc links in error message to navigate user | +| 1.0.3 | 2023-03-14 | [24043](https://github.com/airbytehq/airbyte/pull/24043) | Do not emit incomplete records for `user_insights` stream | +| 1.0.2 | 2023-03-14 | [24042](https://github.com/airbytehq/airbyte/pull/24042) | Test publish flow | +| 1.0.1 | 2023-01-19 | [21602](https://github.com/airbytehq/airbyte/pull/21602) | Handle abnormally large state values | +| 1.0.0 | 2022-09-23 | [17110](https://github.com/airbytehq/airbyte/pull/17110) | Remove custom read function and migrate to per-stream state | +| 0.1.11 | 2022-09-08 | [16428](https://github.com/airbytehq/airbyte/pull/16428) | Fix requests metrics for Reels media product type | +| 0.1.10 | 2022-09-05 | [16340](https://github.com/airbytehq/airbyte/pull/16340) | Update to latest version of the CDK (v0.1.81) | +| 0.1.9 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification | +| 0.1.8 | 2021-08-11 | [5354](https://github.com/airbytehq/airbyte/pull/5354) | Added check for empty state and fixed tests | +| 0.1.7 | 2021-07-19 | [4805](https://github.com/airbytehq/airbyte/pull/4805) | Add support for previous `STATE` format | +| 0.1.6 | 2021-07-07 | [4210](https://github.com/airbytehq/airbyte/pull/4210) | Refactor connector to use CDK: - improve error handling - fix sync fail with HTTP status 400 - integrate SAT | From edd37805ae3e30268f5b24b776ec61a3998548aa Mon Sep 17 00:00:00 2001 From: Augustin Date: Tue, 14 Nov 2023 10:26:48 +0100 Subject: [PATCH 031/124] airbyte-ci release: fix auth in workflow (#32489) --- .github/workflows/airbyte-ci-release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/airbyte-ci-release.yml b/.github/workflows/airbyte-ci-release.yml index a70d4dd70078..1d42e96e7d3b 100644 --- a/.github/workflows/airbyte-ci-release.yml +++ b/.github/workflows/airbyte-ci-release.yml @@ -61,8 +61,8 @@ jobs: name: airbyte-ci-${{ matrix.os }}-${{ steps.get_short_sha.outputs.sha }} path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }} - - name: Authenticate to Google Cloud - id: auth + - name: Authenticate to Google Cloud Dev + id: auth_dev uses: google-github-actions/auth@v1 with: credentials_json: "${{ secrets.METADATA_SERVICE_DEV_GCS_CREDENTIALS }}" @@ -91,8 +91,8 @@ jobs: run: | echo "::set-output name=version::$(poetry version --short)" - - name: Authenticate to Google Cloud - id: auth + - name: Authenticate to Google Cloud Prod + id: auth_prod uses: google-github-actions/auth@v1 with: credentials_json: "${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}" From f8b0b3e99e7b741a0c69d2a24e1c81631b5ef0a6 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 14 Nov 2023 11:49:19 +0100 Subject: [PATCH 032/124] File CDK: Improve stream config appearance (#32420) --- .../sources/file_based/config/file_based_stream_config.py | 2 ++ .../unit_tests/sources/file_based/scenarios/csv_scenarios.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/file_based_stream_config.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/file_based_stream_config.py index dedd2df27f33..92b808926b09 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/file_based_stream_config.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/file_based_stream_config.py @@ -26,8 +26,10 @@ class ValidationPolicy(Enum): class FileBasedStreamConfig(BaseModel): name: str = Field(title="Name", description="The name of the stream.") globs: Optional[List[str]] = Field( + default=["**"], title="Globs", description='The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look here.', + order=1, ) legacy_prefix: Optional[str] = Field( title="Legacy Prefix", diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py index d3445062d027..2ca4be8d103a 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py @@ -76,6 +76,8 @@ "description": 'The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look here.', "type": "array", "items": {"type": "string"}, + "order": 1, + "default": ["**"], }, "legacy_prefix": { "title": "Legacy Prefix", From 2c3b093c81e3a3508dddcce6a18db111a246a7e6 Mon Sep 17 00:00:00 2001 From: flash1293 Date: Tue, 14 Nov 2023 10:55:49 +0000 Subject: [PATCH 033/124] =?UTF-8?q?=F0=9F=A4=96=20Bump=20patch=20version?= =?UTF-8?q?=20of=20Python=20CDK?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-cdk/python/.bumpversion.cfg | 2 +- airbyte-cdk/python/CHANGELOG.md | 3 +++ airbyte-cdk/python/Dockerfile | 4 ++-- airbyte-cdk/python/setup.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/airbyte-cdk/python/.bumpversion.cfg b/airbyte-cdk/python/.bumpversion.cfg index 0f180530524d..186cb9760f84 100644 --- a/airbyte-cdk/python/.bumpversion.cfg +++ b/airbyte-cdk/python/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.53.4 +current_version = 0.53.5 commit = False [bumpversion:file:setup.py] diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 0c6625c0d7f9..4c76cd7e510b 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.53.5 +File CDK: Improve stream config appearance + ## 0.53.4 Concurrent CDK: fix futures pruning diff --git a/airbyte-cdk/python/Dockerfile b/airbyte-cdk/python/Dockerfile index eac8ca127452..a966090089a5 100644 --- a/airbyte-cdk/python/Dockerfile +++ b/airbyte-cdk/python/Dockerfile @@ -10,7 +10,7 @@ RUN apk --no-cache upgrade \ && apk --no-cache add tzdata build-base # install airbyte-cdk -RUN pip install --prefix=/install airbyte-cdk==0.53.4 +RUN pip install --prefix=/install airbyte-cdk==0.53.5 # build a clean environment FROM base @@ -32,5 +32,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] # needs to be the same as CDK -LABEL io.airbyte.version=0.53.4 +LABEL io.airbyte.version=0.53.5 LABEL io.airbyte.name=airbyte/source-declarative-manifest diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index b90af9c4862c..538550c201af 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -36,7 +36,7 @@ name="airbyte-cdk", # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be # updated if our semver format changes such as using release candidate versions. - version="0.53.4", + version="0.53.5", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", From f6bc4b8f1a61bbc34519afa6495e7eecef0f865b Mon Sep 17 00:00:00 2001 From: Augustin Date: Tue, 14 Nov 2023 12:28:26 +0100 Subject: [PATCH 034/124] source-s3: remove useless test deps (#32495) --- airbyte-integrations/connectors/source-s3/setup.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-s3/setup.py b/airbyte-integrations/connectors/source-s3/setup.py index 5cf8dd246356..d23a87b38b81 100644 --- a/airbyte-integrations/connectors/source-s3/setup.py +++ b/airbyte-integrations/connectors/source-s3/setup.py @@ -19,9 +19,6 @@ "pytest-mock~=3.6.1", "pytest~=6.1", "pandas==2.0.3", - "psutil", - "pytest-order", - "netifaces~=0.11.0", "docker", ] From be6aebc483b2a5c5de372805de64ed66e17a1039 Mon Sep 17 00:00:00 2001 From: Artem Inzhyyants <36314070+artem1205@users.noreply.github.com> Date: Tue, 14 Nov 2023 16:39:52 +0100 Subject: [PATCH 035/124] =?UTF-8?q?=F0=9F=90=9B=20Source=20Bing=20Ads:=20n?= =?UTF-8?q?ormalize=20numeric=20values=20(#32422)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../acceptance-test-config.yml | 29 ++++- .../integration_tests/configured_catalog.json | 10 ++ .../configured_catalog_full_refresh.json | 120 ------------------ .../integration_tests/expected_records.jsonl | 71 +++++------ .../connectors/source-bing-ads/metadata.yaml | 2 +- .../source_bing_ads/streams.py | 6 +- .../unit_tests/test_reports.py | 15 ++- docs/integrations/sources/bing-ads.md | 1 + 8 files changed, 83 insertions(+), 171 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml index cf9c2c522a94..480c522ac5a8 100644 --- a/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml @@ -50,12 +50,37 @@ acceptance_tests: bypass_reason: "Can not populate; new campaign with link to app needed; feature is not available yet" - name: app_install_ad_labels bypass_reason: "Can not populate; depends on stream app_install_ads" - timeout_seconds: 4800 + #### TODO: remove *_report_monthly after all become populated on December, 1 + - name: account_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: ad_group_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: ad_group_impression_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: ad_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: campaign_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: campaign_impression_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: keyword_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: geographic_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: age_gender_audience_report_monthly + bypass_reason: "Campaign is still in progress" + - name: search_query_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: user_location_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: account_impression_performance_report_monthly + bypass_reason: "Campaign is still in progress" + timeout_seconds: 7200 full_refresh: tests: - config_path: secrets/config.json configured_catalog_path: integration_tests/configured_catalog_full_refresh.json - timeout_seconds: 4800 + timeout_seconds: 7200 incremental: tests: bypass_reason: "SAT doesn't support complex nested states used in incremental report streams" diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog.json index 5c58592ddbbc..cdf122f10f7b 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog.json @@ -156,6 +156,16 @@ "cursor_field": ["TimePeriod"], "destination_sync_mode": "append" }, + { + "stream": { + "name": "geographic_performance_report_daily", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, { "stream": { "name": "geographic_performance_report_weekly", diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog_full_refresh.json b/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog_full_refresh.json index 9eb62db8fddc..a5a62dab8de5 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog_full_refresh.json +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog_full_refresh.json @@ -66,46 +66,6 @@ "cursor_field": ["TimePeriod"], "destination_sync_mode": "append" }, - { - "stream": { - "name": "account_performance_report_monthly", - "json_schema": {}, - "supported_sync_modes": ["incremental", "full_refresh"] - }, - "sync_mode": "incremental", - "cursor_field": ["TimePeriod"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "ad_group_performance_report_monthly", - "json_schema": {}, - "supported_sync_modes": ["incremental", "full_refresh"] - }, - "sync_mode": "incremental", - "cursor_field": ["TimePeriod"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "ad_performance_report_monthly", - "json_schema": {}, - "supported_sync_modes": ["incremental", "full_refresh"] - }, - "sync_mode": "incremental", - "cursor_field": ["TimePeriod"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "geographic_performance_report_monthly", - "json_schema": {}, - "supported_sync_modes": ["incremental", "full_refresh"] - }, - "sync_mode": "incremental", - "cursor_field": ["TimePeriod"], - "destination_sync_mode": "append" - }, { "stream": { "name": "budget_summary_report", @@ -116,76 +76,6 @@ "cursor_field": ["TimePeriod"], "destination_sync_mode": "append" }, - { - "stream": { - "name": "campaign_performance_report_monthly", - "json_schema": {}, - "supported_sync_modes": ["incremental", "full_refresh"] - }, - "sync_mode": "incremental", - "cursor_field": ["TimePeriod"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "campaign_impression_performance_report_monthly", - "json_schema": {}, - "supported_sync_modes": ["incremental", "full_refresh"] - }, - "sync_mode": "incremental", - "cursor_field": ["TimePeriod"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "keyword_performance_report_monthly", - "json_schema": {}, - "supported_sync_modes": ["incremental", "full_refresh"] - }, - "sync_mode": "incremental", - "cursor_field": ["TimePeriod"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "age_gender_audience_report_monthly", - "json_schema": {}, - "supported_sync_modes": ["incremental", "full_refresh"] - }, - "sync_mode": "incremental", - "cursor_field": ["TimePeriod"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "search_query_performance_report_monthly", - "json_schema": {}, - "supported_sync_modes": ["incremental", "full_refresh"] - }, - "sync_mode": "incremental", - "cursor_field": ["TimePeriod"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "user_location_performance_report_monthly", - "json_schema": {}, - "supported_sync_modes": ["incremental", "full_refresh"] - }, - "sync_mode": "incremental", - "cursor_field": ["TimePeriod"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "account_impression_performance_report_monthly", - "json_schema": {}, - "supported_sync_modes": ["incremental", "full_refresh"] - }, - "sync_mode": "incremental", - "cursor_field": ["TimePeriod"], - "destination_sync_mode": "append" - }, { "stream": { "name": "labels", @@ -255,16 +145,6 @@ "sync_mode": "incremental", "cursor_field": ["TimePeriod"], "destination_sync_mode": "append" - }, - { - "stream": { - "name": "ad_group_impression_performance_report_monthly", - "json_schema": {}, - "supported_sync_modes": ["incremental", "full_refresh"] - }, - "sync_mode": "incremental", - "cursor_field": ["TimePeriod"], - "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl index 56deedb7d681..a4154ed8d18f 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl @@ -1,45 +1,32 @@ -{"stream": "ad_groups", "data": {"AdRotation": null, "AudienceAdsBidAdjustment": null, "BiddingScheme": {"Type": "InheritFromParent", "InheritedBidStrategyType": "EnhancedCpc"}, "CpcBid": {"Amount": 0.05}, "EndDate": null, "FinalUrlSuffix": null, "ForwardCompatibilityMap": null, "Id": 1357897480389129, "Language": null, "Name": "Data Integration Tool", "Network": "OwnedAndOperatedAndSyndicatedSearch", "PrivacyStatus": null, "Settings": null, "StartDate": {"Day": 4, "Month": 12, "Year": 2020}, "Status": "Paused", "TrackingUrlTemplate": null, "UrlCustomParameters": null, "AdScheduleUseSearcherTimeZone": false, "AdGroupType": "SearchStandard", "CpvBid": {"Amount": null}, "CpmBid": {"Amount": null}, "CampaignId": 407519039, "AccountId": 180278106, "CustomerId": 251186883}, "emitted_at": 1698928139401} -{"stream": "ads", "data": {"AdFormatPreference": "All", "DevicePreference": 0, "EditorialStatus": "ActiveLimited", "FinalAppUrls": null, "FinalMobileUrls": {"string": ["https://airbyte.io"]}, "FinalUrlSuffix": null, "FinalUrls": {"string": ["https://airbyte.io"]}, "ForwardCompatibilityMap": null, "Id": 84525295496190, "Status": "Active", "TrackingUrlTemplate": null, "Type": "ResponsiveSearch", "UrlCustomParameters": null, "Descriptions": {"AssetLink": [{"Asset": {"Id": 10239221964468, "Name": null, "Type": "TextAsset", "Text": "Open data integration for modern data teams"}, "AssetPerformanceLabel": "Learning", "EditorialStatus": "Active", "PinnedField": null}, {"Asset": {"Id": 10239221964466, "Name": null, "Type": "TextAsset", "Text": "Get your data pipelines running in minutes. With pre-built or custom connectors"}, "AssetPerformanceLabel": "Learning", "EditorialStatus": "Active", "PinnedField": null}]}, "Domain": "airbyte.io", "Headlines": {"AssetLink": [{"Asset": {"Id": 10239221964471, "Name": null, "Type": "TextAsset", "Text": "Data Integration Tool"}, "AssetPerformanceLabel": "Learning", "EditorialStatus": "Active", "PinnedField": null}, {"Asset": {"Id": 10239221964469, "Name": null, "Type": "TextAsset", "Text": "1,000+ Members"}, "AssetPerformanceLabel": "Learning", "EditorialStatus": "Active", "PinnedField": null}, {"Asset": {"Id": 10239221964467, "Name": null, "Type": "TextAsset", "Text": "Data Management Software"}, "AssetPerformanceLabel": "Learning", "EditorialStatus": "Active", "PinnedField": null}]}, "Path1": null, "Path2": null, "AdGroupId": 1352400325389092, "AccountId": 180278106, "CustomerId": 251186883}, "emitted_at": 1698927837499} -{"stream": "campaigns", "data": {"AudienceAdsBidAdjustment": 0, "BiddingScheme": {"Type": "EnhancedCpc"}, "BudgetType": "DailyBudgetStandard", "DailyBudget": 0.1, "ExperimentId": null, "FinalUrlSuffix": null, "ForwardCompatibilityMap": null, "Id": 407519039, "MultimediaAdsBidAdjustment": 40, "Name": "integration-test-campaign", "Status": "Paused", "SubType": null, "TimeZone": "Arizona", "TrackingUrlTemplate": null, "UrlCustomParameters": null, "CampaignType": "Search", "Settings": {"Setting": [{"Type": "TargetSetting", "Details": {"TargetSettingDetail": [{"CriterionTypeGroup": "Audience", "TargetAndBid": false}]}}]}, "BudgetId": null, "Languages": {"string": ["English"]}, "AdScheduleUseSearcherTimeZone": false, "AccountId": 180278106, "CustomerId": 251186883}, "emitted_at": 1698928296177} -{"stream":"accounts","data":{"BillToCustomerId":251186883,"CurrencyCode":"USD","AccountFinancialStatus":"ClearFinancialStatus","Id":180278106,"Language":"English","LastModifiedByUserId":0,"LastModifiedTime":"2023-08-11T03:26:10.277000","Name":"Daxtarity Inc.","Number":"F149GKV5","ParentCustomerId":251186883,"PaymentMethodId":138188746,"PaymentMethodType":"CreditCard","PrimaryUserId":138225488,"AccountLifeCycleStatus":"Active","TimeStamp":"AAAAAH1yyMo=","TimeZone":"Arizona","PauseReason":null,"ForwardCompatibilityMap":null,"LinkedAgencies":null,"SalesHouseCustomerId":null,"TaxInformation":null,"BackUpPaymentInstrumentId":null,"BillingThresholdAmount":null,"BusinessAddress":{"City":"San Francisco","CountryCode":"US","Id":149004358,"Line1":"350 29th avenue","Line2":null,"Line3":null,"Line4":null,"PostalCode":"94121","StateOrProvince":"CA","TimeStamp":null,"BusinessName":"Daxtarity Inc."},"AutoTagType":"Inactive","SoldToPaymentInstrumentId":null,"AccountMode":"Expert"},"emitted_at":1698149762546} -{"stream":"account_performance_report_daily","data":{"AccountId":180278106,"TimePeriod":"2021-06-09","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Smartphone","Network":"Bing and Yahoo! search","DeliveredMatchType":"Broad","DeviceOS":"Android","TopVsOther":"Bing and Yahoo! search - Top","BidMatchType":"Broad","AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","PhoneImpressions":0,"PhoneCalls":0,"Clicks":0,"Ctr":0.0,"Spend":0.0,"Impressions":3,"CostPerConversion":null,"Ptr":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":0,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1698149775393} -{"stream":"account_performance_report_weekly","data":{"AccountId":180278106,"TimePeriod":"2021-06-06","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Bing and Yahoo! search","DeliveredMatchType":"Exact","DeviceOS":"Windows","TopVsOther":"Bing and Yahoo! search - Other","BidMatchType":"Broad","AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","PhoneImpressions":0,"PhoneCalls":0,"Clicks":0,"Ctr":null,"Spend":0.0,"Impressions":0,"CostPerConversion":null,"Ptr":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":7,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1698149785273} -{"stream":"account_performance_report_monthly","data":{"AccountId":180278106,"TimePeriod":"2021-06-01","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Bing and Yahoo! search","DeliveredMatchType":"Exact","DeviceOS":"Windows","TopVsOther":"Bing and Yahoo! search - Top","BidMatchType":"Broad","AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","PhoneImpressions":0,"PhoneCalls":0,"Clicks":0,"Ctr":0.0,"Spend":0.0,"Impressions":11,"CostPerConversion":null,"Ptr":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":30,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1698149794340} -{"stream":"ad_group_performance_report_daily","data":{"AccountId":180278106,"CampaignId":413444833,"AdGroupId":1352400325389092,"TimePeriod":"2021-06-09","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Bing and Yahoo! search","DeliveredMatchType":"Phrase","DeviceOS":"Windows","TopVsOther":"Bing and Yahoo! search - Top","BidMatchType":"Broad","Language":"English","AccountName":"Daxtarity Inc.","CampaignName":"Test 2","CampaignType":"Search & content","AdGroupName":"Airbyte","AdGroupType":"Standard","Impressions":1,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"QualityScore":null,"ExpectedCtr":null,"AdRelevance":null,"LandingPageExperience":null,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"CostPerAssist":null,"CustomParameters":null,"FinalUrlSuffix":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"HistoricalQualityScore":8.0,"HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":3.0},"emitted_at":1698149811393} -{"stream":"ad_group_performance_report_weekly","data":{"AccountId":180278106,"CampaignId":413444833,"AdGroupId":1352400325389092,"TimePeriod":"2021-06-06","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Bing and Yahoo! search","DeliveredMatchType":"Phrase","DeviceOS":"Windows","TopVsOther":"Bing and Yahoo! search - Top","BidMatchType":"Broad","Language":"English","AccountName":"Daxtarity Inc.","CampaignName":"Test 2","CampaignType":"Search & content","AdGroupName":"Airbyte","AdGroupType":"Standard","Impressions":1,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"QualityScore":null,"ExpectedCtr":null,"AdRelevance":null,"LandingPageExperience":null,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"CostPerAssist":null,"CustomParameters":null,"FinalUrlSuffix":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"HistoricalQualityScore":8.0,"HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":3.0},"emitted_at":1698149821559} -{"stream":"ad_group_performance_report_monthly","data":{"AccountId":180278106,"CampaignId":413444833,"AdGroupId":1352400325389092,"TimePeriod":"2021-06-01","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Bing and Yahoo! search","DeliveredMatchType":"Exact","DeviceOS":"Windows","TopVsOther":"Bing and Yahoo! search - Other","BidMatchType":"Broad","Language":"German","AccountName":"Daxtarity Inc.","CampaignName":"Test 2","CampaignType":"Search & content","AdGroupName":"Airbyte","AdGroupType":"Standard","Impressions":1,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"QualityScore":null,"ExpectedCtr":null,"AdRelevance":null,"LandingPageExperience":null,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"CostPerAssist":null,"CustomParameters":null,"FinalUrlSuffix":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"HistoricalQualityScore":8.0,"HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":3.0},"emitted_at":1698149830886} -{"stream":"ad_group_impression_performance_report_daily","data":{"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","AccountId":180278106,"TimePeriod":"2021-08-02","Status":"Active","CampaignName":"Test 2","CampaignId":413444833,"AdGroupName":"Airbyte","AdGroupId":1352400325389092,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":26,"Clicks":1,"Ctr":3.85,"AverageCpc":0.11,"Spend":0.11,"AveragePosition":0.0,"Conversions":0,"ConversionRate":0.0,"CostPerConversion":null,"DeviceType":"Computer","Language":"English","ImpressionSharePercent":10.83,"ImpressionLostToBudgetPercent":0.83,"ImpressionLostToRankAggPercent":88.33,"QualityScore":null,"ExpectedCtr":null,"AdRelevance":null,"LandingPageExperience":null,"HistoricalQualityScore":8,"HistoricalExpectedCtr":2,"HistoricalAdRelevance":3,"HistoricalLandingPageExperience":3,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","CampaignStatus":"Paused","AdGroupLabels":null,"ExactMatchImpressionSharePercent":80.0,"ClickSharePercent":3.33,"AbsoluteTopImpressionSharePercent":1.35,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":91.89,"TopImpressionShareLostToBudgetPercent":0.45,"AbsoluteTopImpressionShareLostToRankPercent":98.2,"AbsoluteTopImpressionShareLostToBudgetPercent":0.45,"TopImpressionSharePercent":7.66,"AbsoluteTopImpressionRatePercent":11.54,"TopImpressionRatePercent":65.38,"BaseCampaignId":413444833,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":0.0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AdGroupType":"Standard","AverageCpm":4.23,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1698788776673} -{"stream":"ad_group_impression_performance_report_weekly","data":{"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","AccountId":180278106,"TimePeriod":"2021-08-01","Status":"Active","CampaignName":"Test 2","CampaignId":413444833,"AdGroupName":"Airbyte","AdGroupId":1352400325389092,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":59,"Clicks":1,"Ctr":1.69,"AverageCpc":0.11,"Spend":0.11,"AveragePosition":0.0,"Conversions":0,"ConversionRate":0.0,"CostPerConversion":null,"DeviceType":"Computer","Language":"English","ImpressionSharePercent":14.46,"ImpressionLostToBudgetPercent":0.49,"ImpressionLostToRankAggPercent":85.05,"QualityScore":null,"ExpectedCtr":null,"AdRelevance":null,"LandingPageExperience":null,"HistoricalQualityScore":8,"HistoricalExpectedCtr":2,"HistoricalAdRelevance":3,"HistoricalLandingPageExperience":3,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","CampaignStatus":"Paused","AdGroupLabels":null,"ExactMatchImpressionSharePercent":87.5,"ClickSharePercent":1.69,"AbsoluteTopImpressionSharePercent":1.55,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":87.6,"TopImpressionShareLostToBudgetPercent":0.26,"AbsoluteTopImpressionShareLostToRankPercent":98.19,"AbsoluteTopImpressionShareLostToBudgetPercent":0.26,"TopImpressionSharePercent":12.14,"AbsoluteTopImpressionRatePercent":10.17,"TopImpressionRatePercent":79.66,"BaseCampaignId":413444833,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":0.0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AdGroupType":"Standard","AverageCpm":1.86,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1698788763314} -{"stream":"ad_group_impression_performance_report_monthly","data":{"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","AccountId":180278106,"TimePeriod":"2021-07-01","Status":"Active","CampaignName":"Test 2","CampaignId":413444833,"AdGroupName":"Airbyte","AdGroupId":1352400325389092,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":2,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"DeviceType":"Computer","Language":"Portuguese","ImpressionSharePercent":null,"ImpressionLostToBudgetPercent":null,"ImpressionLostToRankAggPercent":null,"QualityScore":null,"ExpectedCtr":null,"AdRelevance":null,"LandingPageExperience":null,"HistoricalQualityScore":8,"HistoricalExpectedCtr":2,"HistoricalAdRelevance":3,"HistoricalLandingPageExperience":3,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Bing and Yahoo! search","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","CampaignStatus":"Paused","AdGroupLabels":null,"ExactMatchImpressionSharePercent":null,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":null,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":null,"TopImpressionShareLostToBudgetPercent":null,"AbsoluteTopImpressionShareLostToRankPercent":null,"AbsoluteTopImpressionShareLostToBudgetPercent":null,"TopImpressionSharePercent":null,"AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":100.0,"BaseCampaignId":413444833,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AdGroupType":"Standard","AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1698775265485} -{"stream":"ad_performance_report_daily","data":{"AccountId":180278106,"CampaignId":413444833,"AdGroupId":1352400325389092,"AdId":84525295496190,"TimePeriod":"2021-06-09","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Language":"English","Network":"Bing and Yahoo! search","DeviceOS":"Unknown","TopVsOther":"Bing and Yahoo! search - Other","BidMatchType":"Broad","DeliveredMatchType":"Broad","AccountName":"Daxtarity Inc.","CampaignName":"Test 2","CampaignType":"Search & content","AdGroupName":"Airbyte","Impressions":2,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"DestinationUrl":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"AdDescription":null,"AdDescription2":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1698149848147} -{"stream":"ad_performance_report_weekly","data":{"AccountId":180278106,"CampaignId":413444833,"AdGroupId":1352400325389092,"AdId":84525295496190,"TimePeriod":"2021-06-06","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Language":"German","Network":"Syndicated search partners","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Other","BidMatchType":"Broad","DeliveredMatchType":"Broad","AccountName":"Daxtarity Inc.","CampaignName":"Test 2","CampaignType":"Search & content","AdGroupName":"Airbyte","Impressions":1,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"DestinationUrl":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"AdDescription":null,"AdDescription2":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1698149858191} -{"stream":"ad_performance_report_monthly","data":{"AccountId":180278106,"CampaignId":413444833,"AdGroupId":1352400325389092,"AdId":84525295496190,"TimePeriod":"2021-06-01","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Language":"German","Network":"Bing and Yahoo! search","DeviceOS":"Windows","TopVsOther":"Bing and Yahoo! search - Other","BidMatchType":"Broad","DeliveredMatchType":"Exact","AccountName":"Daxtarity Inc.","CampaignName":"Test 2","CampaignType":"Search & content","AdGroupName":"Airbyte","Impressions":1,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"DestinationUrl":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"AdDescription":null,"AdDescription2":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1698149867383} -{"stream":"budget_summary_report","data":{"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","AccountId":180278106,"CampaignName":"Test 2","CampaignId":413444833,"Date":"6/10/2021","MonthlyBudget":22.8,"DailySpend":0.74,"MonthToDateSpend":1.45},"emitted_at":1698149948646} -{"stream":"campaign_performance_report_daily","data":{"AccountId":180278106,"CampaignId":413444833,"TimePeriod":"2021-06-09","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Bing and Yahoo! search","DeliveredMatchType":"Exact","DeviceOS":"Windows","TopVsOther":"Bing and Yahoo! search - Other","BidMatchType":"Broad","AccountName":"Daxtarity Inc.","CampaignName":"Test 2","CampaignType":"Search & content","CampaignStatus":"Paused","CampaignLabels":"integration-test-label;another label;what a new label","Impressions":0,"Clicks":0,"Ctr":null,"Spend":0.0,"CostPerConversion":null,"QualityScore":null,"AdRelevance":null,"LandingPageExperience":null,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":6,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"BudgetName":null,"BudgetStatus":null,"BudgetAssociationStatus":"Current","HistoricalQualityScore":8.0,"HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":3.0},"emitted_at":1698149965377} -{"stream":"campaign_performance_report_weekly","data":{"AccountId":180278106,"CampaignId":413444833,"TimePeriod":"2021-06-06","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Bing and Yahoo! search","DeliveredMatchType":"Phrase","DeviceOS":"Unknown","TopVsOther":"Bing and Yahoo! search - Top","BidMatchType":"Broad","AccountName":"Daxtarity Inc.","CampaignName":"Test 2","CampaignType":"Search & content","CampaignStatus":"Paused","CampaignLabels":"integration-test-label;another label;what a new label","Impressions":0,"Clicks":0,"Ctr":null,"Spend":0.0,"CostPerConversion":null,"QualityScore":null,"AdRelevance":null,"LandingPageExperience":null,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":3,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"BudgetName":null,"BudgetStatus":null,"BudgetAssociationStatus":"Current","HistoricalQualityScore":8.0,"HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":3.0},"emitted_at":1698149976036} -{"stream":"campaign_performance_report_monthly","data":{"AccountId":180278106,"CampaignId":413444833,"TimePeriod":"2021-06-01","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Bing and Yahoo! search","DeliveredMatchType":"Exact","DeviceOS":"Windows","TopVsOther":"Bing and Yahoo! search - Top","BidMatchType":"Broad","AccountName":"Daxtarity Inc.","CampaignName":"Test 2","CampaignType":"Search & content","CampaignStatus":"Paused","CampaignLabels":"integration-test-label;another label;what a new label","Impressions":11,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"QualityScore":null,"AdRelevance":null,"LandingPageExperience":null,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":30,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"BudgetName":null,"BudgetStatus":null,"BudgetAssociationStatus":"Current","HistoricalQualityScore":8.0,"HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":3.0},"emitted_at":1698149985116} -{"stream":"campaign_impression_performance_report_daily","data":{"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","AccountId":180278106,"TimePeriod":"2021-08-01","CampaignStatus":"Paused","CampaignName":"Test 2","CampaignId":413444833,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":584,"Clicks":12,"Ctr":2.05,"AverageCpc":0.07,"Spend":0.78,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"LowQualityClicks":0,"LowQualityClicksPercent":0.0,"LowQualityImpressions":12,"LowQualityImpressionsPercent":2.01,"LowQualityConversions":0,"LowQualityConversionRate":null,"DeviceType":"Smartphone","ImpressionSharePercent":75.0,"ImpressionLostToBudgetPercent":1.79,"ImpressionLostToRankAggPercent":23.21,"QualityScore":null,"ExpectedCtr":null,"AdRelevance":null,"LandingPageExperience":null,"HistoricalQualityScore":8,"HistoricalExpectedCtr":2,"HistoricalAdRelevance":3,"HistoricalLandingPageExperience":3,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":0,"CampaignLabels":"integration-test-label;another label;what a new label","ExactMatchImpressionSharePercent":96.32,"ClickSharePercent":14.46,"AbsoluteTopImpressionSharePercent":4.17,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":38.85,"TopImpressionShareLostToBudgetPercent":1.83,"AbsoluteTopImpressionShareLostToRankPercent":94.0,"AbsoluteTopImpressionShareLostToBudgetPercent":1.83,"TopImpressionSharePercent":59.32,"AbsoluteTopImpressionRatePercent":5.48,"TopImpressionRatePercent":77.74,"BaseCampaignId":413444833,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AverageCpm":1.34,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1698926408384} -{"stream":"campaign_impression_performance_report_weekly","data":{"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","AccountId":180278106,"TimePeriod":"2021-06-20","CampaignStatus":"Paused","CampaignName":"Test 2","CampaignId":413444833,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":468,"Clicks":6,"Ctr":1.28,"AverageCpc":0.12,"Spend":0.7,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"LowQualityClicks":3,"LowQualityClicksPercent":33.33,"LowQualityImpressions":329,"LowQualityImpressionsPercent":41.28,"LowQualityConversions":0,"LowQualityConversionRate":0.0,"DeviceType":"Computer","ImpressionSharePercent":null,"ImpressionLostToBudgetPercent":null,"ImpressionLostToRankAggPercent":null,"QualityScore":null,"ExpectedCtr":null,"AdRelevance":null,"LandingPageExperience":null,"HistoricalQualityScore":8,"HistoricalExpectedCtr":2,"HistoricalAdRelevance":3,"HistoricalLandingPageExperience":3,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Bing and Yahoo! search","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","LowQualityGeneralClicks":1,"LowQualitySophisticatedClicks":2,"CampaignLabels":"integration-test-label;another label;what a new label","ExactMatchImpressionSharePercent":null,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":null,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":null,"TopImpressionShareLostToBudgetPercent":null,"AbsoluteTopImpressionShareLostToRankPercent":null,"AbsoluteTopImpressionShareLostToBudgetPercent":null,"TopImpressionSharePercent":null,"AbsoluteTopImpressionRatePercent":28.63,"TopImpressionRatePercent":76.92,"BaseCampaignId":413444833,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AverageCpm":1.5,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1698926395903} -{"stream":"campaign_impression_performance_report_monthly","data":{"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","AccountId":180278106,"TimePeriod":"2021-07-01","CampaignStatus":"Paused","CampaignName":"Test 2","CampaignId":413444833,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":38546,"Clicks":201,"Ctr":0.52,"AverageCpc":0.09,"Spend":18.1,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"LowQualityClicks":6,"LowQualityClicksPercent":2.9,"LowQualityImpressions":500,"LowQualityImpressionsPercent":1.28,"LowQualityConversions":0,"LowQualityConversionRate":0.0,"DeviceType":"Smartphone","ImpressionSharePercent":50.38,"ImpressionLostToBudgetPercent":1.68,"ImpressionLostToRankAggPercent":47.94,"QualityScore":null,"ExpectedCtr":null,"AdRelevance":null,"LandingPageExperience":null,"HistoricalQualityScore":8,"HistoricalExpectedCtr":2,"HistoricalAdRelevance":3,"HistoricalLandingPageExperience":3,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","LowQualityGeneralClicks":5,"LowQualitySophisticatedClicks":1,"CampaignLabels":"integration-test-label;another label;what a new label","ExactMatchImpressionSharePercent":88.93,"ClickSharePercent":2.51,"AbsoluteTopImpressionSharePercent":1.49,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":58.7,"TopImpressionShareLostToBudgetPercent":1.63,"AbsoluteTopImpressionShareLostToRankPercent":96.88,"AbsoluteTopImpressionShareLostToBudgetPercent":1.63,"TopImpressionSharePercent":39.67,"AbsoluteTopImpressionRatePercent":2.89,"TopImpressionRatePercent":76.83,"BaseCampaignId":413444833,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AverageCpm":0.47,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1698868217376} -{"stream":"keyword_performance_report_daily","data":{"AccountId":180278106,"CampaignId":413444833,"AdGroupId":1352400325389092,"KeywordId":84525593559627,"AdId":84525295496190,"TimePeriod":"2021-06-14","CurrencyCode":"USD","DeliveredMatchType":"Broad","AdDistribution":"Search","DeviceType":"Smartphone","Language":"English","Network":"Syndicated search partners","DeviceOS":"Android","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Daxtarity Inc.","CampaignName":"Test 2","AdGroupName":"Airbyte","Keyword":"big data integration tools","KeywordStatus":"Active","Impressions":1,"Clicks":0,"Ctr":0.0,"CurrentMaxCpc":0.11,"Spend":0.0,"CostPerConversion":null,"QualityScore":null,"ExpectedCtr":null,"AdRelevance":null,"LandingPageExperience":null,"QualityImpact":0.0,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"Mainline1Bid":null,"MainlineBid":null,"FirstPageBid":null,"FinalUrlSuffix":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"HistoricalQualityScore":7.0,"HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":2.0,"HistoricalLandingPageExperience":3.0},"emitted_at":1698150002220} -{"stream":"keyword_performance_report_weekly","data":{"AccountId":180278106,"CampaignId":413444833,"AdGroupId":1352400325389092,"KeywordId":84525593559627,"AdId":84525295496190,"TimePeriod":"2021-06-13","CurrencyCode":"USD","DeliveredMatchType":"Broad","AdDistribution":"Search","DeviceType":"Smartphone","Language":"English","Network":"Syndicated search partners","DeviceOS":"iOS","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Daxtarity Inc.","CampaignName":"Test 2","AdGroupName":"Airbyte","Keyword":"big data integration tools","KeywordStatus":"Active","Impressions":10,"Clicks":0,"Ctr":0.0,"CurrentMaxCpc":0.11,"Spend":0.0,"CostPerConversion":null,"QualityScore":null,"ExpectedCtr":null,"AdRelevance":null,"LandingPageExperience":null,"QualityImpact":0.0,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"Mainline1Bid":null,"MainlineBid":null,"FirstPageBid":null,"FinalUrlSuffix":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1698150013239} -{"stream":"keyword_performance_report_monthly","data":{"AccountId":180278106,"CampaignId":413444833,"AdGroupId":1352400325389092,"KeywordId":84525593559629,"AdId":84525295496190,"TimePeriod":"2021-08-01","CurrencyCode":"USD","DeliveredMatchType":"Broad","AdDistribution":"Search","DeviceType":"Tablet","Language":"English","Network":"Syndicated search partners","DeviceOS":"Android","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Daxtarity Inc.","CampaignName":"Test 2","AdGroupName":"Airbyte","Keyword":"data integration tools","KeywordStatus":"Active","Impressions":1,"Clicks":0,"Ctr":0.0,"CurrentMaxCpc":0.11,"Spend":0.0,"CostPerConversion":null,"QualityScore":null,"ExpectedCtr":null,"AdRelevance":null,"LandingPageExperience":null,"QualityImpact":0.0,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"Mainline1Bid":null,"MainlineBid":null,"FirstPageBid":null,"FinalUrlSuffix":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1698150022983} -{"stream":"geographic_performance_report_daily","data":{"AccountId":180278106,"CampaignId":413444833,"AdGroupId":1352400325389092,"TimePeriod":"2021-06-09","Country":"United Kingdom","CurrencyCode":"USD","DeliveredMatchType":"Broad","AdDistribution":"Search","DeviceType":"Computer","Language":"English","Network":"Syndicated search partners","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","MetroArea":"London","State":"England","City":"London","AdGroupName":"Airbyte","Ctr":0.0,"ProximityTargetLocation":null,"Radius":"0","Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"LocationType":"Physical location","MostSpecificLocation":"SE13","AccountStatus":"Active","CampaignStatus":"Paused","AdGroupStatus":"Active","County":null,"PostalCode":"SE13","LocationId":"132401","BaseCampaignId":"413444833","AllCostPerConversion":null,"AllReturnOnAdSpend":null,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":"100.00","AllConversionsQualified":"0.00","ViewThroughConversionsQualified":null,"Neighborhood":null,"ViewThroughRevenue":"0.00","CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null,"AssetGroupStatus":null,"Clicks":0,"Spend":0.0,"Impressions":1,"CostPerConversion":null,"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","CampaignName":"Test 2","Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1698153346893} -{"stream":"geographic_performance_report_weekly","data":{"AccountId":180278106,"CampaignId":413444833,"AdGroupId":1352400325389092,"TimePeriod":"2021-06-06","Country":"Australia","CurrencyCode":"USD","DeliveredMatchType":"Broad","AdDistribution":"Search","DeviceType":"Computer","Language":"English","Network":"Bing and Yahoo! search","DeviceOS":"Windows","TopVsOther":"Bing and Yahoo! search - Other","BidMatchType":"Broad","MetroArea":null,"State":"New South Wales","City":null,"AdGroupName":"Airbyte","Ctr":0.0,"ProximityTargetLocation":null,"Radius":"0","Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"LocationType":"Physical location","MostSpecificLocation":"New South Wales","AccountStatus":"Active","CampaignStatus":"Paused","AdGroupStatus":"Active","County":null,"PostalCode":null,"LocationId":"4048","BaseCampaignId":"413444833","AllCostPerConversion":null,"AllReturnOnAdSpend":null,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":"0.00","AllConversionsQualified":"0.00","ViewThroughConversionsQualified":null,"Neighborhood":null,"ViewThroughRevenue":"0.00","CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null,"AssetGroupStatus":null,"Clicks":0,"Spend":0.0,"Impressions":1,"CostPerConversion":null,"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","CampaignName":"Test 2","Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1698149877247} -{"stream":"geographic_performance_report_monthly","data":{"AccountId":180278106,"CampaignId":413444833,"AdGroupId":1352400325389092,"TimePeriod":"2021-06-01","Country":"United Arab Emirates","CurrencyCode":"USD","DeliveredMatchType":"Broad","AdDistribution":"Search","DeviceType":"Computer","Language":"English","Network":"Bing and Yahoo! search","DeviceOS":"Windows","TopVsOther":"Bing and Yahoo! search - Other","BidMatchType":"Broad","MetroArea":null,"State":null,"City":null,"AdGroupName":"Airbyte","Ctr":0.0,"ProximityTargetLocation":null,"Radius":"0","Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"LocationType":"Physical location","MostSpecificLocation":"United Arab Emirates","AccountStatus":"Active","CampaignStatus":"Paused","AdGroupStatus":"Active","County":null,"PostalCode":null,"LocationId":"218","BaseCampaignId":"413444833","AllCostPerConversion":null,"AllReturnOnAdSpend":null,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":"0.00","AllConversionsQualified":"0.00","ViewThroughConversionsQualified":null,"Neighborhood":null,"ViewThroughRevenue":"0.00","CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null,"AssetGroupStatus":null,"Clicks":0,"Spend":0.0,"Impressions":1,"CostPerConversion":null,"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","CampaignName":"Test 2","Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1698149915741} -{"stream":"age_gender_audience_report_daily","data":{"AccountId":180278106,"AgeGroup":"Unknown","Gender":"Unknown","TimePeriod":"2021-06-09","AllConversions":0,"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","CampaignName":"Test 2","CampaignId":413444833,"AdGroupName":"Airbyte","AdGroupId":1352400325389092,"AdDistribution":"Search","Impressions":250,"Clicks":1,"Conversions":0.0,"Spend":0.3,"Revenue":0.0,"ExtendedCost":0.0,"Assists":0,"Language":"English","AccountStatus":"Active","CampaignStatus":"Paused","AdGroupStatus":"Active","BaseCampaignId":"413444833","AllRevenue":0.0,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":9.64,"TopImpressionRatePercent":91.16,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0},"emitted_at":1698150031605} -{"stream":"age_gender_audience_report_weekly","data":{"AccountId":180278106,"AgeGroup":"Unknown","Gender":"Female","TimePeriod":"2021-07-18","AllConversions":0,"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","CampaignName":"Test 2","CampaignId":413444833,"AdGroupName":"Airbyte","AdGroupId":1352400325389092,"AdDistribution":"Search","Impressions":1,"Clicks":0,"Conversions":0.0,"Spend":0.0,"Revenue":0.0,"ExtendedCost":0.0,"Assists":0,"Language":"English","AccountStatus":"Active","CampaignStatus":"Paused","AdGroupStatus":"Active","BaseCampaignId":"413444833","AllRevenue":0.0,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.00,"TopImpressionRatePercent":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0},"emitted_at":1698144728984} -{"stream":"age_gender_audience_report_weekly","data":{"AccountId":180278106,"AgeGroup":"18-24","Gender":"Unknown","TimePeriod":"2021-06-06","AllConversions":0,"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","CampaignName":"Test 2","CampaignId":413444833,"AdGroupName":"Airbyte","AdGroupId":1352400325389092,"AdDistribution":"Search","Impressions":5,"Clicks":0,"Conversions":0.0,"Spend":0.0,"Revenue":0.0,"ExtendedCost":0.0,"Assists":0,"Language":"English","AccountStatus":"Active","CampaignStatus":"Paused","AdGroupStatus":"Active","BaseCampaignId":"413444833","AllRevenue":0.0,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":20.00,"TopImpressionRatePercent":80.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0},"emitted_at":1698144728986} -{"stream":"age_gender_audience_report_monthly","data":{"AccountId":180278106,"AgeGroup":"50-64","Gender":"Female","TimePeriod":"2021-07-01","AllConversions":0,"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","CampaignName":"Test 2","CampaignId":413444833,"AdGroupName":"Airbyte","AdGroupId":1352400325389092,"AdDistribution":"Search","Impressions":3,"Clicks":0,"Conversions":0.0,"Spend":0.0,"Revenue":0.0,"ExtendedCost":0.0,"Assists":0,"Language":"English","AccountStatus":"Active","CampaignStatus":"Paused","AdGroupStatus":"Active","BaseCampaignId":"413444833","AllRevenue":0.0,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":33.33,"TopImpressionRatePercent":33.33,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0},"emitted_at":1698144746216} -{"stream":"search_query_performance_report_daily","data":{"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","AccountId":180278106,"TimePeriod":"2021-07-27","CampaignName":"Test 2","CampaignId":413444833,"AdGroupName":"Airbyte","AdGroupId":1352400325389092,"AdId":84525295496190,"AdType":"Responsive search ad","DestinationUrl":null,"BidMatchType":"Broad","DeliveredMatchType":"Exact (close variant)","CampaignStatus":"Paused","AdStatus":"Active","Impressions":112,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"SearchQuery":"data integration tool","Keyword":"data integration tools","AdGroupCriterionId":null,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"Language":"English","KeywordId":84525593559629,"Network":"Syndicated search partners","TopVsOther":"Syndicated search partners - Top","DeviceType":"Smartphone","DeviceOS":"Android","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","AdGroupStatus":"Active","KeywordStatus":"Active","CampaignType":"Search & content","CustomerId":251186883,"CustomerName":"Daxtarity Inc.","AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":17.86,"TopImpressionRatePercent":100.0,"AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0},"emitted_at":1698172081512} -{"stream":"search_query_performance_report_weekly","data":{"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","AccountId":180278106,"TimePeriod":"2021-07-11","CampaignName":"Test 2","CampaignId":413444833,"AdGroupName":"Airbyte","AdGroupId":1352400325389092,"AdId":84525295496190,"AdType":"Responsive search ad","DestinationUrl":null,"BidMatchType":"Broad","DeliveredMatchType":"Exact (close variant)","CampaignStatus":"Paused","AdStatus":"Active","Impressions":1,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"SearchQuery":"data integration tool","Keyword":"data integration tools","AdGroupCriterionId":null,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"Language":"English","KeywordId":84525593559629,"Network":"Syndicated search partners","TopVsOther":"Syndicated search partners - Top","DeviceType":"Tablet","DeviceOS":"Android","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","AdGroupStatus":"Active","KeywordStatus":"Active","CampaignType":"Search & content","CustomerId":251186883,"CustomerName":"Daxtarity Inc.","AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":100.0,"AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0},"emitted_at":1698172090980} -{"stream":"search_query_performance_report_monthly","data":{"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","AccountId":180278106,"TimePeriod":"2021-07-01","CampaignName":"Test 2","CampaignId":413444833,"AdGroupName":"Airbyte","AdGroupId":1352400325389092,"AdId":84525295496190,"AdType":"Responsive search ad","DestinationUrl":null,"BidMatchType":"Broad","DeliveredMatchType":"Broad","CampaignStatus":"Paused","AdStatus":"Active","Impressions":551,"Clicks":11,"Ctr":2.0,"AverageCpc":0.03,"Spend":0.33,"AveragePosition":0.0,"SearchQuery":"data management platform","Keyword":"data integration tools","AdGroupCriterionId":null,"Conversions":0,"ConversionRate":0.0,"CostPerConversion":null,"Language":"English","KeywordId":84525593559629,"Network":"Syndicated search partners","TopVsOther":"Syndicated search partners - Top","DeviceType":"Smartphone","DeviceOS":"Android","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","AdGroupStatus":"Active","KeywordStatus":"Active","CampaignType":"Search & content","CustomerId":251186883,"CustomerName":"Daxtarity Inc.","AllConversions":0,"AllRevenue":0.0,"AllConversionRate":0.0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":5.99,"TopImpressionRatePercent":100.0,"AverageCpm":0.6,"ConversionsQualified":0.0,"AllConversionsQualified":0.0},"emitted_at":1698172100880} -{"stream":"user_location_performance_report_daily","data":{"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","AccountId":180278106,"TimePeriod":"2021-06-22","CampaignName":"Test 2","CampaignId":413444833,"AdGroupName":"Airbyte","AdGroupId":1352400325389092,"Country":"Philippines","State":null,"MetroArea":null,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":2,"Clicks":2,"Ctr":100.0,"AverageCpc":0.02,"Spend":0.04,"AveragePosition":0.0,"ProximityTargetLocation":null,"Radius":0,"Language":"English","City":null,"QueryIntentCountry":"Philippines","QueryIntentState":null,"QueryIntentCity":null,"QueryIntentDMA":null,"BidMatchType":"Broad","DeliveredMatchType":"Broad","Network":"Bing and Yahoo! search","TopVsOther":"Bing and Yahoo! search - Top","DeviceType":"Smartphone","DeviceOS":"Android","Assists":0,"Conversions":0,"ConversionRate":0.0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerConversion":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"County":null,"PostalCode":null,"QueryIntentCounty":null,"QueryIntentPostalCode":null,"LocationId":149,"QueryIntentLocationId":149,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":0.0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":50.0,"TopImpressionRatePercent":100.0,"AverageCpm":20.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"Neighborhood":null,"QueryIntentNeighborhood":null,"ViewThroughRevenue":0.0,"CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null},"emitted_at":1698264446467} -{"stream":"user_location_performance_report_weekly","data":{"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","AccountId":180278106,"TimePeriod":"2021-06-13","CampaignName":"Test 2","CampaignId":413444833,"AdGroupName":"Airbyte","AdGroupId":1352400325389092,"Country":"Indonesia","State":null,"MetroArea":null,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":22,"Clicks":3,"Ctr":13.64,"AverageCpc":0.09,"Spend":0.26,"AveragePosition":0.0,"ProximityTargetLocation":null,"Radius":0,"Language":"Indonesian","City":null,"QueryIntentCountry":"Indonesia","QueryIntentState":null,"QueryIntentCity":null,"QueryIntentDMA":null,"BidMatchType":"Broad","DeliveredMatchType":"Broad","Network":"Bing and Yahoo! search","TopVsOther":"Bing and Yahoo! search - Top","DeviceType":"Computer","DeviceOS":"Windows","Assists":0,"Conversions":0,"ConversionRate":0.0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerConversion":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"County":null,"PostalCode":null,"QueryIntentCounty":null,"QueryIntentPostalCode":null,"LocationId":91,"QueryIntentLocationId":91,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":0.0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":59.09,"TopImpressionRatePercent":100.0,"AverageCpm":11.82,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"Neighborhood":null,"QueryIntentNeighborhood":null,"ViewThroughRevenue":0.0,"CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null},"emitted_at":1698264629208} -{"stream":"user_location_performance_report_monthly","data":{"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","AccountId":180278106,"TimePeriod":"2021-07-01","CampaignName":"Test 2","CampaignId":413444833,"AdGroupName":"Airbyte","AdGroupId":1352400325389092,"Country":"United States","State":"New York","MetroArea":"New York, NY","CurrencyCode":"USD","AdDistribution":"Search","Impressions":103,"Clicks":3,"Ctr":2.91,"AverageCpc":0.07,"Spend":0.21,"AveragePosition":0.0,"ProximityTargetLocation":null,"Radius":0,"Language":"English","City":"New York","QueryIntentCountry":"United States","QueryIntentState":null,"QueryIntentCity":null,"QueryIntentDMA":null,"BidMatchType":"Broad","DeliveredMatchType":"Phrase","Network":"Syndicated search partners","TopVsOther":"Syndicated search partners - Top","DeviceType":"Smartphone","DeviceOS":"Android","Assists":0,"Conversions":0,"ConversionRate":0.0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerConversion":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"County":null,"PostalCode":null,"QueryIntentCounty":null,"QueryIntentPostalCode":null,"LocationId":59981,"QueryIntentLocationId":190,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":0.0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":3.88,"TopImpressionRatePercent":100.0,"AverageCpm":2.04,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"Neighborhood":null,"QueryIntentNeighborhood":null,"ViewThroughRevenue":0.0,"CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null},"emitted_at":1698264731347} -{"stream":"account_impression_performance_report_daily","data":{"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","AccountId":180278106,"TimePeriod":"2021-06-10","CurrencyCode":"USD","AdDistribution":"Search","Impressions":4,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":0,"LowQualityImpressionsPercent":0.0,"LowQualityConversions":0,"LowQualityConversionRate":null,"DeviceType":"Smartphone","ImpressionSharePercent":null,"ImpressionLostToBudgetPercent":null,"ImpressionLostToRankAggPercent":null,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Bing and Yahoo! search","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":0,"ExactMatchImpressionSharePercent":null,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":null,"TopImpressionShareLostToRankPercent":null,"TopImpressionShareLostToBudgetPercent":null,"AbsoluteTopImpressionShareLostToRankPercent":null,"AbsoluteTopImpressionShareLostToBudgetPercent":null,"TopImpressionSharePercent":null,"AbsoluteTopImpressionRatePercent":75.0,"TopImpressionRatePercent":100.0,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"AverageCpm":0.0,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1698748305676} -{"stream":"account_impression_performance_report_weekly","data":{"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","AccountId":180278106,"TimePeriod":"2021-06-20","CurrencyCode":"USD","AdDistribution":"Search","Impressions":27,"Clicks":3,"Ctr":11.11,"AverageCpc":0.09,"Spend":0.26,"AveragePosition":0.0,"Conversions":0,"ConversionRate":0.0,"CostPerConversion":null,"LowQualityClicks":0,"LowQualityClicksPercent":0.0,"LowQualityImpressions":19,"LowQualityImpressionsPercent":41.3,"LowQualityConversions":0,"LowQualityConversionRate":null,"DeviceType":"Smartphone","ImpressionSharePercent":null,"ImpressionLostToBudgetPercent":null,"ImpressionLostToRankAggPercent":null,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Bing and Yahoo! search","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":0,"ExactMatchImpressionSharePercent":null,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":null,"TopImpressionShareLostToRankPercent":null,"TopImpressionShareLostToBudgetPercent":null,"AbsoluteTopImpressionShareLostToRankPercent":null,"AbsoluteTopImpressionShareLostToBudgetPercent":null,"TopImpressionSharePercent":null,"AbsoluteTopImpressionRatePercent":44.44,"TopImpressionRatePercent":70.37,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":0.0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"AverageCpm":9.63,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1698757489131} -{"stream":"account_impression_performance_report_monthly","data":{"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","AccountId":180278106,"TimePeriod":"2021-07-01","CurrencyCode":"USD","AdDistribution":"Search","Impressions":52,"Clicks":3,"Ctr":5.77,"AverageCpc":0.06,"Spend":0.19,"AveragePosition":0.0,"Conversions":0,"ConversionRate":0.0,"CostPerConversion":null,"LowQualityClicks":1,"LowQualityClicksPercent":25.0,"LowQualityImpressions":62,"LowQualityImpressionsPercent":54.39,"LowQualityConversions":0,"LowQualityConversionRate":0.0,"DeviceType":"Smartphone","ImpressionSharePercent":0.97,"ImpressionLostToBudgetPercent":0.09,"ImpressionLostToRankAggPercent":98.94,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Bing and Yahoo! search","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":1,"ExactMatchImpressionSharePercent":16.67,"ClickSharePercent":0.41,"AbsoluteTopImpressionSharePercent":0.37,"TopImpressionShareLostToRankPercent":98.86,"TopImpressionShareLostToBudgetPercent":0.04,"AbsoluteTopImpressionShareLostToRankPercent":99.58,"AbsoluteTopImpressionShareLostToBudgetPercent":0.04,"TopImpressionSharePercent":1.1,"AbsoluteTopImpressionRatePercent":32.69,"TopImpressionRatePercent":96.15,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":0.0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"AverageCpm":3.65,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1698339857372} +{"stream":"ad_groups","data":{"AdRotation":null,"AudienceAdsBidAdjustment":null,"BiddingScheme":{"Type":"InheritFromParent","InheritedBidStrategyType":"EnhancedCpc"},"CpcBid":{"Amount":2.27},"EndDate":null,"FinalUrlSuffix":null,"ForwardCompatibilityMap":null,"Id":1356799861840328,"Language":null,"Name":"keywords","Network":"OwnedAndOperatedAndSyndicatedSearch","PrivacyStatus":null,"Settings":null,"StartDate":{"Day":7,"Month":11,"Year":2023},"Status":"Active","TrackingUrlTemplate":null,"UrlCustomParameters":null,"AdScheduleUseSearcherTimeZone":false,"AdGroupType":"SearchStandard","CpvBid":{"Amount":null},"CpmBid":{"Amount":null},"CampaignId":531016227,"AccountId":180519267,"CustomerId":251186883},"emitted_at":1699913367220} +{"stream":"ads","data":{"AdFormatPreference":"All","DevicePreference":0,"EditorialStatus":"Active","FinalAppUrls":null,"FinalMobileUrls":null,"FinalUrlSuffix":null,"FinalUrls":{"string":["https://airbyte.com"]},"ForwardCompatibilityMap":null,"Id":84800390693061,"Status":"Active","TrackingUrlTemplate":null,"Type":"ResponsiveSearch","UrlCustomParameters":null,"Descriptions":{"AssetLink":[{"Asset":{"Id":10239363892977,"Name":null,"Type":"TextAsset","Text":"Connect, integrate, and sync data seamlessly with Airbyte's 800+ contributors and growing!"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363892976,"Name":null,"Type":"TextAsset","Text":"Move data like a pro with our powerful tool trusted by 40,000+ engineers worldwide!"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null}]},"Domain":"airbyte.com","Headlines":{"AssetLink":[{"Asset":{"Id":10239363892979,"Name":null,"Type":"TextAsset","Text":"Get synced with Airbyte"},"AssetPerformanceLabel":"Good","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363893384,"Name":null,"Type":"TextAsset","Text":"Data management made easy"},"AssetPerformanceLabel":"Low","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363892978,"Name":null,"Type":"TextAsset","Text":"Connectors for every need"},"AssetPerformanceLabel":"Best","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363892980,"Name":null,"Type":"TextAsset","Text":"Industry-leading connectors"},"AssetPerformanceLabel":"Best","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363893383,"Name":null,"Type":"TextAsset","Text":"Try Airbyte now for free"},"AssetPerformanceLabel":"Good","EditorialStatus":"Active","PinnedField":null}]},"Path1":null,"Path2":null,"AdGroupId":1356799861840328,"AccountId":180519267,"CustomerId":251186883},"emitted_at":1699913377498} +{"stream":"campaigns","data":{"AudienceAdsBidAdjustment":0,"BiddingScheme":{"Type":"EnhancedCpc"},"BudgetType":"DailyBudgetStandard","DailyBudget":2.0,"ExperimentId":null,"FinalUrlSuffix":null,"ForwardCompatibilityMap":null,"Id":531016227,"MultimediaAdsBidAdjustment":40,"Name":"Airbyte test","Status":"Active","SubType":null,"TimeZone":"CentralTimeUSCanada","TrackingUrlTemplate":null,"UrlCustomParameters":null,"CampaignType":"Search","Settings":{"Setting":[{"Type":"TargetSetting","Details":{"TargetSettingDetail":[{"CriterionTypeGroup":"Audience","TargetAndBid":false}]}}]},"BudgetId":null,"Languages":{"string":["English"]},"AdScheduleUseSearcherTimeZone":false,"AccountId":180519267,"CustomerId":251186883},"emitted_at":1699913381852} +{"stream":"accounts","data":{"BillToCustomerId":251186883,"CurrencyCode":"USD","AccountFinancialStatus":"ClearFinancialStatus","Id":180535609,"Language":"English","LastModifiedByUserId":0,"LastModifiedTime":"2023-08-11T08:24:26.603000","Name":"DEMO-ACCOUNT","Number":"F149W3B6","ParentCustomerId":251186883,"PaymentMethodId":null,"PaymentMethodType":null,"PrimaryUserId":138225488,"AccountLifeCycleStatus":"Pause","TimeStamp":"AAAAAH10c1A=","TimeZone":"Santiago","PauseReason":2,"ForwardCompatibilityMap":null,"LinkedAgencies":null,"SalesHouseCustomerId":null,"TaxInformation":null,"BackUpPaymentInstrumentId":null,"BillingThresholdAmount":null,"BusinessAddress":{"City":"San Francisco","CountryCode":"US","Id":149694999,"Line1":"350 29th avenue","Line2":null,"Line3":null,"Line4":null,"PostalCode":"94121","StateOrProvince":"CA","TimeStamp":null,"BusinessName":"Daxtarity Inc."},"AutoTagType":"Inactive","SoldToPaymentInstrumentId":null,"AccountMode":"Expert"},"emitted_at":1699913384475} +{"stream":"account_performance_report_daily","data":{"AccountId":180519267,"TimePeriod":"2023-11-07","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Syndicated search partners","DeliveredMatchType":"Phrase","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","AccountNumber":"F149MJ18","PhoneImpressions":0,"PhoneCalls":0,"Clicks":1,"Ctr":16.67,"Spend":0.33,"Impressions":6,"CostPerConversion":null,"Ptr":null,"Assists":0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"AverageCpc":0.33,"AveragePosition":0.0,"AverageCpm":55.0,"Conversions":0.0,"ConversionRate":0.0,"ConversionsQualified":0.0,"LowQualityClicks":0,"LowQualityClicksPercent":0.0,"LowQualityImpressions":0,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1699953356703} +{"stream":"account_performance_report_weekly","data":{"AccountId":180519267,"TimePeriod":"2023-11-05","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Syndicated search partners","DeliveredMatchType":"Exact","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","AccountNumber":"F149MJ18","PhoneImpressions":0,"PhoneCalls":0,"Clicks":1,"Ctr":8.33,"Spend":0.03,"Impressions":12,"CostPerConversion":null,"Ptr":null,"Assists":0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"AverageCpc":0.03,"AveragePosition":0.0,"AverageCpm":2.5,"Conversions":0.0,"ConversionRate":0.0,"ConversionsQualified":0.0,"LowQualityClicks":0,"LowQualityClicksPercent":0.0,"LowQualityImpressions":0,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1699953382081} +{"stream":"ad_group_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"TimePeriod":"2023-11-07","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Microsoft sites and select traffic","DeliveredMatchType":"Phrase","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - other","BidMatchType":"Broad","Language":"English","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","AdGroupName":"keywords","AdGroupType":"Standard","Impressions":121,"Clicks":1,"Ctr":0.83,"Spend":0.66,"CostPerConversion":null,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"CostPerAssist":null,"CustomParameters":null,"FinalUrlSuffix":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllConversions":0,"AllConversionRate":0.0,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.66,"AveragePosition":0.0,"AverageCpm":5.45,"Conversions":0.0,"ConversionRate":0.0,"ConversionsQualified":0.0,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"HistoricalQualityScore":null,"HistoricalExpectedCtr":null,"HistoricalAdRelevance":null,"HistoricalLandingPageExperience":null},"emitted_at":1699953471148} +{"stream":"ad_group_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"TimePeriod":"2023-11-05","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Syndicated search partners","DeliveredMatchType":"Exact","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","Language":"English","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","AdGroupName":"keywords","AdGroupType":"Standard","Impressions":3,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"CostPerAssist":null,"CustomParameters":null,"FinalUrlSuffix":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"HistoricalQualityScore":5.0,"HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":1.0},"emitted_at":1699953500072} +{"stream":"ad_group_impression_performance_report_daily","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-11-07","Status":"Active","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":1,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"DeviceType":"Computer","Language":"German","ImpressionSharePercent":null,"ImpressionLostToBudgetPercent":null,"ImpressionLostToRankAggPercent":null,"QualityScore":5,"ExpectedCtr":2.0,"AdRelevance":3,"LandingPageExperience":1,"HistoricalQualityScore":null,"HistoricalExpectedCtr":null,"HistoricalAdRelevance":null,"HistoricalLandingPageExperience":null,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","CampaignStatus":"Active","AdGroupLabels":null,"ExactMatchImpressionSharePercent":null,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":null,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":null,"TopImpressionShareLostToBudgetPercent":null,"AbsoluteTopImpressionShareLostToRankPercent":null,"AbsoluteTopImpressionShareLostToBudgetPercent":null,"TopImpressionSharePercent":null,"AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"BaseCampaignId":531016227,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AdGroupType":"Standard","AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1699955374204} +{"stream":"ad_group_impression_performance_report_weekly","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-11-05","Status":"Active","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":3,"Clicks":1,"Ctr":33.33,"AverageCpc":0.08,"Spend":0.08,"AveragePosition":0.0,"Conversions":0,"ConversionRate":0.0,"CostPerConversion":null,"DeviceType":"Computer","Language":"Danish","ImpressionSharePercent":11.11,"ImpressionLostToBudgetPercent":0.0,"ImpressionLostToRankAggPercent":88.89,"QualityScore":5,"ExpectedCtr":2.0,"AdRelevance":3,"LandingPageExperience":1,"HistoricalQualityScore":5,"HistoricalExpectedCtr":2,"HistoricalAdRelevance":3,"HistoricalLandingPageExperience":1,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Microsoft sites and select traffic","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","CampaignStatus":"Active","AdGroupLabels":null,"ExactMatchImpressionSharePercent":null,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":null,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":null,"TopImpressionShareLostToBudgetPercent":null,"AbsoluteTopImpressionShareLostToRankPercent":null,"AbsoluteTopImpressionShareLostToBudgetPercent":null,"TopImpressionSharePercent":null,"AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"BaseCampaignId":531016227,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":0.0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AdGroupType":"Standard","AverageCpm":26.67,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1699955407065} +{"stream":"ad_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"AdId":84800390693061,"TimePeriod":"2023-11-07","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Language":"German","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","DeliveredMatchType":"Phrase","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","AdGroupName":"keywords","Impressions":2,"Clicks":1,"Ctr":50.0,"Spend":0.79,"CostPerConversion":null,"DestinationUrl":null,"Assists":0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"AdDescription":null,"AdDescription2":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"Conversions":0.0,"ConversionRate":0.0,"ConversionsQualified":0.0,"AverageCpc":0.79,"AveragePosition":0.0,"AverageCpm":395.0,"AllConversions":0,"AllConversionRate":0.0,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1699953597299} +{"stream":"ad_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"AdId":84800390693061,"TimePeriod":"2023-11-05","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Language":"Danish","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","DeliveredMatchType":"Phrase","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","AdGroupName":"keywords","Impressions":3,"Clicks":1,"Ctr":33.33,"Spend":0.08,"CostPerConversion":null,"DestinationUrl":null,"Assists":0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"AdDescription":null,"AdDescription2":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"Conversions":0.0,"ConversionRate":0.0,"ConversionsQualified":0.0,"AverageCpc":0.08,"AveragePosition":0.0,"AverageCpm":26.67,"AllConversions":0,"AllConversionRate":0.0,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1699953623795} +{"stream":"budget_summary_report","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"CampaignName":"Airbyte test","CampaignId":531016227,"Date":"11/7/2023","MonthlyBudget":48.0,"DailySpend":3.12,"MonthToDateSpend":3.12},"emitted_at":1699953978176} +{"stream":"campaign_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"TimePeriod":"2023-11-07","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Syndicated search partners","DeliveredMatchType":"Phrase","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Other","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","CampaignStatus":"Active","CampaignLabels":null,"Impressions":0,"Clicks":0,"Ctr":null,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"AdRelevance":3.0,"LandingPageExperience":1.0,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":1,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"BudgetName":null,"BudgetStatus":null,"BudgetAssociationStatus":"Current","HistoricalQualityScore":null,"HistoricalExpectedCtr":null,"HistoricalAdRelevance":null,"HistoricalLandingPageExperience":null},"emitted_at":1699954051067} +{"stream":"campaign_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"TimePeriod":"2023-11-05","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Syndicated search partners","DeliveredMatchType":"Exact","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","CampaignStatus":"Active","CampaignLabels":null,"Impressions":9,"Clicks":1,"Ctr":11.11,"Spend":0.03,"CostPerConversion":null,"QualityScore":5.0,"AdRelevance":3.0,"LandingPageExperience":1.0,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"CustomParameters":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.03,"AveragePosition":0.0,"AverageCpm":3.33,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"LowQualityClicks":0,"LowQualityClicksPercent":0.0,"LowQualityImpressions":0,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"BudgetName":null,"BudgetStatus":null,"BudgetAssociationStatus":"Current","HistoricalQualityScore":5.0,"HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":1.0},"emitted_at":1699954081143} +{"stream":"campaign_impression_performance_report_daily","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-11-07","CampaignStatus":"Active","CampaignName":"Airbyte test","CampaignId":531016227,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":10,"Clicks":1,"Ctr":10.0,"AverageCpc":0.33,"Spend":0.33,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"LowQualityClicks":0,"LowQualityClicksPercent":0.0,"LowQualityImpressions":9,"LowQualityImpressionsPercent":47.37,"LowQualityConversions":0,"LowQualityConversionRate":null,"DeviceType":"Computer","ImpressionSharePercent":3.37,"ImpressionLostToBudgetPercent":85.19,"ImpressionLostToRankAggPercent":11.45,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"HistoricalQualityScore":null,"HistoricalExpectedCtr":null,"HistoricalAdRelevance":null,"HistoricalLandingPageExperience":null,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":0,"CampaignLabels":null,"ExactMatchImpressionSharePercent":null,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":6.02,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":14.63,"TopImpressionShareLostToBudgetPercent":77.24,"AbsoluteTopImpressionShareLostToRankPercent":15.66,"AbsoluteTopImpressionShareLostToBudgetPercent":78.31,"TopImpressionSharePercent":8.13,"AbsoluteTopImpressionRatePercent":50.0,"TopImpressionRatePercent":100.0,"BaseCampaignId":531016227,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AverageCpm":33.0,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1699954182626} +{"stream":"campaign_impression_performance_report_weekly","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-11-05","CampaignStatus":"Active","CampaignName":"Airbyte test","CampaignId":531016227,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":10,"Clicks":1,"Ctr":10.0,"AverageCpc":0.33,"Spend":0.33,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"LowQualityClicks":0,"LowQualityClicksPercent":0.0,"LowQualityImpressions":9,"LowQualityImpressionsPercent":47.37,"LowQualityConversions":0,"LowQualityConversionRate":null,"DeviceType":"Computer","ImpressionSharePercent":10.87,"ImpressionLostToBudgetPercent":17.05,"ImpressionLostToRankAggPercent":72.08,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"HistoricalQualityScore":null,"HistoricalExpectedCtr":null,"HistoricalAdRelevance":null,"HistoricalLandingPageExperience":null,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":0,"CampaignLabels":null,"ExactMatchImpressionSharePercent":29.07,"ClickSharePercent":2.89,"AbsoluteTopImpressionSharePercent":8.88,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":76.51,"TopImpressionShareLostToBudgetPercent":9.99,"AbsoluteTopImpressionShareLostToRankPercent":81.99,"AbsoluteTopImpressionShareLostToBudgetPercent":9.13,"TopImpressionSharePercent":13.5,"AbsoluteTopImpressionRatePercent":50.0,"TopImpressionRatePercent":100.0,"BaseCampaignId":531016227,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AverageCpm":33.0,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1699954211223} +{"stream":"keyword_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"KeywordId":84801135055365,"AdId":84800390693061,"TimePeriod":"2023-11-07","CurrencyCode":"USD","DeliveredMatchType":"Phrase","AdDistribution":"Search","DeviceType":"Computer","Language":"German","Network":"Syndicated search partners","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","Keyword":"connector","KeywordStatus":"Active","Impressions":1,"Clicks":0,"Ctr":0.0,"CurrentMaxCpc":2.27,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"QualityImpact":0.0,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"Mainline1Bid":null,"MainlineBid":1.2,"FirstPageBid":0.56,"FinalUrlSuffix":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"HistoricalQualityScore":null,"HistoricalExpectedCtr":null,"HistoricalAdRelevance":null,"HistoricalLandingPageExperience":null},"emitted_at":1699954312976} +{"stream":"keyword_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"KeywordId":84801135055365,"AdId":84800390693061,"TimePeriod":"2023-11-05","CurrencyCode":"USD","DeliveredMatchType":"Exact","AdDistribution":"Search","DeviceType":"Computer","Language":"English","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","Keyword":"connector","KeywordStatus":"Active","Impressions":2,"Clicks":0,"Ctr":0.0,"CurrentMaxCpc":2.27,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"QualityImpact":0.0,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"Mainline1Bid":null,"MainlineBid":1.2,"FirstPageBid":0.56,"FinalUrlSuffix":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1699954348243} +{"stream":"geographic_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"TimePeriod":"2023-11-07","Country":"Australia","CurrencyCode":"USD","DeliveredMatchType":"Broad","AdDistribution":"Search","DeviceType":"Computer","Language":"English","Network":"Syndicated search partners","DeviceOS":"Windows","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","MetroArea":null,"State":"New South Wales","City":null,"AdGroupName":"keywords","Ctr":0.0,"ProximityTargetLocation":null,"Radius":"0","Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"LocationType":"Physical location","MostSpecificLocation":"2000","AccountStatus":"Active","CampaignStatus":"Active","AdGroupStatus":"Active","County":null,"PostalCode":"2000","LocationId":"122395","BaseCampaignId":"531016227","AllCostPerConversion":null,"AllReturnOnAdSpend":null,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":"100.00","AllConversionsQualified":"0.00","ViewThroughConversionsQualified":null,"Neighborhood":null,"ViewThroughRevenue":"0.00","CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null,"AssetGroupStatus":null,"Clicks":0,"Spend":0.0,"Impressions":1,"CostPerConversion":null,"AccountName":"Airbyte","AccountNumber":"F149MJ18","CampaignName":"Airbyte test","Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1699956863587} +{"stream":"geographic_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"TimePeriod":"2023-11-05","Country":"Argentina","CurrencyCode":"USD","DeliveredMatchType":"Exact","AdDistribution":"Search","DeviceType":"Computer","Language":"Spanish","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","MetroArea":null,"State":"Buenos Aires Province","City":null,"AdGroupName":"keywords","Ctr":0.0,"ProximityTargetLocation":null,"Radius":"0","Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"LocationType":"Physical location","MostSpecificLocation":"Buenos Aires Province","AccountStatus":"Active","CampaignStatus":"Active","AdGroupStatus":"Active","County":null,"PostalCode":null,"LocationId":"141965","BaseCampaignId":"531016227","AllCostPerConversion":null,"AllReturnOnAdSpend":null,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":"100.00","AllConversionsQualified":"0.00","ViewThroughConversionsQualified":null,"Neighborhood":null,"ViewThroughRevenue":"0.00","CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null,"AssetGroupStatus":null,"Clicks":0,"Spend":0.0,"Impressions":1,"CostPerConversion":null,"AccountName":"Airbyte","AccountNumber":"F149MJ18","CampaignName":"Airbyte test","Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1699953673210} +{"stream":"age_gender_audience_report_daily","data":{"AccountId":180519267,"AgeGroup":"Unknown","Gender":"Unknown","TimePeriod":"2023-11-07","AllConversions":0,"AccountName":"Airbyte","AccountNumber":"F149MJ18","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"AdDistribution":"Search","Impressions":3,"Clicks":1,"Conversions":0.0,"Spend":0.79,"Revenue":0.0,"ExtendedCost":0.0,"Assists":0,"Language":"German","AccountStatus":"Active","CampaignStatus":"Active","AdGroupStatus":"Active","BaseCampaignId":"531016227","AllRevenue":0.0,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":33.33,"TopImpressionRatePercent":100.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0},"emitted_at":1699954406862} +{"stream":"age_gender_audience_report_weekly","data":{"AccountId":180519267,"AgeGroup":"18-24","Gender":"Unknown","TimePeriod":"2023-11-05","AllConversions":0,"AccountName":"Airbyte","AccountNumber":"F149MJ18","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"AdDistribution":"Search","Impressions":1,"Clicks":0,"Conversions":0.0,"Spend":0.0,"Revenue":0.0,"ExtendedCost":0.0,"Assists":0,"Language":"English","AccountStatus":"Active","CampaignStatus":"Active","AdGroupStatus":"Active","BaseCampaignId":"531016227","AllRevenue":0.0,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0},"emitted_at":1699954427029} +{"stream":"search_query_performance_report_daily","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-11-07","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"AdId":84800390693061,"AdType":"Responsive search ad","DestinationUrl":null,"BidMatchType":"Broad","DeliveredMatchType":"Phrase","CampaignStatus":"Active","AdStatus":"Active","Impressions":1,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"SearchQuery":"16 pin dlc connector","Keyword":"connector","AdGroupCriterionId":null,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"Language":"English","KeywordId":84801135055365,"Network":"Microsoft sites and select traffic","TopVsOther":"Microsoft sites and select traffic - top","DeviceType":"Computer","DeviceOS":"Windows","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","AdGroupStatus":"Active","KeywordStatus":"Active","CampaignType":"Search & content","CustomerId":251186883,"CustomerName":"Daxtarity Inc.","AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":100.0,"AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0},"emitted_at":1699954517804} +{"stream":"search_query_performance_report_weekly","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-11-05","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"AdId":84800390693061,"AdType":"Responsive search ad","DestinationUrl":null,"BidMatchType":"Broad","DeliveredMatchType":"Exact","CampaignStatus":"Active","AdStatus":"Active","Impressions":3,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"SearchQuery":"airbyte","Keyword":"Airbyte","AdGroupCriterionId":null,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"Language":"English","KeywordId":84801135055370,"Network":"Syndicated search partners","TopVsOther":"Syndicated search partners - Top","DeviceType":"Computer","DeviceOS":"Unknown","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","AdGroupStatus":"Active","KeywordStatus":"Active","CampaignType":"Search & content","CustomerId":251186883,"CustomerName":"Daxtarity Inc.","AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0},"emitted_at":1699954546030} +{"stream":"user_location_performance_report_daily","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-11-07","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"Country":"Australia","State":"New South Wales","MetroArea":null,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":1,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"ProximityTargetLocation":null,"Radius":0,"Language":"English","City":null,"QueryIntentCountry":"Australia","QueryIntentState":null,"QueryIntentCity":null,"QueryIntentDMA":null,"BidMatchType":"Broad","DeliveredMatchType":"Broad","Network":"Syndicated search partners","TopVsOther":"Syndicated search partners - Top","DeviceType":"Computer","DeviceOS":"Windows","Assists":0,"Conversions":0,"ConversionRate":null,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerConversion":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"County":null,"PostalCode":"2000","QueryIntentCounty":null,"QueryIntentPostalCode":null,"LocationId":122395,"QueryIntentLocationId":9,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":100.0,"AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"Neighborhood":null,"QueryIntentNeighborhood":null,"ViewThroughRevenue":0.0,"CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null},"emitted_at":1699954599053} +{"stream":"user_location_performance_report_weekly","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-11-05","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"Country":"Argentina","State":null,"MetroArea":null,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":1,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"ProximityTargetLocation":null,"Radius":0,"Language":"Spanish","City":null,"QueryIntentCountry":"Argentina","QueryIntentState":null,"QueryIntentCity":null,"QueryIntentDMA":null,"BidMatchType":"Broad","DeliveredMatchType":"Phrase","Network":"Syndicated search partners","TopVsOther":"Syndicated search partners - Top","DeviceType":"Computer","DeviceOS":"Unknown","Assists":0,"Conversions":0,"ConversionRate":null,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerConversion":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"County":null,"PostalCode":null,"QueryIntentCounty":null,"QueryIntentPostalCode":null,"LocationId":8,"QueryIntentLocationId":8,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"Neighborhood":null,"QueryIntentNeighborhood":null,"ViewThroughRevenue":0.0,"CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null},"emitted_at":1699954842196} +{"stream":"account_impression_performance_report_daily","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-11-07","CurrencyCode":"USD","AdDistribution":"Search","Impressions":10,"Clicks":1,"Ctr":10.0,"AverageCpc":0.33,"Spend":0.33,"AveragePosition":0.0,"Conversions":0,"ConversionRate":0.0,"CostPerConversion":null,"LowQualityClicks":0,"LowQualityClicksPercent":0.0,"LowQualityImpressions":9,"LowQualityImpressionsPercent":47.37,"LowQualityConversions":0,"LowQualityConversionRate":null,"DeviceType":"Computer","ImpressionSharePercent":3.37,"ImpressionLostToBudgetPercent":85.19,"ImpressionLostToRankAggPercent":11.45,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":0,"ExactMatchImpressionSharePercent":null,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":6.02,"TopImpressionShareLostToRankPercent":14.63,"TopImpressionShareLostToBudgetPercent":77.24,"AbsoluteTopImpressionShareLostToRankPercent":15.66,"AbsoluteTopImpressionShareLostToBudgetPercent":78.31,"TopImpressionSharePercent":8.13,"AbsoluteTopImpressionRatePercent":50.0,"TopImpressionRatePercent":100.0,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":0.0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"AverageCpm":33.0,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1699955144374} +{"stream":"account_impression_performance_report_weekly","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-11-05","CurrencyCode":"USD","AdDistribution":"Search","Impressions":457,"Clicks":5,"Ctr":1.09,"AverageCpc":0.38,"Spend":1.88,"AveragePosition":0.0,"Conversions":0,"ConversionRate":0.0,"CostPerConversion":null,"LowQualityClicks":6,"LowQualityClicksPercent":54.55,"LowQualityImpressions":183,"LowQualityImpressionsPercent":28.59,"LowQualityConversions":0,"LowQualityConversionRate":0.0,"DeviceType":"Computer","ImpressionSharePercent":10.87,"ImpressionLostToBudgetPercent":17.05,"ImpressionLostToRankAggPercent":72.08,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","LowQualityGeneralClicks":4,"LowQualitySophisticatedClicks":2,"ExactMatchImpressionSharePercent":29.07,"ClickSharePercent":2.89,"AbsoluteTopImpressionSharePercent":8.88,"TopImpressionShareLostToRankPercent":76.51,"TopImpressionShareLostToBudgetPercent":9.99,"AbsoluteTopImpressionShareLostToRankPercent":81.99,"AbsoluteTopImpressionShareLostToBudgetPercent":9.13,"TopImpressionSharePercent":13.5,"AbsoluteTopImpressionRatePercent":39.61,"TopImpressionRatePercent":81.62,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":0.0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"AverageCpm":4.11,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1699955205307} {"stream":"labels","data":{"Status":"Active","Id":10239203506495,"Client Id":null,"Modified Time":"04/27/2023 17:16:49.170","Description":null,"Label":"what a new label","Color":"#B0B20F","Account Id":180278106},"emitted_at":1698687185070} {"stream":"campaign_labels","data":{"Status":"Active","Id":10239203506494,"Parent Id":413444833,"Campaign":null,"Client Id":null,"Modified Time":"04/27/2023 17:57:38.110","Account Id":180278106},"emitted_at":1698687214960} {"stream":"keyword_labels","data":{"Status":"Active","Id":10239203506495,"Parent Id":84868925026027,"Client Id":null,"Modified Time":"04/27/2023 17:22:52.733","Account Id":180278106},"emitted_at":1698687224964} diff --git a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml index 10fa87f954ae..4200d85c8cd9 100644 --- a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml @@ -16,7 +16,7 @@ data: connectorSubtype: api connectorType: source definitionId: 47f25999-dd5e-4636-8c39-e7cea2453331 - dockerImageTag: 1.12.0 + dockerImageTag: 1.12.1 dockerRepository: airbyte/source-bing-ads documentationUrl: https://docs.airbyte.com/integrations/sources/bing-ads githubIssueLabel: source-bing-ads diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/streams.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/streams.py index 8194aeec6292..342a2c827ddd 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/streams.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/streams.py @@ -190,8 +190,7 @@ def parse_response(self, response: sudsobject.Object, **kwargs: Mapping[str, Any yield from [] - @staticmethod - def get_column_value(row: _RowReportRecord, column: str) -> Union[str, None, int, float]: + def get_column_value(self, row: _RowReportRecord, column: str) -> Union[str, None, int, float]: """ Reads field value from row and transforms: 1. empty values to logical None @@ -202,7 +201,8 @@ def get_column_value(row: _RowReportRecord, column: str) -> Union[str, None, int return None if "%" in value: value = value.replace("%", "") - + if value and set(self.get_json_schema()["properties"].get(column, {}).get("type")) & {"integer", "number"}: + value = value.replace(",", "") return value diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py index 40dc8b852b0c..3c7ecc853430 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py @@ -42,18 +42,27 @@ def __init__(self) -> None: def test_get_column_value(): + config = { + "developer_token": "developer_token", + "client_id": "client_id", + "refresh_token": "refresh_token", + "reports_start_date": "2020-01-01T00:00:00Z", + } + test_report = GeographicPerformanceReportDaily(client=Mock(), config=config) + row_values = _RowValues( - {"AccountId": 1, "AverageCpc": 3, "AdGroupId": 2, "AccountName": 5, "Spend": 4}, - {3: "11.5", 1: "33", 2: "--", 5: "123456789", 4: "120.3%"}, + {"AccountId": 1, "AverageCpc": 3, "AdGroupId": 2, "AccountName": 5, "Spend": 4, "AllRevenue": 6, "Assists": 7}, + {3: "11.5", 1: "33", 2: "--", 5: "123456789", 4: "120.3%", 6: "123,456,789.23", 7: "123,456,789"}, ) record = _RowReportRecord(row_values) - test_report = TestReport() assert test_report.get_column_value(record, "AccountId") == "33" assert test_report.get_column_value(record, "AverageCpc") == "11.5" assert test_report.get_column_value(record, "AdGroupId") is None assert test_report.get_column_value(record, "AccountName") == "123456789" assert test_report.get_column_value(record, "Spend") == "120.3" + assert test_report.get_column_value(record, "AllRevenue") == "123456789.23" + assert test_report.get_column_value(record, "Assists") == "123456789" def test_get_updated_state_init_state(): diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md index a269d461261a..7f4d3212ef88 100644 --- a/docs/integrations/sources/bing-ads.md +++ b/docs/integrations/sources/bing-ads.md @@ -168,6 +168,7 @@ The Bing Ads API limits the number of requests for all Microsoft Advertising cli | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------| +| 1.12.1 | 2023-11-10 | [32422](https://github.com/airbytehq/airbyte/pull/32422) | Normalize numeric values in reports | | 1.12.0 | 2023-11-09 | [32340](https://github.com/airbytehq/airbyte/pull/32340) | Remove default start date in favor of Time Period - Last Year and This Year, if start date is not provided | | 1.11.0 | 2023-11-06 | [32201](https://github.com/airbytehq/airbyte/pull/32201) | Skip broken CSV report files | | 1.10.0 | 2023-11-06 | [32148](https://github.com/airbytehq/airbyte/pull/32148) | Add new fields to stream Ads: "BusinessName", "CallToAction", "Headline", "Images", "Videos", "Text" | From 35862d9477bbd34e65828dc0ce9e097faecb6412 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 14 Nov 2023 16:49:20 +0100 Subject: [PATCH 036/124] Google Drive: Fix shared drives (#32429) Co-authored-by: flash1293 --- .../acceptance-test-config.yml | 8 +++- .../integration_tests/abnormal_state.json | 26 ++++++------ .../integration_tests/expected_records.jsonl | 18 ++++---- .../integration_tests/spec.json | 41 ++++++++++++++----- .../source-google-drive/metadata.yaml | 2 +- .../connectors/source-google-drive/setup.py | 2 +- .../source_google_drive/spec.py | 7 +++- .../source_google_drive/stream_reader.py | 7 +++- docs/integrations/sources/google-drive.md | 1 + 9 files changed, 74 insertions(+), 38 deletions(-) diff --git a/airbyte-integrations/connectors/source-google-drive/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-drive/acceptance-test-config.yml index 3ed802d16e22..5714d50eb620 100644 --- a/airbyte-integrations/connectors/source-google-drive/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-drive/acceptance-test-config.yml @@ -4,7 +4,13 @@ acceptance_tests: - config_path: secrets/config.json expect_records: path: integration_tests/expected_records.jsonl - exact_order: true + exact_order: false + timeout_seconds: 1800 + expect_trace_message_on_failure: false + - config_path: secrets/oauth_config.json + expect_records: + path: integration_tests/expected_records.jsonl + exact_order: false timeout_seconds: 1800 expect_trace_message_on_failure: false diff --git a/airbyte-integrations/connectors/source-google-drive/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-google-drive/integration_tests/abnormal_state.json index 8a7cc94f2bbb..49a1209da298 100644 --- a/airbyte-integrations/connectors/source-google-drive/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-google-drive/integration_tests/abnormal_state.json @@ -2,15 +2,15 @@ { "type": "STREAM", "stream": { + "stream_descriptor": { + "name": "test" + }, "stream_state": { "history": { - "test.jsonl": "2023-10-16T15:16:07.574000Z", - "subfolder/test2.jsonl": "2023-10-19T10:43:57.000000Z" + "test.jsonl": "2023-10-16T06:16:06.000000Z", + "subfolder/test2.jsonl": "2023-10-19T01:43:56.000000Z" }, - "_ab_source_file_last_modified": "2023-10-19T10:43:57.000000Z_subfolder/test2.jsonl" - }, - "stream_descriptor": { - "name": "test" + "_ab_source_file_last_modified": "2023-10-19T01:43:56.000000Z_subfolder/test2.jsonl" } } }, @@ -21,14 +21,14 @@ "name": "test_unstructured" }, "stream_state": { - "_ab_source_file_last_modified": "2023-10-27T10:26:09.076000Z_testdoc_presentation.pdf", "history": { - "testdoc_google": "2023-10-27T09:27:54.134000Z", - "testdoc_docx.docx": "2023-10-27T09:45:54.919000Z", - "testdoc_pdf.pdf": "2023-10-27T09:45:59.945000Z", - "testdoc_ocr_pdf.pdf": "2023-10-27T09:46:05.349000Z", - "testdoc_presentation": "2023-10-27T10:26:09.076000Z" - } + "testdoc_docx.docx": "2023-10-27T00:45:54.000000Z", + "testdoc_pdf.pdf": "2023-10-27T00:45:58.000000Z", + "testdoc_ocr_pdf.pdf": "2023-10-27T00:46:04.000000Z", + "testdoc_google": "2023-11-10T13:46:18.551000Z", + "testdoc_presentation": "2023-11-10T13:49:06.640000Z" + }, + "_ab_source_file_last_modified": "2023-11-10T13:49:06.640000Z_testdoc_presentation" } } } diff --git a/airbyte-integrations/connectors/source-google-drive/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-google-drive/integration_tests/expected_records.jsonl index 77e061a23dfd..e3df40fb7e17 100644 --- a/airbyte-integrations/connectors/source-google-drive/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-google-drive/integration_tests/expected_records.jsonl @@ -1,9 +1,9 @@ -{"stream": "test", "data": {"x": 999, "_ab_source_file_last_modified": "2023-10-16T15:16:07.574000Z", "_ab_source_file_url": "test.jsonl"}, "emitted_at": 162727468000} -{"stream": "test", "data": {"x": 9999, "_ab_source_file_last_modified": "2023-10-16T15:16:07.574000Z", "_ab_source_file_url": "test.jsonl"}, "emitted_at": 162727468000} -{"stream": "test", "data": {"y": 9999, "_ab_source_file_last_modified": "2023-10-19T10:43:57.000000Z", "_ab_source_file_url": "subfolder/test2.jsonl"}, "emitted_at": 162727468000} -{"stream": "test", "data": {"y": 123, "_ab_source_file_last_modified": "2023-10-19T10:43:57.000000Z", "_ab_source_file_url": "subfolder/test2.jsonl"}, "emitted_at": 162727468000} -{"stream": "test_unstructured", "data": {"content": "# Heading\n\nThis is the content which is not just a single word", "document_key": "testdoc_google", "_ab_source_file_last_modified": "2023-10-27T09:27:54.134000Z", "_ab_source_file_url": "testdoc_google"}, "emitted_at": 1698400261074} -{"stream": "test_unstructured", "data": {"content": "# Heading\n\nThis is the content which is not just a single word", "document_key": "testdoc_docx.docx", "_ab_source_file_last_modified": "2023-10-27T09:45:54.919000Z", "_ab_source_file_url": "testdoc_docx.docx"}, "emitted_at": 1698400261867} -{"stream": "test_unstructured", "data": {"content": "# Heading\n\nThis is the content which is not just a single word", "document_key": "testdoc_pdf.pdf", "_ab_source_file_last_modified": "2023-10-27T09:45:59.945000Z", "_ab_source_file_url": "testdoc_pdf.pdf"}, "emitted_at": 1698400264556} -{"stream": "test_unstructured", "data": {"content": "This is a test", "document_key": "testdoc_ocr_pdf.pdf", "_ab_source_file_last_modified": "2023-10-27T09:46:05.349000Z", "_ab_source_file_url": "testdoc_ocr_pdf.pdf"}, "emitted_at": 1698400267184} -{"stream": "test_unstructured", "data": {"content": "This is a test", "document_key": "testdoc_presentation", "_ab_source_file_last_modified": "2023-10-27T10:26:09.076000Z", "_ab_source_file_url": "testdoc_presentation"}, "emitted_at": 1698402779268} \ No newline at end of file +{"stream": "test", "data": {"x": 999, "_ab_source_file_last_modified": "2023-10-16T06:16:06.000000Z", "_ab_source_file_url": "test.jsonl"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"x": 9999, "_ab_source_file_last_modified": "2023-10-16T06:16:06.000000Z", "_ab_source_file_url": "test.jsonl"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"y": 9999, "_ab_source_file_last_modified": "2023-10-19T01:43:56.000000Z", "_ab_source_file_url": "subfolder/test2.jsonl"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"y": 123, "_ab_source_file_last_modified": "2023-10-19T01:43:56.000000Z", "_ab_source_file_url": "subfolder/test2.jsonl"}, "emitted_at": 162727468000} +{"stream": "test_unstructured", "data": {"content": "# Heading\n\nThis is the content which is not just a single word", "document_key": "testdoc_docx.docx", "_ab_source_file_last_modified": "2023-10-27T00:45:54.000000Z", "_ab_source_file_url": "testdoc_docx.docx"}, "emitted_at": 1698400261867} +{"stream": "test_unstructured", "data": {"content": "# Heading\n\nThis is the content which is not just a single word", "document_key": "testdoc_pdf.pdf", "_ab_source_file_last_modified": "2023-10-27T00:45:58.000000Z", "_ab_source_file_url": "testdoc_pdf.pdf"}, "emitted_at": 1698400264556} +{"stream": "test_unstructured", "data": {"content": "This is a test", "document_key": "testdoc_ocr_pdf.pdf", "_ab_source_file_last_modified": "2023-10-27T00:46:04.000000Z", "_ab_source_file_url": "testdoc_ocr_pdf.pdf"}, "emitted_at": 1698400267184} +{"stream": "test_unstructured", "data": {"content": "# Heading\n\nThis is the content which is not just a single word", "document_key": "testdoc_google", "_ab_source_file_last_modified": "2023-11-10T13:46:18.551000Z", "_ab_source_file_url": "testdoc_google"}, "emitted_at": 1698400261074} +{"stream": "test_unstructured", "data": {"content": "This is a test", "document_key": "testdoc_presentation", "_ab_source_file_last_modified": "2023-11-10T13:49:06.640000Z", "_ab_source_file_url": "testdoc_presentation"}, "emitted_at": 1698402779268} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-google-drive/integration_tests/spec.json b/airbyte-integrations/connectors/source-google-drive/integration_tests/spec.json index 95ecc1a0b4b3..0b148260c015 100644 --- a/airbyte-integrations/connectors/source-google-drive/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-google-drive/integration_tests/spec.json @@ -31,6 +31,8 @@ }, "globs": { "title": "Globs", + "default": ["**"], + "order": 1, "description": "The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look here.", "type": "array", "items": { @@ -80,7 +82,8 @@ "default": false, "type": "boolean" } - } + }, + "required": ["filetype"] }, { "title": "CSV Format", @@ -166,7 +169,8 @@ "const": "From CSV", "type": "string" } - } + }, + "required": ["header_definition_type"] }, { "title": "Autogenerated", @@ -178,7 +182,8 @@ "const": "Autogenerated", "type": "string" } - } + }, + "required": ["header_definition_type"] }, { "title": "User Provided", @@ -199,7 +204,7 @@ } } }, - "required": ["column_names"] + "required": ["column_names", "header_definition_type"] } ], "type": "object" @@ -224,7 +229,8 @@ }, "uniqueItems": true } - } + }, + "required": ["filetype"] }, { "title": "Jsonl Format", @@ -236,7 +242,8 @@ "const": "jsonl", "type": "string" } - } + }, + "required": ["filetype"] }, { "title": "Parquet Format", @@ -254,7 +261,8 @@ "default": false, "type": "boolean" } - } + }, + "required": ["filetype"] }, { "title": "Document File Type Format (Experimental)", @@ -265,9 +273,17 @@ "default": "unstructured", "const": "unstructured", "type": "string" + }, + "skip_unprocessable_file_types": { + "type": "boolean", + "default": true, + "title": "Skip Unprocessable File Types", + "description": "If true, skip files that cannot be parsed because of their file type and log a warning. If false, fail the sync. Corrupted files with valid file types will still result in a failed sync.", + "always_show": true } }, - "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file." + "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file.", + "required": ["filetype"] } ] }, @@ -325,7 +341,12 @@ "type": "string" } }, - "required": ["client_id", "client_secret", "refresh_token"] + "required": [ + "client_id", + "client_secret", + "refresh_token", + "auth_type" + ] }, { "title": "Service Account Key Authentication", @@ -345,7 +366,7 @@ "type": "string" } }, - "required": ["service_account_info"] + "required": ["service_account_info", "auth_type"] } ] } diff --git a/airbyte-integrations/connectors/source-google-drive/metadata.yaml b/airbyte-integrations/connectors/source-google-drive/metadata.yaml index 4559f077b1e1..f7eb6998beff 100644 --- a/airbyte-integrations/connectors/source-google-drive/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-drive/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: file connectorType: source definitionId: 9f8dda77-1048-4368-815b-269bf54ee9b8 - dockerImageTag: 0.0.1 + dockerImageTag: 0.0.2 dockerRepository: airbyte/source-google-drive githubIssueLabel: source-google-drive icon: google-drive.svg diff --git a/airbyte-integrations/connectors/source-google-drive/setup.py b/airbyte-integrations/connectors/source-google-drive/setup.py index 5928a62fc48d..af0e32a9949e 100644 --- a/airbyte-integrations/connectors/source-google-drive/setup.py +++ b/airbyte-integrations/connectors/source-google-drive/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[file-based]>=0.52.9", + "airbyte-cdk[file-based]>=0.53.5", "google-api-python-client==2.104.0", "google-auth-httplib2==0.1.1", "google-auth-oauthlib==1.1.0", diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py index 47809289db15..7c2a60b27b82 100644 --- a/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py @@ -7,12 +7,14 @@ import dpath.util from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec +from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig from pydantic import BaseModel, Field class OAuthCredentials(BaseModel): - class Config: + class Config(OneOfOptionConfig): title = "Authenticate via Google (OAuth)" + discriminator = "auth_type" auth_type: Literal["Client"] = Field("Client", const=True) client_id: str = Field( @@ -33,8 +35,9 @@ class Config: class ServiceAccountCredentials(BaseModel): - class Config: + class Config(OneOfOptionConfig): title = "Service Account Key Authentication" + discriminator = "auth_type" auth_type: Literal["Service"] = Field("Service", const=True) service_account_info: str = Field( diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py index ae33a54ab015..2b6fa5f9cccc 100644 --- a/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py @@ -95,8 +95,13 @@ def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: lo while len(folder_id_queue) > 0: (path, folder_id) = folder_id_queue.pop() # fetch all files in this folder (1000 is the max page size) + # supportsAllDrives and includeItemsFromAllDrives are required to access files in shared drives request = service.files().list( - q=f"'{folder_id}' in parents", pageSize=1000, fields="nextPageToken, files(id, name, modifiedTime, mimeType)" + q=f"'{folder_id}' in parents", + pageSize=1000, + fields="nextPageToken, files(id, name, modifiedTime, mimeType)", + supportsAllDrives=True, + includeItemsFromAllDrives=True, ) while True: results = request.execute() diff --git a/docs/integrations/sources/google-drive.md b/docs/integrations/sources/google-drive.md index aedb783b2c7e..cff121b4e696 100644 --- a/docs/integrations/sources/google-drive.md +++ b/docs/integrations/sources/google-drive.md @@ -247,5 +247,6 @@ Before parsing each document, the connector exports Google Document files to Doc | Version | Date | Pull Request | Subject | |---------|------------|----------------------------------------------------------|-----------------------------------------------------------------------------------| +| 0.0.2 | 2023-11-02 | [31458](https://github.com/airbytehq/airbyte/pull/31458) | Allow syncs on shared drives | | 0.0.1 | 2023-11-02 | [31458](https://github.com/airbytehq/airbyte/pull/31458) | Initial Google Drive source | From b3396626ee8d1e2282217dc9239cde3556a754b3 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 14 Nov 2023 16:53:27 +0100 Subject: [PATCH 037/124] S3, Azure Blob Storage, GCS, Pinecone, Weaviate, Milvus, Chroma, Qdrant: Update CDK to improve spec generation (#32357) Co-authored-by: flash1293 --- .../connectors/destination-chroma/Dockerfile | 2 +- .../destination-chroma/metadata.yaml | 2 +- .../connectors/destination-chroma/setup.py | 2 +- .../destination_milvus/config.py | 18 ++++++----- .../integration_tests/spec.json | 23 +++++++------ .../destination-milvus/metadata.yaml | 2 +- .../connectors/destination-milvus/setup.py | 2 +- .../destination_pinecone/config.py | 2 +- .../integration_tests/spec.json | 22 ++++++------- .../destination-pinecone/metadata.yaml | 2 +- .../connectors/destination-pinecone/setup.py | 2 +- .../connectors/destination-qdrant/Dockerfile | 2 +- .../destination-qdrant/metadata.yaml | 2 +- .../connectors/destination-qdrant/setup.py | 2 +- .../destination_weaviate/config.py | 25 ++++++++------- .../integration_tests/spec.json | 22 ++++++++----- .../destination-weaviate/metadata.yaml | 2 +- .../connectors/destination-weaviate/setup.py | 2 +- .../integration_tests/spec.json | 32 ++++++++++++++----- .../source-azure-blob-storage/metadata.yaml | 2 +- .../source-azure-blob-storage/setup.py | 2 +- .../connectors/source-gcs/Dockerfile | 17 ---------- .../source-gcs/integration_tests/spec.json | 11 ++++--- .../connectors/source-gcs/metadata.yaml | 4 ++- .../connectors/source-gcs/setup.py | 6 ++-- .../integration_tests/cloud_spec.json | 30 +++++++++++++---- .../source-s3/integration_tests/spec.json | 32 ++++++++++++++----- .../connectors/source-s3/metadata.yaml | 2 +- .../connectors/source-s3/setup.py | 2 +- docs/integrations/destinations/chroma.md | 1 + docs/integrations/destinations/milvus.md | 1 + docs/integrations/destinations/pinecone.md | 1 + docs/integrations/destinations/qdrant.md | 1 + docs/integrations/destinations/weaviate.md | 1 + .../sources/azure-blob-storage.md | 2 ++ docs/integrations/sources/gcs.md | 1 + docs/integrations/sources/s3.md | 1 + 37 files changed, 169 insertions(+), 116 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-gcs/Dockerfile diff --git a/airbyte-integrations/connectors/destination-chroma/Dockerfile b/airbyte-integrations/connectors/destination-chroma/Dockerfile index 7c36bf9c5ee5..da9d91f9af9f 100644 --- a/airbyte-integrations/connectors/destination-chroma/Dockerfile +++ b/airbyte-integrations/connectors/destination-chroma/Dockerfile @@ -41,5 +41,5 @@ COPY destination_chroma ./destination_chroma ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.0.5 +LABEL io.airbyte.version=0.0.6 LABEL io.airbyte.name=airbyte/destination-chroma diff --git a/airbyte-integrations/connectors/destination-chroma/metadata.yaml b/airbyte-integrations/connectors/destination-chroma/metadata.yaml index b3b8aa6633a3..3e57dfca3ab6 100644 --- a/airbyte-integrations/connectors/destination-chroma/metadata.yaml +++ b/airbyte-integrations/connectors/destination-chroma/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: vectorstore connectorType: destination definitionId: 0b75218b-f702-4a28-85ac-34d3d84c0fc2 - dockerImageTag: 0.0.5 + dockerImageTag: 0.0.6 dockerRepository: airbyte/destination-chroma githubIssueLabel: destination-chroma icon: chroma.svg diff --git a/airbyte-integrations/connectors/destination-chroma/setup.py b/airbyte-integrations/connectors/destination-chroma/setup.py index e97865af7889..07c0cffe1f3d 100644 --- a/airbyte-integrations/connectors/destination-chroma/setup.py +++ b/airbyte-integrations/connectors/destination-chroma/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[vector-db-based]==0.51.41", + "airbyte-cdk[vector-db-based]==0.53.3", "chromadb", ] diff --git a/airbyte-integrations/connectors/destination-milvus/destination_milvus/config.py b/airbyte-integrations/connectors/destination-milvus/destination_milvus/config.py index 7aa9ed5a62a2..a79bb6ac5b94 100644 --- a/airbyte-integrations/connectors/destination-milvus/destination_milvus/config.py +++ b/airbyte-integrations/connectors/destination-milvus/destination_milvus/config.py @@ -14,6 +14,7 @@ OpenAIEmbeddingConfigModel, ProcessingConfigModel, ) +from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig from airbyte_cdk.utils.spec_schema_transformations import resolve_refs from pydantic import BaseModel, Field @@ -23,28 +24,29 @@ class UsernamePasswordAuth(BaseModel): username: str = Field(..., title="Username", description="Username for the Milvus instance", order=1) password: str = Field(..., title="Password", description="Password for the Milvus instance", airbyte_secret=True, order=2) - class Config: + class Config(OneOfOptionConfig): title = "Username/Password" - schema_extra = {"description": "Authenticate using username and password (suitable for self-managed Milvus clusters)"} + description = "Authenticate using username and password (suitable for self-managed Milvus clusters)" + discriminator = "mode" class NoAuth(BaseModel): mode: Literal["no_auth"] = Field("no_auth", const=True) - class Config: + class Config(OneOfOptionConfig): title = "No auth" - schema_extra = { - "description": "Do not authenticate (suitable for locally running test clusters, do not use for clusters with public IP addresses)" - } + description = "Do not authenticate (suitable for locally running test clusters, do not use for clusters with public IP addresses)" + discriminator = "mode" class TokenAuth(BaseModel): mode: Literal["token"] = Field("token", const=True) token: str = Field(..., title="API Token", description="API Token for the Milvus instance", airbyte_secret=True) - class Config: + class Config(OneOfOptionConfig): title = "API Token" - schema_extra = {"description": "Authenticate using an API token (suitable for Zilliz Cloud)"} + description = "Authenticate using an API token (suitable for Zilliz Cloud)" + discriminator = "mode" class MilvusIndexingConfigModel(BaseModel): diff --git a/airbyte-integrations/connectors/destination-milvus/integration_tests/spec.json b/airbyte-integrations/connectors/destination-milvus/integration_tests/spec.json index fc79a1c93e8b..a30902f4cae7 100644 --- a/airbyte-integrations/connectors/destination-milvus/integration_tests/spec.json +++ b/airbyte-integrations/connectors/destination-milvus/integration_tests/spec.json @@ -11,6 +11,7 @@ "chunk_size": { "title": "Chunk size", "description": "Size of chunks in tokens to store in vector store (make sure it is not too big for the context if your LLM)", + "minimum": 1, "maximum": 8191, "type": "integer" }, @@ -91,6 +92,7 @@ "type": "boolean" } }, + "required": ["mode"], "description": "Split the text by the list of separators until the chunk size is reached, using the earlier mentioned separators where possible. This is useful for splitting text fields by paragraphs, sentences, words, etc." }, { @@ -113,6 +115,7 @@ "type": "integer" } }, + "required": ["mode"], "description": "Split the text by Markdown headers down to the specified header level. If the chunk size fits multiple sections, they will be combined into a single chunk." }, { @@ -150,7 +153,7 @@ "type": "string" } }, - "required": ["language"], + "required": ["language", "mode"], "description": "Split the text by suitable delimiters based on the programming language. This is useful for splitting code into chunks." } ] @@ -182,7 +185,7 @@ "type": "string" } }, - "required": ["openai_key"], + "required": ["openai_key", "mode"], "description": "Use the OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." }, { @@ -202,7 +205,7 @@ "type": "string" } }, - "required": ["cohere_key"], + "required": ["cohere_key", "mode"], "description": "Use the Cohere API to embed text." }, { @@ -217,6 +220,7 @@ "type": "string" } }, + "required": ["mode"], "description": "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs." }, { @@ -243,7 +247,7 @@ "type": "integer" } }, - "required": ["field_name", "dimensions"], + "required": ["field_name", "dimensions", "mode"], "description": "Use a field in the record as the embedding. This is useful if you already have an embedding for your data and want to store it in the vector store." }, { @@ -276,7 +280,7 @@ "type": "string" } }, - "required": ["openai_key", "api_base", "deployment"], + "required": ["openai_key", "api_base", "deployment", "mode"], "description": "Use the Azure-hosted OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." }, { @@ -316,7 +320,7 @@ "type": "integer" } }, - "required": ["base_url", "dimensions"], + "required": ["base_url", "dimensions", "mode"], "description": "Use a service that's compatible with the OpenAI API to embed text." } ] @@ -372,7 +376,7 @@ "type": "string" } }, - "required": ["token"], + "required": ["token", "mode"], "description": "Authenticate using an API token (suitable for Zilliz Cloud)" }, { @@ -400,7 +404,7 @@ "type": "string" } }, - "required": ["username", "password"], + "required": ["username", "password", "mode"], "description": "Authenticate using username and password (suitable for self-managed Milvus clusters)" }, { @@ -415,7 +419,8 @@ "type": "string" } }, - "description": "Do not authenticate (suitable for locally running test clusters, do not use for clusters with public IP addresses)" + "description": "Do not authenticate (suitable for locally running test clusters, do not use for clusters with public IP addresses)", + "required": ["mode"] } ] }, diff --git a/airbyte-integrations/connectors/destination-milvus/metadata.yaml b/airbyte-integrations/connectors/destination-milvus/metadata.yaml index 9424f86b7640..bf7b68027e9f 100644 --- a/airbyte-integrations/connectors/destination-milvus/metadata.yaml +++ b/airbyte-integrations/connectors/destination-milvus/metadata.yaml @@ -22,7 +22,7 @@ data: connectorSubtype: vectorstore connectorType: destination definitionId: 65de8962-48c9-11ee-be56-0242ac120002 - dockerImageTag: 0.0.8 + dockerImageTag: 0.0.9 dockerRepository: airbyte/destination-milvus githubIssueLabel: destination-milvus icon: milvus.svg diff --git a/airbyte-integrations/connectors/destination-milvus/setup.py b/airbyte-integrations/connectors/destination-milvus/setup.py index e0a6f216cb3b..fdefdea83039 100644 --- a/airbyte-integrations/connectors/destination-milvus/setup.py +++ b/airbyte-integrations/connectors/destination-milvus/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.51.41", "pymilvus==2.3.0"] +MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.53.3", "pymilvus==2.3.0"] TEST_REQUIREMENTS = ["pytest~=6.2"] diff --git a/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/config.py b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/config.py index f33f29103797..dd2314e752e6 100644 --- a/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/config.py +++ b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/config.py @@ -62,7 +62,7 @@ class Config: @staticmethod def remove_discriminator(schema: dict) -> None: """pydantic adds "discriminator" to the schema for oneOfs, which is not treated right by the platform as we inline all references""" - dpath.util.delete(schema, "properties/*/discriminator") + dpath.util.delete(schema, "properties/**/discriminator") @classmethod def schema(cls): diff --git a/airbyte-integrations/connectors/destination-pinecone/integration_tests/spec.json b/airbyte-integrations/connectors/destination-pinecone/integration_tests/spec.json index 22873f556522..3eb78ab20f1b 100644 --- a/airbyte-integrations/connectors/destination-pinecone/integration_tests/spec.json +++ b/airbyte-integrations/connectors/destination-pinecone/integration_tests/spec.json @@ -53,7 +53,7 @@ "type": "string" } }, - "required": ["openai_key"], + "required": ["openai_key", "mode"], "description": "Use the OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." }, { @@ -73,7 +73,7 @@ "type": "string" } }, - "required": ["cohere_key"], + "required": ["cohere_key", "mode"], "description": "Use the Cohere API to embed text." }, { @@ -88,6 +88,7 @@ "type": "string" } }, + "required": ["mode"], "description": "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs." }, { @@ -120,7 +121,7 @@ "type": "string" } }, - "required": ["openai_key", "api_base", "deployment"], + "required": ["openai_key", "api_base", "deployment", "mode"], "description": "Use the Azure-hosted OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." }, { @@ -160,7 +161,7 @@ "type": "integer" } }, - "required": ["base_url", "dimensions"], + "required": ["base_url", "dimensions", "mode"], "description": "Use a service that's compatible with the OpenAI API to embed text." } ] @@ -172,6 +173,7 @@ "chunk_size": { "title": "Chunk size", "description": "Size of chunks in tokens to store in vector store (make sure it is not too big for the context if your LLM)", + "minimum": 1, "maximum": 8191, "type": "integer" }, @@ -226,14 +228,6 @@ "title": "Text splitter", "description": "Split text fields into chunks based on the specified method.", "type": "object", - "discriminator": { - "propertyName": "mode", - "mapping": { - "separator": "#/definitions/SeparatorSplitterConfigModel", - "markdown": "#/definitions/MarkdownHeaderSplitterConfigModel", - "code": "#/definitions/CodeSplitterConfigModel" - } - }, "oneOf": [ { "title": "By Separator", @@ -260,6 +254,7 @@ "type": "boolean" } }, + "required": ["mode"], "description": "Split the text by the list of separators until the chunk size is reached, using the earlier mentioned separators where possible. This is useful for splitting text fields by paragraphs, sentences, words, etc." }, { @@ -282,6 +277,7 @@ "type": "integer" } }, + "required": ["mode"], "description": "Split the text by Markdown headers down to the specified header level. If the chunk size fits multiple sections, they will be combined into a single chunk." }, { @@ -319,7 +315,7 @@ "type": "string" } }, - "required": ["language"], + "required": ["language", "mode"], "description": "Split the text by suitable delimiters based on the programming language. This is useful for splitting code into chunks." } ] diff --git a/airbyte-integrations/connectors/destination-pinecone/metadata.yaml b/airbyte-integrations/connectors/destination-pinecone/metadata.yaml index ce7c17f2ed52..e59d256bf308 100644 --- a/airbyte-integrations/connectors/destination-pinecone/metadata.yaml +++ b/airbyte-integrations/connectors/destination-pinecone/metadata.yaml @@ -13,7 +13,7 @@ data: connectorSubtype: vectorstore connectorType: destination definitionId: 3d2b6f84-7f0d-4e3f-a5e5-7c7d4b50eabd - dockerImageTag: 0.0.19 + dockerImageTag: 0.0.20 dockerRepository: airbyte/destination-pinecone documentationUrl: https://docs.airbyte.com/integrations/destinations/pinecone githubIssueLabel: destination-pinecone diff --git a/airbyte-integrations/connectors/destination-pinecone/setup.py b/airbyte-integrations/connectors/destination-pinecone/setup.py index 8d8f8fc00019..5ca0d26832b8 100644 --- a/airbyte-integrations/connectors/destination-pinecone/setup.py +++ b/airbyte-integrations/connectors/destination-pinecone/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[vector-db-based]==0.51.41", + "airbyte-cdk[vector-db-based]==0.53.3", "pinecone-client[grpc]", ] diff --git a/airbyte-integrations/connectors/destination-qdrant/Dockerfile b/airbyte-integrations/connectors/destination-qdrant/Dockerfile index 3c8d1d7dbdd3..ff82da252dac 100644 --- a/airbyte-integrations/connectors/destination-qdrant/Dockerfile +++ b/airbyte-integrations/connectors/destination-qdrant/Dockerfile @@ -41,5 +41,5 @@ COPY destination_qdrant ./destination_qdrant ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.0.6 +LABEL io.airbyte.version=0.0.7 LABEL io.airbyte.name=airbyte/destination-qdrant diff --git a/airbyte-integrations/connectors/destination-qdrant/metadata.yaml b/airbyte-integrations/connectors/destination-qdrant/metadata.yaml index 82f267ed5e8e..29d3047f24c3 100644 --- a/airbyte-integrations/connectors/destination-qdrant/metadata.yaml +++ b/airbyte-integrations/connectors/destination-qdrant/metadata.yaml @@ -20,7 +20,7 @@ data: connectorSubtype: vectorstore connectorType: destination definitionId: 6eb1198a-6d38-43e5-aaaa-dccd8f71db2b - dockerImageTag: 0.0.6 + dockerImageTag: 0.0.7 dockerRepository: airbyte/destination-qdrant githubIssueLabel: destination-qdrant icon: qdrant.svg diff --git a/airbyte-integrations/connectors/destination-qdrant/setup.py b/airbyte-integrations/connectors/destination-qdrant/setup.py index 6124b07e378f..c5b16e110ee9 100644 --- a/airbyte-integrations/connectors/destination-qdrant/setup.py +++ b/airbyte-integrations/connectors/destination-qdrant/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.51.41", "qdrant-client", "fastembed"] +MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.53.3", "qdrant-client", "fastembed"] TEST_REQUIREMENTS = ["pytest~=6.2"] diff --git a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/config.py b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/config.py index 935e07e18711..19ee77f8a423 100644 --- a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/config.py +++ b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/config.py @@ -14,6 +14,7 @@ OpenAIEmbeddingConfigModel, ProcessingConfigModel, ) +from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig from airbyte_cdk.utils.spec_schema_transformations import resolve_refs from pydantic import BaseModel, Field @@ -23,28 +24,29 @@ class UsernamePasswordAuth(BaseModel): username: str = Field(..., title="Username", description="Username for the Weaviate cluster", order=1) password: str = Field(..., title="Password", description="Password for the Weaviate cluster", airbyte_secret=True, order=2) - class Config: + class Config(OneOfOptionConfig): title = "Username/Password" - schema_extra = {"description": "Authenticate using username and password (suitable for self-managed Weaviate clusters)"} + description = "Authenticate using username and password (suitable for self-managed Weaviate clusters)" + discriminator = "mode" class NoAuth(BaseModel): mode: Literal["no_auth"] = Field("no_auth", const=True) - class Config: + class Config(OneOfOptionConfig): title = "No Authentication" - schema_extra = { - "description": "Do not authenticate (suitable for locally running test clusters, do not use for clusters with public IP addresses)" - } + description = "Do not authenticate (suitable for locally running test clusters, do not use for clusters with public IP addresses)" + discriminator = "mode" class TokenAuth(BaseModel): mode: Literal["token"] = Field("token", const=True) token: str = Field(..., title="API Token", description="API Token for the Weaviate instance", airbyte_secret=True) - class Config: + class Config(OneOfOptionConfig): title = "API Token" - schema_extra = {"description": "Authenticate using an API token (suitable for Weaviate Cloud)"} + description = "Authenticate using an API token (suitable for Weaviate Cloud)" + discriminator = "mode" class Header(BaseModel): @@ -98,11 +100,10 @@ class Config: class NoEmbeddingConfigModel(BaseModel): mode: Literal["no_embedding"] = Field("no_embedding", const=True) - class Config: + class Config(OneOfOptionConfig): title = "No external embedding" - schema_extra = { - "description": "Do not calculate and pass embeddings to Weaviate. Suitable for clusters with configured vectorizers to calculate embeddings within Weaviate or for classes that should only support regular text search." - } + description = "Do not calculate and pass embeddings to Weaviate. Suitable for clusters with configured vectorizers to calculate embeddings within Weaviate or for classes that should only support regular text search." + discriminator = "mode" class ConfigModel(BaseModel): diff --git a/airbyte-integrations/connectors/destination-weaviate/integration_tests/spec.json b/airbyte-integrations/connectors/destination-weaviate/integration_tests/spec.json index 71c7dd56b255..6b2d3c4d86c3 100644 --- a/airbyte-integrations/connectors/destination-weaviate/integration_tests/spec.json +++ b/airbyte-integrations/connectors/destination-weaviate/integration_tests/spec.json @@ -11,6 +11,7 @@ "chunk_size": { "title": "Chunk size", "description": "Size of chunks in tokens to store in vector store (make sure it is not too big for the context if your LLM)", + "minimum": 1, "maximum": 8191, "type": "integer" }, @@ -91,6 +92,7 @@ "type": "boolean" } }, + "required": ["mode"], "description": "Split the text by the list of separators until the chunk size is reached, using the earlier mentioned separators where possible. This is useful for splitting text fields by paragraphs, sentences, words, etc." }, { @@ -113,6 +115,7 @@ "type": "integer" } }, + "required": ["mode"], "description": "Split the text by Markdown headers down to the specified header level. If the chunk size fits multiple sections, they will be combined into a single chunk." }, { @@ -150,7 +153,7 @@ "type": "string" } }, - "required": ["language"], + "required": ["language", "mode"], "description": "Split the text by suitable delimiters based on the programming language. This is useful for splitting code into chunks." } ] @@ -177,6 +180,7 @@ "type": "string" } }, + "required": ["mode"], "description": "Do not calculate and pass embeddings to Weaviate. Suitable for clusters with configured vectorizers to calculate embeddings within Weaviate or for classes that should only support regular text search." }, { @@ -209,7 +213,7 @@ "type": "string" } }, - "required": ["openai_key", "api_base", "deployment"], + "required": ["openai_key", "api_base", "deployment", "mode"], "description": "Use the Azure-hosted OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." }, { @@ -229,7 +233,7 @@ "type": "string" } }, - "required": ["openai_key"], + "required": ["openai_key", "mode"], "description": "Use the OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." }, { @@ -249,7 +253,7 @@ "type": "string" } }, - "required": ["cohere_key"], + "required": ["cohere_key", "mode"], "description": "Use the Cohere API to embed text." }, { @@ -276,7 +280,7 @@ "type": "integer" } }, - "required": ["field_name", "dimensions"], + "required": ["field_name", "dimensions", "mode"], "description": "Use a field in the record as the embedding. This is useful if you already have an embedding for your data and want to store it in the vector store." }, { @@ -291,6 +295,7 @@ "type": "string" } }, + "required": ["mode"], "description": "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs." }, { @@ -330,7 +335,7 @@ "type": "integer" } }, - "required": ["base_url", "dimensions"], + "required": ["base_url", "dimensions", "mode"], "description": "Use a service that's compatible with the OpenAI API to embed text." } ] @@ -370,7 +375,7 @@ "type": "string" } }, - "required": ["token"], + "required": ["token", "mode"], "description": "Authenticate using an API token (suitable for Weaviate Cloud)" }, { @@ -398,7 +403,7 @@ "type": "string" } }, - "required": ["username", "password"], + "required": ["username", "password", "mode"], "description": "Authenticate using username and password (suitable for self-managed Weaviate clusters)" }, { @@ -413,6 +418,7 @@ "type": "string" } }, + "required": ["mode"], "description": "Do not authenticate (suitable for locally running test clusters, do not use for clusters with public IP addresses)" } ] diff --git a/airbyte-integrations/connectors/destination-weaviate/metadata.yaml b/airbyte-integrations/connectors/destination-weaviate/metadata.yaml index 13833b203591..a885f0d025d0 100644 --- a/airbyte-integrations/connectors/destination-weaviate/metadata.yaml +++ b/airbyte-integrations/connectors/destination-weaviate/metadata.yaml @@ -13,7 +13,7 @@ data: connectorSubtype: vectorstore connectorType: destination definitionId: 7b7d7a0d-954c-45a0-bcfc-39a634b97736 - dockerImageTag: 0.2.8 + dockerImageTag: 0.2.9 dockerRepository: airbyte/destination-weaviate documentationUrl: https://docs.airbyte.com/integrations/destinations/weaviate githubIssueLabel: destination-weaviate diff --git a/airbyte-integrations/connectors/destination-weaviate/setup.py b/airbyte-integrations/connectors/destination-weaviate/setup.py index 366c5abc4fe7..fffa30e12001 100644 --- a/airbyte-integrations/connectors/destination-weaviate/setup.py +++ b/airbyte-integrations/connectors/destination-weaviate/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.51.41", "weaviate-client==3.25.2"] +MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.53.3", "weaviate-client==3.25.2"] TEST_REQUIREMENTS = ["pytest~=6.2", "docker", "pytest-docker"] diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/spec.json b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/spec.json index 1ecff3aa29ed..9f2b2a75a61f 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/spec.json @@ -31,6 +31,8 @@ }, "globs": { "title": "Globs", + "default": ["**"], + "order": 1, "description": "The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look here.", "type": "array", "items": { @@ -86,7 +88,8 @@ "default": false, "type": "boolean" } - } + }, + "required": ["filetype"] }, { "title": "CSV Format", @@ -172,7 +175,8 @@ "const": "From CSV", "type": "string" } - } + }, + "required": ["header_definition_type"] }, { "title": "Autogenerated", @@ -184,7 +188,8 @@ "const": "Autogenerated", "type": "string" } - } + }, + "required": ["header_definition_type"] }, { "title": "User Provided", @@ -205,7 +210,7 @@ } } }, - "required": ["column_names"] + "required": ["column_names", "header_definition_type"] } ], "type": "object" @@ -237,7 +242,8 @@ "airbyte_hidden": true, "enum": ["None", "Primitive Types Only"] } - } + }, + "required": ["filetype"] }, { "title": "Jsonl Format", @@ -249,7 +255,8 @@ "const": "jsonl", "type": "string" } - } + }, + "required": ["filetype"] }, { "title": "Parquet Format", @@ -267,7 +274,8 @@ "default": false, "type": "boolean" } - } + }, + "required": ["filetype"] }, { "title": "Document File Type Format (Experimental)", @@ -278,9 +286,17 @@ "default": "unstructured", "const": "unstructured", "type": "string" + }, + "skip_unprocessable_file_types": { + "type": "boolean", + "default": true, + "title": "Skip Unprocessable File Types", + "description": "If true, skip files that cannot be parsed because of their file type and log a warning. If false, fail the sync. Corrupted files with valid file types will still result in a failed sync.", + "always_show": true } }, - "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file." + "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file.", + "required": ["filetype"] } ] }, diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml b/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml index 2a60ff3eb9dc..fb96d5cf2fad 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml +++ b/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: file connectorType: source definitionId: fdaaba68-4875-4ed9-8fcd-4ae1e0a25093 - dockerImageTag: 0.2.2 + dockerImageTag: 0.2.3 dockerRepository: airbyte/source-azure-blob-storage documentationUrl: https://docs.airbyte.com/integrations/sources/azure-blob-storage githubIssueLabel: source-azure-blob-storage diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/setup.py b/airbyte-integrations/connectors/source-azure-blob-storage/setup.py index b2211beb81bb..efb01f6a4af2 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/setup.py +++ b/airbyte-integrations/connectors/source-azure-blob-storage/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[file-based]>=0.52.7", + "airbyte-cdk[file-based]>=0.53.5", "smart_open[azure]", "pytz", ] diff --git a/airbyte-integrations/connectors/source-gcs/Dockerfile b/airbyte-integrations/connectors/source-gcs/Dockerfile deleted file mode 100644 index e620ea6148df..000000000000 --- a/airbyte-integrations/connectors/source-gcs/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" - -WORKDIR /airbyte/integration_code -COPY source_gcs ./source_gcs -COPY setup.py ./ -COPY main.py ./ -RUN pip install . - -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.3.0 -LABEL io.airbyte.name=airbyte/source-gcs diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/spec.json b/airbyte-integrations/connectors/source-gcs/integration_tests/spec.json index 49c035192ebc..fdcde42c9e8c 100644 --- a/airbyte-integrations/connectors/source-gcs/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/spec.json @@ -157,7 +157,8 @@ "const": "From CSV", "type": "string" } - } + }, + "required": ["header_definition_type"] }, { "title": "Autogenerated", @@ -169,7 +170,8 @@ "const": "Autogenerated", "type": "string" } - } + }, + "required": ["header_definition_type"] }, { "title": "User Provided", @@ -190,7 +192,7 @@ } } }, - "required": ["column_names"] + "required": ["column_names", "header_definition_type"] } ], "type": "object" @@ -222,7 +224,8 @@ "airbyte_hidden": true, "enum": ["None", "Primitive Types Only"] } - } + }, + "required": ["filetype"] } ] }, diff --git a/airbyte-integrations/connectors/source-gcs/metadata.yaml b/airbyte-integrations/connectors/source-gcs/metadata.yaml index 5b93da60a465..c531579684c5 100644 --- a/airbyte-integrations/connectors/source-gcs/metadata.yaml +++ b/airbyte-integrations/connectors/source-gcs/metadata.yaml @@ -2,10 +2,12 @@ data: ab_internal: ql: 200 sl: 100 + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: file connectorType: source definitionId: 2a8c41ae-8c23-4be0-a73f-2ab10ca1a820 - dockerImageTag: 0.3.0 + dockerImageTag: 0.3.1 dockerRepository: airbyte/source-gcs documentationUrl: https://docs.airbyte.com/integrations/sources/gcs githubIssueLabel: source-gcs diff --git a/airbyte-integrations/connectors/source-gcs/setup.py b/airbyte-integrations/connectors/source-gcs/setup.py index 28a0e40a874a..70d0679e6bdb 100644 --- a/airbyte-integrations/connectors/source-gcs/setup.py +++ b/airbyte-integrations/connectors/source-gcs/setup.py @@ -6,12 +6,10 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk>=0.51.17", + "airbyte-cdk[file-based]>=0.53.5", "google-cloud-storage==2.12.0", - "pandas==1.5.3", - "pyarrow==12.0.1", - "fastavro==1.4.11", "smart-open[s3]==5.1.0", + "pandas==1.5.3", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/cloud_spec.json b/airbyte-integrations/connectors/source-s3/integration_tests/cloud_spec.json index a6a87e6d431c..9ba3e1f4fb48 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/cloud_spec.json +++ b/airbyte-integrations/connectors/source-s3/integration_tests/cloud_spec.json @@ -31,6 +31,8 @@ }, "globs": { "title": "Globs", + "default": ["**"], + "order": 1, "description": "The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look here.", "type": "array", "items": { @@ -86,7 +88,8 @@ "default": false, "type": "boolean" } - } + }, + "required": ["filetype"] }, { "title": "CSV Format", @@ -172,7 +175,8 @@ "const": "From CSV", "type": "string" } - } + }, + "required": ["header_definition_type"] }, { "title": "Autogenerated", @@ -184,7 +188,8 @@ "const": "Autogenerated", "type": "string" } - } + }, + "required": ["header_definition_type"] }, { "title": "User Provided", @@ -205,7 +210,7 @@ } } }, - "required": ["column_names"] + "required": ["column_names", "header_definition_type"] } ], "type": "object" @@ -237,7 +242,8 @@ "airbyte_hidden": true, "enum": ["None", "Primitive Types Only"] } - } + }, + "required": ["filetype"] }, { "title": "Jsonl Format", @@ -249,7 +255,8 @@ "const": "jsonl", "type": "string" } - } + }, + "required": ["filetype"] }, { "title": "Parquet Format", @@ -267,7 +274,8 @@ "default": false, "type": "boolean" } - } + }, + "required": ["filetype"] }, { "title": "Document File Type Format (Experimental)", @@ -278,8 +286,16 @@ "default": "unstructured", "const": "unstructured", "type": "string" + }, + "skip_unprocessable_file_types": { + "type": "boolean", + "default": true, + "title": "Skip Unprocessable File Types", + "description": "If true, skip files that cannot be parsed because of their file type and log a warning. If false, fail the sync. Corrupted files with valid file types will still result in a failed sync.", + "always_show": true } }, + "required": ["filetype"], "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file." } ] diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/spec.json b/airbyte-integrations/connectors/source-s3/integration_tests/spec.json index 525f71b61488..8a4f04ac247b 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-s3/integration_tests/spec.json @@ -31,6 +31,8 @@ }, "globs": { "title": "Globs", + "default": ["**"], + "order": 1, "description": "The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look here.", "type": "array", "items": { @@ -86,7 +88,8 @@ "default": false, "type": "boolean" } - } + }, + "required": ["filetype"] }, { "title": "CSV Format", @@ -172,7 +175,8 @@ "const": "From CSV", "type": "string" } - } + }, + "required": ["header_definition_type"] }, { "title": "Autogenerated", @@ -184,7 +188,8 @@ "const": "Autogenerated", "type": "string" } - } + }, + "required": ["header_definition_type"] }, { "title": "User Provided", @@ -205,7 +210,7 @@ } } }, - "required": ["column_names"] + "required": ["column_names", "header_definition_type"] } ], "type": "object" @@ -237,7 +242,8 @@ "airbyte_hidden": true, "enum": ["None", "Primitive Types Only"] } - } + }, + "required": ["filetype"] }, { "title": "Jsonl Format", @@ -249,7 +255,8 @@ "const": "jsonl", "type": "string" } - } + }, + "required": ["filetype"] }, { "title": "Parquet Format", @@ -267,7 +274,8 @@ "default": false, "type": "boolean" } - } + }, + "required": ["filetype"] }, { "title": "Document File Type Format (Experimental)", @@ -278,9 +286,17 @@ "default": "unstructured", "const": "unstructured", "type": "string" + }, + "skip_unprocessable_file_types": { + "type": "boolean", + "default": true, + "title": "Skip Unprocessable File Types", + "description": "If true, skip files that cannot be parsed because of their file type and log a warning. If false, fail the sync. Corrupted files with valid file types will still result in a failed sync.", + "always_show": true } }, - "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file." + "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file.", + "required": ["filetype"] } ] }, diff --git a/airbyte-integrations/connectors/source-s3/metadata.yaml b/airbyte-integrations/connectors/source-s3/metadata.yaml index 2daa8c6e4f8b..27c20f59f30f 100644 --- a/airbyte-integrations/connectors/source-s3/metadata.yaml +++ b/airbyte-integrations/connectors/source-s3/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: file connectorType: source definitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2 - dockerImageTag: 4.2.0 + dockerImageTag: 4.2.1 dockerRepository: airbyte/source-s3 documentationUrl: https://docs.airbyte.com/integrations/sources/s3 githubIssueLabel: source-s3 diff --git a/airbyte-integrations/connectors/source-s3/setup.py b/airbyte-integrations/connectors/source-s3/setup.py index d23a87b38b81..a0bfb240328c 100644 --- a/airbyte-integrations/connectors/source-s3/setup.py +++ b/airbyte-integrations/connectors/source-s3/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[file-based]>=0.52.7", + "airbyte-cdk[file-based]>=0.53.5", "smart-open[s3]==5.1.0", "wcmatch==8.4", "dill==0.3.4", diff --git a/docs/integrations/destinations/chroma.md b/docs/integrations/destinations/chroma.md index eb8cb85e9910..bd7357ffc70f 100644 --- a/docs/integrations/destinations/chroma.md +++ b/docs/integrations/destinations/chroma.md @@ -75,6 +75,7 @@ You should now have all the requirements needed to configure Chroma as a destina | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :----------------------------------------- | +| 0.0.6 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | | 0.0.5 | 2023-10-23 | [#31563](https://github.com/airbytehq/airbyte/pull/31563) | Add field mapping option | | 0.0.4 | 2023-10-15 | [#31329](https://github.com/airbytehq/airbyte/pull/31329) | Add OpenAI-compatible embedder option | | 0.0.3 | 2023-10-04 | [#31075](https://github.com/airbytehq/airbyte/pull/31075) | Fix OpenAI embedder batch size | diff --git a/docs/integrations/destinations/milvus.md b/docs/integrations/destinations/milvus.md index 7615374b2a8e..f5de8b2e04ad 100644 --- a/docs/integrations/destinations/milvus.md +++ b/docs/integrations/destinations/milvus.md @@ -109,6 +109,7 @@ vector_store.similarity_search("test") | Version | Date | Pull Request | Subject | |:--------| :--------- |:--------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.0.9 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | | 0.0.8 | 2023-11-08 | [#31563](https://github.com/airbytehq/airbyte/pull/32262) | Auto-create collection if it doesn't exist | | 0.0.7 | 2023-10-23 | [#31563](https://github.com/airbytehq/airbyte/pull/31563) | Add field mapping option | | 0.0.6 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | diff --git a/docs/integrations/destinations/pinecone.md b/docs/integrations/destinations/pinecone.md index 48719fa033fd..6142e7935fcc 100644 --- a/docs/integrations/destinations/pinecone.md +++ b/docs/integrations/destinations/pinecone.md @@ -74,6 +74,7 @@ OpenAI and Fake embeddings produce vectors with 1536 dimensions, and the Cohere | Version | Date | Pull Request | Subject | |:--------| :--------- |:--------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.0.20 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | | 0.0.19 | 2023-10-20 | [#31329](https://github.com/airbytehq/airbyte/pull/31373) | Improve error messages | | 0.0.18 | 2023-10-20 | [#31329](https://github.com/airbytehq/airbyte/pull/31373) | Add support for namespaces and fix index cleaning when namespace is defined | | 0.0.17 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | diff --git a/docs/integrations/destinations/qdrant.md b/docs/integrations/destinations/qdrant.md index 0c249f3c11de..549f26ada6d6 100644 --- a/docs/integrations/destinations/qdrant.md +++ b/docs/integrations/destinations/qdrant.md @@ -70,6 +70,7 @@ You should now have all the requirements needed to configure Qdrant as a destina | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :----------------------------------------- | +| 0.0.7 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | | 0.0.6 | 2023-10-23 | [#31563](https://github.com/airbytehq/airbyte/pull/31563) | Add field mapping option | | 0.0.5 | 2023-10-15 | [#31329](https://github.com/airbytehq/airbyte/pull/31329) | Add OpenAI-compatible embedder option | | 0.0.4 | 2023-10-04 | [#31075](https://github.com/airbytehq/airbyte/pull/31075) | Fix OpenAI embedder batch size | diff --git a/docs/integrations/destinations/weaviate.md b/docs/integrations/destinations/weaviate.md index 41fdbfdea592..0d914d136619 100644 --- a/docs/integrations/destinations/weaviate.md +++ b/docs/integrations/destinations/weaviate.md @@ -83,6 +83,7 @@ As properties have to start will a lowercase letter in Weaviate, field names mig | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------- | +| 0.2.9 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | | 0.2.8 | 2023-11-03 | [#32134](https://github.com/airbytehq/airbyte/pull/32134) | Improve test coverage | | 0.2.7 | 2023-11-03 | [#32134](https://github.com/airbytehq/airbyte/pull/32134) | Upgrade weaviate client library | | 0.2.6 | 2023-11-01 | [#32038](https://github.com/airbytehq/airbyte/pull/32038) | Retry failed object loads | diff --git a/docs/integrations/sources/azure-blob-storage.md b/docs/integrations/sources/azure-blob-storage.md index 782eeef1e562..02942fa6ee0b 100644 --- a/docs/integrations/sources/azure-blob-storage.md +++ b/docs/integrations/sources/azure-blob-storage.md @@ -175,6 +175,7 @@ The Avro parser uses the [Fastavro library](https://fastavro.readthedocs.io/en/l There are currently no options for JSONL parsing. + ### Document File Type Format (Experimental) :::warning @@ -192,6 +193,7 @@ To perform the text extraction from PDF and Docx files, the connector uses the [ | Version | Date | Pull Request | Subject | |:--------|:-----------|:------------------------------------------------|:------------------------------------------------------------------------| +| 0.2.3 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | | 0.2.2 | 2023-10-30 | [31904](https://github.com/airbytehq/airbyte/pull/31904) | Update CDK to support document file types | | 0.2.1 | 2023-10-18 | [31543](https://github.com/airbytehq/airbyte/pull/31543) | Base image migration: remove Dockerfile and use the python-connector-base image | | 0.2.0 | 2023-10-10 | https://github.com/airbytehq/airbyte/pull/31336 | Migrate to File-based CDK. Add support of CSV, Parquet and Avro files | diff --git a/docs/integrations/sources/gcs.md b/docs/integrations/sources/gcs.md index e127c9e44cec..0fd9e627d4e7 100644 --- a/docs/integrations/sources/gcs.md +++ b/docs/integrations/sources/gcs.md @@ -37,6 +37,7 @@ Use the service account ID from above, grant read access to your target bucket. | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------| +| 0.3.1 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | | 0.3.0 | 2023-10-11 | [31212](https://github.com/airbytehq/airbyte/pull/31212) | Migrated to file based CDK | | 0.2.0 | 2023-06-26 | [27725](https://github.com/airbytehq/airbyte/pull/27725) | License Update: Elv2 | | 0.1.0 | 2023-02-16 | [23186](https://github.com/airbytehq/airbyte/pull/23186) | New Source: GCS | diff --git a/docs/integrations/sources/s3.md b/docs/integrations/sources/s3.md index 9ed244e9e836..bc5652a0c934 100644 --- a/docs/integrations/sources/s3.md +++ b/docs/integrations/sources/s3.md @@ -256,6 +256,7 @@ To perform the text extraction from PDF and Docx files, the connector uses the [ | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------| +| 4.2.1 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | | 4.2.0 | 2023-11-02 | [32109](https://github.com/airbytehq/airbyte/pull/32109) | Fix docs; add HTTPS validation for S3 endpoint; fix coverage | | 4.1.4 | 2023-10-30 | [31904](https://github.com/airbytehq/airbyte/pull/31904) | Update CDK | | 4.1.3 | 2023-10-25 | [31654](https://github.com/airbytehq/airbyte/pull/31654) | Reduce image size | From e0a568804c5f4296bfef7ea32453c8e03c61e676 Mon Sep 17 00:00:00 2001 From: Augustin Date: Tue, 14 Nov 2023 17:02:04 +0100 Subject: [PATCH 038/124] qa-check: tolerate http string in expected records (#32499) --- .../connector_ops/connector_ops/qa_checks.py | 10 +++++++++- airbyte-ci/connectors/connector_ops/pyproject.toml | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py b/airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py index b1eb15ecbdca..99bd8e1786b0 100644 --- a/airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py +++ b/airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py @@ -170,7 +170,15 @@ def read_all_files_in_directory( ".hypothesis", } -IGNORED_FILENAME_PATTERN_FOR_HTTPS_CHECKS = {"*Test.java", "*.jar", "*.pyc", "*.gz", "*.svg"} +IGNORED_FILENAME_PATTERN_FOR_HTTPS_CHECKS = { + "*Test.java", + "*.jar", + "*.pyc", + "*.gz", + "*.svg", + "expected_records.jsonl", + "expected_records.json", +} IGNORED_URLS_PREFIX = { "http://json-schema.org", "http://localhost", diff --git a/airbyte-ci/connectors/connector_ops/pyproject.toml b/airbyte-ci/connectors/connector_ops/pyproject.toml index a191022adf58..f4e5460b2493 100644 --- a/airbyte-ci/connectors/connector_ops/pyproject.toml +++ b/airbyte-ci/connectors/connector_ops/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "connector_ops" -version = "0.3.2" +version = "0.3.3" description = "Packaged maintained by the connector operations team to perform CI for connectors" authors = ["Airbyte "] From 9b94d5e8e56d4236f6a77a9ace94841569dfe274 Mon Sep 17 00:00:00 2001 From: Augustin Date: Tue, 14 Nov 2023 17:04:15 +0100 Subject: [PATCH 039/124] format: `airbyte-ci format fix all` on PR / `airbyte-ci format check all` on master (#32491) --- .github/workflows/format_check.yml | 59 +++++++++++++++++++ .../workflows/{format.yml => format_fix.yml} | 43 ++++---------- 2 files changed, 70 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/format_check.yml rename .github/workflows/{format.yml => format_fix.yml} (60%) diff --git a/.github/workflows/format_check.yml b/.github/workflows/format_check.yml new file mode 100644 index 000000000000..290eb5662fd9 --- /dev/null +++ b/.github/workflows/format_check.yml @@ -0,0 +1,59 @@ +name: Check for formatting errors on head ref + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref }} + +on: + workflow_dispatch: + push: + branches: + - master +jobs: + format-check: + runs-on: "conn-prod-xlarge-runner" + name: "Check for formatting errors on ${{ github.head_ref }}" + timeout-minutes: 40 + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + + - name: Run airbyte-ci format check + id: airbyte_ci_format_check_all + uses: ./.github/actions/run-dagger-pipeline + continue-on-error: true + with: + context: "master" + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + subcommand: "format check all" + + # This is helpful in the case that we change a previously committed generated file to be ignored by git. + - name: Remove any files that have been gitignored + run: git ls-files -i -c --exclude-from=.gitignore | xargs -r git rm --cached + + - name: Match GitHub User to Slack User + id: match-github-to-slack-user + uses: ./.github/actions/match-github-to-slack-user + env: + AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }} + GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Format Failure on Master Slack Channel + if: steps.airbyte_ci_format_check_all.outcome == 'failure' + uses: abinoda/slack-action@master + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + with: + args: >- + {\"channel\":\"C03BEADRPNY\", \"blocks\":[ + {\"type\":\"divider\"}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"Formatting is broken on master! :bangbang: \n\n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}> \n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: :octavia-shocked: \n\"}}, + {\"type\":\"divider\"}]} diff --git a/.github/workflows/format.yml b/.github/workflows/format_fix.yml similarity index 60% rename from .github/workflows/format.yml rename to .github/workflows/format_fix.yml index e826618861d5..b1b2c31562aa 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format_fix.yml @@ -1,4 +1,4 @@ -name: Check for formatting errors +name: Automatic Formatting on PRs concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -7,9 +7,6 @@ concurrency: on: workflow_dispatch: - push: - branches: - - master pull_request: jobs: format-fix: @@ -44,7 +41,7 @@ jobs: - '**/*' - '!**/*.md' - - name: Run airbyte-ci format fix + - name: Run airbyte-ci format fix all uses: ./.github/actions/run-dagger-pipeline with: context: "pull_request" @@ -68,31 +65,13 @@ jobs: commit_user_name: Octavia Squidington III commit_user_email: octavia-squidington-iii@users.noreply.github.com - notify-failure-slack-channel: - name: "Notify Slack Channel on Build Failures" - runs-on: ubuntu-latest - needs: - - format-fix - if: ${{ failure() && github.ref == 'refs/heads/master' }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Match GitHub User to Slack User - id: match-github-to-slack-user - uses: ./.github/actions/match-github-to-slack-user - env: - AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }} - GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Format Failure on Master Slack Channel - uses: abinoda/slack-action@master - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + - name: Run airbyte-ci format check all + uses: ./.github/actions/run-dagger-pipeline with: - args: >- - {\"channel\":\"C03BEADRPNY\", \"blocks\":[ - {\"type\":\"divider\"}, - {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"Formatting is broken on master! :bangbang: \n\n\"}}, - {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, - {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}> \n\"}}, - {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: :octavia-shocked: \n\"}}, - {\"type\":\"divider\"}]} + context: "pull_request" + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + subcommand: "format check all" From a94cfdfa31bdb1e9e435ea9c437b34f067c4d11b Mon Sep 17 00:00:00 2001 From: Aazam Thakur <59562284+aazam-gh@users.noreply.github.com> Date: Tue, 14 Nov 2023 21:53:45 +0530 Subject: [PATCH 040/124] =?UTF-8?q?=20=E2=9C=A8=20Source=20Insightly=20:?= =?UTF-8?q?=20Migrate=20Python=20CDK=20to=20Low-code=20CDK=20=20(#31162)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Marcos Marx Co-authored-by: marcosmarxm Co-authored-by: Sajarin Co-authored-by: sajarin --- .../connectors/source-insightly/Dockerfile | 4 +- .../connectors/source-insightly/README.md | 87 ++-- .../{unit_tests => }/__init__.py | 0 .../acceptance-test-config.yml | 36 ++ .../connectors/source-insightly/bootstrap.md | 13 - .../integration_tests/abnormal_state.json | 134 +++++- .../integration_tests/configured_catalog.json | 418 ++++++++++++++++- .../integration_tests/sample_state.json | 2 +- .../connectors/source-insightly/metadata.yaml | 19 +- .../connectors/source-insightly/setup.py | 7 +- .../source_insightly/manifest.yaml | 424 ++++++++++++++++++ .../source_insightly/schemas/contacts.json | 7 + .../schemas/opportunities.json | 11 +- .../schemas/organisations.json | 7 + .../source_insightly/schemas/projects.json | 7 + .../source_insightly/schemas/tasks.json | 4 + .../source_insightly/schemas/teams.json | 3 +- .../source_insightly/source.py | 412 +---------------- .../source_insightly/spec.json | 25 -- .../unit_tests/test_incremental_streams.py | 81 ---- .../unit_tests/test_source.py | 56 --- .../unit_tests/test_streams.py | 126 ------ docs/integrations/sources/insightly.md | 1 + 23 files changed, 1117 insertions(+), 767 deletions(-) rename airbyte-integrations/connectors/source-insightly/{unit_tests => }/__init__.py (100%) delete mode 100644 airbyte-integrations/connectors/source-insightly/bootstrap.md create mode 100644 airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml delete mode 100644 airbyte-integrations/connectors/source-insightly/source_insightly/spec.json delete mode 100644 airbyte-integrations/connectors/source-insightly/unit_tests/test_incremental_streams.py delete mode 100644 airbyte-integrations/connectors/source-insightly/unit_tests/test_source.py delete mode 100644 airbyte-integrations/connectors/source-insightly/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-insightly/Dockerfile b/airbyte-integrations/connectors/source-insightly/Dockerfile index c4ac2c8fb73a..6a744a678910 100644 --- a/airbyte-integrations/connectors/source-insightly/Dockerfile +++ b/airbyte-integrations/connectors/source-insightly/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.13-alpine3.15 as base +FROM python:3.9.11-alpine3.15 as base # build and load all requirements FROM base as builder @@ -34,5 +34,5 @@ COPY source_insightly ./source_insightly ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-insightly diff --git a/airbyte-integrations/connectors/source-insightly/README.md b/airbyte-integrations/connectors/source-insightly/README.md index efbb1f171fb5..92b977856b8b 100644 --- a/airbyte-integrations/connectors/source-insightly/README.md +++ b/airbyte-integrations/connectors/source-insightly/README.md @@ -1,37 +1,23 @@ # Insightly Source -This is the repository for the Insightly source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/insightly). +This is the repository for the Insightly configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/insightly). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** +#### Building via Gradle -#### Minimum Python version required `= 3.9.0` +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` +To build using Gradle, from the Airbyte repository root, run: -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: ``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' +./gradlew :airbyte-integrations:connectors:source-insightly:build ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/insightly) + +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/insightly) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_insightly/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. @@ -39,19 +25,12 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source insightly test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image - #### Build + **Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** + ```bash airbyte-ci connectors --name=source-insightly build ``` @@ -59,12 +38,15 @@ airbyte-ci connectors --name=source-insightly build An image will be built with the tag `airbyte/source-insightly:dev`. **Via `docker build`:** + ```bash docker build -t airbyte/source-insightly:dev . ``` #### Run + Then run any of the connector commands as follows: + ``` docker run --rm airbyte/source-insightly:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-insightly:dev check --config /secrets/config.json @@ -73,23 +55,61 @@ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integrat ``` ## Testing + +<<<<<<< HEAD + +#### Acceptance Tests + +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with Docker, run: + +``` +./acceptance-test-docker.sh +``` + +### Using gradle to run tests + +All commands should be run from airbyte project root. +To run unit tests: + +``` +./gradlew :airbyte-integrations:connectors:source-insightly:unitTest +``` + +To run acceptance and custom integration tests: + +``` +./gradlew :airbyte-integrations:connectors:source-insightly:integrationTest +``` + +======= You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): + ```bash airbyte-ci connectors --name=source-insightly test ``` ### Customizing acceptance Tests + Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +> > > > > > > master + ## Dependency Management + All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list + +- required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +- required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector + You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? + 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-insightly test` 2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). 3. Make sure the `metadata.yaml` content is up to date. @@ -97,4 +117,3 @@ You've checked out the repo, implemented a million dollar feature, and you're re 5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). 6. Pat yourself on the back for being an awesome contributor. 7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - diff --git a/airbyte-integrations/connectors/source-insightly/unit_tests/__init__.py b/airbyte-integrations/connectors/source-insightly/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-insightly/unit_tests/__init__.py rename to airbyte-integrations/connectors/source-insightly/__init__.py diff --git a/airbyte-integrations/connectors/source-insightly/acceptance-test-config.yml b/airbyte-integrations/connectors/source-insightly/acceptance-test-config.yml index a7de796e79a5..b0f10cdef373 100644 --- a/airbyte-integrations/connectors/source-insightly/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-insightly/acceptance-test-config.yml @@ -18,10 +18,46 @@ acceptance_tests: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: emails + bypass_reason: "no data for this stream in our sandbox account" + - name: events + bypass_reason: "no data for this stream in our sandbox account" + - name: milestones + bypass_reason: "no data for this stream in our sandbox account" + - name: notes + bypass_reason: "no data for this stream in our sandbox account" + - name: opportunity_categories + bypass_reason: "no data for this stream in our sandbox account" + - name: project_categories + bypass_reason: "no data for this stream in our sandbox account" + - name: knowledge_article_categories + bypass_reason: "current sandbox account does not have permissions to access this stream (403 error)" + - name: knowledge_article_folders + bypass_reason: "current sandbox account does not have permissions to access this stream (403 error)" + - name: knowledge_articles + bypass_reason: "current sandbox account does not have permissions to access this stream (403 error)" + - name: lead_sources + bypass_reason: "current sandbox account does not have permissions to access this stream (403 error)" + - name: lead_statuses + bypass_reason: "current sandbox account does not have permissions to access this stream (403 error)" + - name: prospects + bypass_reason: "current sandbox account does not have permissions to access this stream (403 error)" + full_refresh: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + ignored_fields: + contacts: + - name: "IMAGE_URL" + bypass_reason: "image url is a dynamic s3 url with a changing expiration date in the query parameter" + organisations: + - name: "IMAGE_URL" + bypass_reason: "image url is a dynamic s3 url with a changing expiration date in the query parameter" + users: + - name: "USER_CURRENCY" + bypass_reason: "this can change between sequential reads" incremental: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-insightly/bootstrap.md b/airbyte-integrations/connectors/source-insightly/bootstrap.md deleted file mode 100644 index d52b29577dea..000000000000 --- a/airbyte-integrations/connectors/source-insightly/bootstrap.md +++ /dev/null @@ -1,13 +0,0 @@ -# Insightly -OpenWeather is an online service offering an API to retrieve historical, current and forecasted weather data over the globe. - -### Auth -API calls are authenticated through an API key. An API key can be retrieved from Insightly User Settings page in the API section. - -### Rate limits -The API has different rate limits for different account types. Keep that in mind when syncing large amounts of data: -* Free/Gratis - 1,000 requests/day/instance -* Legacy plans - 20,000 requests/day/instance -* Plus - 40,000 requests/day/instance -* Professional - 60,000 requests/day/instance -* Enterprise - 100,000 requests/day/instance diff --git a/airbyte-integrations/connectors/source-insightly/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-insightly/integration_tests/abnormal_state.json index c06e9d0a75c0..f49326b9fd77 100644 --- a/airbyte-integrations/connectors/source-insightly/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-insightly/integration_tests/abnormal_state.json @@ -1,4 +1,136 @@ [ + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "contacts" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "events" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "knowledge_article_categories" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "knowledge_article_folders" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "knowledge_articles" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "milestones" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "notes" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "opportunities" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "organisations" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "projects" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "prospects" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "tasks" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, { "type": "STREAM", "stream": { @@ -6,7 +138,7 @@ "name": "users" }, "stream_state": { - "DATE_UPDATED_UTC": "2122-10-17T19:10:14+00:00" + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" } } } diff --git a/airbyte-integrations/connectors/source-insightly/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-insightly/integration_tests/configured_catalog.json index 0873c8cb593f..60107bacf103 100644 --- a/airbyte-integrations/connectors/source-insightly/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-insightly/integration_tests/configured_catalog.json @@ -1,26 +1,424 @@ { "streams": [ { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "team_members", + "default_cursor_field": null, "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["MEMBER_USER_ID"]] + "name": "activity_sets", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["ACTIVITYSET_ID"]], + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "users", + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "contacts", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["CONTACT_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "countries", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["COUNTRY_NAME"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "currencies", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["CURRENCY_CODE"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "emails", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["EMAIL_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "events", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["EVENT_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "knowledge_article_categories", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["CATEGORY_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "knowledge_article_folders", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["FOLDER_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "knowledge_articles", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["ARTICLE_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "lead_sources", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["LEAD_SOURCE_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "lead_statuses", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["LEAD_STATUS_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "milestones", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["MILESTONE_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "notes", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["NOTE_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "opportunities", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["OPPORTUNITY_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "opportunity_categories", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["CATEGORY_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "opportunity_state_reasons", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["STATE_REASON_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "name": "organisations", + "namespace": null, "source_defined_cursor": true, + "source_defined_primary_key": [["ORGANISATION_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "pipelines", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["PIPELINE_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "pipeline_stages", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["STAGE_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "project_categories", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["CATEGORY_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { "default_cursor_field": ["DATE_UPDATED_UTC"], - "source_defined_primary_key": [["USER_ID"]] + "json_schema": {}, + "name": "projects", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["PROJECT_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "prospects", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["PROSPECT_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "relationships", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["RELATIONSHIP_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "task_categories", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["CATEGORY_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "tasks", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["TASK_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "team_members", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["MEMBER_USER_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "teams", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["TEAM_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "users", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["USER_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "incremental", - "destination_sync_mode": "append" + "sync_mode": "full_refresh" } ] } diff --git a/airbyte-integrations/connectors/source-insightly/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-insightly/integration_tests/sample_state.json index 3127937e0f0e..672363e87f6e 100644 --- a/airbyte-integrations/connectors/source-insightly/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-insightly/integration_tests/sample_state.json @@ -6,7 +6,7 @@ "name": "users" }, "stream_state": { - "DATE_UPDATED_UTC": "2022-10-17T19:10:14+00:00" + "DATE_UPDATED_UTC": "2022-10-17 19:10:14" } } } diff --git a/airbyte-integrations/connectors/source-insightly/metadata.yaml b/airbyte-integrations/connectors/source-insightly/metadata.yaml index 06202981ce95..321d751ffa7a 100644 --- a/airbyte-integrations/connectors/source-insightly/metadata.yaml +++ b/airbyte-integrations/connectors/source-insightly/metadata.yaml @@ -1,24 +1,27 @@ data: + allowedHosts: + hosts: + - TODO # Please change to the hostname of the source. + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: 38f84314-fe6a-4257-97be-a8dcd942d693 - dockerImageTag: 0.1.3 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-insightly githubIssueLabel: source-insightly icon: insightly.svg license: MIT name: Insightly - registries: - cloud: - enabled: true - oss: - enabled: true releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/insightly tags: - - language:python + - language:lowcode ab_internal: sl: 100 ql: 100 - supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-insightly/setup.py b/airbyte-integrations/connectors/source-insightly/setup.py index a8c4637c1342..a3c070098791 100644 --- a/airbyte-integrations/connectors/source-insightly/setup.py +++ b/airbyte-integrations/connectors/source-insightly/setup.py @@ -5,14 +5,11 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", - "pendulum==2.1.2", -] +MAIN_REQUIREMENTS = ["airbyte-cdk"] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", ] diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml b/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml new file mode 100644 index 000000000000..5f28969be084 --- /dev/null +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml @@ -0,0 +1,424 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + + requester: + type: HttpRequester + url_base: "https://api.na1.insightly.com/v3.1/" + http_method: "GET" + authenticator: + type: BasicHttpAuthenticator + username: "{{ config['token'] }}" + request_parameters: + count_total: "True" + updated_after_utc: "{{ stream_state['DATE_UPDATED_UTC'] }}" + error_handler: + type: "CompositeErrorHandler" + error_handlers: + - response_filters: + - http_codes: [403] + action: IGNORE #ignore 403 errors for knowledge_article streams, lead streams and prospects + - response_filters: + - http_codes: [429] + action: RETRY + backoff_strategies: + - type: "ConstantBackoffStrategy" + backoff_time_in_seconds: 6.0 + + date_incremental_sync: + type: DatetimeBasedCursor + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + datetime: "{{ now_utc() }}" + datetime_format: "%Y-%m-%d %H:%M:%S.%f+00:00" + datetime_format: "%Y-%m-%d %H:%M:%S" + cursor_granularity: PT1S + step: P30D # Step should reflect the actual time granularity you need + cursor_field: "DATE_UPDATED_UTC" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: DefaultPaginator + pagination_strategy: + type: "OffsetIncrement" + page_size: 500 + page_token_option: + type: RequestOption + inject_into: "request_parameter" + field_name: "skip" + page_size_option: + inject_into: "request_parameter" + field_name: "top" + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + base_incremental_stream: + incremental_sync: + $ref: "#/definitions/date_incremental_sync" + retriever: + $ref: "#/definitions/retriever" + + activity_sets_stream: + $ref: "#/definitions/base_stream" + name: "activity_sets" + primary_key: "ACTIVITYSET_ID" + $parameters: + path: "/ActivitySets" + + contacts_stream: + $ref: "#/definitions/base_incremental_stream" + name: "contacts" + primary_key: "CONTACT_ID" + $parameters: + path: "/Contacts/Search" + + countries_stream: + $ref: "#/definitions/base_stream" + name: "countries" + primary_key: "COUNTRY_NAME" + $parameters: + path: "/Countries" + + currencies_stream: + $ref: "#/definitions/base_stream" + name: "currencies" + primary_key: "CURRENCY_CODE" + $parameters: + path: "/Currencies" + + emails_stream: + $ref: "#/definitions/base_stream" + name: "emails" + primary_key: "EMAIL_ID" + $parameters: + path: "/Emails/Search" + + events_stream: + $ref: "#/definitions/base_incremental_stream" + name: "events" + primary_key: "EVENT_ID" + $parameters: + path: "/Events/Search" + + knowledge_article_categories_stream: + $ref: "#/definitions/base_incremental_stream" + name: "knowledge_article_categories" + primary_key: "CATEGORY_ID" + $parameters: + path: "/KnowledgeArticleCategory/Search" + + knowledge_article_folders_stream: + $ref: "#/definitions/base_incremental_stream" + name: "knowledge_article_folders" + primary_key: "FOLDER_ID" + $parameters: + path: "/KnowledgeArticleFolder/Search" + + knowledge_articles_stream: + $ref: "#/definitions/base_incremental_stream" + name: "knowledge_articles" + primary_key: "ARTICLE_ID" + $parameters: + path: "/KnowledgeArticle/Search" + + leads_stream: + $ref: "#/definitions/base_incremental_stream" + name: "leads" + primary_key: "LEAD_ID" + $parameters: + path: "/Leads/Search" + + lead_sources_stream: + $ref: "#/definitions/base_stream" + name: "lead_sources" + primary_key: "LEAD_SOURCE_ID" + $parameters: + path: "/LeadSources" + + lead_statuses_stream: + $ref: "#/definitions/base_stream" + name: "lead_statuses" + primary_key: "LEAD_STATUS_ID" + $parameters: + path: "/LeadStatuses" + + milestones_stream: + $ref: "#/definitions/base_incremental_stream" + name: "milestones" + primary_key: "MILESTONE_ID" + $parameters: + path: "/Milestones/Search" + + notes_stream: + $ref: "#/definitions/base_incremental_stream" + name: "notes" + primary_key: "NOTE_ID" + $parameters: + path: "/Notes/Search" + + opportunities_stream: + $ref: "#/definitions/base_incremental_stream" + name: "opportunities" + primary_key: "OPPORTUNITY_ID" + $parameters: + path: "/Opportunities/Search" + + opportunity_categories_stream: + $ref: "#/definitions/base_stream" + name: "opportunity_categories" + primary_key: "CATEGORY_ID" + $parameters: + path: "/OpportunityCategories" + + opportunity_products_stream: + $ref: "#/definitions/base_incremental_stream" + name: "opportunity_products" + primary_key: "OPPORTUNITY_ITEM_ID" + $parameters: + path: "/OpportunityLineItem/Search" + + opportunity_state_reasons_stream: + $ref: "#/definitions/base_stream" + name: "opportunity_state_reasons" + primary_key: "STATE_REASON_ID" + $parameters: + path: "/OpportunityStateReasons" + + organisations_stream: + $ref: "#/definitions/base_incremental_stream" + name: "organisations" + primary_key: "ORGANISATION_ID" + $parameters: + path: "/Organisations/Search" + + pipelines_stream: + $ref: "#/definitions/base_stream" + name: "pipelines" + primary_key: "PIPELINE_ID" + $parameters: + path: "/Pipelines" + + pipeline_stages_stream: + $ref: "#/definitions/base_stream" + name: "pipeline_stages" + primary_key: "STAGE_ID" + $parameters: + path: "/PipelineStages" + + pricebook_entries_stream: + $ref: "#/definitions/base_incremental_stream" + name: "price_book_entries" + primary_key: "PRICEBOOK_ENTRY_ID" + $parameters: + path: "/PricebookEntry/Search" + + pricebooks_stream: + $ref: "#/definitions/base_incremental_stream" + name: "price_books" + primary_key: "PRICEBOOK_ID" + $parameters: + path: "/Pricebook/Search" + + products_stream: + $ref: "#/definitions/base_incremental_stream" + name: "products" + primary_key: "PRODUCT_ID" + $parameters: + path: "/Product/Search" + + project_categories_stream: + $ref: "#/definitions/base_stream" + name: "project_categories" + primary_key: "CATEGORY_ID" + $parameters: + path: "/ProjectCategories" + + projects_stream: + $ref: "#/definitions/base_incremental_stream" + name: "projects" + primary_key: "PROJECT_ID" + $parameters: + path: "/Projects/Search" + + prospects_stream: + $ref: "#/definitions/base_incremental_stream" + name: "prospects" + primary_key: "PROSPECT_ID" + $parameters: + path: "/Prospect/Search" + + quote_products_stream: + $ref: "#/definitions/base_incremental_stream" + name: "quote_products" + primary_key: "QUOTATION_ITEM_ID" + $parameters: + path: "/QuotationLineItem/Search" + + quotes_stream: + $ref: "#/definitions/base_incremental_stream" + name: "quotes" + primary_key: "QUOTE_ID" + $parameters: + path: "/Quotation/Search" + + relationships_stream: + $ref: "#/definitions/base_stream" + name: "relationships" + primary_key: "RELATIONSHIP_ID" + $parameters: + path: "/Relationships" + + tags_stream: + $ref: "#/definitions/base_stream" + name: "tags" + primary_key: "TAG_NAME" + $parameters: + path: "/Tags" + + task_categories_stream: + $ref: "#/definitions/base_stream" + name: "task_categories" + primary_key: "CATEGORY_ID" + $parameters: + path: "/TaskCategories" + + tasks_stream: + $ref: "#/definitions/base_incremental_stream" + name: "tasks" + primary_key: "TASK_ID" + $parameters: + path: "/Tasks/Search" + + team_members_stream: + $ref: "#/definitions/base_stream" + name: "team_members" + primary_key: "MEMBER_USER_ID" + $parameters: + path: "/TeamMembers" + + teams_stream: + $ref: "#/definitions/base_stream" + name: "teams" + primary_key: "TEAM_ID" + $parameters: + path: "/Teams" + + tickets_stream: + $ref: "#/definitions/base_incremental_stream" + name: "tickets" + primary_key: "TICKET_ID" + $parameters: + path: "/Ticket/Search" + + users_stream: + $ref: "#/definitions/base_incremental_stream" + name: "users" + primary_key: "USER_ID" + $parameters: + path: "/Users/Search" + +streams: + - "#/definitions/activity_sets_stream" + - "#/definitions/contacts_stream" + - "#/definitions/countries_stream" + - "#/definitions/currencies_stream" + - "#/definitions/emails_stream" + - "#/definitions/events_stream" + - "#/definitions/knowledge_article_categories_stream" + - "#/definitions/knowledge_article_folders_stream" + - "#/definitions/knowledge_articles_stream" + # - "#/definitions/leads_stream" + - "#/definitions/lead_sources_stream" + - "#/definitions/lead_statuses_stream" + - "#/definitions/milestones_stream" + - "#/definitions/notes_stream" + - "#/definitions/opportunities_stream" + - "#/definitions/opportunity_categories_stream" + # - "#/definitions/opportunity_products_stream" + - "#/definitions/opportunity_state_reasons_stream" + - "#/definitions/organisations_stream" + - "#/definitions/pipelines_stream" + - "#/definitions/pipeline_stages_stream" + # - "#/definitions/pricebook_entries_stream" + # - "#/definitions/pricebooks_stream" + # - "#/definitions/products_stream" + - "#/definitions/project_categories_stream" + - "#/definitions/projects_stream" + - "#/definitions/prospects_stream" + # - "#/definitions/quote_products_stream" + # - "#/definitions/quotes_stream" + - "#/definitions/relationships_stream" + # - "#/definitions/tags_stream" + - "#/definitions/task_categories_stream" + - "#/definitions/tasks_stream" + - "#/definitions/team_members_stream" + - "#/definitions/teams_stream" + # - "#/definitions/tickets_stream" + - "#/definitions/users_stream" + +check: + type: CheckStream + stream_names: + - "activity_sets" + - "contacts" + # - "countries" + # - "currencies" + # - "opportunities" + # - "opportunity_state_reasons" + # - "organizations" + # - "pipelines" + # - "pipeline_stages" + # - "projects" + # - "relationships" + # - "task_categories" + # - "tasks" + # - "team_members" + # - "teams" + # - "users" + +spec: + type: Spec + documentationUrl: https://docs.airbyte.com/integrations/sources/insightly + connection_specification: + "$schema": http://json-schema.org/draft-07/schema# + title: Insightly Spec + type: object + required: + - token + - start_date + additionalProperties: true + properties: + token: + type: + - string + - "null" + title: API Token + description: Your Insightly API token. + airbyte_secret: true + start_date: + type: + - string + - "null" + title: Start Date + description: + The date from which you'd like to replicate data for Insightly + in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will + be replicated. Note that it will be used only for incremental streams. + examples: + - "2021-03-01T00:00:00Z" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/contacts.json b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/contacts.json index dd0b5039fe01..6b6fd0672c44 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/contacts.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, "type": "object", "properties": { "CONTACT_ID": { @@ -114,6 +115,12 @@ "TITLE": { "type": ["string", "null"] }, + "VISIBLE_TEAM_ID": { + "type": ["integer", "null"] + }, + "VISIBLE_TO": { + "type": ["string", "null"] + }, "EMAIL_OPTED_OUT": { "type": ["boolean", "null"] }, diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/opportunities.json b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/opportunities.json index e154e26090d2..ca20e33d36b8 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/opportunities.json +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/opportunities.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": true, "type": "object", "properties": { "OPPORTUNITY_ID": { @@ -27,7 +28,7 @@ "type": ["string", "null"] }, "BID_AMOUNT": { - "type": ["integer", "null"] + "type": ["number", "null"] }, "BID_TYPE": { "type": ["string", "null"] @@ -48,7 +49,7 @@ "format": "date-time" }, "OPPORTUNITY_VALUE": { - "type": ["integer", "null"] + "type": ["number", "null"] }, "PROBABILITY": { "type": ["integer", "null"] @@ -83,6 +84,12 @@ "PRICEBOOK_ID": { "type": ["integer", "null"] }, + "VISIBLE_TEAM_ID": { + "type": ["integer", "null"] + }, + "VISIBLE_TO": { + "type": ["string", "null"] + }, "CUSTOMFIELDS": { "type": "array", "items": { diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/organisations.json b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/organisations.json index b2dd05169110..9f7c844de6ad 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/organisations.json +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/organisations.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": true, "type": "object", "properties": { "ORGANISATION_ID": { @@ -84,6 +85,12 @@ "SOCIAL_TWITTER": { "type": ["string", "null"] }, + "VISIBLE_TEAM_ID": { + "type": ["integer", "null"] + }, + "VISIBLE_TO": { + "type": ["string", "null"] + }, "CUSTOMFIELDS": { "type": "array", "items": { diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/projects.json b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/projects.json index fa91ca651ef4..d11b04949b52 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/projects.json +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/projects.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": true, "type": "object", "properties": { "PROJECT_ID": { @@ -60,6 +61,12 @@ "RESPONSIBLE_USER_ID": { "type": ["integer", "null"] }, + "VISIBLE_TEAM_ID": { + "type": ["integer", "null"] + }, + "VISIBLE_TO": { + "type": ["string", "null"] + }, "CUSTOMFIELDS": { "type": "array", "items": { diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/tasks.json b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/tasks.json index 6b41220e9e10..752a64002f8f 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/tasks.json +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/tasks.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": true, "type": "object", "properties": { "TASK_ID": { @@ -34,6 +35,9 @@ "PERCENT_COMPLETE": { "type": ["integer", "null"] }, + "PUBLICLY_VISIBLE": { + "type": ["null", "boolean"] + }, "START_DATE": { "type": ["string", "null"], "format": "date-time" diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/teams.json b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/teams.json index b8454b8b393e..367acda41e53 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/teams.json +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/teams.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, "type": "object", "properties": { "TEAM_ID": { @@ -20,7 +21,7 @@ "format": "date-time" }, "TEAMMEMBERS": { - "type": "object" + "type": "array" } } } diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/source.py b/airbyte-integrations/connectors/source-insightly/source_insightly/source.py index 68133dfb1df3..29fa855efc5b 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/source.py +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/source.py @@ -2,409 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from abc import ABC -from datetime import datetime -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple -from urllib.parse import parse_qs, urlparse +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import pendulum -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import BasicHttpAuthenticator -from requests.auth import AuthBase +WARNING: Do not modify this file. +""" -PAGE_SIZE = 500 -BASE_URL = "https://api.insightly.com/v3.1/" - -# Basic full refresh stream -class InsightlyStream(HttpStream, ABC): - total_count: int = 0 - page_size: Optional[int] = PAGE_SIZE - - url_base = BASE_URL - - def __init__(self, authenticator: AuthBase, start_date: str = None, **kwargs): - self.start_date = start_date - super().__init__(authenticator=authenticator) - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - parsed = urlparse(response.request.url) - previous_skip = parse_qs(parsed.query)["skip"][0] - new_skip = int(previous_skip) + self.page_size - return new_skip if new_skip <= self.total_count else None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return { - "count_total": True, - "top": self.page_size, - "skip": next_page_token or 0, - } - - def request_headers(self, **kwargs) -> Mapping[str, Any]: - return {"Accept": "application/json"} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - self.total_count = int(response.headers.get("X-Total-Count", 0)) - results = response.json() - yield from results - - -class ActivitySets(InsightlyStream): - primary_key = "ACTIVITYSET_ID" - - def path(self, **kwargs) -> str: - return "ActivitySets" - - -class Countries(InsightlyStream): - primary_key = "COUNTRY_NAME" - - def path(self, **kwargs) -> str: - return "Countries" - - -class Currencies(InsightlyStream): - primary_key = "CURRENCY_CODE" - - def path(self, **kwargs) -> str: - return "Currencies" - - -class Emails(InsightlyStream): - primary_key = "EMAIL_ID" - - def path(self, **kwargs) -> str: - return "Emails" - - -class LeadSources(InsightlyStream): - primary_key = "LEAD_SOURCE_ID" - - def path(self, **kwargs) -> str: - return "LeadSources" - - -class LeadStatuses(InsightlyStream): - primary_key = "LEAD_STATUS_ID" - - def path(self, **kwargs) -> str: - return "LeadStatuses" - - -class OpportunityCategories(InsightlyStream): - primary_key = "CATEGORY_ID" - - def path(self, **kwargs) -> str: - return "OpportunityCategories" - - -class OpportunityStateReasons(InsightlyStream): - primary_key = "STATE_REASON_ID" - - def path(self, **kwargs) -> str: - return "OpportunityStateReasons" - - -class Pipelines(InsightlyStream): - primary_key = "PIPELINE_ID" - - def path(self, **kwargs) -> str: - return "Pipelines" - - -class PipelineStages(InsightlyStream): - primary_key = "STAGE_ID" - - def path(self, **kwargs) -> str: - return "PipelineStages" - - -class ProjectCategories(InsightlyStream): - primary_key = "CATEGORY_ID" - - def path(self, **kwargs) -> str: - return "ProjectCategories" - - -class Relationships(InsightlyStream): - primary_key = "RELATIONSHIP_ID" - - def path(self, **kwargs) -> str: - return "Relationships" - - -class Tags(InsightlyStream): - primary_key = "TAG_NAME" - - def path(self, **kwargs) -> str: - return "Tags" - - -class TaskCategories(InsightlyStream): - primary_key = "CATEGORY_ID" - - def path(self, **kwargs) -> str: - return "TaskCategories" - - -class TeamMembers(InsightlyStream): - primary_key = "MEMBER_USER_ID" - - def path(self, **kwargs) -> str: - return "TeamMembers" - - -class Teams(InsightlyStream): - primary_key = "TEAM_ID" - - def path(self, **kwargs) -> str: - return "Teams" - - -class IncrementalInsightlyStream(InsightlyStream, ABC): - """Insighlty incremental stream using `updated_after_utc` filter""" - - cursor_field = "DATE_UPDATED_UTC" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - - start_datetime = pendulum.parse(self.start_date) - cursor_datetime = stream_state.get(self.cursor_field) - if cursor_datetime: - if isinstance(cursor_datetime, datetime): - start_datetime = cursor_datetime - else: - start_datetime = pendulum.parse(cursor_datetime) - - # subtract 1 second to make the incremental request inclusive - start_datetime = start_datetime.subtract(seconds=1) - - params.update({"updated_after_utc": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")}) - return params - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - record_time = pendulum.parse(latest_record[self.cursor_field]) - current_state = current_stream_state.get(self.cursor_field) - if current_state: - current_state = current_state if isinstance(current_state, datetime) else pendulum.parse(current_state) - - current_stream_state[self.cursor_field] = max(record_time, current_state) if current_state else record_time - return current_stream_state - - -class Contacts(IncrementalInsightlyStream): - primary_key = "CONTACT_ID" - - def path(self, **kwargs) -> str: - return "Contacts/Search" - - -class Events(IncrementalInsightlyStream): - primary_key = "EVENT_ID" - - def path(self, **kwargs) -> str: - return "Events/Search" - - -class KnowledgeArticleCategories(IncrementalInsightlyStream): - primary_key = "CATEGORY_ID" - - def path(self, **kwargs) -> str: - return "KnowledgeArticleCategory/Search" - - -class KnowledgeArticleFolders(IncrementalInsightlyStream): - primary_key = "FOLDER_ID" - - def path(self, **kwargs) -> str: - return "KnowledgeArticleFolder/Search" - - -class KnowledgeArticles(IncrementalInsightlyStream): - primary_key = "ARTICLE_ID" - - def path(self, **kwargs) -> str: - return "KnowledgeArticle/Search" - - -class Leads(IncrementalInsightlyStream): - primary_key = "LEAD_ID" - - def path(self, **kwargs) -> str: - return "Leads/Search" - - -class Milestones(IncrementalInsightlyStream): - primary_key = "MILESTONE_ID" - - def path(self, **kwargs) -> str: - return "Milestones/Search" - - -class Notes(IncrementalInsightlyStream): - primary_key = "NOTE_ID" - - def path(self, **kwargs) -> str: - return "Notes/Search" - - -class Opportunities(IncrementalInsightlyStream): - primary_key = "OPPORTUNITY_ID" - - def path(self, **kwargs) -> str: - return "Opportunities/Search" - - -class OpportunityProducts(IncrementalInsightlyStream): - primary_key = "OPPORTUNITY_ITEM_ID" - - def path(self, **kwargs) -> str: - return "OpportunityLineItem/Search" - - -class Organisations(IncrementalInsightlyStream): - primary_key = "ORGANISATION_ID" - - def path(self, **kwargs) -> str: - return "Organisations/Search" - - -class PricebookEntries(IncrementalInsightlyStream): - primary_key = "PRICEBOOK_ENTRY_ID" - - def path(self, **kwargs) -> str: - return "PricebookEntry/Search" - - -class Pricebooks(IncrementalInsightlyStream): - primary_key = "PRICEBOOK_ID" - - def path(self, **kwargs) -> str: - return "Pricebook/Search" - - -class Products(IncrementalInsightlyStream): - primary_key = "PRODUCT_ID" - - def path(self, **kwargs) -> str: - return "Product/Search" - - -class Projects(IncrementalInsightlyStream): - primary_key = "PROJECT_ID" - - def path(self, **kwargs) -> str: - return "Projects/Search" - - -class Prospects(IncrementalInsightlyStream): - primary_key = "PROSPECT_ID" - - def path(self, **kwargs) -> str: - return "Prospect/Search" - - -class QuoteProducts(IncrementalInsightlyStream): - primary_key = "QUOTATION_ITEM_ID" - - def path(self, **kwargs) -> str: - return "QuotationLineItem/Search" - - -class Quotes(IncrementalInsightlyStream): - primary_key = "QUOTE_ID" - - def path(self, **kwargs) -> str: - return "Quotation/Search" - - -class Tasks(IncrementalInsightlyStream): - primary_key = "TASK_ID" - - def path(self, **kwargs) -> str: - return "Tasks/Search" - - -class Tickets(IncrementalInsightlyStream): - primary_key = "TICKET_ID" - - def path(self, **kwargs) -> str: - return "Ticket/Search" - - -class Users(IncrementalInsightlyStream): - primary_key = "USER_ID" - - def path(self, **kwargs) -> str: - return "Users/Search" - - -# Source -class SourceInsightly(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - token = config.get("token") - response = requests.get(f"{BASE_URL}Instance", auth=(token, "")) - response.raise_for_status() - - result = response.json() - logger.info(result) - - return True, None - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - - auth = BasicHttpAuthenticator(username=config.get("token"), password="") - return [ - ActivitySets(authenticator=auth, **config), - Contacts(authenticator=auth, **config), - Countries(authenticator=auth, **config), - Currencies(authenticator=auth, **config), - Emails(authenticator=auth, **config), - Events(authenticator=auth, **config), - KnowledgeArticleCategories(authenticator=auth, **config), - KnowledgeArticleFolders(authenticator=auth, **config), - KnowledgeArticles(authenticator=auth, **config), - LeadSources(authenticator=auth, **config), - LeadStatuses(authenticator=auth, **config), - Leads(authenticator=auth, **config), - Milestones(authenticator=auth, **config), - Notes(authenticator=auth, **config), - Opportunities(authenticator=auth, **config), - OpportunityCategories(authenticator=auth, **config), - OpportunityProducts(authenticator=auth, **config), - OpportunityStateReasons(authenticator=auth, **config), - Organisations(authenticator=auth, **config), - PipelineStages(authenticator=auth, **config), - Pipelines(authenticator=auth, **config), - PricebookEntries(authenticator=auth, **config), - Pricebooks(authenticator=auth, **config), - Products(authenticator=auth, **config), - ProjectCategories(authenticator=auth, **config), - Projects(authenticator=auth, **config), - Prospects(authenticator=auth, **config), - QuoteProducts(authenticator=auth, **config), - Quotes(authenticator=auth, **config), - Relationships(authenticator=auth, **config), - Tags(authenticator=auth, **config), - TaskCategories(authenticator=auth, **config), - Tasks(authenticator=auth, **config), - TeamMembers(authenticator=auth, **config), - Teams(authenticator=auth, **config), - Tickets(authenticator=auth, **config), - Users(authenticator=auth, **config), - ] +# Declarative Source +class SourceInsightly(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/spec.json b/airbyte-integrations/connectors/source-insightly/source_insightly/spec.json deleted file mode 100644 index a21504f3e553..000000000000 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/spec.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/insightly", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Insightly Spec", - "type": "object", - "required": ["token", "start_date"], - "additionalProperties": true, - "properties": { - "token": { - "type": ["string", "null"], - "title": "API Token", - "description": "Your Insightly API token.", - "airbyte_secret": true - }, - "start_date": { - "type": ["string", "null"], - "title": "Start Date", - "description": "The date from which you'd like to replicate data for Insightly in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated. Note that it will be used only for incremental streams.", - "examples": ["2021-03-01T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - } - } - } -} diff --git a/airbyte-integrations/connectors/source-insightly/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-insightly/unit_tests/test_incremental_streams.py deleted file mode 100644 index eb854c3bae84..000000000000 --- a/airbyte-integrations/connectors/source-insightly/unit_tests/test_incremental_streams.py +++ /dev/null @@ -1,81 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import pendulum -from airbyte_cdk.sources.streams.http.auth import BasicHttpAuthenticator -from pytest import fixture -from source_insightly.source import IncrementalInsightlyStream - -start_date = "2021-01-01T00:00:00Z" -authenticator = BasicHttpAuthenticator(username="test", password="") - - -@fixture -def patch_incremental_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(IncrementalInsightlyStream, "path", "v0/example_endpoint") - mocker.patch.object(IncrementalInsightlyStream, "primary_key", "test_primary_key") - mocker.patch.object(IncrementalInsightlyStream, "__abstractmethods__", set()) - - -def test_cursor_field(patch_incremental_base_class): - stream = IncrementalInsightlyStream(authenticator=authenticator, start_date=start_date) - expected_cursor_field = "DATE_UPDATED_UTC" - assert stream.cursor_field == expected_cursor_field - - -def test_incremental_params(patch_incremental_base_class): - """ - After talking to the insightly team we learned that the DATE_UPDATED_UTC - cursor is exclusive. Subtracting 1 second from the previous state makes it inclusive. - """ - stream = IncrementalInsightlyStream(authenticator=authenticator, start_date=start_date) - inputs = { - "stream_slice": None, - "stream_state": {"DATE_UPDATED_UTC": pendulum.datetime(2023, 5, 15, 18, 12, 44, tz="UTC")}, - "next_page_token": None, - } - expected_params = { - "count_total": True, - "skip": 0, - "top": 500, - "updated_after_utc": "2023-05-15T18:12:43Z", # 1 second subtracted from stream_state - } - assert stream.request_params(**inputs) == expected_params - - -def test_get_updated_state(patch_incremental_base_class): - stream = IncrementalInsightlyStream(authenticator=authenticator, start_date=start_date) - inputs = { - "current_stream_state": {"DATE_UPDATED_UTC": "2021-01-01T00:00:00Z"}, - "latest_record": {"DATE_UPDATED_UTC": "2021-02-01T00:00:00Z"}, - } - expected_state = {"DATE_UPDATED_UTC": pendulum.datetime(2021, 2, 1, 0, 0, 0, tz="UTC")} - assert stream.get_updated_state(**inputs) == expected_state - - -def test_get_updated_state_no_current_state(patch_incremental_base_class): - stream = IncrementalInsightlyStream(authenticator=authenticator, start_date=start_date) - inputs = {"current_stream_state": {}, "latest_record": {"DATE_UPDATED_UTC": "2021-01-01T00:00:00Z"}} - expected_state = {"DATE_UPDATED_UTC": pendulum.datetime(2021, 1, 1, 0, 0, 0, tz="UTC")} - assert stream.get_updated_state(**inputs) == expected_state - - -def test_supports_incremental(patch_incremental_base_class, mocker): - mocker.patch.object(IncrementalInsightlyStream, "cursor_field", "dummy_field") - stream = IncrementalInsightlyStream(authenticator=authenticator, start_date=start_date) - assert stream.supports_incremental - - -def test_source_defined_cursor(patch_incremental_base_class): - stream = IncrementalInsightlyStream(authenticator=authenticator, start_date=start_date) - assert stream.source_defined_cursor - - -def test_stream_checkpoint_interval(patch_incremental_base_class): - stream = IncrementalInsightlyStream(authenticator=authenticator, start_date=start_date) - # TODO: replace this with your expected checkpoint interval - expected_checkpoint_interval = None - assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-insightly/unit_tests/test_source.py b/airbyte-integrations/connectors/source-insightly/unit_tests/test_source.py deleted file mode 100644 index 4e9f2c408756..000000000000 --- a/airbyte-integrations/connectors/source-insightly/unit_tests/test_source.py +++ /dev/null @@ -1,56 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock, patch - -from source_insightly.source import SourceInsightly - - -class MockResponse: - def __init__(self, json_data, status_code): - self.json_data = json_data - self.status_code = status_code - - def json(self): - return self.json_data - - def raise_for_status(self): - if self.status_code != 200: - raise Exception("Bad things happened") - - -def mocked_requests_get(fail=False): - def wrapper(*args, **kwargs): - if fail: - return MockResponse(None, 404) - - return MockResponse( - {"INSTANCE_NAME": "bossco", "INSTANCE_SUBDOMAIN": None, "PLAN_NAME": "Gratis", "NEW_USER_EXPERIENCE_ENABLED": True}, 200 - ) - - return wrapper - - -@patch("requests.get", side_effect=mocked_requests_get()) -def test_check_connection(mocker): - source = SourceInsightly() - logger_mock, config_mock = MagicMock(), MagicMock() - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -@patch("requests.get", side_effect=mocked_requests_get(fail=True)) -def test_check_connection_fail(mocker): - source = SourceInsightly() - logger_mock, config_mock = MagicMock(), MagicMock() - assert source.check_connection(logger_mock, config_mock)[0] is False - assert source.check_connection(logger_mock, config_mock)[1] is not None - - -def test_streams(mocker): - source = SourceInsightly() - config_mock = MagicMock() - streams = source.streams(config_mock) - - expected_streams_number = 37 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-insightly/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-insightly/unit_tests/test_streams.py deleted file mode 100644 index e44895f5db26..000000000000 --- a/airbyte-integrations/connectors/source-insightly/unit_tests/test_streams.py +++ /dev/null @@ -1,126 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from airbyte_cdk.sources.streams.http.auth import BasicHttpAuthenticator -from source_insightly.source import InsightlyStream - -authenticator = BasicHttpAuthenticator(username="test", password="") - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(InsightlyStream, "path", "v0/example_endpoint") - mocker.patch.object(InsightlyStream, "primary_key", "test_primary_key") - mocker.patch.object(InsightlyStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {"count_total": True, "skip": 0, "top": 500} - assert stream.request_params(**inputs) == expected_params - - -def test_request_param_with_next_page_token(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": 1000} - expected_params = {"count_total": True, "skip": 1000, "top": 500} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - stream.total_count = 10000 - - request = MagicMock() - request.url = "https://api.insight.ly/v0/example_endpoint?count_total=True&skip=0&top=500" - response = MagicMock() - response.status_code = HTTPStatus.OK - response.request = request - - inputs = {"response": response} - expected_token = 500 - assert stream.next_page_token(**inputs) == expected_token - - -def test_next_page_token_last_records(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - stream.total_count = 2100 - - request = MagicMock() - request.url = "https://api.insight.ly/v0/example_endpoint?count_total=True&skip=1500&top=500" - response = MagicMock() - response.status_code = HTTPStatus.OK - response.request = request - - inputs = {"response": response} - expected_token = 2000 - assert stream.next_page_token(**inputs) == expected_token - - -def test_next_page_token_no_more_records(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - stream.total_count = 1000 - - request = MagicMock() - request.url = "https://api.insight.ly/v0/example_endpoint?count_total=True&skip=1000&top=500" - response = MagicMock() - response.status_code = HTTPStatus.OK - response.request = request - - inputs = {"response": response} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_parse_response(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - - response = MagicMock() - response.json = MagicMock(return_value=[{"data_field": [{"keys": ["keys"]}]}]) - - inputs = {"stream_state": "test_stream_state", "response": response} - expected_parsed_object = response.json()[0] - assert next(stream.parse_response(**inputs)) == expected_parsed_object - - -def test_request_headers(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {"Accept": "application/json"} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = InsightlyStream(authenticator=authenticator) - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = InsightlyStream(authenticator=authenticator) - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/docs/integrations/sources/insightly.md b/docs/integrations/sources/insightly.md index b51287925972..24c5aa71dba9 100644 --- a/docs/integrations/sources/insightly.md +++ b/docs/integrations/sources/insightly.md @@ -71,6 +71,7 @@ The connector is restricted by Insightly [requests limitation](https://api.na1.i | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------- | +| 0.2.0 | 2023-10-23 |[31162](https://github.com/airbytehq/airbyte/pull/31162) | Migrate to low-code framework | | 0.1.3 | 2023-05-15 |[26079](https://github.com/airbytehq/airbyte/pull/26079) | Make incremental syncs timestamp inclusive | | 0.1.2 | 2023-03-23 |[24422](https://github.com/airbytehq/airbyte/pull/24422) | Fix incremental timedelta causing missing records | | 0.1.1 | 2022-11-11 |[19356](https://github.com/airbytehq/airbyte/pull/19356) | Fix state date parse bug | From 93bc6f10ef3063f8ac0fdeee896430bd0ad81e26 Mon Sep 17 00:00:00 2001 From: Maxime Carbonneau-Leclerc Date: Tue, 14 Nov 2023 12:11:13 -0500 Subject: [PATCH 041/124] =?UTF-8?q?=E2=9C=A8=20Source=20Stripe:=20have=20a?= =?UTF-8?q?ll=20full=5Frefresh=20stream=20syncs=20be=20concurrent=20(#3247?= =?UTF-8?q?3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../connectors/source-stripe/main.py | 4 +- .../connectors/source-stripe/metadata.yaml | 2 +- .../source-stripe/source_stripe/source.py | 47 ++++++++++--------- .../source-stripe/unit_tests/conftest.py | 2 +- .../source-stripe/unit_tests/test_source.py | 43 +++++++++++++++-- docs/integrations/sources/stripe.md | 1 + 6 files changed, 70 insertions(+), 29 deletions(-) diff --git a/airbyte-integrations/connectors/source-stripe/main.py b/airbyte-integrations/connectors/source-stripe/main.py index 5ed89fbb10d4..2e2166e50213 100644 --- a/airbyte-integrations/connectors/source-stripe/main.py +++ b/airbyte-integrations/connectors/source-stripe/main.py @@ -14,9 +14,9 @@ def _get_source(args: List[str]): - catalog = AirbyteEntrypoint.extract_catalog(args) + catalog_path = AirbyteEntrypoint.extract_catalog(args) try: - return SourceStripe(catalog) + return SourceStripe(SourceStripe.read_catalog(catalog_path) if catalog_path else None) except Exception as error: print( AirbyteMessage( diff --git a/airbyte-integrations/connectors/source-stripe/metadata.yaml b/airbyte-integrations/connectors/source-stripe/metadata.yaml index bc5f90db6dea..0c34e7b7c27d 100644 --- a/airbyte-integrations/connectors/source-stripe/metadata.yaml +++ b/airbyte-integrations/connectors/source-stripe/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: e094cb9a-26de-4645-8761-65c0c425d1de - dockerImageTag: 4.5.2 + dockerImageTag: 4.5.3 dockerRepository: airbyte/source-stripe documentationUrl: https://docs.airbyte.com/integrations/sources/stripe githubIssueLabel: source-stripe diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py index 9ebd06df8026..e72e4dd5398b 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py @@ -9,7 +9,7 @@ import stripe from airbyte_cdk import AirbyteLogger from airbyte_cdk.entrypoint import logger as entrypoint_logger -from airbyte_cdk.models import FailureType, SyncMode +from airbyte_cdk.models import ConfiguredAirbyteCatalog, FailureType from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.message.repository import InMemoryMessageRepository from airbyte_cdk.sources.streams import Stream @@ -17,6 +17,7 @@ from airbyte_cdk.sources.streams.concurrent.cursor import NoopCursor from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from airbyte_protocol.models import SyncMode from source_stripe.streams import ( CheckoutSessionsLineItems, CreatedCursorIncrementalStripeStream, @@ -38,15 +39,17 @@ class SourceStripe(AbstractSource): - def __init__(self, catalog_path: Optional[str] = None, **kwargs): + def __init__(self, catalog: Optional[ConfiguredAirbyteCatalog], **kwargs): super().__init__(**kwargs) - if catalog_path: - catalog = self.read_catalog(catalog_path) - # Only use concurrent cdk if all streams are running in full_refresh - all_sync_mode_are_full_refresh = all(stream.sync_mode == SyncMode.full_refresh for stream in catalog.streams) - self._use_concurrent_cdk = all_sync_mode_are_full_refresh + if catalog: + self._streams_configured_as_full_refresh = { + configured_stream.stream.name + for configured_stream in catalog.streams + if configured_stream.sync_mode == SyncMode.full_refresh + } else: - self._use_concurrent_cdk = False + # things will NOT be executed concurrently + self._streams_configured_as_full_refresh = set() message_repository = InMemoryMessageRepository(entrypoint_logger.level) @@ -437,17 +440,19 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: **args, ), ] - if self._use_concurrent_cdk: - # We cap the number of workers to avoid hitting the Stripe rate limit - # The limit can be removed or increased once we have proper rate limiting - concurrency_level = min(config.get("num_workers", 2), _MAX_CONCURRENCY) - streams[0].logger.info(f"Using concurrent cdk with concurrency level {concurrency_level}") - # The state is known to be empty because concurrent CDK is currently only used for full refresh - state = {} - cursor = NoopCursor() - return [ - StreamFacade.create_from_stream(stream, self, entrypoint_logger, concurrency_level, state, cursor) for stream in streams - ] - else: - return streams + # We cap the number of workers to avoid hitting the Stripe rate limit + # The limit can be removed or increased once we have proper rate limiting + concurrency_level = min(config.get("num_workers", 2), _MAX_CONCURRENCY) + streams[0].logger.info(f"Using concurrent cdk with concurrency level {concurrency_level}") + + return [ + StreamFacade.create_from_stream(stream, self, entrypoint_logger, concurrency_level, self._create_empty_state(), NoopCursor()) + if stream.name in self._streams_configured_as_full_refresh + else stream + for stream in streams + ] + + def _create_empty_state(self) -> MutableMapping[str, Any]: + # The state is known to be empty because concurrent CDK is currently only used for full refresh + return {} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py b/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py index 5eb8e515068a..5dc33d2f8ddb 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py @@ -39,7 +39,7 @@ def stream_by_name(config): from source_stripe.source import SourceStripe def mocker(stream_name, source_config=config): - source = SourceStripe() + source = SourceStripe(None) streams = source.streams(source_config) for stream in streams: if stream.name == stream_name: diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py index 13d7e15f9fc4..61b226b5da83 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py @@ -9,10 +9,37 @@ import pytest import source_stripe import stripe +from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode +from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade from airbyte_cdk.utils import AirbyteTracedException from source_stripe import SourceStripe logger = logging.getLogger("airbyte") +_ANY_CATALOG = ConfiguredAirbyteCatalog.parse_obj({"streams": []}) + + +class CatalogBuilder: + def __init__(self) -> None: + self._streams = [] + + def with_stream(self, name: str, sync_mode: SyncMode) -> "CatalogBuilder": + self._streams.append( + { + "stream": { + "name": name, + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]], + }, + "primary_key": [["id"]], + "sync_mode": sync_mode.name, + "destination_sync_mode": "overwrite", + } + ) + return self + + def build(self) -> ConfiguredAirbyteCatalog: + return ConfiguredAirbyteCatalog.parse_obj({"streams": self._streams}) def _a_valid_config(): @@ -21,11 +48,11 @@ def _a_valid_config(): @patch.object(source_stripe.source, "stripe") def test_source_check_connection_ok(mocked_client, config): - assert SourceStripe().check_connection(logger, config=config) == (True, None) + assert SourceStripe(_ANY_CATALOG).check_connection(logger, config=config) == (True, None) def test_streams_are_unique(config): - stream_names = [s.name for s in SourceStripe().streams(config=config)] + stream_names = [s.name for s in SourceStripe(_ANY_CATALOG).streams(config=config)] assert len(stream_names) == len(set(stream_names)) == 46 @@ -42,7 +69,7 @@ def test_streams_are_unique(config): def test_config_validation(mocked_client, input_config, expected_error_msg): context = pytest.raises(AirbyteTracedException, match=expected_error_msg) if expected_error_msg else does_not_raise() with context: - SourceStripe().check_connection(logger, config=input_config) + SourceStripe(_ANY_CATALOG).check_connection(logger, config=input_config) @pytest.mark.parametrize( @@ -55,5 +82,13 @@ def test_config_validation(mocked_client, input_config, expected_error_msg): @patch.object(source_stripe.source.stripe, "Account") def test_given_stripe_error_when_check_connection_then_connection_not_available(mocked_client, exception): mocked_client.retrieve.side_effect = exception - is_available, _ = SourceStripe().check_connection(logger, config=_a_valid_config()) + is_available, _ = SourceStripe(_ANY_CATALOG).check_connection(logger, config=_a_valid_config()) assert not is_available + + +def test_when_streams_return_full_refresh_as_concurrent(): + streams = SourceStripe( + CatalogBuilder().with_stream("bank_accounts", SyncMode.full_refresh).with_stream("customers", SyncMode.incremental).build() + ).streams(_a_valid_config()) + + assert len(list(filter(lambda stream: isinstance(stream, StreamFacade), streams))) == 1 diff --git a/docs/integrations/sources/stripe.md b/docs/integrations/sources/stripe.md index 30c101fac0d8..30e1967bf371 100644 --- a/docs/integrations/sources/stripe.md +++ b/docs/integrations/sources/stripe.md @@ -192,6 +192,7 @@ The Stripe connector should not run into Stripe API limitations under normal usa | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| 4.5.3 | 2023-11-14 | [32473](https://github.com/airbytehq/airbyte/pull/32473/) | Have all full_refresh stream syncs be concurrent | | 4.5.2 | 2023-11-03 | [32146](https://github.com/airbytehq/airbyte/pull/32146/) | Fix multiple BankAccount issues | | 4.5.1 | 2023-11-01 | [32056](https://github.com/airbytehq/airbyte/pull/32056/) | Use CDK version 0.52.8 | | 4.5.0 | 2023-10-25 | [31327](https://github.com/airbytehq/airbyte/pull/31327/) | Use concurrent CDK when running in full-refresh | From f65569e7099e5ab1e65f856b7e683a6b8264a85b Mon Sep 17 00:00:00 2001 From: Erica D'Souza <93952107+erica-airbyte@users.noreply.github.com> Date: Tue, 14 Nov 2023 11:08:34 -0700 Subject: [PATCH 042/124] Update contact-support.md --- docs/operator-guides/contact-support.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/operator-guides/contact-support.md b/docs/operator-guides/contact-support.md index e5a97d4d1351..d90b59272924 100644 --- a/docs/operator-guides/contact-support.md +++ b/docs/operator-guides/contact-support.md @@ -18,9 +18,9 @@ If you require personalized support, reach out to our sales team to inquire abou If you have questions about connector setup, error resolution, or want to report a bug, Airbyte Support is available to assist you. We recommend checking [our documentation](https://docs.airbyte.com/) and searching our [Help Center](https://support.airbyte.com/hc/en-us) before opening a support ticket. -If you couldn't find the information you need in our docs or Help Center, open a ticket within the Airbyte Cloud platform by selecting the "Support" icon in the lower left navigation bar. Alternatively, you can submit a ticket through our [Help Center](https://support.airbyte.com/hc/en-us) by completing an Airbyte Cloud Support Request. Our team is online and availible to assist from 7AM - 7PM Eastern. +If you couldn't find the information you need in our docs or Help Center, open a ticket within the Airbyte Cloud platform by selecting the "Support" icon in the lower left navigation bar. Alternatively, you can submit a ticket through our [Help Center](https://support.airbyte.com/hc/en-us) by completing an Airbyte Cloud Support Request. Our team is online and availible to assist from 7AM - 7PM Eastern. -If you're unsure about the supported connectors, refer to our [Connector Catalog](https://docs.airbyte.com/integrations/) & [Connector Support Levels](https://docs.airbyte.com/project-overview/product-support-levels/). +**If you're unsure about the supported connectors, refer to our [Connector Support Levels](https://docs.airbyte.com/project-overview/product-support-levels/) & [Connector Catalog](https://docs.airbyte.com/integrations/).** For account or credit-related inquiries, contact our [sales team](https://airbyte.com/talk-to-sales). @@ -34,6 +34,8 @@ If you're running Airbyte Open Source with Airbyte Enterprise or have an OSS sup Before opening a support ticket, we recommend consulting [our documentation](https://docs.airbyte.com/) and searching our [Help Center](https://support.airbyte.com/hc/en-us). If your question remains unanswered, please submit a ticket through our Help Center. We suggest creating an [Airbyte Help Center account](https://airbyte1416.zendesk.com/auth/v2/login/signin?return_to=https%3A%2F%2Fsupport.airbyte.com%2Fhc%2Fen-us&theme=hc&locale=en-us&brand_id=15365055240347&auth_origin=15365055240347%2Ctrue%2Ctrue) to access your organization's support requests. Our team is online and availible to assist from 7AM - 7PM Eastern. +**Connector support is based on certification status of the connector.** Please see our [Connector Support Levels](https://docs.airbyte.com/project-overview/product-support-levels) if you have any questions on support provided for one of your connectors. + Submitting a Pull Request for review? * Be sure to follow our [contribution guidelines](https://docs.airbyte.com/contributing-to-airbyte/) laid out here on our doc. Highlights include: From dc48c4f29a0b72868d965e8c0dbf827c2828ec66 Mon Sep 17 00:00:00 2001 From: Daryna Ishchenko <80129833+darynaishchenko@users.noreply.github.com> Date: Tue, 14 Nov 2023 20:38:27 +0200 Subject: [PATCH 043/124] :sparkles: Source Bing Ads: custom reports (#32306) Co-authored-by: darynaishchenko --- .../connectors/source-bing-ads/metadata.yaml | 2 +- .../source-bing-ads/source_bing_ads/client.py | 2 +- .../source-bing-ads/source_bing_ads/source.py | 41 ++- .../source-bing-ads/source_bing_ads/spec.json | 99 +++++++ .../source_bing_ads/streams.py | 98 ++++++- .../source-bing-ads/unit_tests/test_client.py | 13 + .../source-bing-ads/unit_tests/test_source.py | 244 +++++++++++++++++- docs/integrations/sources/bing-ads.md | 28 +- 8 files changed, 516 insertions(+), 11 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml index 4200d85c8cd9..2fa6e2d659d9 100644 --- a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml @@ -16,7 +16,7 @@ data: connectorSubtype: api connectorType: source definitionId: 47f25999-dd5e-4636-8c39-e7cea2453331 - dockerImageTag: 1.12.1 + dockerImageTag: 1.13.0 dockerRepository: airbyte/source-bing-ads documentationUrl: https://docs.airbyte.com/integrations/sources/bing-ads githubIssueLabel: source-bing-ads diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py index c07e4d158f9d..afd7862a93f9 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py @@ -37,7 +37,7 @@ class Client: # https://docs.microsoft.com/en-us/advertising/guides/services-protocol?view=bingads-13#throttling # https://docs.microsoft.com/en-us/advertising/guides/operation-error-codes?view=bingads-13 retry_on_codes: Iterator[str] = ["117", "207", "4204", "109", "0"] - max_retries: int = 10 + max_retries: int = 5 # A backoff factor to apply between attempts after the second try # {retry_factor} * (2 ** ({number of total retries} - 1)) retry_factor: int = 15 diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py index f4ef320b3855..9ff7a581479c 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # from itertools import product -from typing import Any, List, Mapping, Tuple +from typing import Any, List, Mapping, Optional, Tuple from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import FailureType, SyncMode @@ -41,6 +41,7 @@ AgeGenderAudienceReportWeekly, AppInstallAdLabels, AppInstallAds, + BingAdsReportingServiceStream, BudgetSummaryReport, CampaignImpressionPerformanceReportDaily, CampaignImpressionPerformanceReportHourly, @@ -52,6 +53,7 @@ CampaignPerformanceReportMonthly, CampaignPerformanceReportWeekly, Campaigns, + CustomReport, GeographicPerformanceReportDaily, GeographicPerformanceReportHourly, GeographicPerformanceReportMonthly, @@ -83,17 +85,49 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> try: client = Client(**config) account_ids = {str(account["Id"]) for account in Accounts(client, config).read_records(SyncMode.full_refresh)} + self.validate_custom_reposts(config, client) if account_ids: return True, None else: raise AirbyteTracedException( - message="Config validation error: You don't have accounts assigned to this user.", + message="Config validation error: You don't have accounts assigned to this user. Please verify your developer token.", internal_message="You don't have accounts assigned to this user.", failure_type=FailureType.config_error, ) except Exception as error: return False, error + def validate_custom_reposts(self, config: Mapping[str, Any], client: Client): + custom_reports = self.get_custom_reports(config, client) + for custom_report in custom_reports: + is_valid, reason = custom_report.validate_report_configuration() + if not is_valid: + raise AirbyteTracedException( + message=f"Config validation error: {custom_report.name}: {reason}", + internal_message=f"{custom_report.name}: {reason}", + failure_type=FailureType.config_error, + ) + + def _clear_reporting_object_name(self, report_object: str) -> str: + # reporting mixin adds it + if report_object.endswith("Request"): + return report_object.replace("Request", "") + return report_object + + def get_custom_reports(self, config: Mapping[str, Any], client: Client) -> List[Optional[Stream]]: + return [ + type( + report["name"], + (CustomReport,), + { + "report_name": self._clear_reporting_object_name(report["reporting_object"]), + "custom_report_columns": report["report_columns"], + "report_aggregation": report["report_aggregation"], + }, + )(client, config) + for report in config.get("custom_reports", []) + ] + def streams(self, config: Mapping[str, Any]) -> List[Stream]: client = Client(**config) streams = [ @@ -127,4 +161,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: ) report_aggregation = ("Hourly", "Daily", "Weekly", "Monthly") streams.extend([eval(f"{report}{aggregation}")(client, config) for (report, aggregation) in product(reports, report_aggregation)]) + + custom_reports = self.get_custom_reports(config, client) + streams.extend(custom_reports) return streams diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json index e7314b86a022..4461a72fd7dd 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json @@ -63,6 +63,105 @@ "minimum": 0, "maximum": 90, "order": 6 + }, + "custom_reports": { + "title": "Custom Reports", + "description": "You can add your Custom Bing Ads report by creating one.", + "order": 7, + "type": "array", + "items": { + "title": "Custom Report Config", + "type": "object", + "properties": { + "name": { + "title": "Report Name", + "description": "The name of the custom report, this name would be used as stream name", + "type": "string", + "examples": [ + "Account Performance", + "AdDynamicTextPerformanceReport", + "custom report" + ] + }, + "reporting_object": { + "title": "Reporting Data Object", + "description": "The name of the the object derives from the ReportRequest object. You can find it in Bing Ads Api docs - Reporting API - Reporting Data Objects.", + "type": "string", + "enum": [ + "AccountPerformanceReportRequest", + "AdDynamicTextPerformanceReportRequest", + "AdExtensionByAdReportRequest", + "AdExtensionByKeywordReportRequest", + "AdExtensionDetailReportRequest", + "AdGroupPerformanceReportRequest", + "AdPerformanceReportRequest", + "AgeGenderAudienceReportRequest", + "AudiencePerformanceReportRequest", + "CallDetailReportRequest", + "CampaignPerformanceReportRequest", + "ConversionPerformanceReportRequest", + "DestinationUrlPerformanceReportRequest", + "DSAAutoTargetPerformanceReportRequest", + "DSACategoryPerformanceReportRequest", + "DSASearchQueryPerformanceReportRequest", + "GeographicPerformanceReportRequest", + "GoalsAndFunnelsReportRequest", + "HotelDimensionPerformanceReportRequest", + "HotelGroupPerformanceReportRequest", + "KeywordPerformanceReportRequest", + "NegativeKeywordConflictReportRequest", + "ProductDimensionPerformanceReportRequest", + "ProductMatchCountReportRequest", + "ProductNegativeKeywordConflictReportRequest", + "ProductPartitionPerformanceReportRequest", + "ProductPartitionUnitPerformanceReportRequest", + "ProductSearchQueryPerformanceReportRequest", + "ProfessionalDemographicsAudienceReportRequest", + "PublisherUsagePerformanceReportRequest", + "SearchCampaignChangeHistoryReportRequest", + "SearchQueryPerformanceReportRequest", + "ShareOfVoiceReportRequest", + "UserLocationPerformanceReportRequest" + ] + }, + "report_columns": { + "title": "Columns", + "description": "A list of available report object columns. You can find it in description of reporting object that you want to add to custom report.", + "type": "array", + "items": { + "description": "Name of report column.", + "type": "string" + }, + "minItems": 1 + }, + "report_aggregation": { + "title": "Aggregation", + "description": "A list of available aggregations.", + "type": "string", + "items": { + "title": "ValidEnums", + "description": "An enumeration of aggregations.", + "enum": [ + "Hourly", + "Daily", + "Weekly", + "Monthly", + "DayOfWeek", + "HourOfDay", + "WeeklyStartingMonday", + "Summary" + ] + }, + "default": ["Hourly"] + } + }, + "required": [ + "name", + "reporting_object", + "report_columns", + "report_aggregation" + ] + } } } }, diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/streams.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/streams.py index 342a2c827ddd..ecbb4cb450f8 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/streams.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/streams.py @@ -2,12 +2,15 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # import os +import re import ssl import time +import xml.etree.ElementTree as ET from abc import ABC, abstractmethod from datetime import timezone from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union from urllib.error import URLError +from urllib.parse import urlparse import _csv import pandas as pd @@ -16,6 +19,7 @@ from airbyte_cdk.sources.streams import IncrementalMixin, Stream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from bingads.service_client import ServiceClient +from bingads.v13.internal.reporting.row_report import _RowReport from bingads.v13.internal.reporting.row_report_iterator import _RowReportRecord from bingads.v13.reporting.reporting_service_manager import ReportingServiceManager from numpy import nan @@ -32,7 +36,7 @@ PerformanceReportsMixin, ReportsMixin, ) -from suds import sudsobject +from suds import WebFault, sudsobject class BingAdsBaseStream(Stream, ABC): @@ -1260,3 +1264,95 @@ class UserLocationPerformanceReportWeekly(UserLocationPerformanceReport): class UserLocationPerformanceReportMonthly(UserLocationPerformanceReport): report_aggregation = "Monthly" + + +class CustomReport(PerformanceReportsMixin, BingAdsReportingServiceStream, ABC): + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + custom_report_columns = [] + report_schema_name = None + primary_key = None + + @property + def cursor_field(self) -> Union[str, List[str]]: + # Summary aggregation doesn't include TimePeriod field + if self.report_aggregation != "Summary": + return "TimePeriod" + + @property + def report_columns(self): + # adding common and default columns + if "AccountId" not in self.custom_report_columns: + self.custom_report_columns.append("AccountId") + if self.cursor_field and self.cursor_field not in self.custom_report_columns: + self.custom_report_columns.append(self.cursor_field) + return list(frozenset(self.custom_report_columns)) + + def get_json_schema(self) -> Mapping[str, Any]: + columns_schema = {col: {"type": ["null", "string"]} for col in self.report_columns} + schema: Mapping[str, Any] = { + "$schema": "https://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": True, + "properties": columns_schema, + } + return schema + + def validate_report_configuration(self) -> Tuple[bool, str]: + # gets /bingads/v13/proxies/production/reporting_service.xml + reporting_service_file = self.client.get_service(self.service_name)._get_service_info_dict(self.client.api_version)[ + ("reporting", self.client.environment) + ] + tree = ET.parse(urlparse(reporting_service_file).path) + request_object = tree.find(f".//{{*}}complexType[@name='{self.report_name}Request']") + + report_object_columns = self._get_object_columns(request_object, tree) + is_custom_cols_in_report_object_cols = all(x in report_object_columns for x in self.custom_report_columns) + + if not is_custom_cols_in_report_object_cols: + return False, ( + f"Reporting Columns are invalid. Columns that you provided don't belong to Reporting Data Object Columns:" + f" {self.custom_report_columns}. Please ensure it is correct in Bing Ads Docs." + ) + + return True, "" + + def _clear_namespace(self, type: str) -> str: + return re.sub(r"^[a-z]+:", "", type) + + def _get_object_columns(self, request_el: ET.Element, tree: ET.ElementTree) -> List[str]: + column_el = request_el.find(".//{*}element[@name='Columns']") + array_of_columns_name = self._clear_namespace(column_el.get("type")) + + array_of_columns_elements = tree.find(f".//{{*}}complexType[@name='{array_of_columns_name}']") + inner_array_of_columns_elements = array_of_columns_elements.find(".//{*}element") + column_el_name = self._clear_namespace(inner_array_of_columns_elements.get("type")) + + column_el = tree.find(f".//{{*}}simpleType[@name='{column_el_name}']") + column_enum_items = column_el.findall(".//{*}enumeration") + column_enum_items_values = [el.get("value") for el in column_enum_items] + return column_enum_items_values + + def get_report_record_timestamp(self, datestring: str) -> int: + """ + Parse report date field based on aggregation type + """ + if not self.report_aggregation: + date = pendulum.from_format(datestring, "M/D/YYYY") + else: + if self.report_aggregation in ["DayOfWeek", "HourOfDay"]: + return int(datestring) + if self.report_aggregation == "Hourly": + date = pendulum.from_format(datestring, "YYYY-MM-DD|H") + else: + date = pendulum.parse(datestring) + + return date.int_timestamp + + def send_request(self, params: Mapping[str, Any], customer_id: str, account_id: str) -> _RowReport: + try: + return super().send_request(params, customer_id, account_id) + except WebFault as e: + self.logger.error( + f"Could not sync custom report {self.name}: Please validate your column and aggregation configuration. " + f"Error form server: [{e.fault.faultstring}]" + ) diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_client.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_client.py index cad3d70b7c30..96b2d5777f57 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_client.py @@ -12,6 +12,7 @@ import source_bing_ads.client from airbyte_cdk.utils import AirbyteTracedException from bingads.authorization import AuthorizationData, OAuthTokens +from bingads.v13.bulk import BulkServiceManager from bingads.v13.reporting.exceptions import ReportingDownloadException from suds import sudsobject @@ -176,3 +177,15 @@ def test_bulk_service_manager(patched_request_tokens): client = source_bing_ads.client.Client("tenant_id", "2020-01-01", client_id="client_id", refresh_token="refresh_token") service = client._bulk_service_manager() assert (service._poll_interval_in_milliseconds, service._environment) == (5000, client.environment) + + +def test_get_bulk_entity(requests_mock): + requests_mock.post( + "https://login.microsoftonline.com/tenant_id/oauth2/v2.0/token", + status_code=200, + json={"access_token": "test", "expires_in": "9000", "refresh_token": "test"}, + ) + client = source_bing_ads.client.Client("tenant_id", "2020-01-01", client_id="client_id", refresh_token="refresh_token") + with patch.object(BulkServiceManager, "download_file", return_value="file.csv"): + bulk_entity = client.get_bulk_entity(data_scope=["EntityData"], download_entities=["AppInstallAds"]) + assert bulk_entity == "file.csv" diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py index f0f69c63faad..379eb5afacb1 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py @@ -2,14 +2,20 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import xml.etree.ElementTree as ET from pathlib import Path -from unittest.mock import patch +from unittest.mock import MagicMock, patch +from urllib.parse import urlparse import pytest import source_bing_ads from airbyte_cdk.models import SyncMode +from airbyte_cdk.utils import AirbyteTracedException +from bingads.service_info import SERVICE_INFO_DICT_V13 +from source_bing_ads.reports import ReportsMixin from source_bing_ads.source import SourceBingAds from source_bing_ads.streams import AccountPerformanceReportMonthly, Accounts, AdGroups, Ads, AppInstallAds, Campaigns +from suds import WebFault @pytest.fixture(name="config") @@ -25,6 +31,44 @@ def config_fixture(): } +@pytest.fixture(name="config_with_custom_reports") +def config_with_custom_reports_fixture(): + """Generates streams settings with custom reports from a config file""" + return { + "tenant_id": "common", + "developer_token": "fake_developer_token", + "refresh_token": "fake_refresh_token", + "client_id": "fake_client_id", + "reports_start_date": "2020-01-01", + "lookback_window": 0, + "custom_reports": [ + { + "name": "my test custom report", + "reporting_object": "DSAAutoTargetPerformanceReport", + "report_columns": [ + "AbsoluteTopImpressionRatePercent", + "AccountId", + "AccountName", + "AccountNumber", + "AccountStatus", + "AdDistribution", + "AdGroupId", + "AdGroupName", + "AdGroupStatus", + "AdId", + "AllConversionRate", + "AllConversions", + "AllConversionsQualified", + "AllCostPerConversion", + "AllReturnOnAdSpend", + "AllRevenue", + ], + "report_aggregation": "Weekly", + } + ], + } + + @pytest.fixture(name="logger_mock") def logger_mock_fixture(): return patch("source_bing_ads.source.AirbyteLogger") @@ -47,7 +91,9 @@ def test_source_check_connection_failed_user_do_not_have_accounts(mocked_client, with patch.object(Accounts, "read_records", return_value=[]): connected, reason = SourceBingAds().check_connection(logger_mock, config=config) assert connected is False - assert reason.message == "Config validation error: You don't have accounts assigned to this user." + assert ( + reason.message == "Config validation error: You don't have accounts assigned to this user. Please verify your developer token." + ) def test_source_check_connection_failed_invalid_creds(config, logger_mock): @@ -60,6 +106,56 @@ def test_source_check_connection_failed_invalid_creds(config, logger_mock): ) +@patch.object(source_bing_ads.source, "Client") +def test_validate_custom_reposts(mocked_client, config_with_custom_reports, logger_mock): + reporting_service_mock = MagicMock() + reporting_service_mock._get_service_info_dict.return_value = SERVICE_INFO_DICT_V13 + mocked_client.get_service.return_value = reporting_service_mock + mocked_client.environment = "production" + res = SourceBingAds().validate_custom_reposts(config=config_with_custom_reports, client=mocked_client) + assert res is None + + +@patch.object(source_bing_ads.source, "Client") +def test_validate_custom_reposts_failed_invalid_report_columns(mocked_client, config_with_custom_reports, logger_mock): + reporting_service_mock = MagicMock() + reporting_service_mock._get_service_info_dict.return_value = SERVICE_INFO_DICT_V13 + mocked_client.get_service.return_value = reporting_service_mock + mocked_client.environment = "production" + config_with_custom_reports["custom_reports"][0]["report_columns"] = ["TimePeriod", "NonExistingColumn", "ConversionRate"] + + with pytest.raises(AirbyteTracedException) as e: + SourceBingAds().validate_custom_reposts(config=config_with_custom_reports, client=mocked_client) + assert e.value.internal_message == ( + "my test custom report: Reporting Columns are invalid. " + "Columns that you provided don't belong to Reporting Data Object Columns:" + " ['TimePeriod', 'NonExistingColumn', 'ConversionRate']. " + "Please ensure it is correct in Bing Ads Docs." + ) + assert ( + "Config validation error: my test custom report: Reporting Columns are " + "invalid. Columns that you provided don't belong to Reporting Data Object " + "Columns: ['TimePeriod', 'NonExistingColumn', 'ConversionRate']. Please " + "ensure it is correct in Bing Ads Docs." + ) in e.value.message + + +@patch.object(source_bing_ads.source, "Client") +def test_get_custom_reports(mocked_client, config_with_custom_reports): + custom_reports = SourceBingAds().get_custom_reports(config_with_custom_reports, mocked_client) + assert isinstance(custom_reports, list) + assert custom_reports[0].report_name == "DSAAutoTargetPerformanceReport" + assert custom_reports[0].report_aggregation == "Weekly" + assert "AccountId" in custom_reports[0].custom_report_columns + + +def test_clear_reporting_object_name(): + reporting_object = SourceBingAds()._clear_reporting_object_name("DSAAutoTargetPerformanceReportRequest") + assert reporting_object == "DSAAutoTargetPerformanceReport" + reporting_object = SourceBingAds()._clear_reporting_object_name("DSAAutoTargetPerformanceReport") + assert reporting_object == "DSAAutoTargetPerformanceReport" + + @patch.object(source_bing_ads.source, "Client") def test_campaigns_request_params(mocked_client, config): @@ -262,3 +358,147 @@ def test_transform(mocked_client, config): "EditorialStatus": "ActiveLimited", "FinalAppUrls": None, } + + +@patch.object(source_bing_ads.source, "Client") +def test_custom_report_clear_namespace(mocked_client, config_with_custom_reports, logger_mock): + custom_report = SourceBingAds().get_custom_reports(config_with_custom_reports, mocked_client)[0] + assert custom_report._clear_namespace("tns:ReportAggregation") == "ReportAggregation" + + +@patch.object(source_bing_ads.source, "Client") +def test_custom_report_get_object_columns(mocked_client, config_with_custom_reports, logger_mock): + reporting_service_mock = MagicMock() + reporting_service_mock._get_service_info_dict.return_value = SERVICE_INFO_DICT_V13 + mocked_client.get_service.return_value = reporting_service_mock + mocked_client.environment = "production" + + custom_report = SourceBingAds().get_custom_reports(config_with_custom_reports, mocked_client)[0] + + tree = ET.parse(urlparse(SERVICE_INFO_DICT_V13[("reporting", mocked_client.environment)]).path) + request_object = tree.find(f".//{{*}}complexType[@name='{custom_report.report_name}Request']") + + assert custom_report._get_object_columns(request_object, tree) == [ + "TimePeriod", + "AccountId", + "AccountName", + "AccountNumber", + "AccountStatus", + "CampaignId", + "CampaignName", + "CampaignStatus", + "AdGroupId", + "AdGroupName", + "AdGroupStatus", + "AdDistribution", + "Language", + "Network", + "TopVsOther", + "DeviceType", + "DeviceOS", + "BidStrategyType", + "TrackingTemplate", + "CustomParameters", + "DynamicAdTargetId", + "DynamicAdTarget", + "DynamicAdTargetStatus", + "WebsiteCoverage", + "Impressions", + "Clicks", + "Ctr", + "AverageCpc", + "Spend", + "AveragePosition", + "Conversions", + "ConversionRate", + "CostPerConversion", + "Assists", + "Revenue", + "ReturnOnAdSpend", + "CostPerAssist", + "RevenuePerConversion", + "RevenuePerAssist", + "AllConversions", + "AllRevenue", + "AllConversionRate", + "AllCostPerConversion", + "AllReturnOnAdSpend", + "AllRevenuePerConversion", + "ViewThroughConversions", + "Goal", + "GoalType", + "AbsoluteTopImpressionRatePercent", + "TopImpressionRatePercent", + "AverageCpm", + "ConversionsQualified", + "AllConversionsQualified", + "ViewThroughConversionsQualified", + "AdId", + "ViewThroughRevenue", + ] + + +@patch.object(source_bing_ads.source, "Client") +def test_custom_report_send_request(mocked_client, config_with_custom_reports, logger_mock, caplog): + class Fault: + faultstring = "Invalid Client Data" + + custom_report = SourceBingAds().get_custom_reports(config_with_custom_reports, mocked_client)[0] + with patch.object(ReportsMixin, "send_request", side_effect=WebFault(fault=Fault(), document=None)): + custom_report.send_request(params={}, customer_id="13131313", account_id="800800808") + assert ( + "Could not sync custom report my test custom report: Please validate your column and aggregation configuration. " + "Error form server: [Invalid Client Data]" + ) in caplog.text + + +@pytest.mark.parametrize( + "aggregation,datastring,expected", + ( + ( + "DayOfWeek", + "1", + 1, + ), + ( + "HourOfDay", + "20", + 20, + ), + ( + "Hourly", + "2022-11-13|10", + 1668333600, + ), + ( + "Hourly", + "2022-11-13|10", + 1668333600, + ), + ( + "Daily", + "2022-11-13", + 1668297600, + ), + ( + "Weekly", + "2022-11-13", + 1668297600, + ), + ( + "Monthly", + "2022-11-13", + 1668297600, + ), + ( + "WeeklyStartingMonday", + "2022-11-13", + 1668297600, + ), + ), +) +@patch.object(source_bing_ads.source, "Client") +def test_custom_report_get_report_record_timestamp(mocked_client, config_with_custom_reports, aggregation, datastring, expected): + custom_report = SourceBingAds().get_custom_reports(config_with_custom_reports, mocked_client)[0] + custom_report.report_aggregation = aggregation + assert custom_report.get_report_record_timestamp(datastring) == expected diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md index 7f4d3212ef88..0986d6250757 100644 --- a/docs/integrations/sources/bing-ads.md +++ b/docs/integrations/sources/bing-ads.md @@ -53,9 +53,14 @@ The tenant is used in the authentication URL, for example: `https://login.micros 6. Add the developer token from [Step 1](#step-1-set-up-bing-ads). 7. For **Reports Replication Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data from previous and current calendar years. 8. For **Lookback window** (also known as attribution or conversion window) enter the number of **days** to look into the past. If your conversion window has an hours/minutes granularity, round it up to the number of days exceeding. If you're not using performance report streams in incremental mode and Reports Start Date is not provided, let it with 0 default value. -9. Click **Authenticate your Bing Ads account**. -10. Log in and authorize the Bing Ads account. -11. Click **Set up source**. +9. For *Custom Reports* - see [custom reports](#custom-reports) section, list of custom reports object: + 1. For *Report Name* enter the name that you want for your custom report. + 2. For *Reporting Data Object* add the Bing Ads Reporting Object that you want to sync in the custom report. + 3. For *Columns* add list columns of Reporting Data Object that you want to see in the custom report. + 4. For *Aggregation* add time aggregation. See [report aggregation](#report-aggregation) section. +10. Click **Authenticate your Bing Ads account**. +11. Log in and authorize the Bing Ads account. +12. Click **Set up source**. @@ -70,7 +75,13 @@ The tenant is used in the authentication URL, for example: `https://login.micros 6. Enter the **Client ID**, **Client Secret**, **Refresh Token**, and **Developer Token** from [Step 1](#step-1-set-up-bing-ads). 7. For **Reports Replication Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data from previous and current calendar years. 8. For **Lookback window** (also known as attribution or conversion window) enter the number of **days** to look into the past. If your conversion window has an hours/minutes granularity, round it up to the number of days exceeding. If you're not using performance report streams in incremental mode and Reports Start Date is not provided, let it with 0 default value. -9. Click **Set up source**. +9. For *Custom Reports* - see [custom reports](#custom-reports) section: + 1. For *Report Name* enter the name that you want for your custom report. + 2. For *Reporting Data Object* add the Bing Ads Reporting Object that you want to sync in the custom report. + 3. For *Columns* add columns of Reporting Data Object that you want to see in the custom report. + 4. For *Aggregation* select time aggregation. See [report aggregation](#report-aggregation) section. + +10. Click **Set up source**. ## Supported sync modes @@ -152,6 +163,14 @@ The Bing Ads source connector supports the following streams. For more informati - [Search Query Performance Report Weekly](https://learn.microsoft.com/en-us/advertising/reporting-service/searchqueryperformancereportrequest?view=bingads-13) - [Search Query Performance Report Monthly](https://learn.microsoft.com/en-us/advertising/reporting-service/searchqueryperformancereportrequest?view=bingads-13) +### Custom Reports +You can build your own report by providing: +- *Report Name* - name of the stream +- *Reporting Data Object* - Bing Ads reporting data object that you can find [here](https://learn.microsoft.com/en-us/advertising/reporting-service/reporting-data-objects?view=bingads-13). All data object with ending ReportRequest can be used as data object in custom reports. +- *Columns* - Reporting object columns that you want to sync. You can find it on ReportRequest data object page by clicking the ...ReportColumn link in [Bing Ads docs](https://learn.microsoft.com/en-us/advertising/reporting-service/reporting-value-sets?view=bingads-13). +The report must include the Required Columns (you can find it under list of all columns of reporting object) at a minimum. As a general rule, each report must include at least one attribute column and at least one non-impression share performance statistics column. Be careful you can't add extra columns that not specified in Bing Ads docs and not all fields can be skipped. +- *Aggregation* - Hourly, Daily, Weekly, Monthly, DayOfWeek, HourOfDay, WeeklyStartingMonday, Summary. See [report aggregation](#report-aggregation). + ### Report aggregation All reports synced by this connector can be [aggregated](https://docs.microsoft.com/en-us/advertising/reporting-service/reportaggregation?view=bingads-13) using hourly, daily, weekly, or monthly time windows. @@ -168,6 +187,7 @@ The Bing Ads API limits the number of requests for all Microsoft Advertising cli | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------| +| 1.13.0 | 2023-11-13 | [32306](https://github.com/airbytehq/airbyte/pull/32306) | Add Custom reports and decrease backoff max tries number | | 1.12.1 | 2023-11-10 | [32422](https://github.com/airbytehq/airbyte/pull/32422) | Normalize numeric values in reports | | 1.12.0 | 2023-11-09 | [32340](https://github.com/airbytehq/airbyte/pull/32340) | Remove default start date in favor of Time Period - Last Year and This Year, if start date is not provided | | 1.11.0 | 2023-11-06 | [32201](https://github.com/airbytehq/airbyte/pull/32201) | Skip broken CSV report files | From d53dcb4c735b52eb17a41c1b3fae4d87500be569 Mon Sep 17 00:00:00 2001 From: Christo Grabowski <108154848+ChristoGrab@users.noreply.github.com> Date: Tue, 14 Nov 2023 14:05:36 -0500 Subject: [PATCH 044/124] Source Mailchimp: Add expected records (#32226) --- .../source-mailchimp/acceptance-test-config.yml | 11 +++++++++-- .../integration_tests/expected_records.jsonl | 7 +++++++ .../source-mailchimp/integration_tests/state.json | 8 ++++---- .../connectors/source-mailchimp/metadata.yaml | 2 +- .../source_mailchimp/schemas/reports.json | 2 +- .../schemas/shared/campaignType.json | 9 ++++++++- .../source-mailchimp/source_mailchimp/streams.py | 13 +++++++++++++ docs/integrations/sources/mailchimp.md | 3 ++- 8 files changed, 45 insertions(+), 10 deletions(-) create mode 100644 airbyte-integrations/connectors/source-mailchimp/integration_tests/expected_records.jsonl diff --git a/airbyte-integrations/connectors/source-mailchimp/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mailchimp/acceptance-test-config.yml index d8dcf296b434..c7b7594f3190 100644 --- a/airbyte-integrations/connectors/source-mailchimp/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-mailchimp/acceptance-test-config.yml @@ -28,12 +28,19 @@ acceptance_tests: tests: - config_path: "secrets/config.json" expect_records: - bypass_reason: "Risk to disclose internal data. Need to set up a sandbox account - https://github.com/airbytehq/airbyte/issues/20726" + path: "integration_tests/expected_records.jsonl" fail_on_extra_columns: false + empty_streams: + - name: "automations" + bypass_reason: "Cannot seed in free sandbox account, need to upgrade to paid account." - config_path: "secrets/config_oauth.json" expect_records: - bypass_reason: "Risk to disclose internal data. Need to set up a sandbox account - https://github.com/airbytehq/airbyte/issues/20726" + path: "integration_tests/expected_records.jsonl" + extra_records: True fail_on_extra_columns: false + empty_streams: + - name: "automations" + bypass_reason: "Cannot seed in free sandbox account, need to upgrade to paid account." incremental: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-mailchimp/integration_tests/expected_records.jsonl new file mode 100644 index 000000000000..96b38c9c28eb --- /dev/null +++ b/airbyte-integrations/connectors/source-mailchimp/integration_tests/expected_records.jsonl @@ -0,0 +1,7 @@ +{"stream": "campaigns", "data": {"id": "7847cdaeff", "web_id": 13701472, "type": "regular", "create_time": "2023-11-06T20:12:07+00:00", "archive_url": "http://eepurl.com/iDnTtY", "long_archive_url": "https://mailchi.mp/5e0065d29854/invitation-to-unsubscribe", "status": "sent", "emails_sent": 1, "send_time": "2023-11-06T20:17:44+00:00", "content_type": "multichannel", "needs_block_refresh": false, "resendable": false, "recipients": {"list_id": "16d6ec4ffc", "list_is_active": true, "list_name": "Airbyte", "segment_text": "

Contacts that match any of the following conditions:

  1. Tags contact is tagged Unsubscriber
For a total of 1 emails sent.", "recipient_count": 1, "segment_opts": {"saved_segment_id": 14351532, "match": "any", "conditions": [{"condition_type": "StaticSegment", "field": "static_segment", "op": "static_is", "value": 14351532}]}}, "settings": {"subject_line": "Invitation to Unsubscribe", "title": "Invitation to unsubscribe", "from_name": "yurii", "reply_to": "integration-test+yurii@airbyte.io", "use_conversation": false, "to_name": "", "folder_id": "", "authenticate": true, "auto_footer": false, "inline_css": false, "auto_tweet": false, "fb_comments": true, "timewarp": false, "template_id": 13, "drag_and_drop": false}, "tracking": {"opens": true, "html_clicks": true, "text_clicks": false, "goal_tracking": false, "ecomm360": false, "google_analytics": "", "clicktale": ""}, "report_summary": {"opens": 2, "unique_opens": 1, "open_rate": 1, "clicks": 0, "subscriber_clicks": 0, "click_rate": 0, "ecommerce": {"total_orders": 0, "total_spent": 0, "total_revenue": 0}}, "delivery_status": {"enabled": false}, "_links": [{"rel": "parent", "href": "https://us10.api.mailchimp.com/3.0/campaigns", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Campaigns/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Campaigns/Collection.json"}, {"rel": "self", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Campaigns/Response.json"}, {"rel": "delete", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff", "method": "DELETE"}, {"rel": "send", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/actions/send", "method": "POST"}, {"rel": "cancel_send", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/actions/cancel-send", "method": "POST"}, {"rel": "feedback", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/feedback", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Campaigns/Feedback/CollectionResponse.json"}, {"rel": "content", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/content", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Campaigns/Content/Response.json"}, {"rel": "send_checklist", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/send-checklist", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Campaigns/Checklist/Response.json"}, {"rel": "pause", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/actions/pause", "method": "POST"}, {"rel": "resume", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/actions/resume", "method": "POST"}, {"rel": "replicate", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/actions/replicate", "method": "POST"}, {"rel": "create_resend", "href": "https://us10.api.mailchimp.com/3.0/campaigns/7847cdaeff/actions/create-resend", "method": "POST"}]}, "emitted_at": 1699461462352} +{"stream": "email_activity", "data": {"campaign_id": "7847cdaeff", "list_id": "16d6ec4ffc", "list_is_active": true, "email_id": "11273c9a5dc6ae6c5aaccfb77b2addfb", "email_address": "AirbyteMailchimpUser@gmail.com", "_links": [{"rel": "parent", "href": "https://us10.api.mailchimp.com/3.0/reports/7847cdaeff/email-activity", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/EmailActivity/CollectionResponse.json"}, {"rel": "self", "href": "https://us10.api.mailchimp.com/3.0/reports/7847cdaeff/email-activity/11273c9a5dc6ae6c5aaccfb77b2addfb", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/EmailActivity/Response.json"}, {"rel": "member", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/members/11273c9a5dc6ae6c5aaccfb77b2addfb", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Members/Response.json"}], "action": "open", "timestamp": "2023-11-06T20:17:57+00:00", "ip": "74.125.212.231"}, "emitted_at": 1699457307719} +{"stream": "list_members", "data": {"id": "458f50b08c829a8ab901d3f8f88df914", "email_address": "integration-test+Thomas@airbyte.io", "unique_email_id": "42d6d67d11", "contact_id": "475a8f7f7b5087d7be924c9b331c8316", "full_name": "Thomas", "web_id": 546044608, "email_type": "html", "status": "unsubscribed", "unsubscribe_reason": "N/A (Unsubscribed by admin)", "consents_to_one_to_one_messaging": true, "merge_fields": {"FNAME": "Thomas", "LNAME": "", "ADDRESS": "", "PHONE": "", "BIRTHDAY": ""}, "interests": {"bbbb369575": false, "97bbc1227a": false, "d802d794f8": false, "b35e48738e": false, "44d2c158e3": false, "29f73b8209": false, "2010f3c101": false, "75f1cb79fd": false, "aa2fd02c59": false, "f7b60a3c3d": false, "7733d60f61": false, "cc454d76d6": false, "797533254b": false, "9ea08b864b": false, "e2e5fdcac9": false, "8eccc648d6": false, "a7c814599e": false, "20ef45c5d3": false, "1824f5d1a5": false, "644f34517f": false, "c57e1a9ff6": false, "b97fee61c8": false, "b9d16768e3": false, "810348679c": false, "43ebb04472": false, "73ee7c1d1b": false, "045738fa17": false, "0a7cbd4449": false, "fef00a4695": false, "4a19201dc9": false, "571a80ed60": false}, "stats": {"avg_open_rate": 1, "avg_click_rate": 1}, "ip_signup": "", "timestamp_signup": "", "ip_opt": "93.73.161.112", "timestamp_opt": "2022-12-27T08:34:39+00:00", "member_rating": 2, "last_changed": "2023-11-03T20:53:12+00:00", "language": "", "vip": false, "email_client": "", "location": {"latitude": 0, "longitude": 0, "gmtoff": 0, "dstoff": 0, "country_code": "", "timezone": "", "region": ""}, "source": "Import", "tags_count": 0, "tags": [], "list_id": "16d6ec4ffc"}, "emitted_at": 1699302001460} +{"stream": "lists", "data": {"id": "16d6ec4ffc", "web_id": 903380, "name": "Airbyte", "contact": {"company": "Airbyte", "address1": "kyiv", "address2": "", "city": "Kiev", "state": "30", "zip": "04200", "country": "UA", "phone": ""}, "permission_reminder": "You are receiving this email because you opted in via our website.", "use_archive_bar": true, "campaign_defaults": {"from_name": "yurii", "from_email": "integration-test+yurii@airbyte.io", "subject": "", "language": "en"}, "notify_on_subscribe": "", "notify_on_unsubscribe": "", "date_created": "2022-12-27T07:56:47+00:00", "list_rating": 0, "email_type_option": false, "subscribe_url_short": "http://eepurl.com/ihg3RD", "subscribe_url_long": "https://airbyte.us10.list-manage.com/subscribe?u=caf9055242d41edd9215d1898&id=16d6ec4ffc", "beamer_address": "us10-d527bd96ba-6d1a9988db@inbound.mailchimp.com", "visibility": "prv", "double_optin": false, "has_welcome": false, "marketing_permissions": false, "modules": [], "stats": {"member_count": 47, "unsubscribe_count": 4, "cleaned_count": 0, "member_count_since_send": 0, "unsubscribe_count_since_send": 1, "cleaned_count_since_send": 0, "campaign_count": 6, "campaign_last_sent": "2022-12-27T08:37:53+00:00", "merge_field_count": 5, "avg_sub_rate": 0, "avg_unsub_rate": 1, "target_sub_rate": 1, "open_rate": 100, "click_rate": 64.70588235294117, "last_sub_date": "2022-12-27T08:34:39+00:00", "last_unsub_date": "2023-11-06T20:18:01+00:00"}, "_links": [{"rel": "self", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Response.json"}, {"rel": "parent", "href": "https://us10.api.mailchimp.com/3.0/lists", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/Collection.json"}, {"rel": "update", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc", "method": "PATCH", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Response.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/PATCH.json"}, {"rel": "batch-sub-unsub-members", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc", "method": "POST", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/BatchPOST-Response.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/BatchPOST.json"}, {"rel": "delete", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc", "method": "DELETE"}, {"rel": "abuse-reports", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/abuse-reports", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Abuse/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/Abuse/Collection.json"}, {"rel": "activity", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/activity", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Activity/Response.json"}, {"rel": "clients", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/clients", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Clients/Response.json"}, {"rel": "growth-history", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/growth-history", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Growth/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/Growth/Collection.json"}, {"rel": "interest-categories", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/interest-categories", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/InterestCategories/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/InterestCategories/Collection.json"}, {"rel": "members", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/members", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Members/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/Members/Collection.json"}, {"rel": "merge-fields", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/merge-fields", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/MergeFields/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/MergeFields/Collection.json"}, {"rel": "segments", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/segments", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Segments/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/Segments/Collection.json"}, {"rel": "webhooks", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/webhooks", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Webhooks/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/Webhooks/Collection.json"}, {"rel": "signup-forms", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/signup-forms", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/SignupForms/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/SignupForms/Collection.json"}, {"rel": "locations", "href": "https://us10.api.mailchimp.com/3.0/lists/16d6ec4ffc/locations", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Lists/Locations/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Lists/Locations/Collection.json"}]}, "emitted_at": 1699626450570} +{"stream": "reports", "data": {"id": "a79651273b", "campaign_title": "Untitled", "type": "regular", "list_id": "16d6ec4ffc", "list_is_active": true, "list_name": "Airbyte", "subject_line": "Airbyte Test", "preview_text": "", "emails_sent": 50, "abuse_reports": 0, "unsubscribed": 0, "send_time": "2022-12-27T08:36:55+00:00", "bounces": {"hard_bounces": 0, "soft_bounces": 0, "syntax_errors": 0}, "forwards": {"forwards_count": 0, "forwards_opens": 0}, "opens": {"opens_total": 412, "unique_opens": 50, "open_rate": 1, "last_open": "2023-01-09T10:07:54+00:00"}, "clicks": {"clicks_total": 48, "unique_clicks": 47, "unique_subscriber_clicks": 33, "click_rate": 0.66, "last_click": "2022-12-27T15:28:11+00:00"}, "facebook_likes": {"recipient_likes": 0, "unique_likes": 0, "facebook_likes": 0}, "list_stats": {"sub_rate": 0, "unsub_rate": 1, "open_rate": 100, "click_rate": 64.70588235294117}, "timeseries": [{"timestamp": "2022-12-27T08:00:00+00:00", "emails_sent": 50, "unique_opens": 6, "recipients_clicks": 1}, {"timestamp": "2022-12-27T09:00:00+00:00", "emails_sent": 0, "unique_opens": 43, "recipients_clicks": 0}, {"timestamp": "2022-12-27T10:00:00+00:00", "emails_sent": 0, "unique_opens": 1, "recipients_clicks": 3}, {"timestamp": "2022-12-27T11:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 11}, {"timestamp": "2022-12-27T12:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 10}, {"timestamp": "2022-12-27T13:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 3}, {"timestamp": "2022-12-27T14:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 2}, {"timestamp": "2022-12-27T15:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 3}, {"timestamp": "2022-12-27T16:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T17:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T18:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T19:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T20:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T21:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T22:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T23:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T00:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T01:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T02:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T03:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T04:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T05:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T06:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T07:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}], "ecommerce": {"total_orders": 0, "total_spent": 0, "total_revenue": 0, "currency_code": "USD"}, "delivery_status": {"enabled": false}, "_links": [{"rel": "parent", "href": "https://us10.api.mailchimp.com/3.0/reports", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/CollectionResponse.json", "schema": "https://us10.api.mailchimp.com/schema/3.0/Paths/Reports/Collection.json"}, {"rel": "self", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/Response.json"}, {"rel": "campaign", "href": "https://us10.api.mailchimp.com/3.0/campaigns/a79651273b", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Campaigns/Response.json"}, {"rel": "sub-reports", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/sub-reports", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/Sub/Response.json"}, {"rel": "abuse-reports", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/abuse-reports", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/Abuse/CollectionResponse.json"}, {"rel": "advice", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/advice", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/Advice/Response.json"}, {"rel": "open-details", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/open-details", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/OpenDetails/CollectionResponse.json"}, {"rel": "click-details", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/click-details", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/ClickDetails/CollectionResponse.json"}, {"rel": "domain-performance", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/domain-performance", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/DomainPerformance/Response.json"}, {"rel": "eepurl", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/eepurl", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/Eepurl/CollectionResponse.json"}, {"rel": "email-activity", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/email-activity", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/EmailActivity/CollectionResponse.json"}, {"rel": "locations", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/locations", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/Locations/Response.json"}, {"rel": "sent-to", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/sent-to", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/SentTo/CollectionResponse.json"}, {"rel": "unsubscribed", "href": "https://us10.api.mailchimp.com/3.0/reports/a79651273b/unsubscribed", "method": "GET", "targetSchema": "https://us10.api.mailchimp.com/schema/3.0/Definitions/Reports/Unsubs/CollectionResponse.json"}]}, "emitted_at": 1699627079113} +{"stream": "segments", "data": {"id": 13506132, "name": "Influencer", "member_count": 3, "type": "static", "created_at": "2022-12-27T08:33:35+00:00", "updated_at": "2022-12-27T08:33:35+00:00", "list_id": "16d6ec4ffc"}, "emitted_at": 1699302003309} +{"stream": "unsubscribes", "data": {"email_id": "11273c9a5dc6ae6c5aaccfb77b2addfb", "email_address": "AirbyteMailchimpUser@gmail.com", "merge_fields": {"FNAME": "Joe", "LNAME": "Barry", "ADDRESS": {"addr1": "109 Barry St", "addr2": "", "city": "Gary", "state": "IN", "zip": "46401", "country": "US"}, "PHONE": "", "BIRTHDAY": ""}, "vip": false, "timestamp": "2023-11-06T20:18:01+00:00", "reason": "Did not signup for list", "campaign_id": "7847cdaeff", "list_id": "16d6ec4ffc", "list_is_active": true}, "emitted_at": 1699302005437} diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/state.json b/airbyte-integrations/connectors/source-mailchimp/integration_tests/state.json index adbf183cd3c7..3cc3a67b4573 100644 --- a/airbyte-integrations/connectors/source-mailchimp/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-mailchimp/integration_tests/state.json @@ -24,7 +24,7 @@ "type": "STREAM", "stream": { "stream_state": { - "00f14fbfd8": { "timestamp": "2230-11-23T05:42:10+00:00" } + "7847cdaeff": { "timestamp": "2230-11-23T05:42:10+00:00" } }, "stream_descriptor": { "name": "email_activity" } } @@ -33,7 +33,7 @@ "type": "STREAM", "stream": { "stream_state": { - "f59b426832": { "last_changed": "2230-02-26T05:42:10+00:00" } + "16d6ec4ffc": { "last_changed": "2230-02-26T05:42:10+00:00" } }, "stream_descriptor": { "name": "list_members" } } @@ -49,7 +49,7 @@ "type": "STREAM", "stream": { "stream_state": { - "f59b426832": { "updated_at": "2230-02-26T05:42:10+00:00" } + "16d6ec4ffc": { "updated_at": "2230-02-26T05:42:10+00:00" } }, "stream_descriptor": { "name": "segments" } } @@ -58,7 +58,7 @@ "type": "STREAM", "stream": { "stream_state": { - "00f14fbfd8": { "timestamp": "2231-09-26T05:42:10+00:00" } + "7847cdaeff": { "timestamp": "2231-09-26T05:42:10+00:00" } }, "stream_descriptor": { "name": "unsubscribes" } } diff --git a/airbyte-integrations/connectors/source-mailchimp/metadata.yaml b/airbyte-integrations/connectors/source-mailchimp/metadata.yaml index f0d412ca20ad..3500b5a94acb 100644 --- a/airbyte-integrations/connectors/source-mailchimp/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailchimp/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: b03a9f3e-22a5-11eb-adc1-0242ac120002 - dockerImageTag: 0.8.0 + dockerImageTag: 0.8.1 dockerRepository: airbyte/source-mailchimp documentationUrl: https://docs.airbyte.com/integrations/sources/mailchimp githubIssueLabel: source-mailchimp diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/reports.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/reports.json index d94cb73ad0df..fe6dcf6a599b 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/reports.json +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/reports.json @@ -171,7 +171,7 @@ "description": "The number of unique clicks divided by the total number of successful deliveries." }, "last_click": { - "type": "string", + "type": ["null", "string"], "format": "date-time", "title": "Last Click", "description": "The date and time of the last recorded click for the campaign in ISO 8601 format." diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/shared/campaignType.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/shared/campaignType.json index 0171d498babc..4e7ad8a5978f 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/shared/campaignType.json +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/shared/campaignType.json @@ -2,5 +2,12 @@ "type": "string", "title": "Campaign Type", "description": "There are four types of [campaigns](https://mailchimp.com/help/getting-started-with-campaigns/) you can create in Mailchimp. A/B Split campaigns have been deprecated and variate campaigns should be used instead.", - "enum": ["regular", "plaintext", "absplit", "rss", "variate"] + "enum": [ + "automation-email", + "regular", + "plaintext", + "absplit", + "rss", + "variate" + ] } diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py index 2a0de882f9a0..d5239e7f3ecf 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py @@ -277,6 +277,19 @@ class Reports(IncrementalMailChimpStream): def path(self, **kwargs) -> str: return "reports" + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + + response = super().parse_response(response, **kwargs) + + # In some cases, the 'last_click' field is returned as an empty string, + # which causes validation errors on the `date-time` format. + # To avoid this, we remove the field if it is empty. + for record in response: + clicks = record.get("clicks", {}) + if not clicks.get("last_click"): + clicks.pop("last_click", None) + yield record + class Segments(MailChimpListSubStream): """ diff --git a/docs/integrations/sources/mailchimp.md b/docs/integrations/sources/mailchimp.md index 4b579defe748..3b5c0fef42de 100644 --- a/docs/integrations/sources/mailchimp.md +++ b/docs/integrations/sources/mailchimp.md @@ -76,7 +76,8 @@ Now that you have set up the Mailchimp source connector, check out the following | Version | Date | Pull Request | Subject | |---------|------------|----------------------------------------------------------|----------------------------------------------------------------------------| -| 0.8.0 | 2023-11-01 | [32032](https://github.com/airbytehq/airbyte/pull/32032) | Add ListMembers stream +| 0.8.1 | 2023-11-06 | [32226](https://github.com/airbytehq/airbyte/pull/32226) | Unmute expected records test after data anonymisation | +| 0.8.0 | 2023-11-01 | [32032](https://github.com/airbytehq/airbyte/pull/32032) | Add ListMembers stream | | 0.7.0 | 2023-10-27 | [31940](https://github.com/airbytehq/airbyte/pull/31940) | Implement availability strategy | | 0.6.0 | 2023-10-27 | [31922](https://github.com/airbytehq/airbyte/pull/31922) | Add Segments stream | | 0.5.0 | 2023-10-20 | [31675](https://github.com/airbytehq/airbyte/pull/31675) | Add Unsubscribes stream | From 6e777dd2e6f755815828f784f0b148cb704e5b05 Mon Sep 17 00:00:00 2001 From: Erica D'Souza <93952107+erica-airbyte@users.noreply.github.com> Date: Tue, 14 Nov 2023 12:07:03 -0700 Subject: [PATCH 045/124] Update contact-support.md --- docs/operator-guides/contact-support.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/operator-guides/contact-support.md b/docs/operator-guides/contact-support.md index d90b59272924..db42a9aef36f 100644 --- a/docs/operator-guides/contact-support.md +++ b/docs/operator-guides/contact-support.md @@ -28,6 +28,8 @@ If you don't see a connector you need, you can submit a [connector request](http To stay updated on Airbyte's future plans, take a look at [our roadmap](https://github.com/orgs/airbytehq/projects/37/views/1). +Please be sure to sign up for Airbyte with your company email address, as we do not support personal accounts. + ## Airbyte Enterprise (self-hosted) Support If you're running Airbyte Open Source with Airbyte Enterprise or have an OSS support package, we're here to help you with upgrading Airbyte versions, debugging connector issues, or troubleshooting schema changes. From ef95ef614a658a4e558c0608b31a785993d23907 Mon Sep 17 00:00:00 2001 From: Christo Grabowski <108154848+ChristoGrab@users.noreply.github.com> Date: Tue, 14 Nov 2023 14:18:40 -0500 Subject: [PATCH 046/124] =?UTF-8?q?=F0=9F=90=9B=20Source=20Mailchimp:=20Co?= =?UTF-8?q?nnection=20Check=20Error=20Handling=20(#32466)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../connectors/source-mailchimp/metadata.yaml | 2 +- .../source_mailchimp/source.py | 35 +++++++++++++--- .../unit_tests/test_source.py | 41 +++++++++++++++---- docs/integrations/sources/mailchimp.md | 3 +- 4 files changed, 65 insertions(+), 16 deletions(-) diff --git a/airbyte-integrations/connectors/source-mailchimp/metadata.yaml b/airbyte-integrations/connectors/source-mailchimp/metadata.yaml index 3500b5a94acb..85bb1badfd2e 100644 --- a/airbyte-integrations/connectors/source-mailchimp/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailchimp/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: b03a9f3e-22a5-11eb-adc1-0242ac120002 - dockerImageTag: 0.8.1 + dockerImageTag: 0.8.2 dockerRepository: airbyte/source-mailchimp documentationUrl: https://docs.airbyte.com/integrations/sources/mailchimp githubIssueLabel: source-mailchimp diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py index d31559c358b0..f6d4ed7a75aa 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py @@ -18,14 +18,26 @@ class MailChimpAuthenticator: @staticmethod - def get_server_prefix(access_token: str) -> str: + def get_oauth_data_center(access_token: str) -> str: + """ + Every Mailchimp API request must be sent to a specific data center. + The data center is already embedded in API keys, but not OAuth access tokens. + This method retrieves the data center for OAuth credentials. + """ try: response = requests.get( "https://login.mailchimp.com/oauth2/metadata", headers={"Authorization": "OAuth {}".format(access_token)} ) + + # Requests to this endpoint will return a 200 status code even if the access token is invalid. + error = response.json().get("error") + if error == "invalid_token": + raise ValueError("The access token you provided was invalid. Please check your credentials and try again.") return response.json()["dc"] + + # Handle any other exceptions that may occur. except Exception as e: - raise Exception(f"Cannot retrieve server_prefix for you account. \n {repr(e)}") + raise Exception(f"An error occured while retrieving the data center for your account. \n {repr(e)}") def get_auth(self, config: Mapping[str, Any]) -> AuthBase: authorization = config.get("credentials", {}) @@ -35,7 +47,7 @@ def get_auth(self, config: Mapping[str, Any]) -> AuthBase: # See https://mailchimp.com/developer/marketing/docs/fundamentals/#api-structure apikey = authorization.get("apikey") or config.get("apikey") if not apikey: - raise Exception("No apikey in creds") + raise Exception("Please provide a valid API key for authentication.") auth_string = f"anystring:{apikey}".encode("utf8") b64_encoded = base64.b64encode(auth_string).decode("utf8") auth = TokenAuthenticator(token=b64_encoded, auth_method="Basic") @@ -44,7 +56,7 @@ def get_auth(self, config: Mapping[str, Any]) -> AuthBase: elif auth_type == "oauth2.0": access_token = authorization["access_token"] auth = TokenAuthenticator(token=access_token, auth_method="Bearer") - auth.data_center = self.get_server_prefix(access_token) + auth.data_center = self.get_oauth_data_center(access_token) else: raise Exception(f"Invalid auth type: {auth_type}") @@ -56,8 +68,21 @@ class SourceMailchimp(AbstractSource): def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: try: authenticator = MailChimpAuthenticator().get_auth(config) - requests.get(f"https://{authenticator.data_center}.api.mailchimp.com/3.0/ping", headers=authenticator.get_auth_header()) + response = requests.get( + f"https://{authenticator.data_center}.api.mailchimp.com/3.0/ping", headers=authenticator.get_auth_header() + ) + + # A successful response will return a simple JSON object with a single key: health_status. + # Otherwise, errors are returned as a JSON object with keys: + # {type, title, status, detail, instance} + + if not response.json().get("health_status"): + error_title = response.json().get("title", "Unknown Error") + error_details = response.json().get("details", "An unknown error occurred. Please verify your credentials and try again.") + return False, f"Encountered an error while connecting to Mailchimp. Type: {error_title}. Details: {error_details}" return True, None + + # Handle any other exceptions that may occur. except Exception as e: return False, repr(e) diff --git a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_source.py b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_source.py index cde22d0ef4ef..0e56d333da62 100644 --- a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_source.py @@ -5,7 +5,6 @@ import logging import pytest -import requests from source_mailchimp.source import MailChimpAuthenticator, SourceMailchimp logger = logging.getLogger("airbyte") @@ -13,7 +12,7 @@ def test_check_connection_ok(requests_mock, config, data_center): responses = [ - {"json": [], "status_code": 200}, + {"json": {"health_status": "Everything's Chimpy!"}}, ] requests_mock.register_uri("GET", f"https://{data_center}.api.mailchimp.com/3.0/ping", responses) ok, error_msg = SourceMailchimp().check_connection(logger, config=config) @@ -22,30 +21,54 @@ def test_check_connection_ok(requests_mock, config, data_center): assert not error_msg -def test_check_connection_error(requests_mock, config, data_center): - requests_mock.register_uri("GET", f"https://{data_center}.api.mailchimp.com/3.0/ping", body=requests.ConnectionError()) +@pytest.mark.parametrize( + "response, expected_message", + [ + ( + { + "json": { + "title": "API Key Invalid", + "details": "Your API key may be invalid, or you've attempted to access the wrong datacenter.", + } + }, + "Encountered an error while connecting to Mailchimp. Type: API Key Invalid. Details: Your API key may be invalid, or you've attempted to access the wrong datacenter.", + ), + ( + {"json": {"title": "Forbidden", "details": "You don't have permission to access this resource."}}, + "Encountered an error while connecting to Mailchimp. Type: Forbidden. Details: You don't have permission to access this resource.", + ), + ( + {"json": {}}, + "Encountered an error while connecting to Mailchimp. Type: Unknown Error. Details: An unknown error occurred. Please verify your credentials and try again.", + ), + ], + ids=["API Key Invalid", "Forbidden", "Unknown Error"], +) +def test_check_connection_error(requests_mock, config, data_center, response, expected_message): + requests_mock.register_uri("GET", f"https://{data_center}.api.mailchimp.com/3.0/ping", json=response["json"]) ok, error_msg = SourceMailchimp().check_connection(logger, config=config) assert not ok - assert error_msg + assert error_msg == expected_message -def test_get_server_prefix_ok(requests_mock, access_token, data_center): +def test_get_oauth_data_center_ok(requests_mock, access_token, data_center): responses = [ {"json": {"dc": data_center}, "status_code": 200}, ] requests_mock.register_uri("GET", "https://login.mailchimp.com/oauth2/metadata", responses) - assert MailChimpAuthenticator().get_server_prefix(access_token) == data_center + assert MailChimpAuthenticator().get_oauth_data_center(access_token) == data_center -def test_get_server_prefix_exception(requests_mock, access_token, data_center): +def test_get_oauth_data_center_exception(requests_mock, access_token): responses = [ {"json": {}, "status_code": 200}, + {"json": {"error": "invalid_token"}, "status_code": 200}, {"status_code": 403}, ] requests_mock.register_uri("GET", "https://login.mailchimp.com/oauth2/metadata", responses) with pytest.raises(Exception): - MailChimpAuthenticator().get_server_prefix(access_token) + MailChimpAuthenticator().get_oauth_data_center(access_token) def test_oauth_config(requests_mock, oauth_config, data_center): diff --git a/docs/integrations/sources/mailchimp.md b/docs/integrations/sources/mailchimp.md index 3b5c0fef42de..d3d2551a8a9c 100644 --- a/docs/integrations/sources/mailchimp.md +++ b/docs/integrations/sources/mailchimp.md @@ -76,7 +76,8 @@ Now that you have set up the Mailchimp source connector, check out the following | Version | Date | Pull Request | Subject | |---------|------------|----------------------------------------------------------|----------------------------------------------------------------------------| -| 0.8.1 | 2023-11-06 | [32226](https://github.com/airbytehq/airbyte/pull/32226) | Unmute expected records test after data anonymisation | +| 0.8.2 | 2023-11-13 | [32466](https://github.com/airbytehq/airbyte/pull/32466) | Improve error handling during connection check | +| 0.8.1 | 2023-11-06 | [32226](https://github.com/airbytehq/airbyte/pull/32226) | Unmute expected records test after data anonymisation | | 0.8.0 | 2023-11-01 | [32032](https://github.com/airbytehq/airbyte/pull/32032) | Add ListMembers stream | | 0.7.0 | 2023-10-27 | [31940](https://github.com/airbytehq/airbyte/pull/31940) | Implement availability strategy | | 0.6.0 | 2023-10-27 | [31922](https://github.com/airbytehq/airbyte/pull/31922) | Add Segments stream | From 203e01e435d6f9f8dfa1ef04f2bc3882a02fde6d Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Tue, 14 Nov 2023 22:04:55 +0200 Subject: [PATCH 047/124] CDK: increase `maximum_attempts_to_acquire` to avoid crashing in acquire_call (#32516) Co-authored-by: Eugene Kulak --- airbyte-cdk/python/airbyte_cdk/sources/streams/call_rate.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/call_rate.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/call_rate.py index 905489dfbb4f..eb33754504ee 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/call_rate.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/call_rate.py @@ -375,11 +375,12 @@ def update_from_response(self, request: Any, response: Any) -> None: class APIBudget(AbstractAPIBudget): """Default APIBudget implementation""" - def __init__(self, policies: list[AbstractCallRatePolicy], maximum_attempts_to_acquire: int = 10) -> None: + def __init__(self, policies: list[AbstractCallRatePolicy], maximum_attempts_to_acquire: int = 100000) -> None: """Constructor :param policies: list of policies in this budget - :param maximum_attempts_to_acquire: number of attempts before throwing hit ratelimit exception + :param maximum_attempts_to_acquire: number of attempts before throwing hit ratelimit exception, we put some big number here + to avoid situations when many threads compete with each other for a few lots over a significant amount of time """ self._policies = policies From 49683828a4274c02484860e6570e9f0491b08b69 Mon Sep 17 00:00:00 2001 From: keu Date: Tue, 14 Nov 2023 20:14:13 +0000 Subject: [PATCH 048/124] =?UTF-8?q?=F0=9F=A4=96=20Bump=20patch=20version?= =?UTF-8?q?=20of=20Python=20CDK?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-cdk/python/.bumpversion.cfg | 2 +- airbyte-cdk/python/CHANGELOG.md | 3 +++ airbyte-cdk/python/Dockerfile | 4 ++-- airbyte-cdk/python/setup.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/airbyte-cdk/python/.bumpversion.cfg b/airbyte-cdk/python/.bumpversion.cfg index 186cb9760f84..e597dfab9070 100644 --- a/airbyte-cdk/python/.bumpversion.cfg +++ b/airbyte-cdk/python/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.53.5 +current_version = 0.53.6 commit = False [bumpversion:file:setup.py] diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 4c76cd7e510b..68398eb18ee1 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.53.6 +Increase maximum_attempts_to_acquire to avoid crashing in acquire_call + ## 0.53.5 File CDK: Improve stream config appearance diff --git a/airbyte-cdk/python/Dockerfile b/airbyte-cdk/python/Dockerfile index a966090089a5..4cca0b6e34a5 100644 --- a/airbyte-cdk/python/Dockerfile +++ b/airbyte-cdk/python/Dockerfile @@ -10,7 +10,7 @@ RUN apk --no-cache upgrade \ && apk --no-cache add tzdata build-base # install airbyte-cdk -RUN pip install --prefix=/install airbyte-cdk==0.53.5 +RUN pip install --prefix=/install airbyte-cdk==0.53.6 # build a clean environment FROM base @@ -32,5 +32,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] # needs to be the same as CDK -LABEL io.airbyte.version=0.53.5 +LABEL io.airbyte.version=0.53.6 LABEL io.airbyte.name=airbyte/source-declarative-manifest diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 538550c201af..14b770867eb8 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -36,7 +36,7 @@ name="airbyte-cdk", # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be # updated if our semver format changes such as using release candidate versions. - version="0.53.5", + version="0.53.6", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", From 75e71e33788a0b02a0f177a4d729453db355f660 Mon Sep 17 00:00:00 2001 From: rijumone Date: Wed, 15 Nov 2023 02:03:07 +0530 Subject: [PATCH 049/124] Update airbyte-protocol.md (#32490) --- docs/understanding-airbyte/airbyte-protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/understanding-airbyte/airbyte-protocol.md b/docs/understanding-airbyte/airbyte-protocol.md index edf7c7f15c06..17c742722882 100644 --- a/docs/understanding-airbyte/airbyte-protocol.md +++ b/docs/understanding-airbyte/airbyte-protocol.md @@ -701,7 +701,7 @@ AirbyteGlobalState: ### AirbyteConnectionStatus Message -This message reports whether an Actor was able to connect to its underlying data store with all the permissions it needs to succeed. The goal is that if a successful stat is returned, that the user should be confident that using that Actor will succeed. The depth of the verification is not specified in the protocol. More robust verification is preferred but going to deep can create undesired performance tradeoffs +This message reports whether an Actor was able to connect to its underlying data store with all the permissions it needs to succeed. The goal is that if a successful stat is returned, that the user should be confident that using that Actor will succeed. The depth of the verification is not specified in the protocol. More robust verification is preferred but going too deep can create undesired performance tradeoffs. ```yaml AirbyteConnectionStatus: From 75ff8e1875f1d2d5d09449ab458cb90bcd4d0fc6 Mon Sep 17 00:00:00 2001 From: Marcus Hof <13001502+MarconLP@users.noreply.github.com> Date: Tue, 14 Nov 2023 21:34:04 +0100 Subject: [PATCH 050/124] update posthog rate limit docs (#32460) --- docs/integrations/sources/posthog.md | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/integrations/sources/posthog.md b/docs/integrations/sources/posthog.md index dd617aeffa0f..2a8c431b2d1c 100644 --- a/docs/integrations/sources/posthog.md +++ b/docs/integrations/sources/posthog.md @@ -44,10 +44,25 @@ This page contains the setup guide and reference information for the PostHog sou - [Insights](https://posthog.com/docs/api/insights) - [Persons](https://posthog.com/docs/api/people) -### Performance considerations +### Rate limiting -The PostHog API doesn't have any known request limitation. -Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. +Private `GET`, `POST`, `PATCH`, `DELETE` endpoints are rate limited. Public POST-only endpoints are **not** rate limited. A rule of thumb for whether rate limits apply is if the personal API key is used for authentication. + +There are separate limits for different kinds of resources. + +- For all analytics endpoints (such as calculating insights, retrieving persons, or retrieving session recordings), the rate limits are `240/minute` and `1200/hour`. + +- The [HogQL query](/docs/hogql#api-access) endpoint (`/api/project/:id/query`) has a rate limit of `120/hour`. + +- For the rest of the create, read, update, and delete endpoints, the rate limits are `480/minute` and `4800/hour`. + +- For Public POST-only endpoints like event capture (`/capture`) and feature flag evaluation (`/decide`), there are no rate limits. + +These limits apply to **the entire team** (i.e. all users within your PostHog organization). For example, if a script requesting feature flag metadata hits the rate limit, and another user, using a different personal API key, makes a single request to the persons API, this gets rate limited as well. + +For large or regular exports of events, use [batch exports](/docs/cdp/batch-exports). + +> **Want to use the PostHog API beyond these limits?** Email us at [customers@posthog.com](to:customers@posthog.com). ## Changelog From 81f4a432b282854c2c8e88abe28fab539e436fd8 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Tue, 14 Nov 2023 18:30:53 -0300 Subject: [PATCH 051/124] Docs: fix posthog (#32522) --- docs/integrations/sources/posthog.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/integrations/sources/posthog.md b/docs/integrations/sources/posthog.md index 2a8c431b2d1c..cecd0fe2736f 100644 --- a/docs/integrations/sources/posthog.md +++ b/docs/integrations/sources/posthog.md @@ -52,7 +52,7 @@ There are separate limits for different kinds of resources. - For all analytics endpoints (such as calculating insights, retrieving persons, or retrieving session recordings), the rate limits are `240/minute` and `1200/hour`. -- The [HogQL query](/docs/hogql#api-access) endpoint (`/api/project/:id/query`) has a rate limit of `120/hour`. +- The [HogQL query](https://posthog.com/docs/hogql#api-access) endpoint (`/api/project/:id/query`) has a rate limit of `120/hour`. - For the rest of the create, read, update, and delete endpoints, the rate limits are `480/minute` and `4800/hour`. @@ -60,9 +60,9 @@ There are separate limits for different kinds of resources. These limits apply to **the entire team** (i.e. all users within your PostHog organization). For example, if a script requesting feature flag metadata hits the rate limit, and another user, using a different personal API key, makes a single request to the persons API, this gets rate limited as well. -For large or regular exports of events, use [batch exports](/docs/cdp/batch-exports). +For large or regular exports of events, use [batch exports](https://posthog.com/docs/cdp). -> **Want to use the PostHog API beyond these limits?** Email us at [customers@posthog.com](to:customers@posthog.com). +Want to use the PostHog API beyond these limits? Email Posthog at `customers@posthog.com`. ## Changelog From 92516b7f711d05b614fde0c82ae6a004ae8644bf Mon Sep 17 00:00:00 2001 From: Xiaohan Song Date: Wed, 15 Nov 2023 06:33:29 +0800 Subject: [PATCH 052/124] Load connector secrets in performance harness to pass acceptance test (#32480) --- .github/workflows/connector-performance-command.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/connector-performance-command.yml b/.github/workflows/connector-performance-command.yml index c86c798841f0..3ed30a4ceb77 100644 --- a/.github/workflows/connector-performance-command.yml +++ b/.github/workflows/connector-performance-command.yml @@ -137,7 +137,7 @@ jobs: id: regex uses: AsasInnab/regex-action@v1 with: - regex_pattern: "^((connectors|bases)/)?[a-zA-Z0-9-_]+$" + regex_pattern: "^(connectors/)?[a-zA-Z0-9-_]+$" regex_flags: "i" # required to be set for this plugin search_string: ${{ inputs.connector }} - name: Validate input workflow format @@ -179,6 +179,8 @@ jobs: run: | export PATH="$PATH:/root/.local/bin" ci_credentials connectors-performance/$HARNESS_TYPE write-to-storage + connector_name=$(echo ${{ inputs.connector }} | sed 's,.*/,,') + ci_credentials connectors-performance/$connector_name write-to-storage env: GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} HARNESS_TYPE: ${{ steps.which-harness.outputs.harness_type }} @@ -195,7 +197,7 @@ jobs: run: | echo "Building... ${{inputs.connector}}" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY # this is a blank line - connector_name=$(echo ${{ inputs.connector }} | cut -d / -f 2) + connector_name=$(echo ${{ inputs.connector }} | sed 's,.*/,,') echo "Running ./gradlew :airbyte-integrations:connectors:$connector_name:build -x check" ./gradlew :airbyte-integrations:connectors:$connector_name:build -x check env: From c4c1333974cca8d220da40b6ecd37be267bbc970 Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Tue, 14 Nov 2023 16:59:14 -0800 Subject: [PATCH 053/124] [docs] Show metadata above header, remove ID (#32525) --- docusaurus/src/css/custom.css | 20 +++++----- docusaurus/src/remark/docsHeaderDecoration.js | 38 +++++++++++-------- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/docusaurus/src/css/custom.css b/docusaurus/src/css/custom.css index 374c3285ff4a..56563f0b9d24 100644 --- a/docusaurus/src/css/custom.css +++ b/docusaurus/src/css/custom.css @@ -72,7 +72,7 @@ html[data-theme="dark"] .docusaurus-highlight-code-line { } .header-github-link::before { - content: ''; + content: ""; width: 24px; height: 24px; display: flex; @@ -80,7 +80,7 @@ html[data-theme="dark"] .docusaurus-highlight-code-line { no-repeat; } -[data-theme='dark'] .header-github-link::before { +[data-theme="dark"] .header-github-link::before { background: url("data:image/svg+xml,%3Csvg viewBox='0 0 24 24' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath fill='white' d='M12 .297c-6.63 0-12 5.373-12 12 0 5.303 3.438 9.8 8.205 11.385.6.113.82-.258.82-.577 0-.285-.01-1.04-.015-2.04-3.338.724-4.042-1.61-4.042-1.61C4.422 18.07 3.633 17.7 3.633 17.7c-1.087-.744.084-.729.084-.729 1.205.084 1.838 1.236 1.838 1.236 1.07 1.835 2.809 1.305 3.495.998.108-.776.417-1.305.76-1.605-2.665-.3-5.466-1.332-5.466-5.93 0-1.31.465-2.38 1.235-3.22-.135-.303-.54-1.523.105-3.176 0 0 1.005-.322 3.3 1.23.96-.267 1.98-.399 3-.405 1.02.006 2.04.138 3 .405 2.28-1.552 3.285-1.23 3.285-1.23.645 1.653.24 2.873.12 3.176.765.84 1.23 1.91 1.23 3.22 0 4.61-2.805 5.625-5.475 5.92.42.36.81 1.096.81 2.22 0 1.606-.015 2.896-.015 3.286 0 .315.21.69.825.57C20.565 22.092 24 17.592 24 12.297c0-6.627-5.373-12-12-12'/%3E%3C/svg%3E") no-repeat; } @@ -123,7 +123,7 @@ html[data-theme="dark"] .docusaurus-highlight-code-line { .navbar__category { font-weight: 700; font-size: 0.8em; - padding: .4em 0 .4em .4em; + padding: 0.4em 0 0.4em 0.4em; margin-top: 1.1em; color: var(--docsearch-text-color); background-color: var(--ifm-hover-overlay); @@ -147,7 +147,7 @@ html[data-theme="dark"] .docusaurus-highlight-code-line { height: 10px; width: 10px; border-radius: 100%; - background: #D1D1D1; + background: #d1d1d1; transition: background 0.3s ease; } @@ -157,11 +157,11 @@ html[data-theme="dark"] .docusaurus-highlight-code-line { .cloudStatusLink.status-up::before, .cloudStatusLink.status-undermaintenance::before { - background: #00B093; + background: #00b093; } .cloudStatusLink.status-hasissues::before { - background: #ED8936; + background: #ed8936; } .codeBlockContainer_I0IT { @@ -190,8 +190,8 @@ The variables for them have been added to :root at the top of this file */ .menu__link--active:not(.menu__link--sublist), .menu__list-item-collapsible--active, .menu__list-item-collapsible--active:hover { - background: var(--color-active-nav-item-background); - color: var(--color-active-nav-item-text); + background: var(--color-active-nav-item-background); + color: var(--color-active-nav-item-text); } .connectorIcon { @@ -208,7 +208,7 @@ The variables for them have been added to :root at the top of this file */ font-size: smaller; margin: 0; padding: 5px; - margin-bottom: 5px; + margin-bottom: 15px; } .connectorMetadata > div { @@ -249,4 +249,4 @@ The variables for them have been added to :root at the top of this file */ [data-theme="dark"] .connectorMetadata .availability .unavailable { color: var(--ifm-color-emphasis-400); -} \ No newline at end of file +} diff --git a/docusaurus/src/remark/docsHeaderDecoration.js b/docusaurus/src/remark/docsHeaderDecoration.js index 57a237e7cae0..d627da204bf4 100644 --- a/docusaurus/src/remark/docsHeaderDecoration.js +++ b/docusaurus/src/remark/docsHeaderDecoration.js @@ -9,7 +9,7 @@ const REGISTRY_URL = const fetchCatalog = async () => { console.log("Fetching connector registry..."); - const json = await fetch(REGISTRY_URL).then(resp => resp.json()); + const json = await fetch(REGISTRY_URL).then((resp) => resp.json()); console.log(`fetched ${json.length} connectors form registry`); return json; }; @@ -61,36 +61,38 @@ const buildConnectorHTMLContent = ( ) => { // note - you can't have any leading whitespace here const htmlContent = `
-
- -

${originalTitle}

-
-
Availability
- ${registryEntry.is_cloud ? CHECK_ICON : CROSS_ICON} Airbyte Cloud - ${registryEntry.is_oss ? CHECK_ICON : CROSS_ICON} Airbyte OSS + ${ + registryEntry.is_cloud ? CHECK_ICON : CROSS_ICON + } Airbyte Cloud + ${ + registryEntry.is_oss ? CHECK_ICON : CROSS_ICON + } Airbyte OSS
Latest Version
${escape(registryEntry.dockerImageTag_oss)}
-
-
Definition Id
-
${escape(registryEntry.definitionId)}
-
+ +
+ +

${originalTitle}

+
`; return htmlContent; @@ -112,7 +114,11 @@ const isDocsPage = (vfile) => { }; const escape = (string) => { - return string.replace(/&/g, '&').replace(/"/g, '"').replace(//g, '>'); + return string + .replace(/&/g, "&") + .replace(/"/g, """) + .replace(//g, ">"); }; const capitalizeFirstLetter = (string) => { From 6ae6e8084a8c1946e002567d36c91e50c9f84aa1 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Wed, 15 Nov 2023 08:22:28 +0200 Subject: [PATCH 054/124] :bug: Source Amazon Seller Partner: remove AWS signature (#32508) --- .../metadata.yaml | 2 +- .../source-amazon-seller-partner/setup.py | 2 +- .../source_amazon_seller_partner/auth.py | 85 ------------------- .../source_amazon_seller_partner/source.py | 41 +-------- .../source_amazon_seller_partner/spec.json | 41 ++------- .../source_amazon_seller_partner/streams.py | 77 +++++------------ .../unit_tests/test_finance_streams.py | 17 ---- .../unit_tests/test_order_items_stream.py | 9 -- .../test_reports_stream_sales_and_traffic.py | 9 -- .../test_reports_streams_rate_limits.py | 66 -------------- .../test_reports_streams_settlement_report.py | 9 -- .../unit_tests/test_source.py | 46 +--------- .../unit_tests/test_transform_function.py | 9 -- .../sources/amazon-seller-partner.md | 4 +- 14 files changed, 37 insertions(+), 380 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_rate_limits.py diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml index aa6a34bd1760..83c764ab41f1 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: api connectorType: source definitionId: e55879a8-0ef8-4557-abcf-ab34c53ec460 - dockerImageTag: 1.6.1 + dockerImageTag: 1.6.2 dockerRepository: airbyte/source-amazon-seller-partner documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-seller-partner githubIssueLabel: source-amazon-seller-partner diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py b/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py index 2e5ca2d1e6d5..af80eec8c453 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "boto3~=1.16", "pendulum~=2.1", "pycryptodome~=3.10", "xmltodict~=0.12"] +MAIN_REQUIREMENTS = ["airbyte-cdk", "pendulum~=2.1", "pycryptodome~=3.10", "xmltodict~=0.12"] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py index 26b0dd29ae58..fd0dc7e33b7b 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py @@ -2,16 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import hashlib -import hmac -import urllib.parse from typing import Any, Mapping -from urllib.parse import urlparse import pendulum -import requests from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator -from requests.auth import AuthBase class AWSAuthenticator(Oauth2Authenticator): @@ -27,82 +21,3 @@ def get_auth_header(self) -> Mapping[str, Any]: "x-amz-access-token": self.get_access_token(), "x-amz-date": pendulum.now("utc").strftime("%Y%m%dT%H%M%SZ"), } - - -class AWSSignature(AuthBase): - """Source from https://github.com/saleweaver/python-amazon-sp-api/blob/master/sp_api/base/aws_sig_v4.py""" - - def __init__(self, service: str, aws_access_key_id: str, aws_secret_access_key: str, aws_session_token: str, region: str): - self.service = service - self.aws_access_key_id = aws_access_key_id - self.aws_secret_access_key = aws_secret_access_key - self.aws_session_token = aws_session_token - self.region = region - - @staticmethod - def _sign_msg(key: bytes, msg: str) -> bytes: - """Sign message using key""" - return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest() - - def _get_authorization_header(self, prepared_request: requests.PreparedRequest) -> str: - current_ts = pendulum.now("utc") - url_parsed = urlparse(prepared_request.url) - uri = urllib.parse.quote(url_parsed.path) - host = url_parsed.hostname - - amz_date = current_ts.strftime("%Y%m%dT%H%M%SZ") - datestamp = current_ts.strftime("%Y%m%d") - - # sort query parameters alphabetically - if len(url_parsed.query) > 0: - split_query_parameters = list(map(lambda param: param.split("="), url_parsed.query.split("&"))) - ordered_query_parameters = sorted(split_query_parameters, key=lambda param: (param[0], param[1])) - else: - ordered_query_parameters = list() - - canonical_querystring = "&".join(map(lambda param: "=".join(param), ordered_query_parameters)) - - headers_to_sign = {"host": host, "x-amz-date": amz_date} - if self.aws_session_token: - headers_to_sign["x-amz-security-token"] = self.aws_session_token - - ordered_headers = dict(sorted(headers_to_sign.items(), key=lambda h: h[0])) - canonical_headers = "".join(map(lambda h: ":".join(h) + "\n", ordered_headers.items())) - signed_headers = ";".join(ordered_headers.keys()) - - if prepared_request.method == "GET": - payload_hash = hashlib.sha256("".encode("utf-8")).hexdigest() - else: - if prepared_request.body: - payload_hash = hashlib.sha256(prepared_request.body.encode("utf-8")).hexdigest() - else: - payload_hash = hashlib.sha256("".encode("utf-8")).hexdigest() - - canonical_request = "\n".join( - [prepared_request.method, uri, canonical_querystring, canonical_headers, signed_headers, payload_hash] - ) - - credential_scope = "/".join([datestamp, self.region, self.service, "aws4_request"]) - string_to_sign = "\n".join( - ["AWS4-HMAC-SHA256", amz_date, credential_scope, hashlib.sha256(canonical_request.encode("utf-8")).hexdigest()] - ) - - datestamp_signed = self._sign_msg(("AWS4" + self.aws_secret_access_key).encode("utf-8"), datestamp) - region_signed = self._sign_msg(datestamp_signed, self.region) - service_signed = self._sign_msg(region_signed, self.service) - aws4_request_signed = self._sign_msg(service_signed, "aws4_request") - signature = hmac.new(aws4_request_signed, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest() - - authorization_header = "AWS4-HMAC-SHA256 Credential={}/{}, SignedHeaders={}, Signature={}".format( - self.aws_access_key_id, credential_scope, signed_headers, signature - ) - return authorization_header - - def __call__(self, prepared_request: requests.PreparedRequest) -> requests.PreparedRequest: - prepared_request.headers.update( - { - "authorization": self._get_authorization_header(prepared_request), - "x-amz-security-token": self.aws_session_token, - } - ) - return prepared_request diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py index 5f9bb1c5b7b0..2ca326c5f47f 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py @@ -4,12 +4,11 @@ from typing import Any, List, Mapping, Tuple -import boto3 from airbyte_cdk.logger import AirbyteLogger from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream -from source_amazon_seller_partner.auth import AWSAuthenticator, AWSSignature +from source_amazon_seller_partner.auth import AWSAuthenticator from source_amazon_seller_partner.constants import get_marketplaces from source_amazon_seller_partner.streams import ( BrandAnalyticsAlternatePurchaseReports, @@ -71,17 +70,7 @@ class SourceAmazonSellerPartner(AbstractSource): def _get_stream_kwargs(self, config: Mapping[str, Any]) -> Mapping[str, Any]: - endpoint, marketplace_id, region = get_marketplaces(config.get("aws_environment"))[config.get("region")] - - sts_credentials = self.get_sts_credentials(config) - role_creds = sts_credentials["Credentials"] - aws_signature = AWSSignature( - service="execute-api", - aws_access_key_id=role_creds.get("AccessKeyId"), - aws_secret_access_key=role_creds.get("SecretAccessKey"), - aws_session_token=role_creds.get("SessionToken"), - region=region, - ) + endpoint, marketplace_id, _ = get_marketplaces(config.get("aws_environment"))[config.get("region")] auth = AWSAuthenticator( token_refresh_endpoint="https://api.amazon.com/auth/o2/token", client_id=config.get("lwa_app_id"), @@ -93,7 +82,6 @@ def _get_stream_kwargs(self, config: Mapping[str, Any]) -> Mapping[str, Any]: stream_kwargs = { "url_base": endpoint, "authenticator": auth, - "aws_signature": aws_signature, "replication_start_date": config.get("replication_start_date"), "marketplace_id": marketplace_id, "period_in_days": config.get("period_in_days", 90), @@ -104,31 +92,6 @@ def _get_stream_kwargs(self, config: Mapping[str, Any]) -> Mapping[str, Any]: } return stream_kwargs - @staticmethod - def get_sts_credentials(config: Mapping[str, Any]) -> dict: - """ - We can only use a IAM User arn entity or a IAM Role entity. - If we use an IAM user arn entity in the connector configuration we need to get the credentials directly from the boto3 sts client - If we use an IAM role arn entity we need to invoke the assume_role from the boto3 sts client to get the credentials related to that role - - :param config: - """ - boto3_client = boto3.client( - "sts", aws_access_key_id=config.get("aws_access_key"), aws_secret_access_key=config.get("aws_secret_key") - ) - - if config.get("role_arn") is None: - return boto3_client.get_session_token() - - *_, arn_resource = config.get("role_arn").split(":") - if arn_resource.startswith("user"): - sts_credentials = boto3_client.get_session_token() - elif arn_resource.startswith("role"): - sts_credentials = boto3_client.assume_role(RoleArn=config.get("role_arn"), RoleSessionName="guid") - else: - raise ValueError("Invalid ARN, your ARN is not for a user or a role") - return sts_credentials - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: """ Check connection to Amazon SP API by requesting the Orders endpoint diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json index c3a53a712dda..afcd6279342f 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json @@ -5,9 +5,7 @@ "title": "Amazon Seller Partner Spec", "type": "object", "required": [ - "aws_access_key", "aws_environment", - "aws_secret_key", "region", "lwa_app_id", "lwa_client_secret", @@ -61,31 +59,10 @@ "type": "string", "order": 2 }, - "aws_access_key": { - "title": "AWS Access Key", - "description": "Specifies the AWS access key used as part of the credentials to authenticate the user.", - "airbyte_secret": true, - "order": 3, - "type": "string" - }, - "aws_secret_key": { - "title": "AWS Secret Access Key", - "description": "Specifies the AWS secret key used as part of the credentials to authenticate the user.", - "airbyte_secret": true, - "order": 4, - "type": "string" - }, - "role_arn": { - "title": "Role ARN", - "description": "Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. (Needs permission to 'Assume Role' STS).", - "airbyte_secret": true, - "order": 5, - "type": "string" - }, "lwa_app_id": { "title": "LWA Client Id", "description": "Your Login with Amazon Client ID.", - "order": 6, + "order": 3, "airbyte_secret": true, "type": "string" }, @@ -93,14 +70,14 @@ "title": "LWA Client Secret", "description": "Your Login with Amazon Client Secret.", "airbyte_secret": true, - "order": 7, + "order": 4, "type": "string" }, "refresh_token": { "title": "Refresh Token", "description": "The Refresh Token obtained via OAuth flow authorization.", "airbyte_secret": true, - "order": 8, + "order": 5, "type": "string" }, "replication_start_date": { @@ -108,7 +85,7 @@ "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "examples": ["2017-01-25T00:00:00Z"], - "order": 9, + "order": 6, "type": "string", "format": "date-time" }, @@ -117,7 +94,7 @@ "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data after this date will not be replicated.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$|^$", "examples": ["2017-01-25T00:00:00Z"], - "order": 10, + "order": 7, "type": "string", "format": "date-time" }, @@ -127,7 +104,7 @@ "description": "Will be used for stream slicing for initial full_refresh sync when no updated state is present for reports that support sliced incremental sync.", "default": 90, "minimum": 1, - "order": 11 + "order": 8 }, "report_options": { "title": "Report Options", @@ -136,7 +113,7 @@ "{\"GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT\": {\"reportPeriod\": \"WEEK\"}}", "{\"GET_SOME_REPORT\": {\"custom\": \"true\"}}" ], - "order": 12, + "order": 9, "type": "string" }, "max_wait_seconds": { @@ -144,7 +121,7 @@ "description": "Sometimes report can take up to 30 minutes to generate. This will set the limit for how long to wait for a successful report.", "default": 500, "examples": ["500", "1980"], - "order": 13, + "order": 10, "minimum": 1, "type": "integer" }, @@ -155,7 +132,7 @@ "{\"GET_SALES_AND_TRAFFIC_REPORT\": {\"availability_sla_days\": 3}}", "{\"GET_SOME_REPORT\": {\"custom\": \"true\"}}" ], - "order": 14, + "order": 11, "type": "string" } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py index 6fe4216f8490..38304231378d 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py @@ -9,21 +9,15 @@ from abc import ABC, abstractmethod from io import StringIO from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Union -from urllib.parse import urljoin import pendulum import requests import xmltodict from airbyte_cdk.entrypoint import logger from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator -from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, RequestBodyException -from airbyte_cdk.sources.streams.http.http import BODY_REQUEST_METHODS from airbyte_cdk.sources.streams.http.rate_limiting import default_backoff_handler from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from source_amazon_seller_partner.auth import AWSSignature REPORTS_API_VERSION = "2021-06-30" # 2020-09-04 ORDERS_API_VERSION = "v0" @@ -39,7 +33,6 @@ class AmazonSPStream(HttpStream, ABC): def __init__( self, url_base: str, - aws_signature: AWSSignature, replication_start_date: str, marketplace_id: str, period_in_days: Optional[int], @@ -56,7 +49,6 @@ def __init__( self._replication_start_date = replication_start_date self._replication_end_date = replication_end_date self.marketplace_id = marketplace_id - self._session.auth = aws_signature @property def url_base(self) -> str: @@ -139,7 +131,7 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late return {self.cursor_field: latest_benchmark} -class ReportsAmazonSPStream(Stream, ABC): +class ReportsAmazonSPStream(HttpStream, ABC): """ API docs: https://github.com/amzn/selling-partner-api-docs/blob/main/references/reports-api/reports_2020-09-04.md API model: https://github.com/amzn/selling-partner-api-models/blob/main/models/reports-api-model/reports_2020-09-04.json @@ -167,7 +159,6 @@ class ReportsAmazonSPStream(Stream, ABC): def __init__( self, url_base: str, - aws_signature: AWSSignature, replication_start_date: str, marketplace_id: str, period_in_days: Optional[int], @@ -175,12 +166,11 @@ def __init__( max_wait_seconds: Optional[int], replication_end_date: Optional[str], advanced_stream_options: Optional[str], - authenticator: HttpAuthenticator = None, + *args, + **kwargs, ): - self._authenticator = authenticator - self._session = requests.Session() + super().__init__(*args, **kwargs) self._url_base = url_base.rstrip("/") + "/" - self._session.auth = aws_signature self._replication_start_date = replication_start_date self._replication_end_date = replication_end_date self.marketplace_id = marketplace_id @@ -188,16 +178,24 @@ def __init__( self._report_options = report_options or "{}" self.max_wait_seconds = max_wait_seconds self._advanced_stream_options = dict() + self._http_method = "GET" if advanced_stream_options is not None: self._advanced_stream_options = json_lib.loads(advanced_stream_options) + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + @property - def url_base(self) -> str: - return self._url_base + def http_method(self) -> str: + return self._http_method + + @http_method.setter + def http_method(self, value: str): + self._http_method = value @property - def authenticator(self) -> HttpAuthenticator: - return self._authenticator + def url_base(self) -> str: + return self._url_base def request_params(self) -> MutableMapping[str, Any]: return {"MarketplaceIds": self.marketplace_id} @@ -208,37 +206,6 @@ def request_headers(self) -> Mapping[str, Any]: def path(self, document_id: str) -> str: return f"{self.path_prefix}/documents/{document_id}" - def should_retry(self, response: requests.Response) -> bool: - return response.status_code == 429 or 500 <= response.status_code < 600 - - @default_backoff_handler(max_tries=5, factor=5) - def _send_request(self, request: requests.PreparedRequest) -> requests.Response: - response: requests.Response = self._session.send(request) - if self.should_retry(response): - raise DefaultBackoffException(request=request, response=response) - else: - response.raise_for_status() - return response - - def _create_prepared_request( - self, path: str, http_method: str = "GET", headers: Mapping = None, params: Mapping = None, json: Any = None, data: Any = None - ) -> requests.PreparedRequest: - """ - Override to make http_method configurable per method call - """ - args = {"method": http_method, "url": urljoin(self.url_base, path), "headers": headers, "params": params} - if http_method.upper() in BODY_REQUEST_METHODS: - if json and data: - raise RequestBodyException( - "At the same time only one of the 'request_body_data' and 'request_body_json' functions can return data" - ) - elif json: - args["json"] = json - elif data: - args["data"] = data - - return self._session.prepare_request(requests.Request(**args)) - def _report_data( self, sync_mode: SyncMode, @@ -261,13 +228,14 @@ def _create_report( ) -> Mapping[str, Any]: request_headers = self.request_headers() report_data = self._report_data(sync_mode, cursor_field, stream_slice, stream_state) + self.http_method = "POST" create_report_request = self._create_prepared_request( - http_method="POST", path=f"{self.path_prefix}/reports", headers=dict(request_headers, **self.authenticator.get_auth_header()), data=json_lib.dumps(report_data), ) - report_response = self._send_request(create_report_request) + report_response = self._send_request(create_report_request, {}) + self.http_method = "GET" # rollback return report_response.json() def _retrieve_report(self, report_id: str) -> Mapping[str, Any]: @@ -276,7 +244,7 @@ def _retrieve_report(self, report_id: str) -> Mapping[str, Any]: path=f"{self.path_prefix}/reports/{report_id}", headers=dict(request_headers, **self.authenticator.get_auth_header()), ) - retrieve_report_response = self._send_request(retrieve_report_request) + retrieve_report_response = self._send_request(retrieve_report_request, {}) report_payload = retrieve_report_response.json() return report_payload @@ -373,7 +341,7 @@ def read_records( headers=dict(request_headers, **self.authenticator.get_auth_header()), params=self.request_params(), ) - response = self._send_request(request) + response = self._send_request(request, {}) yield from self.parse_response(response, stream_state, stream_slice) elif is_fatal: raise Exception(f"The report for stream '{self.name}' was aborted due to a fatal error") @@ -1224,12 +1192,11 @@ def stream_slices( request_headers = self.request_headers() get_reports = self._create_prepared_request( - http_method="GET", path=f"{self.path_prefix}/reports", headers=dict(request_headers, **self.authenticator.get_auth_header()), params=params, ) - report_response = self._send_request(get_reports) + report_response = self._send_request(get_reports, {}) response = report_response.json() data = response.get("reports", list()) records = [e.get("reportId") for e in data if e and e.get("reportId") not in unique_records] diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py index bf0ad2cbcc0b..7621df97a0f2 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py @@ -5,7 +5,6 @@ import pendulum import pytest import requests -from source_amazon_seller_partner.auth import AWSSignature from source_amazon_seller_partner.streams import ListFinancialEventGroups, ListFinancialEvents list_financial_event_groups_data = { @@ -96,16 +95,8 @@ @pytest.fixture def list_financial_event_groups_stream(): def _internal(start_date: str = START_DATE_1, end_date: str = END_DATE_1): - aws_signature = AWSSignature( - service="execute-api", - aws_access_key_id="AccessKeyId", - aws_secret_access_key="SecretAccessKey", - aws_session_token="SessionToken", - region="US", - ) stream = ListFinancialEventGroups( url_base="https://test.url", - aws_signature=aws_signature, replication_start_date=start_date, replication_end_date=end_date, marketplace_id="id", @@ -123,16 +114,8 @@ def _internal(start_date: str = START_DATE_1, end_date: str = END_DATE_1): @pytest.fixture def list_financial_events_stream(): def _internal(start_date: str = START_DATE_1, end_date: str = END_DATE_1): - aws_signature = AWSSignature( - service="execute-api", - aws_access_key_id="AccessKeyId", - aws_secret_access_key="SecretAccessKey", - aws_session_token="SessionToken", - region="US", - ) stream = ListFinancialEvents( url_base="https://test.url", - aws_signature=aws_signature, replication_start_date=start_date, replication_end_date=end_date, marketplace_id="id", diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_order_items_stream.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_order_items_stream.py index b445e2d3eea1..0c4a80a1bcf4 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_order_items_stream.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_order_items_stream.py @@ -4,7 +4,6 @@ import pytest import requests -from source_amazon_seller_partner.auth import AWSSignature from source_amazon_seller_partner.streams import OrderItems list_order_items_payload_data = { @@ -31,16 +30,8 @@ @pytest.fixture def order_items_stream(): def _internal(): - aws_signature = AWSSignature( - service="execute-api", - aws_access_key_id="AccessKeyId", - aws_secret_access_key="SecretAccessKey", - aws_session_token="SessionToken", - region="US", - ) stream = OrderItems( url_base="https://test.url", - aws_signature=aws_signature, replication_start_date="2023-08-08T00:00:00Z", replication_end_date=None, marketplace_id="id", diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_stream_sales_and_traffic.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_stream_sales_and_traffic.py index 6167fea7c10f..c255aa92eee6 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_stream_sales_and_traffic.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_stream_sales_and_traffic.py @@ -2,7 +2,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from source_amazon_seller_partner.auth import AWSSignature from source_amazon_seller_partner.streams import SellerAnalyticsSalesAndTrafficReports START_DATE_1 = "2023-02-05T00:00:00Z" @@ -10,16 +9,8 @@ def test_stream_uses_advanced_options(): - aws_signature = AWSSignature( - service="execute-api", - aws_access_key_id="AccessKeyId", - aws_secret_access_key="SecretAccessKey", - aws_session_token="SessionToken", - region="US", - ) stream = SellerAnalyticsSalesAndTrafficReports( url_base="https://test.url", - aws_signature=aws_signature, replication_start_date=START_DATE_1, replication_end_date=END_DATE_1, marketplace_id="id", diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_rate_limits.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_rate_limits.py deleted file mode 100644 index fec6e1b7326a..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_rate_limits.py +++ /dev/null @@ -1,66 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest -import requests -from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException -from source_amazon_seller_partner.auth import AWSSignature -from source_amazon_seller_partner.streams import MerchantListingsReports - - -@pytest.fixture -def reports_stream(): - aws_signature = AWSSignature( - service="execute-api", - aws_access_key_id="AccessKeyId", - aws_secret_access_key="SecretAccessKey", - aws_session_token="SessionToken", - region="US", - ) - stream = MerchantListingsReports( - url_base="https://test.url", - aws_signature=aws_signature, - replication_start_date="2017-01-25T00:00:00Z", - replication_end_date="2017-02-25T00:00:00Z", - marketplace_id="id", - authenticator=None, - period_in_days=0, - report_options=None, - advanced_stream_options=None, - max_wait_seconds=500, - ) - return stream - - -def test_reports_stream_should_retry(mocker, reports_stream): - response = requests.Response() - response.status_code = 429 - mocker.patch.object(requests.Session, "send", return_value=response) - should_retry = reports_stream.should_retry(response=response) - - assert should_retry is True - - -def test_reports_stream_send_request(mocker, reports_stream): - response = requests.Response() - response.status_code = 200 - mocker.patch.object(requests.Session, "send", return_value=response) - - assert response == reports_stream._send_request(request=requests.PreparedRequest()) - - -def test_reports_stream_send_request_backoff_exception(mocker, caplog, reports_stream): - mocker.patch("time.sleep", lambda x: None) - response = requests.Response() - response.status_code = 429 - mocker.patch.object(requests.Session, "send", return_value=response) - - with pytest.raises(DefaultBackoffException): - reports_stream._send_request(request=requests.PreparedRequest()) - - assert "Backing off _send_request(...) for 5.0s" in caplog.text - assert "Backing off _send_request(...) for 10.0s" in caplog.text - assert "Backing off _send_request(...) for 20.0s" in caplog.text - assert "Backing off _send_request(...) for 40.0s" in caplog.text - assert "Giving up _send_request(...) after 5 tries" in caplog.text diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py index b94b2d30b381..a1f64cf10b64 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py @@ -5,7 +5,6 @@ import pytest import requests from airbyte_cdk.models import SyncMode -from source_amazon_seller_partner.auth import AWSSignature from source_amazon_seller_partner.streams import FlatFileSettlementV2Reports START_DATE_1 = "2022-05-25T00:00:00Z" @@ -78,16 +77,8 @@ @pytest.fixture def settlement_reports_stream(): def _internal(start_date: str = START_DATE_1, end_date: str = END_DATE_1): - aws_signature = AWSSignature( - service="execute-api", - aws_access_key_id="AccessKeyId", - aws_secret_access_key="SecretAccessKey", - aws_session_token="SessionToken", - region="US", - ) stream = FlatFileSettlementV2Reports( url_base="https://test.url", - aws_signature=aws_signature, replication_start_date=start_date, replication_end_date=end_date, marketplace_id="id", diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py index 4c24d0fccd65..cfd02fd04ecd 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py @@ -2,12 +2,9 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from unittest.mock import MagicMock - import pytest from airbyte_cdk.sources.streams import Stream from source_amazon_seller_partner import SourceAmazonSellerPartner -from source_amazon_seller_partner.source import boto3 @pytest.fixture @@ -24,52 +21,11 @@ def connector_config(): "app_id": "amzn1.sp.solution.2cfa6ca8-2c35-123-456-78910", "lwa_app_id": "amzn1.application-oa2-client.abc123", "lwa_client_secret": "abc123", - "aws_access_key": "aws_access_key", - "aws_secret_key": "aws_secret_key", - "role_arn": "arn:aws:iam::123456789098:role/some-role", "aws_environment": "SANDBOX", "region": "US", } -@pytest.fixture -def sts_credentials(): - return { - "Credentials": { - "AccessKeyId": "foo", - "SecretAccessKey": "bar", - "SessionToken": "foobar", - } - } - - -@pytest.fixture -def mock_boto_client(mocker, sts_credentials): - boto_client = MagicMock() - mocker.patch.object(boto3, "client", return_value=boto_client) - boto_client.assume_role.return_value = sts_credentials - boto_client.get_session_token.return_value = sts_credentials - return boto_client - - -def test_streams(connector_source, connector_config, mock_boto_client): +def test_streams(connector_source, connector_config): for stream in connector_source.streams(connector_config): assert isinstance(stream, Stream) - - -@pytest.mark.parametrize("arn", ("arn:aws:iam::123456789098:user/some-user", "arn:aws:iam::123456789098:role/some-role")) -def test_stream_with_good_iam_arn_value(mock_boto_client, connector_source, connector_config, arn): - connector_config["role_arn"] = arn - result = connector_source.get_sts_credentials(connector_config) - assert "Credentials" in result - if "user" in arn: - mock_boto_client.get_session_token.assert_called_once() - if "role" in arn: - mock_boto_client.assume_role.assert_called_once_with(RoleArn=arn, RoleSessionName="guid") - - -def test_stream_with_bad_iam_arn_value(connector_source, connector_config, mock_boto_client): - connector_config["role_arn"] = "bad-arn" - with pytest.raises(ValueError) as e: - connector_source.get_sts_credentials(connector_config) - assert "Invalid" in e.message diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_transform_function.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_transform_function.py index 9f4df656aded..27d7ca2d3b36 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_transform_function.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_transform_function.py @@ -3,21 +3,12 @@ # import pytest -from source_amazon_seller_partner.auth import AWSSignature from source_amazon_seller_partner.streams import SellerFeedbackReports def reports_stream(marketplace_id): - aws_signature = AWSSignature( - service="execute-api", - aws_access_key_id="AccessKeyId", - aws_secret_access_key="SecretAccessKey", - aws_session_token="SessionToken", - region="Mars", - ) stream = SellerFeedbackReports( url_base="https://test.url", - aws_signature=aws_signature, replication_start_date="2010-01-25T00:00:00Z", replication_end_date="2017-02-25T00:00:00Z", marketplace_id=marketplace_id, diff --git a/docs/integrations/sources/amazon-seller-partner.md b/docs/integrations/sources/amazon-seller-partner.md index de3abffd3caf..3841fa3e416a 100644 --- a/docs/integrations/sources/amazon-seller-partner.md +++ b/docs/integrations/sources/amazon-seller-partner.md @@ -6,9 +6,6 @@ This page guides you through the process of setting up the Amazon Seller Partner - AWS Environment - AWS Region -- AWS Access Key -- AWS Secret Key -- Role ARN - LWA Client ID (LWA App ID)** - LWA Client Secret** - Refresh token** @@ -129,6 +126,7 @@ So, for any value that exceeds the limit, the `period_in_days` will be automatic | Version | Date | Pull Request | Subject | |:---------|:-----------|:--------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `1.6.2` | 2023-11-14 | [\#32508](https://github.com/airbytehq/airbyte/pull/32508) | Do not use AWS signature as it is no longer required by the Amazon API | | `1.6.1` | 2023-11-13 | [\#32457](https://github.com/airbytehq/airbyte/pull/32457) | Fix report decompression | | `1.6.0` | 2023-11-09 | [\#32259](https://github.com/airbytehq/airbyte/pull/32259) | mark "aws_secret_key" and "aws_access_key" as required in specification; update schema for stream `Orders` | | `1.5.1` | 2023-08-18 | [\#29255](https://github.com/airbytehq/airbyte/pull/29255) | role_arn is optional on UI but not really on the backend blocking connector set up using oauth | From af402ca36707d5098d2835731eb22c642bf355bd Mon Sep 17 00:00:00 2001 From: Daryna Ishchenko <80129833+darynaishchenko@users.noreply.github.com> Date: Wed, 15 Nov 2023 12:53:40 +0200 Subject: [PATCH 055/124] Source Zendesk Support: connector health: updated expected records for ticket mertics stream (#32540) --- .../integration_tests/expected_records.jsonl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl index 615d50bb3ebc..c20d59cbd37a 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl @@ -43,7 +43,7 @@ {"stream": "ticket_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_fields/360002833116.json", "id": 360002833116, "type": "status", "title": "Status", "raw_title": "Status", "description": "Request status", "raw_description": "Request status", "position": 3, "active": true, "required": false, "collapsed_for_agents": false, "regexp_for_validation": null, "title_in_portal": "Status", "raw_title_in_portal": "Status", "visible_in_portal": false, "editable_in_portal": false, "required_in_portal": false, "tag": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z", "removable": false, "key": null, "agent_description": null, "system_field_options": [{"name": "Open", "value": "open"}, {"name": "Pending", "value": "pending"}, {"name": "Solved", "value": "solved"}], "sub_type_id": 0}, "emitted_at": 1697714860085} {"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/8154457562767.json", "id": 8154457562767, "ticket_id": 154, "created_at": "2023-10-17T14:24:53Z", "updated_at": "2023-10-17T14:28:25Z", "group_stations": 2, "assignee_stations": 1, "reopens": 0, "replies": 1, "assignee_updated_at": "2023-10-17T14:27:52Z", "requester_updated_at": "2023-10-17T14:26:45Z", "status_updated_at": "2023-11-06T15:01:40Z", "initially_assigned_at": "2023-10-17T14:26:33Z", "assigned_at": "2023-10-17T14:26:33Z", "solved_at": "2023-10-17T14:27:52Z", "latest_comment_added_at": "2023-10-17T14:28:25Z", "reply_time_in_minutes": {"calendar": 4, "business": 0}, "first_resolution_time_in_minutes": {"calendar": 3, "business": 0}, "full_resolution_time_in_minutes": {"calendar": 3, "business": 0}, "agent_wait_time_in_minutes": {"calendar": 0, "business": 0}, "requester_wait_time_in_minutes": {"calendar": 3, "business": 0}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "reply_time_in_seconds": {"calendar": 212}, "custom_status_updated_at": "2023-10-17T14:27:52Z"}, "emitted_at": 1699646404810} {"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7283000498191.json", "id": 7283000498191, "ticket_id": 153, "created_at": "2023-06-26T11:31:48Z", "updated_at": "2023-06-26T12:13:42Z", "group_stations": 2, "assignee_stations": 2, "reopens": 0, "replies": 0, "assignee_updated_at": "2023-06-26T11:31:48Z", "requester_updated_at": "2023-06-26T11:31:48Z", "status_updated_at": "2023-06-26T11:31:48Z", "initially_assigned_at": "2023-06-26T11:31:48Z", "assigned_at": "2023-06-26T12:13:42Z", "solved_at": null, "latest_comment_added_at": "2023-06-26T11:31:48Z", "reply_time_in_minutes": {"calendar": null, "business": null}, "first_resolution_time_in_minutes": {"calendar": null, "business": null}, "full_resolution_time_in_minutes": {"calendar": null, "business": null}, "agent_wait_time_in_minutes": {"calendar": null, "business": null}, "requester_wait_time_in_minutes": {"calendar": null, "business": null}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "custom_status_updated_at": "2023-06-26T11:31:48Z"}, "emitted_at": 1699646404810} -{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7282909551759.json", "id": 7282909551759, "ticket_id": 152, "created_at": "2023-06-26T11:10:33Z", "updated_at": "2023-06-26T11:25:43Z", "group_stations": 1, "assignee_stations": 1, "reopens": 0, "replies": 1, "assignee_updated_at": "2023-06-26T11:25:43Z", "requester_updated_at": "2023-06-26T11:10:33Z", "status_updated_at": "2023-07-16T12:01:39Z", "initially_assigned_at": "2023-06-26T11:10:33Z", "assigned_at": "2023-06-26T11:10:33Z", "solved_at": "2023-06-26T11:25:43Z", "latest_comment_added_at": "2023-06-26T11:21:06Z", "reply_time_in_minutes": {"calendar": 11, "business": 0}, "first_resolution_time_in_minutes": {"calendar": 15, "business": 0}, "full_resolution_time_in_minutes": {"calendar": 15, "business": 0}, "agent_wait_time_in_minutes": {"calendar": 15, "business": 0}, "requester_wait_time_in_minutes": {"calendar": 0, "business": 0}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "custom_status_updated_at": "2023-06-26T11:25:43Z"}, "emitted_at": 1699646404810} +{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7282901696015.json", "id": 7282901696015, "ticket_id": 151, "created_at": "2023-06-26T11:09:33Z", "updated_at": "2023-06-26T12:03:38Z", "group_stations": 1, "assignee_stations": 1, "reopens": 0, "replies": 1, "assignee_updated_at": "2023-06-26T12:03:37Z", "requester_updated_at": "2023-06-26T11:09:33Z", "status_updated_at": "2023-06-26T11:09:33Z", "initially_assigned_at": "2023-06-26T11:09:33Z", "assigned_at": "2023-06-26T11:09:33Z", "solved_at": null, "latest_comment_added_at": "2023-06-26T12:03:37Z", "reply_time_in_minutes": {"calendar": 54, "business": 0}, "first_resolution_time_in_minutes": {"calendar": null, "business": null}, "full_resolution_time_in_minutes": {"calendar": null, "business": null}, "agent_wait_time_in_minutes": {"calendar": null, "business": null}, "requester_wait_time_in_minutes": {"calendar": null, "business": null}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "custom_status_updated_at": "2023-06-26T11:09:33Z"}, "emitted_at": 1700040843806} {"stream": "ticket_metric_events", "data": {"id": 4992797383183, "ticket_id": 121, "metric": "agent_work_time", "instance_id": 0, "type": "measure", "time": "2022-06-17T14:49:20Z"}, "emitted_at": 1697714863384} {"stream": "ticket_metric_events", "data": {"id": 4992797383311, "ticket_id": 121, "metric": "pausable_update_time", "instance_id": 0, "type": "measure", "time": "2022-06-17T14:49:20Z"}, "emitted_at": 1697714863386} {"stream": "ticket_metric_events", "data": {"id": 4992797383439, "ticket_id": 121, "metric": "reply_time", "instance_id": 0, "type": "measure", "time": "2022-06-17T14:49:20Z"}, "emitted_at": 1697714863386} From 76da539b5f94058b9bffcee94a5ccdbde2d02ead Mon Sep 17 00:00:00 2001 From: Daryna Ishchenko <80129833+darynaishchenko@users.noreply.github.com> Date: Wed, 15 Nov 2023 12:54:00 +0200 Subject: [PATCH 056/124] Source Klaviyo: updated events expected records (#32504) --- .../integration_tests/expected_records.jsonl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-klaviyo/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-klaviyo/integration_tests/expected_records.jsonl index dc97d4690c72..5ea6bd27e710 100644 --- a/airbyte-integrations/connectors/source-klaviyo/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-klaviyo/integration_tests/expected_records.jsonl @@ -3,11 +3,11 @@ {"stream": "campaigns", "data": {"type": "campaign", "id": "01HE82EVNPCB3YP0TZYNXAPJKQ", "attributes": {"name": "Email Campaign - Nov 2 2023 3:09 PM", "status": "Draft", "archived": false, "channel": "email", "audiences": {"included": [], "excluded": []}, "send_options": {"use_smart_sending": true, "ignore_unsubscribes": false}, "message": "01HE82EVP0TMKNH6RGFE4ED0P6", "tracking_options": {"is_tracking_opens": true, "is_tracking_clicks": true, "is_add_utm": false, "utm_params": []}, "send_strategy": {"method": "static", "options_static": {"datetime": "2023-11-01T22:00:00+00:00", "is_local": false, "send_past_recipients_immediately": null}, "options_throttled": null, "options_sto": null}, "created_at": "2023-11-02T13:09:45.276362+00:00", "scheduled_at": null, "updated_at": "2023-11-02T13:09:45.276403+00:00", "send_time": null}, "relationships": {"tags": {"links": {"self": "https://a.klaviyo.com/api/campaigns/01HE82EVNPCB3YP0TZYNXAPJKQ/relationships/tags/", "related": "https://a.klaviyo.com/api/campaigns/01HE82EVNPCB3YP0TZYNXAPJKQ/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/campaigns/01HE82EVNPCB3YP0TZYNXAPJKQ/"}, "updated_at": "2023-11-02T13:09:45.276403+00:00"}, "emitted_at": 1698930649424} {"stream": "campaigns", "data": {"type": "campaign", "id": "01HE2PASG4GSV564GPXCAW8TFJ", "attributes": {"name": "Email Campaign Archived - Nov 01 2023 12:55 PM", "status": "Sent", "archived": true, "channel": "email", "audiences": {"included": ["RnsiHB", "UXi5Jz"], "excluded": []}, "send_options": {"use_smart_sending": true, "ignore_unsubscribes": false}, "message": "01HE2PASMWR4TBTD7JD79N675K", "tracking_options": {"is_tracking_opens": true, "is_tracking_clicks": true, "is_add_utm": false, "utm_params": []}, "send_strategy": {"method": "static", "options_static": {"datetime": "2023-10-31T11:01:53+00:00", "is_local": false, "send_past_recipients_immediately": null}, "options_throttled": null, "options_sto": null}, "created_at": "2023-10-31T11:01:36.900676+00:00", "scheduled_at": "2023-10-31T11:01:53.122496+00:00", "updated_at": "2023-10-31T11:02:12.888185+00:00", "send_time": "2023-10-31T11:01:53+00:00"}, "relationships": {"tags": {"links": {"self": "https://a.klaviyo.com/api/campaigns/01HE2PASG4GSV564GPXCAW8TFJ/relationships/tags/", "related": "https://a.klaviyo.com/api/campaigns/01HE2PASG4GSV564GPXCAW8TFJ/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/campaigns/01HE2PASG4GSV564GPXCAW8TFJ/"}, "updated_at": "2023-10-31T11:02:12.888185+00:00"}, "emitted_at": 1698930649780} {"stream": "campaigns", "data": {"type": "campaign", "id": "01HEHY2911JYEGQ4EMAWRWMGKE", "attributes": {"name": "Email Campaign Archived 2 - Nov 6 2023 11:05 AM", "status": "Sent", "archived": true, "channel": "email", "audiences": {"included": ["RnsiHB"], "excluded": []}, "send_options": {"use_smart_sending": true, "ignore_unsubscribes": false}, "message": "01HEHY291CNK9TAWBSKJ28P032", "tracking_options": {"is_tracking_opens": true, "is_tracking_clicks": true, "is_add_utm": false, "utm_params": []}, "send_strategy": {"method": "static", "options_static": {"datetime": "2023-11-06T09:06:34+00:00", "is_local": false, "send_past_recipients_immediately": null}, "options_throttled": null, "options_sto": null}, "created_at": "2023-11-06T09:05:22.984339+00:00", "scheduled_at": "2023-11-06T09:06:34.279041+00:00", "updated_at": "2023-11-06T09:07:01.148389+00:00", "send_time": "2023-11-06T09:06:34+00:00"}, "relationships": {"tags": {"links": {"self": "https://a.klaviyo.com/api/campaigns/01HEHY2911JYEGQ4EMAWRWMGKE/relationships/tags/", "related": "https://a.klaviyo.com/api/campaigns/01HEHY2911JYEGQ4EMAWRWMGKE/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/campaigns/01HEHY2911JYEGQ4EMAWRWMGKE/"}, "updated_at": "2023-11-06T09:07:01.148389+00:00"}, "emitted_at": 1699263218286} -{"stream": "events", "data": {"type": "event", "id": "3qvdbYg3", "attributes": {"timestamp": 1621295008, "event_properties": {"$event_id": "1621295008"}, "datetime": "2021-05-17T23:43:28+00:00", "uuid": "adc8d000-b769-11eb-8001-28a6687f81c3"}, "relationships": {"profile": {"data": {"type": "profile", "id": "01F5YBDQE9W7WDSH9KK398CAYX"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdbYg3/relationships/profile/", "related": "https://a.klaviyo.com/api/events/3qvdbYg3/profile/"}}, "metric": {"data": {"type": "metric", "id": "VFFb4u"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdbYg3/relationships/metric/", "related": "https://a.klaviyo.com/api/events/3qvdbYg3/metric/"}}}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdbYg3/"}, "datetime": "2021-05-17T23:43:28+00:00"}, "emitted_at": 1698955720964} -{"stream": "events", "data": {"type": "event", "id": "3qvdgpzF", "attributes": {"timestamp": 1621295124, "event_properties": {"$event_id": "1621295124"}, "datetime": "2021-05-17T23:45:24+00:00", "uuid": "f2ed0200-b769-11eb-8001-76152f6b1c82"}, "relationships": {"profile": {"data": {"type": "profile", "id": "01F5YBGKW1SQN453RM293PHH37"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgpzF/relationships/profile/", "related": "https://a.klaviyo.com/api/events/3qvdgpzF/profile/"}}, "metric": {"data": {"type": "metric", "id": "VFFb4u"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgpzF/relationships/metric/", "related": "https://a.klaviyo.com/api/events/3qvdgpzF/metric/"}}}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgpzF/"}, "datetime": "2021-05-17T23:45:24+00:00"}, "emitted_at": 1698955720965} -{"stream": "events", "data": {"type": "event", "id": "3qvdgr5Z", "attributes": {"timestamp": 1621295124, "event_properties": {"$event_id": "1621295124"}, "datetime": "2021-05-17T23:45:24+00:00", "uuid": "f2ed0200-b769-11eb-8001-b642ddab48ad"}, "relationships": {"profile": {"data": {"type": "profile", "id": "01F5YBGM7J4YD4P6EYK5Q87BG4"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgr5Z/relationships/profile/", "related": "https://a.klaviyo.com/api/events/3qvdgr5Z/profile/"}}, "metric": {"data": {"type": "metric", "id": "VFFb4u"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgr5Z/relationships/metric/", "related": "https://a.klaviyo.com/api/events/3qvdgr5Z/metric/"}}}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgr5Z/"}, "datetime": "2021-05-17T23:45:24+00:00"}, "emitted_at": 1698955720965} -{"stream": "events", "data": {"type": "event", "id": "3qvdgBgK", "attributes": {"timestamp": 1621295124, "event_properties": {"$event_id": "1621295124"}, "datetime": "2021-05-17T23:45:24+00:00", "uuid": "f2ed0200-b769-11eb-8001-2006a2b2b6e7"}, "relationships": {"profile": {"data": {"type": "profile", "id": "01F5YBGMK62AJR0955G7NW6EP7"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgBgK/relationships/profile/", "related": "https://a.klaviyo.com/api/events/3qvdgBgK/profile/"}}, "metric": {"data": {"type": "metric", "id": "VFFb4u"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgBgK/relationships/metric/", "related": "https://a.klaviyo.com/api/events/3qvdgBgK/metric/"}}}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgBgK/"}, "datetime": "2021-05-17T23:45:24+00:00"}, "emitted_at": 1698955720965} -{"stream": "events", "data": {"type": "event", "id": "3qvdgs9P", "attributes": {"timestamp": 1621295125, "event_properties": {"$event_id": "1621295125"}, "datetime": "2021-05-17T23:45:25+00:00", "uuid": "f3859880-b769-11eb-8001-f6a061424b91"}, "relationships": {"profile": {"data": {"type": "profile", "id": "01F5YBGMK62AJR0955G7NW6EP7"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgs9P/relationships/profile/", "related": "https://a.klaviyo.com/api/events/3qvdgs9P/profile/"}}, "metric": {"data": {"type": "metric", "id": "VFFb4u"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgs9P/relationships/metric/", "related": "https://a.klaviyo.com/api/events/3qvdgs9P/metric/"}}}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgs9P/"}, "datetime": "2021-05-17T23:45:25+00:00"}, "emitted_at": 1698955720965} +{"stream": "events", "data": {"type": "event", "id": "3qvdbYg3", "attributes": {"timestamp": 1621295008, "event_properties": {"$event_id": "1621295008"}, "datetime": "2021-05-17 23:43:28+00:00", "uuid": "adc8d000-b769-11eb-8001-28a6687f81c3"}, "relationships": {"profile": {"data": {"type": "profile", "id": "01F5YBDQE9W7WDSH9KK398CAYX"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdbYg3/relationships/profile/", "related": "https://a.klaviyo.com/api/events/3qvdbYg3/profile/"}}, "metric": {"data": {"type": "metric", "id": "VFFb4u"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdbYg3/relationships/metric/", "related": "https://a.klaviyo.com/api/events/3qvdbYg3/metric/"}}}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdbYg3/"}, "datetime": "2021-05-17 23:43:28+00:00"}, "emitted_at": 1699980660456} +{"stream": "events", "data": {"type": "event", "id": "3qvdgpzF", "attributes": {"timestamp": 1621295124, "event_properties": {"$event_id": "1621295124"}, "datetime": "2021-05-17 23:45:24+00:00", "uuid": "f2ed0200-b769-11eb-8001-76152f6b1c82"}, "relationships": {"profile": {"data": {"type": "profile", "id": "01F5YBGKW1SQN453RM293PHH37"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgpzF/relationships/profile/", "related": "https://a.klaviyo.com/api/events/3qvdgpzF/profile/"}}, "metric": {"data": {"type": "metric", "id": "VFFb4u"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgpzF/relationships/metric/", "related": "https://a.klaviyo.com/api/events/3qvdgpzF/metric/"}}}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgpzF/"}, "datetime": "2021-05-17 23:45:24+00:00"}, "emitted_at": 1699980660457} +{"stream": "events", "data": {"type": "event", "id": "3qvdgr5Z", "attributes": {"timestamp": 1621295124, "event_properties": {"$event_id": "1621295124"}, "datetime": "2021-05-17 23:45:24+00:00", "uuid": "f2ed0200-b769-11eb-8001-b642ddab48ad"}, "relationships": {"profile": {"data": {"type": "profile", "id": "01F5YBGM7J4YD4P6EYK5Q87BG4"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgr5Z/relationships/profile/", "related": "https://a.klaviyo.com/api/events/3qvdgr5Z/profile/"}}, "metric": {"data": {"type": "metric", "id": "VFFb4u"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgr5Z/relationships/metric/", "related": "https://a.klaviyo.com/api/events/3qvdgr5Z/metric/"}}}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgr5Z/"}, "datetime": "2021-05-17 23:45:24+00:00"}, "emitted_at": 1699980660457} +{"stream": "events", "data": {"type": "event", "id": "3qvdgBgK", "attributes": {"timestamp": 1621295124, "event_properties": {"$event_id": "1621295124"}, "datetime": "2021-05-17 23:45:24+00:00", "uuid": "f2ed0200-b769-11eb-8001-2006a2b2b6e7"}, "relationships": {"profile": {"data": {"type": "profile", "id": "01F5YBGMK62AJR0955G7NW6EP7"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgBgK/relationships/profile/", "related": "https://a.klaviyo.com/api/events/3qvdgBgK/profile/"}}, "metric": {"data": {"type": "metric", "id": "VFFb4u"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgBgK/relationships/metric/", "related": "https://a.klaviyo.com/api/events/3qvdgBgK/metric/"}}}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgBgK/"}, "datetime": "2021-05-17 23:45:24+00:00"}, "emitted_at": 1699980660457} +{"stream": "events", "data": {"type": "event", "id": "3qvdgs9P", "attributes": {"timestamp": 1621295125, "event_properties": {"$event_id": "1621295125"}, "datetime": "2021-05-17 23:45:25+00:00", "uuid": "f3859880-b769-11eb-8001-f6a061424b91"}, "relationships": {"profile": {"data": {"type": "profile", "id": "01F5YBGMK62AJR0955G7NW6EP7"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgs9P/relationships/profile/", "related": "https://a.klaviyo.com/api/events/3qvdgs9P/profile/"}}, "metric": {"data": {"type": "metric", "id": "VFFb4u"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgs9P/relationships/metric/", "related": "https://a.klaviyo.com/api/events/3qvdgs9P/metric/"}}}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgs9P/"}, "datetime": "2021-05-17 23:45:25+00:00"}, "emitted_at": 1699980660457} {"stream": "global_exclusions", "data": {"type": "profile", "id": "01F5YBGPSXF1N23RBJZ947R1N1", "attributes": {"email": "some.email.that.dont.exist.8@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 8", "last_name": "Last Name 8", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:27+00:00", "updated": "2021-05-17T23:45:27+00:00", "last_event_date": "2021-05-17T23:45:27+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null, "ip": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [{"reason": "USER_SUPPRESSED", "timestamp": "2021-05-18T01:29:51+00:00"}], "list_suppressions": []}}, "sms": null}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/segments/"}}}, "updated": "2021-05-17T23:45:27+00:00"}, "emitted_at": 1663367161413} {"stream": "global_exclusions", "data": {"type": "profile", "id": "01F5YBGQ6X21SSWPGRDK9QK97C", "attributes": {"email": "some.email.that.dont.exist.9@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 9", "last_name": "Last Name 9", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:28+00:00", "updated": "2021-05-17T23:45:30+00:00", "last_event_date": "2021-05-17T23:45:28+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null, "ip": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [{"reason": "USER_SUPPRESSED", "timestamp": "2021-05-18T01:20:01+00:00"}], "list_suppressions": []}}, "sms": null}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/segments/"}}}, "updated": "2021-05-17T23:45:30+00:00"}, "emitted_at": 1663367161413} {"stream": "lists", "data": {"type": "list", "id": "RnsiHB", "attributes": {"name": "Newsletter", "created": "2021-03-31T10:50:36+00:00", "updated": "2021-03-31T10:50:36+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/RnsiHB/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/RnsiHB/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/RnsiHB/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/RnsiHB/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/RnsiHB/"}, "updated": "2021-03-31T10:50:36+00:00"}, "emitted_at": 1698942733516} From 7ae7e421eafd44135eca8883f8629e31b45313fa Mon Sep 17 00:00:00 2001 From: Augustin Date: Wed, 15 Nov 2023 13:16:01 +0100 Subject: [PATCH 057/124] airbyte-ci: use binary in CI (#32497) --- .../actions/run-dagger-pipeline/action.yml | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/actions/run-dagger-pipeline/action.yml b/.github/actions/run-dagger-pipeline/action.yml index 31af4caf9ada..c6645186cccc 100644 --- a/.github/actions/run-dagger-pipeline/action.yml +++ b/.github/actions/run-dagger-pipeline/action.yml @@ -69,6 +69,11 @@ inputs: s3_build_cache_secret_key: description: "Gradle S3 Build Cache AWS secret key" required: false + airbyte_ci_binary_url: + description: "URL to airbyte-ci binary" + required: false + default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci + runs: using: "composite" steps: @@ -89,22 +94,18 @@ runs: id: get-start-timestamp shell: bash run: echo "name=start-timestamp=$(date +%s)" >> $GITHUB_OUTPUT - - name: Install Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: "3.10" - token: ${{ inputs.github_token }} - - name: Install ci-connector-ops package + - name: Install airbyte-ci binary + id: install-airbyte-ci shell: bash run: | - pip install pipx - pipx ensurepath - pipx install airbyte-ci/connectors/pipelines/ + curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin + sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci + sudo chmod +x /usr/local/bin/airbyte-ci - name: Run airbyte-ci shell: bash run: | export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock" - airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }} + airbyte-ci --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }} env: _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k" CI_CONTEXT: "${{ inputs.context }}" From 3d48b92d3a6e23ed0584e89b4746b1ebffb25951 Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk <35109939+tolik0@users.noreply.github.com> Date: Wed, 15 Nov 2023 14:45:11 +0200 Subject: [PATCH 058/124] =?UTF-8?q?=E2=9C=A8=20Source=20Instagram:=20Marke?= =?UTF-8?q?d=20start=5Fdate=20as=20optional;=20set=20max=20retry=20time=20?= =?UTF-8?q?to=2010=20minutes=20(#32494)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../integration_tests/expected_records.jsonl | 14 ++--- .../integration_tests/spec.json | 7 ++- .../connectors/source-instagram/metadata.yaml | 11 +++- .../source-instagram/source_instagram/api.py | 2 +- .../schemas/user_insights.json | 3 ++ .../source_instagram/source.py | 25 ++++++--- .../source_instagram/streams.py | 2 +- .../source-instagram/unit_tests/conftest.py | 4 ++ .../unit_tests/test_source.py | 10 ++-- .../unit_tests/test_streams.py | 10 ++-- docs/integrations/sources/instagram.inapp.md | 17 ------ docs/integrations/sources/instagram.md | 53 +++++++++++-------- 12 files changed, 90 insertions(+), 68 deletions(-) delete mode 100644 docs/integrations/sources/instagram.inapp.md diff --git a/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl index f3dc892818c0..0a60185b5dbf 100644 --- a/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl @@ -1,7 +1,7 @@ -{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_city", "date": "2023-11-13T08:00:00+0000", "value": {"London, England": 7, "Sydney, New South Wales": 19, "Algiers, Algiers Province": 4, "Caracas, Capital District": 4, "Atlanta, Georgia": 5, "S\u00e3o Paulo, S\u00e3o Paulo (state)": 14, "Munich, Bayern": 3, "Rio de Janeiro, Rio de Janeiro (state)": 5, "Hong Kong, Hong Kong": 4, "Berlin, Berlin": 7, "Kolkata, West Bengal": 4, "Tulsa, Oklahoma": 4, "Lagos, Lagos State": 18, "Ahmedabad, Gujarat": 4, "Moscow, Moscow": 5, "Karachi, Sindh": 4, "Bogot\u00e1, Distrito Especial": 5, "Dar es Salaam, Dar es Salaam": 7, "Jakarta, Jakarta": 10, "Accra, Greater Accra Region": 4, "Buenos Aires, Ciudad Aut\u00f3noma de Buenos Aires": 9, "Taipei, Taipei": 3, "Melbourne, Victoria": 7, "Delhi, Delhi": 6, "Gurugram, Haryana": 6, "Kuala Lumpur, Kuala Lumpur": 4, "Los Angeles, California": 5, "Lima, Lima Region": 5, "Istanbul, Istanbul Province": 9, "Chennai, Tamil Nadu": 6, "Abuja, Federal Capital Territory": 6, "Bangkok, Bangkok": 5, "Mexico City, Distrito Federal": 6, "Cape Town, Western Cape": 4, "San Francisco, California": 6, "Tehran, Tehran Province": 4, "New York, New York": 14, "Cairo, Cairo Governorate": 4, "Santiago, Santiago Metropolitan Region": 6, "Dubai, Dubai": 8, "Mumbai, Maharashtra": 8, "Bangalore, Karnataka": 18, "Singapore, Singapore": 6, "Hyderabad, Telangana": 7, "San Diego, California": 6}}, "emitted_at": 1699881329196} -{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_country", "date": "2023-11-13T08:00:00+0000", "value": {"DE": 31, "BD": 4, "HK": 4, "FI": 5, "TW": 5, "RU": 9, "TZ": 8, "UA": 4, "FR": 10, "SA": 8, "BR": 64, "SE": 6, "MA": 6, "SG": 6, "DZ": 6, "ID": 29, "GB": 45, "CA": 24, "US": 264, "EG": 10, "AE": 9, "CH": 7, "IN": 125, "ZA": 16, "IQ": 6, "CL": 9, "IR": 12, "GR": 6, "IT": 19, "MX": 23, "MY": 9, "CO": 11, "ES": 12, "VE": 9, "AR": 23, "TH": 7, "AU": 35, "PE": 5, "PH": 7, "NG": 30, "TN": 6, "PK": 10, "PL": 5, "TR": 10, "NL": 13}}, "emitted_at": 1699881329198} -{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_gender_age", "date": "2023-11-13T08:00:00+0000", "value": {"F.18-24": 11, "F.25-34": 75, "F.35-44": 72, "F.45-54": 17, "F.55-64": 1, "F.65+": 2, "M.13-17": 2, "M.18-24": 49, "M.25-34": 367, "M.35-44": 226, "M.45-54": 83, "M.55-64": 20, "M.65+": 12, "U.18-24": 18, "U.25-34": 66, "U.35-44": 43, "U.45-54": 19, "U.55-64": 5}}, "emitted_at": 1699881329198} -{"stream": "media", "data": {"id": "17884386203808767", "ig_id": "3123724930722523505", "is_comment_enabled": true, "owner": {"id": "17841408147298757"}, "username": "airbytehq", "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/B34BFFBB0614049AD69F066D153FDD8C_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=107&vs=986202625710684_1200838240&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC9CMzRCRkZCQjA2MTQwNDlBRDY5RjA2NkQxNTNGREQ4Q192aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dDQm9HQlV3a2JxUWwtY0JBRnZGTnFBUkdQeHpicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJtDf4euHnbtAFQIoAkMzLBdAUBtDlYEGJRgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfD22CPBcEjf5k7O8xXQHo8O0VkV7ScqVYH4tiXi1gP0cg&oe=6553F09A&_nc_sid=1d576d", "like_count": 9, "permalink": "https://www.instagram.com/reel/CtZs0Y3v2lx/", "media_product_type": "REELS", "timestamp": "2023-06-12T19:20:02+0000", "media_type": "VIDEO", "shortcode": "CtZs0Y3v2lx", "comments_count": 2, "caption": "Terraform Explained Part 1\n.\n.\n.\n#airbyte #dataengineering #tech #terraform #cloud #cloudengineer #coding #reels", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/353022694_609901831117241_2447211336606431614_n.jpg?_nc_cat=100&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=m968I3SdqJAAX9vop_P&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfBhRikvxkwg1fvXBxym8F6XHMvIYpsdij9PgRelILL2Cw&oe=65568E8E", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1699881269892} -{"stream": "media", "data": {"id": "17864256500936159", "ig_id": "3106359072491902976", "is_comment_enabled": true, "owner": {"id": "17841408147298757"}, "username": "airbytehq", "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/BE4F848CC97FBA35A1AE1B1150B989A7_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=110&vs=6290041361087047_1877877688&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC9CRTRGODQ4Q0M5N0ZCQTM1QTFBRTFCMTE1MEI5ODlBN192aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dEaE94aFJJdk1BWGZaWURBQXQyS0FLWWxOSlhicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJrD%2B6LaRwf1AFQIoAkMzLBdARDmZmZmZmhgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfCIURiEF3hTsWaXvIE6ft-jP1WxcZzXQU8cZygCLJFBxQ&oe=65540526&_nc_sid=1d576d", "like_count": 7, "permalink": "https://www.instagram.com/reel/CscAR5EsRgA/", "media_product_type": "REELS", "timestamp": "2023-05-19T20:08:33+0000", "media_type": "VIDEO", "shortcode": "CscAR5EsRgA", "comments_count": 0, "caption": "When and why you should be using Rust for Data Engineering! \n\n#rust #airbyte #coding #programming #tech #dataengineering #data", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/347441626_604256678433845_716271787932876577_n.jpg?_nc_cat=108&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=2ACJfSHiIRkAX8S0ZFU&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfDsFKul5lyAG2EzGOY-RCmm_QmmPhCkDzf4nfp2ylG2dA&oe=65574D99", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1699881269893} -{"stream": "media", "data": {"id": "17964324206288599", "ig_id": "3104241732634871967", "is_comment_enabled": true, "owner": {"id": "17841408147298757"}, "username": "airbytehq", "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/274503D36EA0F6E79A7CF3797A8D5985_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNTc2LmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=106&vs=1336282350269744_3931649106&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC8yNzQ1MDNEMzZFQTBGNkU3OUE3Q0YzNzk3QThENTk4NV92aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dQdzNzaFRId3VlSlBFWURBSDFmTjUzcUNhd0JicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJrDwmtqO44lAFQIoAkMzLBdAIewIMSbpeRgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfAdvwyUYd2Op-U2iwVZkaxB8PtLApdBwMYkp3eP7qdeeQ&oe=655395B2&_nc_sid=1d576d", "like_count": 13, "permalink": "https://www.instagram.com/reel/CsUe2iqpQif/", "media_product_type": "REELS", "timestamp": "2023-05-16T22:01:45+0000", "media_type": "VIDEO", "shortcode": "CsUe2iqpQif", "comments_count": 0, "caption": "We've all been there right? \ud83e\udd23\n\n#airbyte #data #dataengineering #datascience #dataanalytics #tech #softwareengineer", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/347429218_1848940842145573_5975413208994727174_n.jpg?_nc_cat=101&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=3He_36rMQuYAX9Pz0NM&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfAHCtC-iKyfdquGHnCLMwz_yPD5yNroFnaY8qs0etCc6A&oe=65573A3C", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1699881269893} -{"stream": "users", "data": {"id": "17841408147298757", "ig_id": 8070063576, "profile_picture_url": "https://scontent-iev1-1.xx.fbcdn.net/v/t51.2885-15/153169696_890787328349641_8382928081987798464_n.jpg?_nc_cat=111&_nc_sid=7d201b&_nc_ohc=uQq3P1OLNOYAX_JjmNN&_nc_ht=scontent-iev1-1.xx&edm=AL-3X8kEAAAA&oh=00_AfBcZD3D9MiBkz50iVXih1HCe1L52Wz3t1TKVMJv7UEDWg&oe=6556DCFE", "username": "airbytehq", "name": "Jean Lafleur", "biography": "Airbyte is the new open-source data integration platform that consolidates your data into your warehouses.", "website": "https://www.airbyte.io/", "media_count": 258, "followers_count": 1253, "follows_count": 14, "page_id": "144706962067225"}, "emitted_at": 1699881328891} +{"stream": "users", "data": {"id": "17841408147298757", "biography": "Airbyte is the new open-source data integration platform that consolidates your data into your warehouses.", "profile_picture_url": "https://scontent-iev1-1.xx.fbcdn.net/v/t51.2885-15/153169696_890787328349641_8382928081987798464_n.jpg?_nc_cat=111&_nc_sid=7d201b&_nc_ohc=uQq3P1OLNOYAX_JjmNN&_nc_ht=scontent-iev1-1.xx&edm=AL-3X8kEAAAA&oh=00_AfBI4aspXBrxU-bYTD-qnPWh7ex05YFFAOl_24u7JxLYrw&oe=6558D73E", "username": "airbytehq", "followers_count": 1253, "name": "Jean Lafleur", "ig_id": 8070063576, "media_count": 258, "follows_count": 14, "website": "https://www.airbyte.io/", "page_id": "144706962067225"}, "emitted_at": 1700004246764} +{"stream": "media", "data": {"id": "17884386203808767", "media_product_type": "REELS", "shortcode": "CtZs0Y3v2lx", "permalink": "https://www.instagram.com/reel/CtZs0Y3v2lx/", "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/B34BFFBB0614049AD69F066D153FDD8C_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=107&vs=986202625710684_1200838240&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC9CMzRCRkZCQjA2MTQwNDlBRDY5RjA2NkQxNTNGREQ4Q192aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dDQm9HQlV3a2JxUWwtY0JBRnZGTnFBUkdQeHpicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJtDf4euHnbtAFQIoAkMzLBdAUBtDlYEGJRgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfAJ_4aiqzpxj20QL_aetXfmjmA8nRmz27vnAzpiARGK5w&oe=6555EADA&_nc_sid=1d576d", "timestamp": "2023-06-12T19:20:02+0000", "media_type": "VIDEO", "caption": "Terraform Explained Part 1\n.\n.\n.\n#airbyte #dataengineering #tech #terraform #cloud #cloudengineer #coding #reels", "comments_count": 2, "username": "airbytehq", "is_comment_enabled": true, "owner": {"id": "17841408147298757"}, "like_count": 9, "ig_id": "3123724930722523505", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/353022694_609901831117241_2447211336606431614_n.jpg?_nc_cat=100&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=1ZTHPkRhzl8AX-hZcw_&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfAzQkg0OB_775OS9F7QSmHxKMrjBSNFi8Rx24OISWSTTQ&oe=655888CE", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1700004200656} +{"stream": "media", "data": {"id": "17864256500936159", "media_product_type": "REELS", "shortcode": "CscAR5EsRgA", "permalink": "https://www.instagram.com/reel/CscAR5EsRgA/", "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/BE4F848CC97FBA35A1AE1B1150B989A7_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=110&vs=6290041361087047_1877877688&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC9CRTRGODQ4Q0M5N0ZCQTM1QTFBRTFCMTE1MEI5ODlBN192aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dEaE94aFJJdk1BWGZaWURBQXQyS0FLWWxOSlhicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJrD%2B6LaRwf1AFQIoAkMzLBdARDmZmZmZmhgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfB3i72i3aoV6KoK_SkI7W93z4rQLdbYHatg-KzPo0ADCg&oe=655556A6&_nc_sid=1d576d", "timestamp": "2023-05-19T20:08:33+0000", "media_type": "VIDEO", "caption": "When and why you should be using Rust for Data Engineering! \n\n#rust #airbyte #coding #programming #tech #dataengineering #data", "comments_count": 0, "username": "airbytehq", "is_comment_enabled": true, "owner": {"id": "17841408147298757"}, "like_count": 7, "ig_id": "3106359072491902976", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/347441626_604256678433845_716271787932876577_n.jpg?_nc_cat=108&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=2ACJfSHiIRkAX8S0ZFU&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfDJSGXmZXkQnQZmkrVUi4nadhEddZxH5LUNtELipGu4Dw&oe=655947D9", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1700004200657} +{"stream": "media", "data": {"id": "17964324206288599", "media_product_type": "REELS", "shortcode": "CsUe2iqpQif", "permalink": "https://www.instagram.com/reel/CsUe2iqpQif/", "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/274503D36EA0F6E79A7CF3797A8D5985_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNTc2LmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=106&vs=1336282350269744_3931649106&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC8yNzQ1MDNEMzZFQTBGNkU3OUE3Q0YzNzk3QThENTk4NV92aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dQdzNzaFRId3VlSlBFWURBSDFmTjUzcUNhd0JicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJrDwmtqO44lAFQIoAkMzLBdAIewIMSbpeRgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfB1HP_4v5ndbtfq_6eWT0cxo0vqzO9F6mu5ZS-q4IRDzg&oe=65558FF2&_nc_sid=1d576d", "timestamp": "2023-05-16T22:01:45+0000", "media_type": "VIDEO", "caption": "We've all been there right? \ud83e\udd23\n\n#airbyte #data #dataengineering #datascience #dataanalytics #tech #softwareengineer", "comments_count": 0, "username": "airbytehq", "is_comment_enabled": true, "owner": {"id": "17841408147298757"}, "like_count": 13, "ig_id": "3104241732634871967", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/347429218_1848940842145573_5975413208994727174_n.jpg?_nc_cat=101&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=3He_36rMQuYAX9Pz0NM&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfCAgX5HrHW8grC2x_VzJyCf2lUTViJCmwNy0uStHB-YFg&oe=6559347C", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1700004200657} +{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_city", "date": "2023-11-14T08:00:00+0000", "value": {"London, England": 8, "Sydney, New South Wales": 19, "Algiers, Algiers Province": 4, "Casablanca, Grand Casablanca": 4, "S\u00e3o Paulo, S\u00e3o Paulo (state)": 14, "Rio de Janeiro, Rio de Janeiro (state)": 5, "Hong Kong, Hong Kong": 4, "Berlin, Berlin": 7, "Kolkata, West Bengal": 4, "Phoenix, Arizona": 3, "Tulsa, Oklahoma": 4, "Lagos, Lagos State": 18, "Skopje, Municipality of Centar (Skopje)": 4, "Ahmedabad, Gujarat": 4, "Moscow, Moscow": 5, "Karachi, Sindh": 4, "Bogot\u00e1, Distrito Especial": 5, "Dar es Salaam, Dar es Salaam": 7, "Jakarta, Jakarta": 10, "Accra, Greater Accra Region": 4, "Buenos Aires, Ciudad Aut\u00f3noma de Buenos Aires": 9, "Melbourne, Victoria": 7, "Delhi, Delhi": 6, "Gurugram, Haryana": 6, "Kuala Lumpur, Kuala Lumpur": 4, "Los Angeles, California": 5, "Lima, Lima Region": 5, "Istanbul, Istanbul Province": 9, "Abuja, Federal Capital Territory": 5, "Chennai, Tamil Nadu": 6, "Bangkok, Bangkok": 5, "Mexico City, Distrito Federal": 7, "Cape Town, Western Cape": 5, "San Francisco, California": 6, "Greater Noida, Uttar Pradesh": 3, "Tehran, Tehran Province": 4, "New York, New York": 13, "Cairo, Cairo Governorate": 4, "Santiago, Santiago Metropolitan Region": 6, "Dubai, Dubai": 8, "Mumbai, Maharashtra": 8, "Bangalore, Karnataka": 18, "Singapore, Singapore": 6, "Hyderabad, Telangana": 7, "San Diego, California": 6}}, "emitted_at": 1700004246978} +{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_country", "date": "2023-11-14T08:00:00+0000", "value": {"DE": 31, "HK": 4, "FI": 5, "TW": 5, "RU": 9, "TZ": 8, "FR": 11, "SA": 8, "BR": 64, "SE": 6, "MA": 6, "SG": 6, "DZ": 6, "ID": 29, "GB": 45, "CA": 24, "US": 264, "GH": 4, "EG": 10, "AE": 9, "CH": 7, "IN": 125, "ZA": 16, "IQ": 6, "CL": 9, "IR": 12, "GR": 6, "IT": 19, "MX": 24, "MY": 9, "CO": 11, "ES": 12, "VE": 9, "AR": 23, "AT": 4, "TH": 7, "AU": 35, "PE": 5, "PH": 7, "NG": 30, "TN": 6, "PK": 10, "PL": 5, "TR": 10, "NL": 13}}, "emitted_at": 1700004246980} +{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_gender_age", "date": "2023-11-14T08:00:00+0000", "value": {"F.18-24": 11, "F.25-34": 75, "F.35-44": 72, "F.45-54": 17, "F.55-64": 1, "F.65+": 2, "M.13-17": 2, "M.18-24": 50, "M.25-34": 367, "M.35-44": 226, "M.45-54": 83, "M.55-64": 20, "M.65+": 12, "U.18-24": 18, "U.25-34": 67, "U.35-44": 43, "U.45-54": 19, "U.55-64": 5}}, "emitted_at": 1700004246980} diff --git a/airbyte-integrations/connectors/source-instagram/integration_tests/spec.json b/airbyte-integrations/connectors/source-instagram/integration_tests/spec.json index 81d9934b99b8..cfed60215dc4 100644 --- a/airbyte-integrations/connectors/source-instagram/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-instagram/integration_tests/spec.json @@ -7,11 +7,10 @@ "properties": { "start_date": { "title": "Start Date", - "description": "The date from which you'd like to replicate data for User Insights, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", + "description": "The date from which you'd like to replicate data for User Insights, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated. If left blank, the start date will be set to 2 years before the present date.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "examples": ["2017-01-25T00:00:00Z"], - "type": "string", - "format": "date-time" + "type": "string" }, "access_token": { "title": "Access Token", @@ -34,7 +33,7 @@ "type": "string" } }, - "required": ["start_date", "access_token"] + "required": ["access_token"] }, "supportsIncremental": true, "supported_destination_sync_modes": ["append"], diff --git a/airbyte-integrations/connectors/source-instagram/metadata.yaml b/airbyte-integrations/connectors/source-instagram/metadata.yaml index c6a4e09f6df5..2e1403daea66 100644 --- a/airbyte-integrations/connectors/source-instagram/metadata.yaml +++ b/airbyte-integrations/connectors/source-instagram/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6acf6b55-4f1e-4fca-944e-1a3caef8aba8 - dockerImageTag: 1.0.14 + dockerImageTag: 1.0.15 dockerRepository: airbyte/source-instagram githubIssueLabel: source-instagram icon: instagram.svg @@ -19,6 +19,15 @@ data: oss: enabled: true releaseStage: generally_available + suggestedStreams: + streams: + - media + - media_insights + - stories + - user_insights + - story_insights + - users + - user_lifetime_insights documentationUrl: https://docs.airbyte.com/integrations/sources/instagram tags: - language:python diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/api.py b/airbyte-integrations/connectors/source-instagram/source_instagram/api.py index 1f73d0460141..293876dd705e 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/api.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/api.py @@ -17,7 +17,7 @@ from facebook_business.exceptions import FacebookRequestError from source_instagram.common import InstagramAPIException, retry_pattern -backoff_policy = retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5) +backoff_policy = retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5, max_time=600) class MyFacebookAdsApi(FacebookAdsApi): diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_insights.json b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_insights.json index ab81d27d09ea..fe98eafcccbf 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_insights.json +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_insights.json @@ -49,6 +49,9 @@ }, "online_followers": { "type": ["null", "object"] + }, + "email_contacts": { + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/source.py b/airbyte-integrations/connectors/source-instagram/source_instagram/source.py index 6f78da2614c7..73ce1fb5b5f4 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/source.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/source.py @@ -5,6 +5,7 @@ from datetime import datetime from typing import Any, List, Mapping, Optional, Tuple +import pendulum from airbyte_cdk.models import AdvancedAuth, ConnectorSpecification, DestinationSyncMode, OAuthConfigSpecification from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -17,8 +18,8 @@ class ConnectorConfig(BaseModel): class Config: title = "Source Instagram" - start_date: datetime = Field( - description="The date from which you'd like to replicate data for User Insights, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", + start_date: Optional[str] = Field( + description="The date from which you'd like to replicate data for User Insights, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated. If left blank, the start date will be set to 2 years before the present date.", pattern="^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", examples=["2017-01-25T00:00:00Z"], ) @@ -59,8 +60,9 @@ def check_connection(self, logger, config: Mapping[str, Any]) -> Tuple[bool, Any error_msg = None try: - config = ConnectorConfig.parse_obj(config) # FIXME: this will be not need after we fix CDK - api = InstagramAPI(access_token=config.access_token) + self._validate_start_date(config) + + api = InstagramAPI(access_token=config["access_token"]) logger.info(f"Available accounts: {api.accounts}") ok = True except Exception as exc: @@ -68,13 +70,22 @@ def check_connection(self, logger, config: Mapping[str, Any]) -> Tuple[bool, Any return ok, error_msg + def _validate_start_date(self, config): + # If start_date is not found in config, set it to 2 years ago + if not config.get("start_date"): + config["start_date"] = pendulum.now().subtract(years=2).in_timezone("UTC").format("YYYY-MM-DDTHH:mm:ss.SSS[Z]") + else: + if pendulum.parse(config["start_date"]) > pendulum.now(): + raise ValueError("Please fix the start_date parameter in config, it cannot be in the future") + def streams(self, config: Mapping[str, Any]) -> List[Stream]: """Discovery method, returns available streams :param config: A Mapping of the user input configuration as defined in the connector spec. """ - config: ConnectorConfig = ConnectorConfig.parse_obj(config) # FIXME: this will be not need after we fix CDK - api = InstagramAPI(access_token=config.access_token) + api = InstagramAPI(access_token=config["access_token"]) + + self._validate_start_date(config) return [ Media(api=api), @@ -83,7 +94,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: StoryInsights(api=api), Users(api=api), UserLifetimeInsights(api=api), - UserInsights(api=api, start_date=config.start_date), + UserInsights(api=api, start_date=config["start_date"]), ] def spec(self, *args, **kwargs) -> ConnectorSpecification: diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py index e83c64de6fdd..9c07d98bb083 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py @@ -81,7 +81,7 @@ class InstagramIncrementalStream(InstagramStream, IncrementalMixin): def __init__(self, start_date: datetime, **kwargs): super().__init__(**kwargs) - self._start_date = pendulum.instance(start_date) + self._start_date = pendulum.parse(start_date) self._state = {} @property diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py b/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py index 02a48f376cf0..7b9fd1db150e 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py @@ -35,6 +35,10 @@ def config_fixture(): def some_config_fixture(account_id): return {"start_date": "2021-01-23T00:00:00Z", "access_token": "unknown_token"} +@fixture(scope="session", name="some_config_future_date") +def some_config_future_date_fixture(account_id): + return {"start_date": "2030-01-23T00:00:00Z", "access_token": "unknown_token"} + @fixture(name="fb_account_response") def fb_account_response_fixture(account_id, some_config, requests_mock): diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py b/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py index 41b6467f5512..2cdca11b4f8f 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py @@ -31,13 +31,17 @@ def test_check_connection_empty_config(api): assert not ok assert error_msg +def test_check_connection_invalid_config_future_date(api, some_config_future_date): + ok, error_msg = SourceInstagram().check_connection(logger, config=some_config_future_date) -def test_check_connection_invalid_config(api, some_config): + assert not ok + assert error_msg +def test_check_connection_no_date_config(api, some_config): some_config.pop("start_date") ok, error_msg = SourceInstagram().check_connection(logger, config=some_config) - assert not ok - assert error_msg + assert ok + assert not error_msg def test_check_connection_exception(api, config): diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py index cad75b446d38..39fc889e7e48 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py @@ -32,13 +32,13 @@ def test_clear_url(config): def test_state_outdated(api, config): - assert UserInsights(api=api, start_date=datetime.strptime(config["start_date"], "%Y-%m-%dT%H:%M:%S"))._state_has_legacy_format( + assert UserInsights(api=api, start_date=config["start_date"])._state_has_legacy_format( {"state": MagicMock()} ) def test_state_is_not_outdated(api, config): - assert not UserInsights(api=api, start_date=datetime.strptime(config["start_date"], "%Y-%m-%dT%H:%M:%S"))._state_has_legacy_format( + assert not UserInsights(api=api, start_date=config["start_date"])._state_has_legacy_format( {"state": {}} ) @@ -178,7 +178,7 @@ def test_user_read(api, user_data, requests_mock): def test_user_insights_read(api, config, user_insight_data, requests_mock): test_id = "test_id" - stream = UserInsights(api=api, start_date=datetime.strptime(config["start_date"], "%Y-%m-%dT%H:%M:%S")) + stream = UserInsights(api=api, start_date=config["start_date"]) requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{test_id}/insights", [{"json": user_insight_data}]) @@ -275,7 +275,7 @@ def test_user_insights_state(api, user_insights, values, slice_dates, expected): import pendulum # UserInsights stream - stream = UserInsights(api=api, start_date=pendulum.parse("2023-01-01T01:01:01Z")) + stream = UserInsights(api=api, start_date="2023-01-01T01:01:01Z") # Populate the fixute with `values` user_insights(values) # simulate `read_recods` generator job @@ -357,7 +357,7 @@ def test_common_error_retry(error_response, requests_mock, api, account_id): def test_exit_gracefully(api, config, requests_mock, caplog): test_id = "test_id" - stream = UserInsights(api=api, start_date=datetime.strptime(config["start_date"], "%Y-%m-%dT%H:%M:%S")) + stream = UserInsights(api=api, start_date=config["start_date"]) requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{test_id}/insights", json={"data": []}) records = read_incremental(stream, {}) assert not records diff --git a/docs/integrations/sources/instagram.inapp.md b/docs/integrations/sources/instagram.inapp.md deleted file mode 100644 index 6be4f2da558e..000000000000 --- a/docs/integrations/sources/instagram.inapp.md +++ /dev/null @@ -1,17 +0,0 @@ -## Prerequisite - -* [Instagram business account](https://www.facebook.com/business/help/898752960195806) to your Facebook page - -:::info -The Instagram connector syncs data related to Users, Media, and Stories and their insights from the [Instagram Graph API](https://developers.facebook.com/docs/instagram-api/). For performance data related to Instagram Ads, use the Facebook Marketing source. -::: - -## Setup guide - -1. Click Authenticate your Instagram account. -2. Log in and authorize the Instagram account. -3. (Optional) Select a start date date. All data generated after this date will be replicated. If this field is blank, Airbyte will replicate all data. -4. Click Set up source. -​ - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Instagram](https://docs.airbyte.com/integrations/sources/instagram). \ No newline at end of file diff --git a/docs/integrations/sources/instagram.md b/docs/integrations/sources/instagram.md index 1270d59f4cc0..b392c4737016 100644 --- a/docs/integrations/sources/instagram.md +++ b/docs/integrations/sources/instagram.md @@ -1,7 +1,11 @@ # Instagram + + This page contains the setup guide and reference information for the Instagram source connector. + + ## Prerequisites - [Meta for Developers account](https://developers.facebook.com) @@ -42,6 +46,8 @@ This page contains the setup guide and reference information for the Instagram s 8. Click **Set up source**. + + ## Supported sync modes The Instagram source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): @@ -85,26 +91,29 @@ AirbyteRecords are required to conform to the [Airbyte type](https://docs.airbyt ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------| :------------------------------------------------------- |:-------------------------------------------------------------------------------------------------------------------------------| +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------| +| 1.0.15 | 2023-11-14 | [32494](https://github.com/airbytehq/airbyte/pull/32494) | Marked start_date as optional; set max retry time to 10 minutes; add suggested streams | | 1.0.14 | 2023-11-13 | [32423](https://github.com/airbytehq/airbyte/pull/32423) | Capture media_product_type column in media and stories stream | -| 1.0.13 | 2023-11-10 | [32245](https://github.com/airbytehq/airbyte/pull/32245) | Add skipping reading MediaInsights stream if an error code 10 is received | -| 1.0.12 | 2023-11-07 | [32200](https://github.com/airbytehq/airbyte/pull/32200) | The backoff strategy has been updated to make some errors retriable | -| 1.0.11 | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes | -| 1.0.10 | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references | -| 1.0.9 | 2023-07-01 | [27908](https://github.com/airbytehq/airbyte/pull/27908) | Fix bug when `user_lifetime_insights` stream returns `Key Error (end_time)`, refactored `state` to use `IncrementalMixin` | -| 1.0.8 | 2023-05-26 | [26767](https://github.com/airbytehq/airbyte/pull/26767) | Handle permission error for `insights` | -| 1.0.7 | 2023-05-26 | [26656](https://github.com/airbytehq/airbyte/pull/26656) | Remove `authSpecification` from connector specification in favour of `advancedAuth` | -| 1.0.6 | 2023-03-28 | [26599](https://github.com/airbytehq/airbyte/pull/26599) | Handle error for Media posted before business account conversion | -| 1.0.5 | 2023-03-28 | [24634](https://github.com/airbytehq/airbyte/pull/24634) | Add user-friendly message for no instagram_business_accounts case | -| 1.0.4 | 2023-03-15 | [23671](https://github.com/airbytehq/airbyte/pull/23671) | Add info about main permissions in spec and doc links in error message to navigate user | -| 1.0.3 | 2023-03-14 | [24043](https://github.com/airbytehq/airbyte/pull/24043) | Do not emit incomplete records for `user_insights` stream | -| 1.0.2 | 2023-03-14 | [24042](https://github.com/airbytehq/airbyte/pull/24042) | Test publish flow | -| 1.0.1 | 2023-01-19 | [21602](https://github.com/airbytehq/airbyte/pull/21602) | Handle abnormally large state values | -| 1.0.0 | 2022-09-23 | [17110](https://github.com/airbytehq/airbyte/pull/17110) | Remove custom read function and migrate to per-stream state | -| 0.1.11 | 2022-09-08 | [16428](https://github.com/airbytehq/airbyte/pull/16428) | Fix requests metrics for Reels media product type | -| 0.1.10 | 2022-09-05 | [16340](https://github.com/airbytehq/airbyte/pull/16340) | Update to latest version of the CDK (v0.1.81) | -| 0.1.9 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification | -| 0.1.8 | 2021-08-11 | [5354](https://github.com/airbytehq/airbyte/pull/5354) | Added check for empty state and fixed tests | -| 0.1.7 | 2021-07-19 | [4805](https://github.com/airbytehq/airbyte/pull/4805) | Add support for previous `STATE` format | -| 0.1.6 | 2021-07-07 | [4210](https://github.com/airbytehq/airbyte/pull/4210) | Refactor connector to use CDK: - improve error handling - fix sync fail with HTTP status 400 - integrate SAT | +| 1.0.13 | 2023-11-10 | [32245](https://github.com/airbytehq/airbyte/pull/32245) | Add skipping reading MediaInsights stream if an error code 10 is received | +| 1.0.12 | 2023-11-07 | [32200](https://github.com/airbytehq/airbyte/pull/32200) | The backoff strategy has been updated to make some errors retriable | +| 1.0.11 | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes | +| 1.0.10 | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references | +| 1.0.9 | 2023-07-01 | [27908](https://github.com/airbytehq/airbyte/pull/27908) | Fix bug when `user_lifetime_insights` stream returns `Key Error (end_time)`, refactored `state` to use `IncrementalMixin` | +| 1.0.8 | 2023-05-26 | [26767](https://github.com/airbytehq/airbyte/pull/26767) | Handle permission error for `insights` | +| 1.0.7 | 2023-05-26 | [26656](https://github.com/airbytehq/airbyte/pull/26656) | Remove `authSpecification` from connector specification in favour of `advancedAuth` | +| 1.0.6 | 2023-03-28 | [26599](https://github.com/airbytehq/airbyte/pull/26599) | Handle error for Media posted before business account conversion | +| 1.0.5 | 2023-03-28 | [24634](https://github.com/airbytehq/airbyte/pull/24634) | Add user-friendly message for no instagram_business_accounts case | +| 1.0.4 | 2023-03-15 | [23671](https://github.com/airbytehq/airbyte/pull/23671) | Add info about main permissions in spec and doc links in error message to navigate user | +| 1.0.3 | 2023-03-14 | [24043](https://github.com/airbytehq/airbyte/pull/24043) | Do not emit incomplete records for `user_insights` stream | +| 1.0.2 | 2023-03-14 | [24042](https://github.com/airbytehq/airbyte/pull/24042) | Test publish flow | +| 1.0.1 | 2023-01-19 | [21602](https://github.com/airbytehq/airbyte/pull/21602) | Handle abnormally large state values | +| 1.0.0 | 2022-09-23 | [17110](https://github.com/airbytehq/airbyte/pull/17110) | Remove custom read function and migrate to per-stream state | +| 0.1.11 | 2022-09-08 | [16428](https://github.com/airbytehq/airbyte/pull/16428) | Fix requests metrics for Reels media product type | +| 0.1.10 | 2022-09-05 | [16340](https://github.com/airbytehq/airbyte/pull/16340) | Update to latest version of the CDK (v0.1.81) | +| 0.1.9 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification | +| 0.1.8 | 2021-08-11 | [5354](https://github.com/airbytehq/airbyte/pull/5354) | Added check for empty state and fixed tests | +| 0.1.7 | 2021-07-19 | [4805](https://github.com/airbytehq/airbyte/pull/4805) | Add support for previous `STATE` format | +| 0.1.6 | 2021-07-07 | [4210](https://github.com/airbytehq/airbyte/pull/4210) | Refactor connector to use CDK: - improve error handling - fix sync fail with HTTP status 400 - integrate SAT | + + \ No newline at end of file From 2ae2586c0ec4df26be72592bd1974ab787f9bbc4 Mon Sep 17 00:00:00 2001 From: midavadim Date: Wed, 15 Nov 2023 15:13:49 +0200 Subject: [PATCH 059/124] :tada: Source Pinterest - new streams (#32299) Co-authored-by: midavadim Co-authored-by: Augustin Co-authored-by: alafanechere --- .../acceptance-test-config.yml | 63 ++-- .../integration_tests/expected_records.jsonl | 23 +- .../connectors/source-pinterest/metadata.yaml | 2 +- .../source_pinterest/reports/reports.py | 91 +++++ .../source_pinterest/schemas/audiences.json | 75 ++++ .../source_pinterest/schemas/board_pins.json | 3 +- .../schemas/board_section_pins.json | 3 +- .../source_pinterest/schemas/catalogs.json | 23 ++ .../schemas/catalogs_feeds.json | 55 +++ .../schemas/catalogs_product_groups.json | 33 ++ .../schemas/conversion_tags.json | 56 +++ .../schemas/customer_lists.json | 36 ++ .../source_pinterest/schemas/keywords.json | 30 ++ .../source_pinterest/schemas/reports.json | 346 ++++++++++++++++++ .../source_pinterest/source.py | 78 +++- .../source_pinterest/streams.py | 93 ++++- .../unit_tests/test_reports.py | 11 + .../unit_tests/test_source.py | 15 +- .../unit_tests/test_streams.py | 27 ++ docs/integrations/sources/pinterest.md | 109 +++--- 20 files changed, 1069 insertions(+), 103 deletions(-) create mode 100644 airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/audiences.json create mode 100644 airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/catalogs.json create mode 100644 airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/catalogs_feeds.json create mode 100644 airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/catalogs_product_groups.json create mode 100644 airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/conversion_tags.json create mode 100644 airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/customer_lists.json create mode 100644 airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/keywords.json create mode 100644 airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/reports.json diff --git a/airbyte-integrations/connectors/source-pinterest/acceptance-test-config.yml b/airbyte-integrations/connectors/source-pinterest/acceptance-test-config.yml index ab97ad555c2d..768d1d550502 100644 --- a/airbyte-integrations/connectors/source-pinterest/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-pinterest/acceptance-test-config.yml @@ -12,7 +12,7 @@ acceptance_tests: - config_path: secrets/config.json status: succeed - config_path: integration_tests/invalid_config.json - status: exception + status: failed - config_path: secrets/config_oauth.json status: succeed discovery: @@ -23,26 +23,24 @@ acceptance_tests: tests: - config_path: secrets/config.json empty_streams: - - bypass_reason: The stream could return 0 records, because of low rate-limits - name: ad_account_analytics - - bypass_reason: The stream could return 0 records, because of low rate-limits - name: ad_analytics - - bypass_reason: The stream could return 0 records, because of low rate-limits - name: ad_group_analytics - - bypass_reason: The stream could return 0 records, because of low rate-limits - name: ad_groups - - bypass_reason: The stream could return 0 records, because of low rate-limits - name: ads - - bypass_reason: The stream could return 0 records, because of low rate-limits - name: board_section_pins - - bypass_reason: The stream could return 0 records, because of low rate-limits - name: board_sections - - bypass_reason: The stream could return 0 records, because of low rate-limits - name: campaign_analytics - - bypass_reason: The stream could return 0 records, because of low rate-limits - name: campaigns - - bypass_reason: The stream could return 0 records, because of low rate-limits - name: user_account_analytics + - name: conversion_tags + bypass_reason: Not possible to add data + - name: customer_lists + bypass_reason: Not possible to add data + - name: catalogs + bypass_reason: Not possible to add data + - name: catalogs_feeds + bypass_reason: Not possible to add data + - name: catalogs_product_groups + bypass_reason: Not possible to add data + - name: product_group_report + bypass_reason: Not possible to add data + - name: product_group_targeting_report + bypass_reason: Not possible to add data + - name: product_item_report + bypass_reason: Not possible to add data + - name: keyword_report + bypass_reason: Not possible to add data timeout_seconds: 1200 expect_records: path: "integration_tests/expected_records.jsonl" @@ -52,8 +50,27 @@ acceptance_tests: fail_on_extra_columns: false ignored_fields: board_pins: - - name: "links" - bypass_reason: "because it contains non-secure http link, which leads to failed QA tests" + - name: "media" + bypass_reason: "urls may change" + board_section_pins: + - name: "media" + bypass_reason: "urls may change" + ads: + - name: "updated_time" + bypass_reason: "can be updated" + ad_groups: + - name: "updated_time" + bypass_reason: "can be updated" + campaigns: + - name: "updated_time" + bypass_reason: "can be updated" + audiences: + - name: "size" + bypass_reason: "can be changed" + - name: "updated_timestamp" + bypass_reason: "can be changed" + - name: "created_timestamp" + bypass_reason: "can be changed" incremental: tests: - config_path: secrets/config.json diff --git a/airbyte-integrations/connectors/source-pinterest/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-pinterest/integration_tests/expected_records.jsonl index 1215f423eb58..dbba2254d507 100644 --- a/airbyte-integrations/connectors/source-pinterest/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-pinterest/integration_tests/expected_records.jsonl @@ -1,4 +1,23 @@ {"stream": "ad_accounts", "data": {"id": "549761668032", "name": "Airbyte", "owner": {"username": "integrationtest0375", "id": "666744057242074926"}, "country": "US", "currency": "USD", "permissions": ["OWNER"], "created_time": 1603772920, "updated_time": 1623173784}, "emitted_at": 1688461289470} +{"stream": "ad_account_analytics", "data": {"TOTAL_IMPRESSION_FREQUENCY": 1.0, "TOTAL_IMPRESSION_USER": 1.0, "ADVERTISER_ID": "549761668032", "DATE": "2023-10-29", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032"}, "emitted_at": 1699893121669} +{"stream": "ads", "data": {"id": "687218400118", "ad_group_id": "2680068678965", "ad_account_id": "549761668032", "android_deep_link": null, "campaign_id": "626744128956", "carousel_android_deep_links": null, "carousel_destination_urls": null, "carousel_ios_deep_links": null, "click_tracking_url": null, "collection_items_destination_url_template": null, "created_time": 1623245885, "creative_type": "REGULAR", "destination_url": "https://airbyte.io/", "ios_deep_link": null, "is_pin_deleted": false, "is_removable": false, "name": "2021-06-09 | Traffic | Keywords | Data Integration", "pin_id": "666743919837294988", "rejected_reasons": [], "rejection_labels": [], "review_status": "APPROVED", "status": "PAUSED", "summary_status": "PAUSED", "tracking_urls": null, "type": "ad", "updated_time": 1699373013, "view_tracking_url": null, "lead_form_id": null}, "emitted_at": 1699393433303} +{"stream": "ad_analytics", "data": {"PIN_ID": 6.66743919837295e+17, "AD_GROUP_ID": "2680068678993", "AD_GROUP_ENTITY_STATUS": "1", "CAMPAIGN_ENTITY_STATUS": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "TOTAL_IMPRESSION_USER": 1.0, "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "AD_ID": "687218400210", "ADVERTISER_ID": "549761668032", "PIN_PROMOTION_ID": 687218400210.0, "DATE": "2023-10-29", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness"}, "emitted_at": 1699893196846} +{"stream": "ad_groups", "data": {"id": "2680068678965", "created_time": 1623245885.0, "updated_time": 1699373013.0, "start_time": null, "end_time": null, "bid_in_micro_currency": null, "budget_in_micro_currency": null, "campaign_id": "626744128956", "ad_account_id": "549761668032", "auto_targeting_enabled": true, "type": "adgroup", "budget_type": "CBO_ADGROUP", "billable_event": "CLICKTHROUGH", "status": "ACTIVE", "lifetime_frequency_cap": -1.0, "targeting_spec": {"GENDER": ["female", "male", "unknown"], "APPTYPE": ["web", "web_mobile", "iphone", "ipad", "android_mobile", "android_tablet"], "LOCALE": ["cs", "da", "de", "el", "en", "es", "fi", "fr", "hu", "id", "it", "ja", "ko", "nb", "nl", "pl", "pt", "ro", "ru", "sk", "sv", "tr", "uk", "zh"], "TARGETING_STRATEGY": ["CHOOSE_YOUR_OWN"], "LOCATION": ["US"]}, "name": "2021-06-09 | Traffic | Keywords | Data Integration", "placement_group": "ALL", "pacing_delivery_type": "STANDARD", "tracking_urls": null, "conversion_learning_mode_type": null, "summary_status": "COMPLETED", "feed_profile_id": "0", "placement_traffic_type": null, "optimization_goal_metadata": {}, "bid_strategy_type": "AUTOMATIC_BID"}, "emitted_at": 1699393433712} +{"stream": "ad_group_analytics", "data": {"AD_GROUP_ID": "2680068678993", "AD_GROUP_ENTITY_STATUS": "1", "CAMPAIGN_ENTITY_STATUS": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "TOTAL_IMPRESSION_USER": 1.0, "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "ADVERTISER_ID": "549761668032", "DATE": "2023-10-29", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness"}, "emitted_at": 1699893280169} {"stream": "boards", "data": {"media": {"pin_thumbnail_urls": [], "image_cover_url": "https://i.pinimg.com/400x300/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}, "owner": {"username": "integrationtest0375"}, "created_at": "2021-06-08T09:37:18", "board_pins_modified_at": "2021-10-25T11:17:56.715000", "id": "666743988523388559", "collaborator_count": 0, "follower_count": 2, "pin_count": 1, "privacy": "PUBLIC", "name": "business", "description": ""}, "emitted_at": 1680356853019} -{"stream": "board_pins", "data": {"description": "Data Integration", "board_owner": {"username": "integrationtest0375"}, "product_tags": [], "has_been_promoted": true, "created_at": "2021-06-08T09:37:30", "board_id": "666743988523388559", "note": "", "creative_type": "REGULAR", "parent_pin_id": null, "title": "Airbyte", "alt_text": null, "pin_metrics": null, "dominant_color": "#cacafe", "id": "666743919837294988", "is_owner": true, "board_section_id": "5195034916661798218", "link": "http://airbyte.io/", "media": {"media_type": "image", "images": {"150x150": {"width": 150, "height": 150, "url": "https://i.pinimg.com/150x150/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}, "400x300": {"width": 400, "height": 300, "url": "https://i.pinimg.com/400x300/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}, "600x": {"width": 564, "height": 337, "url": "https://i.pinimg.com/564x/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}, "1200x": {"width": 1200, "height": 718, "url": "https://i.pinimg.com/1200x/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}}}, "is_standard": true}, "emitted_at": 1698398201666} -{"stream": "campaign_analytics_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_DAILY_SPEND_CAP": 750000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 3.0, "TOTAL_IMPRESSION_FREQUENCY": 1.5, "TOTAL_IMPRESSION_USER": 2.0, "DATE": "2023-07-14"}, "emitted_at": 1690299367301} +{"stream": "board_pins", "data": {"description": "Data Integration", "board_owner": {"username": "integrationtest0375"}, "product_tags": [], "has_been_promoted": true,"link":"http://airbyte.io/", "created_at": "2021-06-08T09:37:30", "board_id": "666743988523388559", "note": "", "creative_type": "REGULAR", "parent_pin_id": null, "title": "Airbyte", "alt_text": null, "pin_metrics": null, "dominant_color": "#cacafe", "id": "666743919837294988", "is_owner": true, "board_section_id": "5195034916661798218", "is_standard": true}, "emitted_at": 1698398201666} +{"stream": "board_sections", "data": {"name": "Airbyte_board_section_new", "id": "5195035116725909603"}, "emitted_at": 1699893323493} +{"stream": "board_section_pins","data":{"id":"666743919837294988","dominant_color":"#cacafe","pin_metrics":null,"title":"Airbyte","creative_type":"REGULAR","link":"http://airbyte.io/","board_id":"666743988523388559","created_at":"2021-06-08T09:37:30","is_owner":true,"description":"Data Integration","note":"","alt_text":null,"board_section_id":"5195034916661798218","parent_pin_id":null,"product_tags":[],"board_owner":{"username":"integrationtest0375"},"is_standard":true,"has_been_promoted":true},"emitted_at":1699893364884} +{"stream": "campaigns", "data": {"id": "626744128956", "ad_account_id": "549761668032", "name": "2021-06-09 | Traffic | Keywords | Data Integration", "status": "ACTIVE", "objective_type": "CONSIDERATION", "lifetime_spend_cap": 0, "daily_spend_cap": 3000000, "order_line_id": null, "tracking_urls": null, "created_time": 1623245885, "updated_time": 1691447502, "type": "campaign", "is_flexible_daily_budgets": false, "summary_status": "COMPLETED", "is_campaign_budget_optimization": true, "start_time": 1623196800, "end_time": 1624060800}, "emitted_at": 1699393571700} +{"stream": "campaign_analytics", "data": {"TOTAL_IMPRESSION_FREQUENCY": 1.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "TOTAL_IMPRESSION_USER": 1.0, "CAMPAIGN_ENTITY_STATUS": 1.0, "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "ADVERTISER_ID": 549761668032.0, "DATE": "2023-10-29", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness"}, "emitted_at": 1699894065462} +{"stream": "campaign_analytics_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 3.0, "TOTAL_IMPRESSION_FREQUENCY": 1.5, "TOTAL_IMPRESSION_USER": 2.0, "DATE": "2023-07-14"}, "emitted_at": 1690299367301} +{"stream": "campaign_targeting_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "TARGETING_VALUE": "TWOCOLUMN_FEED", "TARGETING_TYPE": "FEED_TYPE", "DATE": "2023-10-29"}, "emitted_at": 1699894287823} +{"stream": "user_account_analytics", "data": {"date": "2023-11-09", "data_status": "READY", "metrics": {"SAVE": 2.0, "OUTBOUND_CLICK_RATE": 0.0043859649122807015, "IMPRESSION": 912.0, "VIDEO_START": 0, "SAVE_RATE": 0.0021929824561403508, "QUARTILE_95_PERCENT_VIEW": 0, "ENGAGEMENT": 22.0, "VIDEO_AVG_WATCH_TIME": 0.0, "ENGAGEMENT_RATE": 0.02412280701754386, "PIN_CLICK": 17, "VIDEO_10S_VIEW": 0, "FULL_SCREEN_PLAY": 0, "CLOSEUP_RATE": 0.017543859649122806, "FULL_SCREEN_PLAYTIME": 0, "VIDEO_V50_WATCH_TIME": 0, "VIDEO_MRC_VIEW": 0, "CLICKTHROUGH": 4.0, "CLICKTHROUGH_RATE": 0.0043859649122807015, "OUTBOUND_CLICK": 4, "CLOSEUP": 16.0, "PIN_CLICK_RATE": 0.01864035087719298}}, "emitted_at": 1699894362486} +{"stream": "keywords", "data": {"archived": false, "id": "2886935172273", "parent_id": "2680068678965", "parent_type": "AD_GROUP", "type": "KEYWORD", "bid": null, "match_type": "BROAD", "value": "data science"}, "emitted_at": 1699393669235} +{"stream": "audiences", "data": {"type": "audience", "id": "2542622254639", "name": "airbyte audience", "ad_account_id": "549761668032", "audience_type": "ENGAGEMENT", "description": "airbyte audience", "status": "TOO_SMALL", "rule": {"engager_type": 1}}, "emitted_at": 1699293090886} +{"stream": "advertizer_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "EENGAGEMENT_RATE": 0.1, "ENGAGEMENT_2": 1.0, "IMPRESSION_2": 10.0, "REPIN_2": 1.0, "TOTAL_ENGAGEMENT": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 5.0, "TOTAL_IMPRESSION_USER": 2.0, "TOTAL_REPIN_RATE": 0.1, "DATE": "2023-02-10"}, "emitted_at": 1699894848024} +{"stream": "advertizer_targeting_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "IMPRESSION_2": 1.0, "TARGETING_VALUE": "Education > Subjects > Science > Applied Science > Technology", "TARGETING_TYPE": "TARGETED_INTEREST", "DATE": "2023-10-29"}, "emitted_at": 1699894982269} +{"stream": "ad_group_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ENTITY_STATUS": "ACTIVE", "AD_GROUP_ID": "2680068678993", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "TOTAL_IMPRESSION_USER": 1.0, "DATE": "2023-10-29"}, "emitted_at": 1699895043538} +{"stream": "ad_group_targeting_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ENTITY_STATUS": "ACTIVE", "AD_GROUP_ID": "2680068678993", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "TARGETING_VALUE": "TWOCOLUMN_FEED", "TARGETING_TYPE": "FEED_TYPE", "DATE": "2023-10-29"}, "emitted_at": 1699895106949} +{"stream": "pin_promotion_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ENTITY_STATUS": "ACTIVE", "AD_GROUP_ID": "2680068678993", "AD_ID": "687218400210", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "PIN_ID": 6.66743919837295e+17, "PIN_PROMOTION_ID": 687218400210.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "TOTAL_IMPRESSION_USER": 1.0, "DATE": "2023-10-29"}, "emitted_at": 1699895200157} +{"stream": "pin_promotion_targeting_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ENTITY_STATUS": "ACTIVE", "AD_GROUP_ID": "2680068678993", "AD_ID": "687218400210", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "PIN_ID": 6.66743919837295e+17, "PIN_PROMOTION_ID": 687218400210.0, "TARGETING_VALUE": "Education > Subjects > Science > Applied Science > Technology", "TARGETING_TYPE": "TARGETED_INTEREST", "DATE": "2023-10-29"}, "emitted_at": 1699895289749} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-pinterest/metadata.yaml b/airbyte-integrations/connectors/source-pinterest/metadata.yaml index 5ce3b8a4ca19..4763dd34d13a 100644 --- a/airbyte-integrations/connectors/source-pinterest/metadata.yaml +++ b/airbyte-integrations/connectors/source-pinterest/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 5cb7e5fe-38c2-11ec-8d3d-0242ac130003 - dockerImageTag: 0.7.1 + dockerImageTag: 0.7.2 dockerRepository: airbyte/source-pinterest connectorBuildOptions: baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py index fcf5437cedd6..98e4809bb296 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py @@ -4,12 +4,15 @@ import json from abc import abstractmethod +from functools import lru_cache from typing import Any, Iterable, List, Mapping, MutableMapping, Optional from urllib.parse import urljoin import backoff import requests from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.core import package_name_from_class +from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader from source_pinterest.streams import PinterestAnalyticsStream from source_pinterest.utils import get_analytics_columns @@ -164,8 +167,96 @@ def _fetch_report_data(self, url: str) -> dict: """Fetch the report data from the given URL.""" return self._http_get(url) + @lru_cache(maxsize=None) + def get_json_schema(self) -> Mapping[str, Any]: + """ + :return: A dict of the JSON schema representing this stream. + + Schema is the same for all *Report and *TargetingReport streams + """ + + return ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("reports") + + +class PinterestAnalyticsTargetingReportStream(PinterestAnalyticsReportStream): + def request_body_json(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> Optional[Mapping]: + """Return the body of the API request in JSON format.""" + columns = get_analytics_columns().split(",") + # remove keys which are lot suitable for targeting report + for odd_value in ["TOTAL_IMPRESSION_FREQUENCY", "TOTAL_IMPRESSION_USER"]: + if odd_value in columns: + columns.remove(odd_value) + columns = ",".join(columns) + return self._construct_request_body(stream_slice["start_date"], stream_slice["end_date"], self.granularity, columns) + class CampaignAnalyticsReport(PinterestAnalyticsReportStream): @property def level(self): return "CAMPAIGN" + + +class CampaignTargetingReport(PinterestAnalyticsTargetingReportStream): + @property + def level(self): + return "CAMPAIGN_TARGETING" + + +class AdvertizerReport(PinterestAnalyticsReportStream): + @property + def level(self): + return "ADVERTISER" + + +class AdvertizerTargetingReport(PinterestAnalyticsTargetingReportStream): + @property + def level(self): + return "ADVERTISER_TARGETING" + + +class AdGroupReport(PinterestAnalyticsReportStream): + @property + def level(self): + return "AD_GROUP" + + +class AdGroupTargetingReport(PinterestAnalyticsTargetingReportStream): + @property + def level(self): + return "AD_GROUP_TARGETING" + + +class PinPromotionReport(PinterestAnalyticsReportStream): + @property + def level(self): + return "PIN_PROMOTION" + + +class PinPromotionTargetingReport(PinterestAnalyticsTargetingReportStream): + @property + def level(self): + return "PIN_PROMOTION_TARGETING" + + +class ProductGroupReport(PinterestAnalyticsReportStream): + @property + def level(self): + return "PRODUCT_GROUP" + + +class ProductGroupTargetingReport(PinterestAnalyticsTargetingReportStream): + @property + def level(self): + return "PRODUCT_GROUP_TARGETING" + + +class ProductItemReport(PinterestAnalyticsReportStream): + @property + def level(self): + return "PRODUCT_ITEM" + + +class KeywordReport(PinterestAnalyticsTargetingReportStream): + @property + def level(self): + return "KEYWORD" diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/audiences.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/audiences.json new file mode 100644 index 000000000000..2ccb1cafad02 --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/audiences.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ad_account_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "audience_type": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "rule": { + "type": ["null", "object"], + "properties": { + "country": { + "type": ["null", "string"] + }, + "customer_list_id": { + "type": ["null", "string"] + }, + "engagement_domain": { + "type": ["null", "array"], + "items": {} + }, + "engagement_type": { + "type": ["null", "string"] + }, + "event": { + "type": ["null", "string"] + }, + "percentage": { + "type": ["null", "integer"] + }, + "prefill": { + "type": ["null", "boolean"] + }, + "retention_days": { + "type": ["null", "integer"] + }, + "visitor_source_id": { + "type": ["null", "string"] + }, + "engager_type": { + "type": ["null", "integer"] + }, + "ad_account_id": { + "type": ["null", "string"] + } + } + }, + "size": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "created_timestamp": { + "type": ["null", "integer"] + }, + "updated_timestamp": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_pins.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_pins.json index a91140d63b3b..2989e890ced2 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_pins.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_pins.json @@ -7,7 +7,8 @@ "type": ["null", "string"] }, "created_at": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "creative_type": { "type": ["null", "string"], diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_section_pins.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_section_pins.json index c19b701e35b7..74bdf144cb8a 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_section_pins.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_section_pins.json @@ -6,7 +6,8 @@ "type": ["null", "string"] }, "created_at": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "link": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/catalogs.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/catalogs.json new file mode 100644 index 000000000000..bf3a1bf06b12 --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/catalogs.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "name": { + "type": ["null", "string"] + }, + "catalog_type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/catalogs_feeds.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/catalogs_feeds.json new file mode 100644 index 000000000000..2474d6bf723e --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/catalogs_feeds.json @@ -0,0 +1,55 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "name": { + "type": ["null", "string"] + }, + "format": { + "type": ["null", "string"] + }, + "catalog_type": { + "type": ["null", "string"] + }, + "location": { + "type": ["null", "string"] + }, + "preferred_processing_schedule": { + "type": ["null", "object"], + "properties": { + "time": { + "type": ["null", "string"] + }, + "timezone": { + "type": ["null", "string"] + } + } + }, + "status": { + "type": ["null", "string"] + }, + "default_currency": { + "type": ["null", "string"] + }, + "default_locale": { + "type": ["null", "string"] + }, + "default_country": { + "type": ["null", "string"] + }, + "default_availability": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/catalogs_product_groups.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/catalogs_product_groups.json new file mode 100644 index 000000000000..0d627d1bf1c0 --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/catalogs_product_groups.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "created_at": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "feed_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "is_featured": { + "type": ["null", "boolean"] + }, + "name": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/conversion_tags.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/conversion_tags.json new file mode 100644 index 000000000000..aa218ebd0bcd --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/conversion_tags.json @@ -0,0 +1,56 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ad_account_id": { + "type": ["null", "string"] + }, + "code_snippet": { + "type": ["null", "string"] + }, + "enhanced_match_status": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "last_fired_time_ms": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "version": { + "type": ["null", "string"] + }, + "configs": { + "type": ["null", "object"], + "properties": { + "aem_enabled": { + "type": ["null", "boolean"] + }, + "md_frequency": { + "type": ["null", "number"] + }, + "aem_fnln_enabled": { + "type": ["null", "boolean"] + }, + "aem_ph_enabled": { + "type": ["null", "boolean"] + }, + "aem_ge_enabled": { + "type": ["null", "boolean"] + }, + "aem_db_enabled": { + "type": ["null", "boolean"] + }, + "aem_loc_enabled": { + "type": ["null", "boolean"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/customer_lists.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/customer_lists.json new file mode 100644 index 000000000000..551d18c2fd8b --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/customer_lists.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ad_account_id": { + "type": ["null", "string"] + }, + "created_time": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "num_batches": { + "type": ["null", "integer"] + }, + "num_removed_user_records": { + "type": ["null", "integer"] + }, + "num_uploaded_user_records": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "updated_time": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/keywords.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/keywords.json new file mode 100644 index 000000000000..8057db2ce275 --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/keywords.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "archived": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "string"] + }, + "parent_id": { + "type": ["null", "string"] + }, + "parent_type": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "bid": { + "type": ["null", "integer"] + }, + "match_type": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/reports.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/reports.json new file mode 100644 index 000000000000..c41983853652 --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/reports.json @@ -0,0 +1,346 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "DATE": { + "type": ["null", "string"], + "format": "date" + }, + "ADVERTISER_ID": { + "type": ["null", "number"] + }, + "AD_ACCOUNT_ID": { + "type": ["string"] + }, + "AD_ID": { + "type": ["null", "string"] + }, + "AD_GROUP_ENTITY_STATUS": { + "type": ["null", "string"] + }, + "AD_GROUP_ID": { + "type": ["null", "string"] + }, + "CAMPAIGN_DAILY_SPEND_CAP": { + "type": ["null", "number"] + }, + "CAMPAIGN_ENTITY_STATUS": { + "type": ["null", "string"] + }, + "CAMPAIGN_ID": { + "type": ["null", "number"] + }, + "CAMPAIGN_LIFETIME_SPEND_CAP": { + "type": ["null", "number"] + }, + "CAMPAIGN_NAME": { + "type": ["null", "string"] + }, + "CHECKOUT_ROAS": { + "type": ["null", "number"] + }, + "CLICKTHROUGH_1": { + "type": ["null", "number"] + }, + "CLICKTHROUGH_1_GROSS": { + "type": ["null", "number"] + }, + "CLICKTHROUGH_2": { + "type": ["null", "number"] + }, + "CPC_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "CPM_IN_DOLLAR": { + "type": ["null", "number"] + }, + "CPM_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "CTR": { + "type": ["null", "number"] + }, + "CTR_2": { + "type": ["null", "number"] + }, + "ECPCV_IN_DOLLAR": { + "type": ["null", "number"] + }, + "ECPCV_P95_IN_DOLLAR": { + "type": ["null", "number"] + }, + "ECPC_IN_DOLLAR": { + "type": ["null", "number"] + }, + "ECPC_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "ECPE_IN_DOLLAR": { + "type": ["null", "number"] + }, + "ECPM_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "ECPV_IN_DOLLAR": { + "type": ["null", "number"] + }, + "ECTR": { + "type": ["null", "number"] + }, + "EENGAGEMENT_RATE": { + "type": ["null", "number"] + }, + "ENGAGEMENT_1": { + "type": ["null", "number"] + }, + "ENGAGEMENT_2": { + "type": ["null", "number"] + }, + "ENGAGEMENT_RATE": { + "type": ["null", "number"] + }, + "IDEA_PIN_PRODUCT_TAG_VISIT_1": { + "type": ["null", "number"] + }, + "IDEA_PIN_PRODUCT_TAG_VISIT_2": { + "type": ["null", "number"] + }, + "IMPRESSION_1": { + "type": ["null", "number"] + }, + "IMPRESSION_1_GROSS": { + "type": ["null", "number"] + }, + "IMPRESSION_2": { + "type": ["null", "number"] + }, + "INAPP_CHECKOUT_COST_PER_ACTION": { + "type": ["null", "number"] + }, + "OUTBOUND_CLICK_1": { + "type": ["null", "number"] + }, + "OUTBOUND_CLICK_2": { + "type": ["null", "number"] + }, + "PAGE_VISIT_COST_PER_ACTION": { + "type": ["null", "number"] + }, + "PAGE_VISIT_ROAS": { + "type": ["null", "number"] + }, + "PAID_IMPRESSION": { + "type": ["null", "number"] + }, + "PIN_ID": { + "type": ["null", "number"] + }, + "PIN_PROMOTION_ID": { + "type": ["null", "number"] + }, + "REPIN_1": { + "type": ["null", "number"] + }, + "REPIN_2": { + "type": ["null", "number"] + }, + "REPIN_RATE": { + "type": ["null", "number"] + }, + "SPEND_IN_DOLLAR": { + "type": ["null", "number"] + }, + "SPEND_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_CLICKTHROUGH": { + "type": ["null", "number"] + }, + "TOTAL_CLICK_ADD_TO_CART": { + "type": ["null", "number"] + }, + "TOTAL_CLICK_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_CLICK_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_CLICK_LEAD": { + "type": ["null", "number"] + }, + "TOTAL_CLICK_SIGNUP": { + "type": ["null", "number"] + }, + "TOTAL_CLICK_SIGNUP_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_CONVERSIONS": { + "type": ["null", "number"] + }, + "TOTAL_CUSTOM": { + "type": ["null", "number"] + }, + "TOTAL_ENGAGEMENT": { + "type": ["null", "number"] + }, + "TOTAL_ENGAGEMENT_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_ENGAGEMENT_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_ENGAGEMENT_LEAD": { + "type": ["null", "number"] + }, + "TOTAL_ENGAGEMENT_SIGNUP": { + "type": ["null", "number"] + }, + "TOTAL_ENGAGEMENT_SIGNUP_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_IDEA_PIN_PRODUCT_TAG_VISIT": { + "type": ["null", "number"] + }, + "TOTAL_IMPRESSION_FREQUENCY": { + "type": ["null", "number"] + }, + "TOTAL_IMPRESSION_USER": { + "type": ["null", "number"] + }, + "TOTAL_LEAD": { + "type": ["null", "number"] + }, + "TOTAL_OFFLINE_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_PAGE_VISIT": { + "type": ["null", "number"] + }, + "TOTAL_REPIN_RATE": { + "type": ["null", "number"] + }, + "TOTAL_SIGNUP": { + "type": ["null", "number"] + }, + "TOTAL_SIGNUP_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_3SEC_VIEWS": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_AVG_WATCHTIME_IN_SECOND": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_MRC_VIEWS": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_P0_COMBINED": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_P100_COMPLETE": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_P25_COMBINED": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_P50_COMBINED": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_P75_COMBINED": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_P95_COMBINED": { + "type": ["null", "number"] + }, + "TOTAL_VIEW_ADD_TO_CART": { + "type": ["null", "number"] + }, + "TOTAL_VIEW_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_VIEW_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_VIEW_LEAD": { + "type": ["null", "number"] + }, + "TOTAL_VIEW_SIGNUP": { + "type": ["null", "number"] + }, + "TOTAL_VIEW_SIGNUP_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_WEB_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_WEB_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_WEB_CLICK_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_WEB_CLICK_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_WEB_ENGAGEMENT_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_WEB_ENGAGEMENT_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_WEB_SESSIONS": { + "type": ["null", "number"] + }, + "TOTAL_WEB_VIEW_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_WEB_VIEW_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "VIDEO_3SEC_VIEWS_2": { + "type": ["null", "number"] + }, + "VIDEO_LENGTH": { + "type": ["null", "number"] + }, + "VIDEO_MRC_VIEWS_2": { + "type": ["null", "number"] + }, + "VIDEO_P0_COMBINED_2": { + "type": ["null", "number"] + }, + "VIDEO_P100_COMPLETE_2": { + "type": ["null", "number"] + }, + "VIDEO_P25_COMBINED_2": { + "type": ["null", "number"] + }, + "VIDEO_P50_COMBINED_2": { + "type": ["null", "number"] + }, + "VIDEO_P75_COMBINED_2": { + "type": ["null", "number"] + }, + "VIDEO_P95_COMBINED_2": { + "type": ["null", "number"] + }, + "WEB_CHECKOUT_COST_PER_ACTION": { + "type": ["null", "number"] + }, + "WEB_CHECKOUT_ROAS": { + "type": ["null", "number"] + }, + "WEB_SESSIONS_1": { + "type": ["null", "number"] + }, + "WEB_SESSIONS_2": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py index e110f15f339b..828e07f22b53 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py @@ -15,6 +15,19 @@ from airbyte_cdk.utils import AirbyteTracedException from source_pinterest.reports import CampaignAnalyticsReport +from .reports.reports import ( + AdGroupReport, + AdGroupTargetingReport, + AdvertizerReport, + AdvertizerTargetingReport, + CampaignTargetingReport, + KeywordReport, + PinPromotionReport, + PinPromotionTargetingReport, + ProductGroupReport, + ProductGroupTargetingReport, + ProductItemReport, +) from .streams import ( AdAccountAnalytics, AdAccounts, @@ -22,12 +35,19 @@ AdGroupAnalytics, AdGroups, Ads, + Audiences, BoardPins, Boards, BoardSectionPins, BoardSections, CampaignAnalytics, Campaigns, + Catalogs, + CatalogsFeeds, + CatalogsProductGroups, + ConversionTags, + CustomerLists, + Keywords, PinterestStream, UserAccountAnalytics, ) @@ -76,11 +96,16 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: config = self._validate_and_transform(config) authenticator = self.get_authenticator(config) url = f"{PinterestStream.url_base}user_account" - auth_headers = {"Accept": "application/json", **authenticator.get_auth_header()} try: + auth_headers = {"Accept": "application/json", **authenticator.get_auth_header()} session = requests.get(url, headers=auth_headers) session.raise_for_status() return True, None + except requests.exceptions.HTTPError as e: + if "401 Client Error: Unauthorized for url" in str(e): + return False, f"Try to re-authenticate because current refresh token is not valid. {e}" + else: + return False, e except requests.exceptions.RequestException as e: return False, e @@ -89,19 +114,44 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: report_config = self._validate_and_transform(config, amount_of_days_allowed_for_lookup=913) config = self._validate_and_transform(config) status = ",".join(config.get("status")) if config.get("status") else None + + ad_accounts = AdAccounts(config) + ads = Ads(ad_accounts, config=config, status_filter=status) + ad_groups = AdGroups(ad_accounts, config=config, status_filter=status) + campaigns = Campaigns(ad_accounts, config=config, status_filter=status) + boards = Boards(config) + board_sections = BoardSections(boards, config=config) return [ - AdAccountAnalytics(AdAccounts(config), config=config), - AdAccounts(config), - AdAnalytics(Ads(AdAccounts(config), with_data_slices=False, config=config), config=config), - AdGroupAnalytics(AdGroups(AdAccounts(config), with_data_slices=False, config=config), config=config), - AdGroups(AdAccounts(config), status_filter=status, config=config), - Ads(AdAccounts(config), status_filter=status, config=config), - BoardPins(Boards(config), config=config), - BoardSectionPins(BoardSections(Boards(config), config=config), config=config), - BoardSections(Boards(config), config=config), - Boards(config), - CampaignAnalytics(Campaigns(AdAccounts(config), with_data_slices=False, config=config), config=config), - CampaignAnalyticsReport(AdAccounts(report_config), config=report_config), - Campaigns(AdAccounts(config), status_filter=status, config=config), + ad_accounts, + AdAccountAnalytics(ad_accounts, config=config), + ads, + AdAnalytics(ads, config=config), + ad_groups, + AdGroupAnalytics(ad_groups, config=config), + boards, + BoardPins(boards, config=config), + board_sections, + BoardSectionPins(board_sections, config=config), + campaigns, + CampaignAnalytics(campaigns, config=config), + CampaignAnalyticsReport(ad_accounts, config=report_config), + CampaignTargetingReport(ad_accounts, config=report_config), UserAccountAnalytics(None, config=config), + Keywords(ad_groups, config=config), + Audiences(ad_accounts, config=config), + ConversionTags(ad_accounts, config=config), + CustomerLists(ad_accounts, config=config), + Catalogs(config=config), + CatalogsFeeds(config=config), + CatalogsProductGroups(config=config), + AdvertizerReport(ad_accounts, config=report_config), + AdvertizerTargetingReport(ad_accounts, config=report_config), + AdGroupReport(ad_accounts, config=report_config), + AdGroupTargetingReport(ad_accounts, config=report_config), + PinPromotionReport(ad_accounts, config=report_config), + PinPromotionTargetingReport(ad_accounts, config=report_config), + ProductGroupReport(ad_accounts, config=report_config), + ProductGroupTargetingReport(ad_accounts, config=report_config), + KeywordReport(ad_accounts, config=report_config), + ProductItemReport(ad_accounts, config=report_config), ] diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py index 906e707b976d..0f30a44e093a 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py @@ -2,16 +2,21 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging from abc import ABC from datetime import datetime -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional +from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional import pendulum import requests from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream +from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from requests import HTTPError from .utils import get_analytics_columns, to_datetime_str @@ -44,10 +49,6 @@ def start_date(self): def window_in_days(self): return 30 # Set window_in_days to 30 days date range - @property - def availability_strategy(self) -> Optional["AvailabilityStrategy"]: - return None - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: next_page = response.json().get("bookmark", {}) if self.data_fields else {} @@ -115,6 +116,53 @@ def path(self, **kwargs) -> str: return "boards" +class Catalogs(PinterestStream): + """Docs: https://developers.pinterest.com/docs/api/v5/#operation/catalogs/list""" + + use_cache = True + + def path(self, **kwargs) -> str: + return "catalogs" + + +class CatalogsFeeds(PinterestStream): + """Docs: https://developers.pinterest.com/docs/api/v5/#operation/feeds/list""" + + use_cache = True + + def path(self, **kwargs) -> str: + return "catalogs/feeds" + + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + # Remove sensitive data + for record in super().parse_response(response, stream_state, **kwargs): + record.pop("credentials", None) + yield record + + +class CatalogsProductGroupsAvailabilityStrategy(HttpAvailabilityStrategy): + def reasons_for_unavailable_status_codes( + self, stream: Stream, logger: logging.Logger, source: Optional[Source], error: HTTPError + ) -> Dict[int, str]: + reasons_for_codes: Dict[int, str] = super().reasons_for_unavailable_status_codes(stream, logger, source, error) + reasons_for_codes[409] = "Can't access catalog product groups because there is no existing catalog." + + return reasons_for_codes + + +class CatalogsProductGroups(PinterestStream): + """Docs: https://developers.pinterest.com/docs/api/v5/#operation/catalogs_product_groups/list""" + + use_cache = True + + def path(self, **kwargs) -> str: + return "catalogs/product_groups" + + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return CatalogsProductGroupsAvailabilityStrategy() + + class AdAccounts(PinterestStream): use_cache = True @@ -137,6 +185,34 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return f"boards/{stream_slice['sub_parent']['parent']['id']}/sections/{stream_slice['parent']['id']}/pins" +class Audiences(PinterestSubStream, PinterestStream): + """Docs: https://developers.pinterest.com/docs/api/v5/#operation/audiences/list""" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"ad_accounts/{stream_slice['parent']['id']}/audiences" + + +class Keywords(PinterestSubStream, PinterestStream): + """Docs: https://developers.pinterest.com/docs/api/v5/#operation/keywords/get""" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"ad_accounts/{stream_slice['parent']['ad_account_id']}/keywords?ad_group_id={stream_slice['parent']['id']}" + + +class ConversionTags(PinterestSubStream, PinterestStream): + """Docs: https://developers.pinterest.com/docs/api/v5/#operation/conversion_tags/list""" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"ad_accounts/{stream_slice['parent']['id']}/conversion_tags" + + +class CustomerLists(PinterestSubStream, PinterestStream): + """Docs: https://developers.pinterest.com/docs/api/v5/#tag/customer_lists""" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"ad_accounts/{stream_slice['parent']['id']}/customer_lists" + + class IncrementalPinterestStream(PinterestStream, ABC): def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: default_value = self.start_date.format("YYYY-MM-DD") @@ -292,7 +368,7 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class Campaigns(ServerSideFilterStream): - def __init__(self, parent: HttpStream, with_data_slices: bool = True, status_filter: str = "", **kwargs): + def __init__(self, parent: HttpStream, with_data_slices: bool = False, status_filter: str = "", **kwargs): super().__init__(parent, with_data_slices, **kwargs) self.status_filter = status_filter @@ -309,11 +385,12 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class AdGroups(ServerSideFilterStream): - def __init__(self, parent: HttpStream, with_data_slices: bool = True, status_filter: str = "", **kwargs): + def __init__(self, parent: HttpStream, with_data_slices: bool = False, status_filter: str = "", **kwargs): super().__init__(parent, with_data_slices, **kwargs) self.status_filter = status_filter def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + print(f"=========== stream_slice: {stream_slice} =====================") params = f"?entity_statuses={self.status_filter}" if self.status_filter else "" return f"ad_accounts/{stream_slice['parent']['id']}/ad_groups{params}" @@ -326,7 +403,7 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class Ads(ServerSideFilterStream): - def __init__(self, parent: HttpStream, with_data_slices: bool = True, status_filter: str = "", **kwargs): + def __init__(self, parent: HttpStream, with_data_slices: bool = False, status_filter: str = "", **kwargs): super().__init__(parent, with_data_slices, **kwargs) self.status_filter = status_filter diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py index 5638ceebe77e..8cc1a4f96057 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py @@ -3,7 +3,9 @@ # import responses +from source_pinterest import SourcePinterest from source_pinterest.utils import get_analytics_columns +from unit_tests.test_source import setup_responses @responses.activate @@ -51,3 +53,12 @@ def test_read_records(analytics_report_stream, date_range): assert next(records) == expected_record assert len(responses.calls) == 3 assert responses.calls[0].request.url == report_request_url + + +@responses.activate +def test_streams(test_config): + setup_responses() + source = SourcePinterest() + streams = source.streams(test_config) + expected_streams_number = 32 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py index 8c3a112c0711..d86620fad40c 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py @@ -40,12 +40,17 @@ def test_check_wrong_date_connection(wrong_date_config): @responses.activate -def test_streams(test_config): - setup_responses() +def test_check_connection_expired_token(test_config): + responses.add( + responses.POST, + "https://api.pinterest.com/v5/oauth/token", + status=401 + ) source = SourcePinterest() - streams = source.streams(test_config) - expected_streams_number = 14 - assert len(streams) == expected_streams_number + logger_mock = MagicMock() + assert source.check_connection(logger_mock, test_config) == (False, + 'Try to re-authenticate because current refresh token is not valid. ' \ + '401 Client Error: Unauthorized for url: https://api.pinterest.com/v5/oauth/token') def test_get_authenticator(test_config): diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py index a13930e3daab..a560986ec482 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py @@ -15,12 +15,19 @@ AdGroupAnalytics, AdGroups, Ads, + Audiences, BoardPins, Boards, BoardSectionPins, BoardSections, CampaignAnalytics, Campaigns, + Catalogs, + CatalogsFeeds, + CatalogsProductGroups, + ConversionTags, + CustomerLists, + Keywords, PinterestStream, PinterestSubStream, UserAccountAnalytics, @@ -64,6 +71,15 @@ def test_parse_response(patch_base_class, test_response, test_current_stream_sta assert next(stream.parse_response(**inputs)) == expected_parsed_object +def test_parse_response_with_sensitive_data(patch_base_class): + """Test that sensitive data is removed""" + stream = CatalogsFeeds(config=MagicMock()) + response = MagicMock() + response.json.return_value = {"items": [{"id": "CatalogsFeeds1", "credentials": {"password": "bla"}}], "bookmark": "string"} + actual_response = list(stream.parse_response(response=response, stream_state=None)) + assert actual_response == [{"id": "CatalogsFeeds1"}] + + def test_request_headers(patch_base_class): stream = PinterestStream(config=MagicMock()) inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} @@ -180,6 +196,17 @@ def test_backoff_on_rate_limit_error(requests_mock, test_response, status_code, {"sub_parent": {"parent": {"id": "234"}}, "parent": {"id": "123"}}, "ad_accounts/234/ads/analytics", ), + (Catalogs(config=MagicMock()), None, "catalogs"), + (CatalogsFeeds(config=MagicMock()), None, "catalogs/feeds"), + (CatalogsProductGroups(config=MagicMock()), None, "catalogs/product_groups"), + ( + Keywords(parent=None, config=MagicMock()), + {"parent": {"id": "234", "ad_account_id": "AD_ACCOUNT_1"}}, + "ad_accounts/AD_ACCOUNT_1/keywords?ad_group_id=234", + ), + (Audiences(parent=None, config=MagicMock()), {"parent": {"id": "AD_ACCOUNT_1"}}, "ad_accounts/AD_ACCOUNT_1/audiences"), + (ConversionTags(parent=None, config=MagicMock()), {"parent": {"id": "AD_ACCOUNT_1"}}, "ad_accounts/AD_ACCOUNT_1/conversion_tags"), + (CustomerLists(parent=None, config=MagicMock()), {"parent": {"id": "AD_ACCOUNT_1"}}, "ad_accounts/AD_ACCOUNT_1/customer_lists"), ], ) def test_path(patch_base_class, stream_cls, slice, expected): diff --git a/docs/integrations/sources/pinterest.md b/docs/integrations/sources/pinterest.md index fa54c2ed62b4..aa599ef5389e 100644 --- a/docs/integrations/sources/pinterest.md +++ b/docs/integrations/sources/pinterest.md @@ -46,57 +46,70 @@ The Pinterest source connector supports the following [sync modes](https://docs. ## Supported Streams - [Account analytics](https://developers.pinterest.com/docs/api/v5/#operation/user_account/analytics) \(Incremental\) -- [Boards](https://developers.pinterest.com/docs/api/v5/#operation/boards/list) \(Full table\) - - [Board sections](https://developers.pinterest.com/docs/api/v5/#operation/board_sections/list) \(Full table\) - - [Pins on board section](https://developers.pinterest.com/docs/api/v5/#operation/board_sections/list_pins) \(Full table\) - - [Pins on board](https://developers.pinterest.com/docs/api/v5/#operation/boards/list_pins) \(Full table\) -- [Ad accounts](https://developers.pinterest.com/docs/api/v5/#operation/ad_accounts/list) \(Full table\) - - [Ad account analytics](https://developers.pinterest.com/docs/api/v5/#operation/ad_account/analytics) \(Incremental\) - - [Campaigns](https://developers.pinterest.com/docs/api/v5/#operation/campaigns/list) \(Incremental\) - - [Campaign analytics](https://developers.pinterest.com/docs/api/v5/#operation/campaigns/list) \(Incremental\) - - [Campaign Analytics Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) - - [Ad groups](https://developers.pinterest.com/docs/api/v5/#operation/ad_groups/list) \(Incremental\) - - [Ad group analytics](https://developers.pinterest.com/docs/api/v5/#operation/ad_groups/analytics) \(Incremental\) - - [Ads](https://developers.pinterest.com/docs/api/v5/#operation/ads/list) \(Incremental\) - - [Ad analytics](https://developers.pinterest.com/docs/api/v5/#operation/ads/analytics) \(Incremental\) +- [Boards](https://developers.pinterest.com/docs/api/v5/#operation/boards/list) \(Full refresh\) +- [Board sections](https://developers.pinterest.com/docs/api/v5/#operation/board_sections/list) \(Full refresh\) +- [Pins on board section](https://developers.pinterest.com/docs/api/v5/#operation/board_sections/list_pins) \(Full refresh\) +- [Pins on board](https://developers.pinterest.com/docs/api/v5/#operation/boards/list_pins) \(Full refresh\) +- [Ad accounts](https://developers.pinterest.com/docs/api/v5/#operation/ad_accounts/list) \(Full refresh\) +- [Ad account analytics](https://developers.pinterest.com/docs/api/v5/#operation/ad_account/analytics) \(Incremental\) +- [Campaigns](https://developers.pinterest.com/docs/api/v5/#operation/campaigns/list) \(Incremental\) +- [Campaign analytics](https://developers.pinterest.com/docs/api/v5/#operation/campaigns/list) \(Incremental\) +- [Campaign Analytics Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) +- [Campaign Targeting Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) +- [Ad Groups](https://developers.pinterest.com/docs/api/v5/#operation/ad_groups/list) \(Incremental\) +- [Ad Group Analytics](https://developers.pinterest.com/docs/api/v5/#operation/ad_groups/analytics) \(Incremental\) +- [Ad Group Report](https://developers.pinterest.com/docs/api/v5/#operation/ad_groups/analytics) \(Incremental\) +- [Ad Group Targeting Report](https://developers.pinterest.com/docs/api/v5/#operation/ad_groups/analytics) \(Incremental\) +- [Ads](https://developers.pinterest.com/docs/api/v5/#operation/ads/list) \(Incremental\) +- [Ad analytics](https://developers.pinterest.com/docs/api/v5/#operation/ads/analytics) \(Incremental\) +- [Catalogs](https://developers.pinterest.com/docs/api/v5/#operation/catalogs/list) \(Full refresh\) +- [Catalogs Feeds](https://developers.pinterest.com/docs/api/v5/#operation/feeds/list) \(Full refresh\) +- [Catalogs Product Groups](https://developers.pinterest.com/docs/api/v5/#operation/catalogs_product_groups/list) \(Full refresh\) +- [Audiences](https://developers.pinterest.com/docs/api/v5/#operation/audiences/list) \(Full refresh\) +- [Keywords](https://developers.pinterest.com/docs/api/v5/#operation/keywords/get) \(Full refresh\) +- [Conversion Tags](https://developers.pinterest.com/docs/api/v5/#operation/conversion_tags/list) \(Full refresh\) +- [Customer Lists](https://developers.pinterest.com/docs/api/v5/#tag/customer_lists) \(Full refresh\) +- [Advertizer Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) +- [Advertizer Targeting Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) +- [Pin Promotion Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) +- [Pin Promotion Targeting Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) +- [Product Group Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) +- [Product Group Targeting Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) +- [Product Item Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) +- [Keyword Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) ## Performance considerations -The connector is restricted by the Pinterest [requests limitation](https://developers.pinterest.com/docs/api/v5/#tag/Rate-limits). - -##### Rate Limits - -- Analytics streams: 300 calls per day / per user \ -- Ad accounts streams (Campaigns, Ad groups, Ads): 1000 calls per min / per user / per app \ -- Boards streams: 10 calls per sec / per user / per app +The connector is restricted by the Pinterest [requests limitation](https://developers.pinterest.com/docs/reference/ratelimits/). ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------| :------------------------------------------------------- |:--------------------------------------------------------------------------------------------------------------| -| 0.7.1 | 2023-11-01 | [32078](https://github.com/airbytehq/airbyte/pull/32078) | handle non json response | -| 0.7.0 | 2023-10-25 | [31876](https://github.com/airbytehq/airbyte/pull/31876) | Migrated to base image, removed token based authentication mthod becuase access_token is valid for 1 day only | -| 0.6.0 | 2023-07-25 | [28672](https://github.com/airbytehq/airbyte/pull/28672) | Add report stream for `CAMPAIGN` level | -| 0.5.3 | 2023-07-05 | [27964](https://github.com/airbytehq/airbyte/pull/27964) | Add `id` field to `owner` field in `ad_accounts` stream | -| 0.5.2 | 2023-06-02 | [26949](https://github.com/airbytehq/airbyte/pull/26949) | Update `BoardPins` stream with `note` property | -| 0.5.1 | 2023-05-11 | [25984](https://github.com/airbytehq/airbyte/pull/25984) | Add pattern for start_date | -| 0.5.0 | 2023-05-17 | [26188](https://github.com/airbytehq/airbyte/pull/26188) | Add `product_tags` field to the `BoardPins` stream | -| 0.4.0 | 2023-05-16 | [26112](https://github.com/airbytehq/airbyte/pull/26112) | Add `is_standard` field to the `BoardPins` stream | -| 0.3.0 | 2023-05-09 | [25915](https://github.com/airbytehq/airbyte/pull/25915) | Add `creative_type` field to the `BoardPins` stream | -| 0.2.6 | 2023-04-26 | [25548](https://github.com/airbytehq/airbyte/pull/25548) | Fix `format` issue for `boards` stream schema for fields with `date-time` | -| 0.2.5 | 2023-04-19 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Update `AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP` to 89 days | -| 0.2.4 | 2023-02-25 | [23457](https://github.com/airbytehq/airbyte/pull/23457) | Add missing columns for analytics streams for pinterest source | -| 0.2.3 | 2023-03-01 | [23649](https://github.com/airbytehq/airbyte/pull/23649) | Fix for `HTTP - 400 Bad Request` when requesting data >= 90 days | -| 0.2.2 | 2023-01-27 | [22020](https://github.com/airbytehq/airbyte/pull/22020) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 0.2.1 | 2022-12-15 | [20532](https://github.com/airbytehq/airbyte/pull/20532) | Bump CDK version | -| 0.2.0 | 2022-12-13 | [20242](https://github.com/airbytehq/airbyte/pull/20242) | Add data-type normalization up to the schemas declared | -| 0.1.9 | 2022-09-06 | [15074](https://github.com/airbytehq/airbyte/pull/15074) | Add filter based on statuses | -| 0.1.8 | 2022-10-21 | [18285](https://github.com/airbytehq/airbyte/pull/18285) | Fix type of `start_date` | -| 0.1.7 | 2022-09-29 | [17387](https://github.com/airbytehq/airbyte/pull/17387) | Set `start_date` dynamically based on API restrictions. | -| 0.1.6 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Use CDK 0.1.89 | -| 0.1.5 | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state | -| 0.1.4 | 2022-09-06 | [16161](https://github.com/airbytehq/airbyte/pull/16161) | Add ability to handle `429 - Too Many Requests` error with respect to `Max Rate Limit Exceeded Error` | -| 0.1.3 | 2022-09-02 | [16271](https://github.com/airbytehq/airbyte/pull/16271) | Add support of `OAuth2.0` authentication method | -| 0.1.2 | 2021-12-22 | [10223](https://github.com/airbytehq/airbyte/pull/10223) | Fix naming of `AD_ID` and `AD_ACCOUNT_ID` fields | -| 0.1.1 | 2021-12-22 | [9043](https://github.com/airbytehq/airbyte/pull/9043) | Update connector fields title/description | -| 0.1.0 | 2021-10-29 | [7493](https://github.com/airbytehq/airbyte/pull/7493) | Release Pinterest CDK Connector | +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :------------------------------------------------------- |:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.7.2 | 2023-11-08 | [32299](https://github.com/airbytehq/airbyte/pull/32299) | added default `AvailabilityStrategy`, fixed bug which cases duplicated requests, added new streams: Catalogs, CatalogsFeeds, CatalogsProductGroups, Audiences, Keywords, ConversionTags, CustomerLists, CampaignTargetingReport, AdvertizerReport, AdvertizerTargetingReport, AdGroupReport, AdGroupTargetingReport, PinPromotionReport, PinPromotionTargetingReport, ProductGroupReport, ProductGroupTargetingReport, ProductItemReport, KeywordReport | +| 0.7.1 | 2023-11-01 | [32078](https://github.com/airbytehq/airbyte/pull/32078) | handle non json response | +| 0.7.0 | 2023-10-25 | [31876](https://github.com/airbytehq/airbyte/pull/31876) | Migrated to base image, removed token based authentication mthod becuase access_token is valid for 1 day only | +| 0.6.0 | 2023-07-25 | [28672](https://github.com/airbytehq/airbyte/pull/28672) | Add report stream for `CAMPAIGN` level | +| 0.5.3 | 2023-07-05 | [27964](https://github.com/airbytehq/airbyte/pull/27964) | Add `id` field to `owner` field in `ad_accounts` stream | +| 0.5.2 | 2023-06-02 | [26949](https://github.com/airbytehq/airbyte/pull/26949) | Update `BoardPins` stream with `note` property | +| 0.5.1 | 2023-05-11 | [25984](https://github.com/airbytehq/airbyte/pull/25984) | Add pattern for start_date | +| 0.5.0 | 2023-05-17 | [26188](https://github.com/airbytehq/airbyte/pull/26188) | Add `product_tags` field to the `BoardPins` stream | +| 0.4.0 | 2023-05-16 | [26112](https://github.com/airbytehq/airbyte/pull/26112) | Add `is_standard` field to the `BoardPins` stream | +| 0.3.0 | 2023-05-09 | [25915](https://github.com/airbytehq/airbyte/pull/25915) | Add `creative_type` field to the `BoardPins` stream | +| 0.2.6 | 2023-04-26 | [25548](https://github.com/airbytehq/airbyte/pull/25548) | Fix `format` issue for `boards` stream schema for fields with `date-time` | +| 0.2.5 | 2023-04-19 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Update `AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP` to 89 days | +| 0.2.4 | 2023-02-25 | [23457](https://github.com/airbytehq/airbyte/pull/23457) | Add missing columns for analytics streams for pinterest source | +| 0.2.3 | 2023-03-01 | [23649](https://github.com/airbytehq/airbyte/pull/23649) | Fix for `HTTP - 400 Bad Request` when requesting data >= 90 days | +| 0.2.2 | 2023-01-27 | [22020](https://github.com/airbytehq/airbyte/pull/22020) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.2.1 | 2022-12-15 | [20532](https://github.com/airbytehq/airbyte/pull/20532) | Bump CDK version | +| 0.2.0 | 2022-12-13 | [20242](https://github.com/airbytehq/airbyte/pull/20242) | Add data-type normalization up to the schemas declared | +| 0.1.9 | 2022-09-06 | [15074](https://github.com/airbytehq/airbyte/pull/15074) | Add filter based on statuses | +| 0.1.8 | 2022-10-21 | [18285](https://github.com/airbytehq/airbyte/pull/18285) | Fix type of `start_date` | +| 0.1.7 | 2022-09-29 | [17387](https://github.com/airbytehq/airbyte/pull/17387) | Set `start_date` dynamically based on API restrictions. | +| 0.1.6 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Use CDK 0.1.89 | +| 0.1.5 | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state | +| 0.1.4 | 2022-09-06 | [16161](https://github.com/airbytehq/airbyte/pull/16161) | Add ability to handle `429 - Too Many Requests` error with respect to `Max Rate Limit Exceeded Error` | +| 0.1.3 | 2022-09-02 | [16271](https://github.com/airbytehq/airbyte/pull/16271) | Add support of `OAuth2.0` authentication method | +| 0.1.2 | 2021-12-22 | [10223](https://github.com/airbytehq/airbyte/pull/10223) | Fix naming of `AD_ID` and `AD_ACCOUNT_ID` fields | +| 0.1.1 | 2021-12-22 | [9043](https://github.com/airbytehq/airbyte/pull/9043) | Update connector fields title/description | +| 0.1.0 | 2021-10-29 | [7493](https://github.com/airbytehq/airbyte/pull/7493) | Release Pinterest CDK Connector | From 6deabcc270763e92b825f1a9d4851d7e9857e5f3 Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Wed, 15 Nov 2023 11:00:42 -0400 Subject: [PATCH 060/124] source-chartmogul: improve v1.0.0 breaking change messaging (#32519) --- .../connectors/source-chartmogul/metadata.yaml | 7 ++----- docs/integrations/sources/chartmogul-migrations.md | 4 ++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-chartmogul/metadata.yaml b/airbyte-integrations/connectors/source-chartmogul/metadata.yaml index 44bbc6075999..42e6a35f9b9f 100644 --- a/airbyte-integrations/connectors/source-chartmogul/metadata.yaml +++ b/airbyte-integrations/connectors/source-chartmogul/metadata.yaml @@ -12,11 +12,8 @@ data: releases: breakingChanges: 1.0.0: - message: > - Refactor and break `customer_count` stream - into multiple streams (daily, weekly, monthly, quarterly). - You need to update your schema and use the new streams. - upgradeDeadline: "2023-11-21" + message: "This version separates the `customer_count` stream into multiple streams (daily, weekly, monthly, quarterly). Users previously using the `customer_count` stream will need to run a reset to enable the new streams and continue syncing." + upgradeDeadline: "2023-11-29" dockerRepository: airbyte/source-chartmogul documentationUrl: https://docs.airbyte.com/integrations/sources/chartmogul githubIssueLabel: source-chartmogul diff --git a/docs/integrations/sources/chartmogul-migrations.md b/docs/integrations/sources/chartmogul-migrations.md index 2cf9e8e1e07f..a0294bf0e818 100644 --- a/docs/integrations/sources/chartmogul-migrations.md +++ b/docs/integrations/sources/chartmogul-migrations.md @@ -2,6 +2,6 @@ ## Upgrading to 1.0.0 -Version 1.0.0 refactors and breaks `customer_count` stream into multiple streams (daily, weekly, monthly, quarterly). +Version 1.0.0 refactors and separates the `customer_count` stream into multiple streams (daily, weekly, monthly, quarterly). -You need to update your schema and use the new streams. +Users that have this stream enabled will need to refresh the schema and run a reset to use the new streams in affected connections to continue syncing. From cd03095170451949359cc06fd45a778f98ceb247 Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Wed, 15 Nov 2023 11:01:03 -0400 Subject: [PATCH 061/124] source-freshsales: improve v1.0.0 breaking change messaging (#32518) --- .../connectors/source-freshsales/metadata.yaml | 4 ++-- docs/integrations/sources/freshsales-migrations.md | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-freshsales/metadata.yaml b/airbyte-integrations/connectors/source-freshsales/metadata.yaml index 03a031b0d259..afb486eb0a95 100644 --- a/airbyte-integrations/connectors/source-freshsales/metadata.yaml +++ b/airbyte-integrations/connectors/source-freshsales/metadata.yaml @@ -12,8 +12,8 @@ data: releases: breakingChanges: 1.0.0: - message: "Large update in schema data types. Need refresh data" - upgradeDeadline: "2023-11-23" + message: "This version migrates the Freshsales connector to our low-code framework for greater maintainability. It also introduces changes to data types across most streams. You will need to run a reset after upgrading to continue syncing data with the connector." + upgradeDeadline: "2023-11-29" dockerRepository: airbyte/source-freshsales documentationUrl: https://docs.airbyte.com/integrations/sources/freshsales githubIssueLabel: source-freshsales diff --git a/docs/integrations/sources/freshsales-migrations.md b/docs/integrations/sources/freshsales-migrations.md index e059dc4e6d3c..42b98fbb668d 100644 --- a/docs/integrations/sources/freshsales-migrations.md +++ b/docs/integrations/sources/freshsales-migrations.md @@ -2,5 +2,6 @@ ## Upgrading to 1.0.0 -All streams data types were updated to match the correct return from the API. -Need refresh schema to match latest updates. +This version migrates the Freshsales connector to our low-code framework for greater maintainability. + +As part of this release, we've also updated data types across streams to match the correct return types from the upstream API. You will need to run a reset on connections using this connector after upgrading to continue syncing. From 33ca41b737909df39240194edb2ca36ded272ce7 Mon Sep 17 00:00:00 2001 From: "Roman Yermilov [GL]" <86300758+roman-yermilov-gl@users.noreply.github.com> Date: Wed, 15 Nov 2023 18:30:25 +0100 Subject: [PATCH 062/124] Source Slack: increase timeout for full refresh CAT (#32548) --- .../connectors/source-slack/acceptance-test-config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml index 9a62ec5ec942..0d248e4409e1 100644 --- a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml @@ -30,6 +30,7 @@ acceptance_tests: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/full_refresh_catalog.json" + timeout_seconds: 4800 incremental: tests: - config_path: "secrets/config.json" From 0f94afe09e8030c781205dfd923f62cf598a7929 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Wed, 15 Nov 2023 10:20:45 -0800 Subject: [PATCH 063/124] :bug: CDK: Apply log level on the entrypoint's logger (#32410) --- airbyte-cdk/python/airbyte_cdk/entrypoint.py | 1 + airbyte-cdk/python/airbyte_cdk/utils/is_cloud_environment.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/python/airbyte_cdk/entrypoint.py b/airbyte-cdk/python/airbyte_cdk/entrypoint.py index d7dfe736b831..a53369d349ec 100644 --- a/airbyte-cdk/python/airbyte_cdk/entrypoint.py +++ b/airbyte-cdk/python/airbyte_cdk/entrypoint.py @@ -87,6 +87,7 @@ def run(self, parsed_args: argparse.Namespace) -> Iterable[str]: if hasattr(parsed_args, "debug") and parsed_args.debug: self.logger.setLevel(logging.DEBUG) + logger.setLevel(logging.DEBUG) self.logger.debug("Debug logs enabled") else: self.logger.setLevel(logging.INFO) diff --git a/airbyte-cdk/python/airbyte_cdk/utils/is_cloud_environment.py b/airbyte-cdk/python/airbyte_cdk/utils/is_cloud_environment.py index 71fc3d0eb249..25b1eee87fad 100644 --- a/airbyte-cdk/python/airbyte_cdk/utils/is_cloud_environment.py +++ b/airbyte-cdk/python/airbyte_cdk/utils/is_cloud_environment.py @@ -7,7 +7,7 @@ CLOUD_DEPLOYMENT_MODE = "cloud" -def is_cloud_environment(): +def is_cloud_environment() -> bool: """ Returns True if the connector is running in a cloud environment, False otherwise. From c7378ba3080e0e0d5e4a6808901a2ed6897c943d Mon Sep 17 00:00:00 2001 From: Lake Mossman Date: Wed, 15 Nov 2023 10:23:37 -0800 Subject: [PATCH 064/124] =?UTF-8?q?=F0=9F=93=9D=20Update=20partition=20doc?= =?UTF-8?q?s=20to=20reflect=20parent=20stream=20/=20parameterized=20reques?= =?UTF-8?q?ts=20UI=20split=20(#32409)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Alex Birdsall --- .../connector-builder-ui/partitioning.md | 71 ++++++++++--------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/docs/connector-development/connector-builder-ui/partitioning.md b/docs/connector-development/connector-builder-ui/partitioning.md index 3ac5afcb862a..57dc0d5d9307 100644 --- a/docs/connector-development/connector-builder-ui/partitioning.md +++ b/docs/connector-development/connector-builder-ui/partitioning.md @@ -12,27 +12,29 @@ There are some cases that require multiple requests to fetch all records as well * If your records are spread out across multiple pages that need to be requested individually if there are too many records, use the Pagination feature. * If your records are spread out over time and multiple requests are necessary to fetch all data (for example one request per day), use the Incremental sync feature. -## Dynamic and static partition routing +## Dynamic and static partitioning -There are three possible sources for the partitions that need to be queried - the connector itself, supplied by the end user when configuring a Source based on the connector, or the API provides the list of partitions on another endpoint (for example the Woocommerce API also includes an `/orders` endpoint that returns all orders). +There are three possible sources for the partitions that need to be queried - the connector itself, supplied by the end user when configuring a Source based on the connector, or the API provides the list of partitions via another endpoint (for example the Woocommerce API also includes an `/orders` endpoint that returns all orders). -The first two options are a "static" form of partition routing (because the partitions won't change as long as the Airbyte configuration isn't changed). The API providing the partitions via one or multiple separate requests is a "dynamic" form of partition routing because the partitions can change any time. +The first two options are a "static" form of partition routing (because the partitions won't change as long as the Airbyte configuration isn't changed). This can be achieved by configuring the Parameterized Requests component in the Connector Builder. -### List partition router +The API providing the partitions via one or multiple separate requests is a "dynamic" form of partitioning because the partitions can change any time. This can be achieved by configuring the Parent Stream partition component in the Connector Builder. -To configure static partitioning, choose the "List" method for the partition router. The following fields have to be configured: -* The "partition values" can either be set to a list of strings, making the partitions part of the connector itself or delegated to a user input so the end user configuring a Source based on the connector can control which partitions to fetch. When using "user input" mode for the partition values, create a user input of type array and reference it as the value using the [placeholder](/connector-development/config-based/understanding-the-yaml-file/reference#variables) value using `{{ config[''] }}` -* The "Current partition value identifier" can be freely choosen and is the identifier of the variable holding the current partition value. It can for example be used in the path of the stream using the `{{ stream_partition. }}` syntax. -* The "Inject partition value into outgoing HTTP request" option allows you to configure how to add the current partition value to the requests +### Parameterized Requests + +To configure static partitioning, enable the `Parameterized Requests` component. The following fields have to be configured: +* The "Parameter Values" can either be set to a list of strings, making the partitions part of the connector itself, or delegated to a user input so the end user configuring a Source based on the connector can control which partitions to fetch. When using "user input" mode for the parameter values, create a user input of type array and reference it as the value using the [placeholder](/connector-development/config-based/understanding-the-yaml-file/reference#variables) value using `{{ config[''] }}` +* The "Current Parameter Value Identifier" can be freely choosen and is the identifier of the variable holding the current parameter value. It can for example be used in the path of the stream using the `{{ stream_partition. }}` syntax. +* The "Inject Parameter Value into outgoing HTTP Request" option allows you to configure how to add the current parameter value to the requests #### Example -To enable static partition routing defined as part of the connector for the [SurveySparrow API](https://developers.surveysparrow.com/rest-apis/response#getV3Responses) responses, the list partition router needs to be configured as following: -* "Partition values" are set to the list of survey ids to fetch -* "Current partition value identifier" is set to `survey` (this is not used for this example) -* "Inject partition value into outgoing HTTP request" is set to `request_parameter` for the field name `survey_id` +To enable static partitioning defined as part of the connector for the [SurveySparrow API](https://developers.surveysparrow.com/rest-apis/response#getV3Responses) responses, the Parameterized Requests component needs to be configured as following: +* "Parameter Values" are set to the list of survey ids to fetch +* "Current Parameter Value Identifier" is set to `survey` (this is not used for this example) +* "Inject Parameter Value into outgoing HTTP Request" is set to `request_parameter` for the field name `survey_id` -When partition values were set to `123`, `456` and `789`, the following requests will be executed: +When parameter values were set to `123`, `456` and `789`, the following requests will be executed: ``` curl -X GET https://api.surveysparrow.com/v3/responses?survey_id=123 curl -X GET https://api.surveysparrow.com/v3/responses?survey_id=456 @@ -40,39 +42,39 @@ curl -X GET https://api.surveysparrow.com/v3/responses?survey_id=789 ``` To enable user-configurable static partitions for the [Woocommerce API](https://woocommerce.github.io/woocommerce-rest-api-docs/#order-notes) order notes, the configuration would look like this: -* Set "Partition values" to "User input" -* In the "Value" input, click the blue user icon and create a new user input +* Set "Parameter Values" to "User input" +* In the "Value" input, click the user icon and create a new user input * Name it `Order IDs`, set type to `array` and click create -* Set "Current partition value identifier" to `order` -* "Inject partition value into outgoing HTTP request" is disabled, because the order id needs to be injected into the path +* Set "Current Parameter Value Identifier" to `order` +* "Inject Parameter Value into outgoing HTTP Request" is disabled, because the order id needs to be injected into the path * In the general section of the stream configuration, the "URL Path" is set to `/orders/{{ stream_partition.order }}/notes` - + -When order IDs were set to `123`, `456` and `789` in the testing values, the following requests will be executed: +When Order IDs were set to `123`, `456` and `789` in the testing values, the following requests will be executed: ``` curl -X GET https://example.com/wp-json/wc/v3/orders/123/notes curl -X GET https://example.com/wp-json/wc/v3/orders/456/notes curl -X GET https://example.com/wp-json/wc/v3/orders/789/notes ``` -### Substream partition router +### Parent Stream -To fetch the list of partitions (in this example surveys or orders) from the API itself, the "Substream" partition router has to be used. It allows you to select another stream of the same connector to serve as the source for partitions to fetch. Each record of the parent stream is used as a partition for the current stream. +To fetch the list of partitions (in this example surveys or orders) from the API itself, the "Parent Stream" component has to be used. It allows you to select another stream of the same connector to serve as the source for partitions to fetch. Each record of the parent stream is used as a partition for the current stream. -The following fields have to be configured to use the substream partition router: -* The "Parent stream" defines the records of which stream should be used as partitions -* The "Parent key" is the property on the parent stream record that should become the partition value (in most cases this is some form of id) -* The "Current partition value identifier" can be freely choosen and is the identifier of the variable holding the current partition value. It can for example be used in the path of the stream using the `{{ stream_partition. }}` [interpolation placeholder](/connector-development/config-based/understanding-the-yaml-file/reference#variables). +The following fields have to be configured to use the Parent Stream component: +* The "Parent Stream" defines the records of which stream should be used as partitions +* The "Parent Key" is the property on the parent stream record that should become the partition value (in most cases this is some form of id) +* The "Current Parent Key Value Identifier" can be freely choosen and is the identifier of the variable holding the current partition value. It can for example be used in the path of the stream using the `{{ stream_partition. }}` [interpolation placeholder](/connector-development/config-based/understanding-the-yaml-file/reference#variables). #### Example -To enable dynamic partition routing for the [Woocommerce API](https://woocommerce.github.io/woocommerce-rest-api-docs/#order-notes) order notes, first an "orders" stream needs to be configured for the `/orders` endpoint to fetch a list of orders. Once this is done, the partition router for responses has be configured like this: -* "Parent key" is set to `id` -* "Current partition value identifier" is set to `order` +To enable dynamic partitioning for the [Woocommerce API](https://woocommerce.github.io/woocommerce-rest-api-docs/#order-notes) order notes, first an orders stream needs to be configured for the `/orders` endpoint to fetch a list of orders. Once this is done, the Parent Stream component for the responses stream has be configured like this: +* "Parent Key" is set to `id` +* "Current Parent Key Value Identifier" is set to `order` * In the general section of the stream configuration, the "URL Path" is set to `/orders/{{ stream_partition.order }}/notes` - + When triggering a sync, the connector will first fetch all records of the orders stream. The records will look like this: ``` @@ -90,9 +92,9 @@ curl -X GET https://example.com/wp-json/wc/v3/orders/789/notes ## Multiple partition routers -It is possible to configure multiple partition routers on a single stream - if this is the case, all possible combinations of partition values are requested separately. +It is possible to configure multiple partitioning mechanisms on a single stream - if this is the case, all possible combinations of partition values are requested separately. -For example, the [Google Pagespeed API](https://developers.google.com/speed/docs/insights/v5/reference/pagespeedapi/runpagespeed) allows to specify the URL and the "strategy" to run an analysis for. To allow a user to trigger an analysis for multiple URLs and strategies at the same time, two list partition routers can be used (one injecting the partition value into the `url` parameter, one injecting it into the `strategy` parameter). +For example, the [Google Pagespeed API](https://developers.google.com/speed/docs/insights/v5/reference/pagespeedapi/runpagespeed) allows to specify the URL and the "strategy" to run an analysis for. To allow a user to trigger an analysis for multiple URLs and strategies at the same time, two Parameterized Request lists can be used (one injecting the parameter value into the `url` parameter, one injecting it into the `strategy` parameter). If a user configures the URLs `example.com` and `example.org` and the strategies `desktop` and `mobile`, then the following requests will be triggered ``` @@ -112,19 +114,20 @@ For example when fetching the order notes via the [Woocommerce API](https://wooc ``` However the order id can be added by taking the following steps: -* Making sure the "Current partition value identifier" is set to `order` +* Making sure the "Current Parameter Value Identifier" is set to `order` * Add an "Add field" transformation with "Path" `order_id` and "Value" `{{ stream_partition.order }}` Using this configuration, the notes record looks like this: ``` { "id": 999, "author": "Jon Doe", "note": "Great product!", "order_id": 123 } ``` + ## Custom parameter injection -Using the "Inject partition value into outgoing HTTP request" option in the partitioning form works for most cases, but sometimes the API has special requirements that can't be handled this way: +Using the "Inject Parameter / Parent Key Value into outgoing HTTP Request" option in the Parameterized Requests and Parent Stream components works for most cases, but sometimes the API has special requirements that can't be handled this way: * The API requires to add a prefix or a suffix to the actual value * Multiple values need to be put together in a single parameter * The value needs to be injected into the URL path * Some conditional logic needs to be applied -To handle these cases, disable injection in the partitioning form and use the generic parameter section at the bottom of the stream configuration form to freely configure query parameters, headers and properties of the JSON body, by using jinja expressions and [available variables](/connector-development/config-based/understanding-the-yaml-file/reference/#/variables). You can also use these variables (like `stream_partition`) as part of the URL path as shown in the Woocommerce example above. +To handle these cases, disable injection in the component and use the generic parameter section at the bottom of the stream configuration form to freely configure query parameters, headers and properties of the JSON body, by using jinja expressions and [available variables](/connector-development/config-based/understanding-the-yaml-file/reference/#/variables). You can also use these variables (like `stream_partition`) as part of the URL path as shown in the Woocommerce example above. From 0e166ebd2412acff736de601483c489d37d5e803 Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Wed, 15 Nov 2023 11:32:48 -0800 Subject: [PATCH 065/124] Async Destinations - Make debug messages less scary (#32526) --- airbyte-cdk/java/airbyte-cdk/README.md | 89 ++++++++++--------- .../destination_async/FlushWorkers.java | 6 +- .../buffers/BufferManager.java | 28 +++--- .../src/main/resources/version.properties | 2 +- .../destination-bigquery/build.gradle | 2 +- .../destination-bigquery/metadata.yaml | 2 +- .../destination-snowflake/build.gradle | 2 +- .../destination-snowflake/metadata.yaml | 2 +- docs/integrations/destinations/bigquery.md | 1 + docs/integrations/destinations/snowflake.md | 33 +++---- 10 files changed, 84 insertions(+), 83 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md index 870b06e675e3..26b1f6dc8378 100644 --- a/airbyte-cdk/java/airbyte-cdk/README.md +++ b/airbyte-cdk/java/airbyte-cdk/README.md @@ -2,21 +2,21 @@ This page will walk through the process of developing with the Java CDK. -* [Developing with the Java CDK](#developing-with-the-java-cdk) - * [Intro to the Java CDK](#intro-to-the-java-cdk) - * [What is included in the Java CDK?](#what-is-included-in-the-java-cdk) - * [How is the CDK published?](#how-is-the-cdk-published) - * [Using the Java CDK](#using-the-java-cdk) - * [Building the CDK](#building-the-cdk) - * [Bumping the CDK version](#bumping-the-cdk-version) - * [Publishing the CDK](#publishing-the-cdk) - * [Developing Connectors with the Java CDK](#developing-connectors-with-the-java-cdk) - * [Referencing the CDK from Java connectors](#referencing-the-cdk-from-java-connectors) - * [Developing a connector alongside the CDK](#developing-a-connector-alongside-the-cdk) - * [Developing a connector against a pinned CDK version](#developing-a-connector-against-a-pinned-cdk-version) - * [Common Debugging Tips](#common-debugging-tips) - * [Changelog](#changelog) - * [Java CDK](#java-cdk) +- [Developing with the Java CDK](#developing-with-the-java-cdk) + - [Intro to the Java CDK](#intro-to-the-java-cdk) + - [What is included in the Java CDK?](#what-is-included-in-the-java-cdk) + - [How is the CDK published?](#how-is-the-cdk-published) + - [Using the Java CDK](#using-the-java-cdk) + - [Building the CDK](#building-the-cdk) + - [Bumping the CDK version](#bumping-the-cdk-version) + - [Publishing the CDK](#publishing-the-cdk) + - [Developing Connectors with the Java CDK](#developing-connectors-with-the-java-cdk) + - [Referencing the CDK from Java connectors](#referencing-the-cdk-from-java-connectors) + - [Developing a connector alongside the CDK](#developing-a-connector-alongside-the-cdk) + - [Developing a connector against a pinned CDK version](#developing-a-connector-against-a-pinned-cdk-version) + - [Common Debugging Tips](#common-debugging-tips) + - [Changelog](#changelog) + - [Java CDK](#java-cdk) ## Intro to the Java CDK @@ -127,7 +127,7 @@ Once you are done developing and testing your CDK changes: 2. After publishing the CDK, update the `useLocalCdk` setting by running `./gradlew :airbyte-integrations:connectors::disableLocalCdkRefs`. to automatically revert `useLocalCdk` to `false`. 3. You can optionally run `./gradlew :airbyte-integrations:connectors::assertNotUsingLocalCdk` to ensure that the project is not using a local CDK reference. -_Note: You can also use `./gradlew assertNotUsingLocalCdk` or `./gradlew disableLocalCdkRefs` to run these tasks on **all** connectors simultaneously._ +_Note: You can also use `./gradlew assertNotUsingLocalCdk` or `./gradlew disableLocalCdkRefs` to run these tasks on **all** connectors simultaneously._ ### Developing a connector against a pinned CDK version @@ -154,31 +154,32 @@ MavenLocal debugging steps: ### Java CDK -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.4.10 | 2023-11-13 | [\#32285](https://github.com/airbytehq/airbyte/pull/32285) | Fix UUID codec ordering for MongoDB connector | -| 0.4.9 | 2023-11-13 | [\#32468](https://github.com/airbytehq/airbyte/pull/32468) | Further error grouping improvements for DV2 connectors | -| 0.4.8 | 2023-11-09 | [\#32377](https://github.com/airbytehq/airbyte/pull/32377) | source-postgres tests: skip dropping database | -| 0.4.7 | 2023-11-08 | [\#31856](https://github.com/airbytehq/airbyte/pull/31856) | source-postgres: support for inifinty date and timestamps | -| 0.4.5 | 2023-11-07 | [\#32112](https://github.com/airbytehq/airbyte/pull/32112) | Async destinations framework: Allow configuring the queue flush threshold | -| 0.4.4 | 2023-11-06 | [\#32119](https://github.com/airbytehq/airbyte/pull/32119) | Add STANDARD UUID codec to MongoDB debezium handler | -| 0.4.2 | 2023-11-06 | [\#32190](https://github.com/airbytehq/airbyte/pull/32190) | Improve error deinterpolation | -| 0.4.1 | 2023-11-02 | [\#32192](https://github.com/airbytehq/airbyte/pull/32192) | Add 's3-destinations' CDK module. | -| 0.4.0 | 2023-11-02 | [\#32050](https://github.com/airbytehq/airbyte/pull/32050) | Fix compiler warnings. | -| 0.3.0 | 2023-11-02 | [\#31983](https://github.com/airbytehq/airbyte/pull/31983) | Add deinterpolation feature to AirbyteExceptionHandler. | -| 0.2.4 | 2023-10-31 | [\#31807](https://github.com/airbytehq/airbyte/pull/31807) | Handle case of debezium update and delete of records in mongodb. | -| 0.2.3 | 2023-10-31 | [\#32022](https://github.com/airbytehq/airbyte/pull/32022) | Update Debezium version from 2.20 -> 2.4.0. | -| 0.2.2 | 2023-10-31 | [\#31976](https://github.com/airbytehq/airbyte/pull/31976) | Debezium tweaks to make tests run faster. | -| 0.2.0 | 2023-10-30 | [\#31960](https://github.com/airbytehq/airbyte/pull/31960) | Hoist top-level gradle subprojects into CDK. | -| 0.1.12 | 2023-10-24 | [\#31674](https://github.com/airbytehq/airbyte/pull/31674) | Fail sync when Debezium does not shut down properly. | -| 0.1.11 | 2023-10-18 | [\#31486](https://github.com/airbytehq/airbyte/pull/31486) | Update constants in AdaptiveSourceRunner. | -| 0.1.9 | 2023-10-12 | [\#31309](https://github.com/airbytehq/airbyte/pull/31309) | Use toPlainString() when handling BigDecimals in PostgresConverter | -| 0.1.8 | 2023-10-11 | [\#31322](https://github.com/airbytehq/airbyte/pull/31322) | Cap log line length to 32KB to prevent loss of records | -| 0.1.7 | 2023-10-10 | [\#31194](https://github.com/airbytehq/airbyte/pull/31194) | Deallocate unused per stream buffer memory when empty | -| 0.1.6 | 2023-10-10 | [\#31083](https://github.com/airbytehq/airbyte/pull/31083) | Fix precision of numeric values in async destinations | -| 0.1.5 | 2023-10-09 | [\#31196](https://github.com/airbytehq/airbyte/pull/31196) | Update typo in CDK (CDN_LSN -> CDC_LSN) | -| 0.1.4 | 2023-10-06 | [\#31139](https://github.com/airbytehq/airbyte/pull/31139) | Reduce async buffer | -| 0.1.1 | 2023-09-28 | [\#30835](https://github.com/airbytehq/airbyte/pull/30835) | JDBC destinations now avoid staging area name collisions by using the raw table name as the stage name. (previously we used the stream name as the stage name) | -| 0.1.0 | 2023-09-27 | [\#30445](https://github.com/airbytehq/airbyte/pull/30445) | First launch, including shared classes for all connectors. | -| 0.0.2 | 2023-08-21 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Version bump only (no other changes). | -| 0.0.1 | 2023-08-08 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Initial release for testing. | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :--------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 0.4.11 | 2023-11-14 | [\#32526](https://github.com/airbytehq/airbyte/pull/32526) | Clean up memory manager logs. | +| 0.4.10 | 2023-11-13 | [\#32285](https://github.com/airbytehq/airbyte/pull/32285) | Fix UUID codec ordering for MongoDB connector | +| 0.4.9 | 2023-11-13 | [\#32468](https://github.com/airbytehq/airbyte/pull/32468) | Further error grouping improvements for DV2 connectors | +| 0.4.8 | 2023-11-09 | [\#32377](https://github.com/airbytehq/airbyte/pull/32377) | source-postgres tests: skip dropping database | +| 0.4.7 | 2023-11-08 | [\#31856](https://github.com/airbytehq/airbyte/pull/31856) | source-postgres: support for inifinty date and timestamps | +| 0.4.5 | 2023-11-07 | [\#32112](https://github.com/airbytehq/airbyte/pull/32112) | Async destinations framework: Allow configuring the queue flush threshold | +| 0.4.4 | 2023-11-06 | [\#32119](https://github.com/airbytehq/airbyte/pull/32119) | Add STANDARD UUID codec to MongoDB debezium handler | +| 0.4.2 | 2023-11-06 | [\#32190](https://github.com/airbytehq/airbyte/pull/32190) | Improve error deinterpolation | +| 0.4.1 | 2023-11-02 | [\#32192](https://github.com/airbytehq/airbyte/pull/32192) | Add 's3-destinations' CDK module. | +| 0.4.0 | 2023-11-02 | [\#32050](https://github.com/airbytehq/airbyte/pull/32050) | Fix compiler warnings. | +| 0.3.0 | 2023-11-02 | [\#31983](https://github.com/airbytehq/airbyte/pull/31983) | Add deinterpolation feature to AirbyteExceptionHandler. | +| 0.2.4 | 2023-10-31 | [\#31807](https://github.com/airbytehq/airbyte/pull/31807) | Handle case of debezium update and delete of records in mongodb. | +| 0.2.3 | 2023-10-31 | [\#32022](https://github.com/airbytehq/airbyte/pull/32022) | Update Debezium version from 2.20 -> 2.4.0. | +| 0.2.2 | 2023-10-31 | [\#31976](https://github.com/airbytehq/airbyte/pull/31976) | Debezium tweaks to make tests run faster. | +| 0.2.0 | 2023-10-30 | [\#31960](https://github.com/airbytehq/airbyte/pull/31960) | Hoist top-level gradle subprojects into CDK. | +| 0.1.12 | 2023-10-24 | [\#31674](https://github.com/airbytehq/airbyte/pull/31674) | Fail sync when Debezium does not shut down properly. | +| 0.1.11 | 2023-10-18 | [\#31486](https://github.com/airbytehq/airbyte/pull/31486) | Update constants in AdaptiveSourceRunner. | +| 0.1.9 | 2023-10-12 | [\#31309](https://github.com/airbytehq/airbyte/pull/31309) | Use toPlainString() when handling BigDecimals in PostgresConverter | +| 0.1.8 | 2023-10-11 | [\#31322](https://github.com/airbytehq/airbyte/pull/31322) | Cap log line length to 32KB to prevent loss of records | +| 0.1.7 | 2023-10-10 | [\#31194](https://github.com/airbytehq/airbyte/pull/31194) | Deallocate unused per stream buffer memory when empty | +| 0.1.6 | 2023-10-10 | [\#31083](https://github.com/airbytehq/airbyte/pull/31083) | Fix precision of numeric values in async destinations | +| 0.1.5 | 2023-10-09 | [\#31196](https://github.com/airbytehq/airbyte/pull/31196) | Update typo in CDK (CDN_LSN -> CDC_LSN) | +| 0.1.4 | 2023-10-06 | [\#31139](https://github.com/airbytehq/airbyte/pull/31139) | Reduce async buffer | +| 0.1.1 | 2023-09-28 | [\#30835](https://github.com/airbytehq/airbyte/pull/30835) | JDBC destinations now avoid staging area name collisions by using the raw table name as the stage name. (previously we used the stream name as the stage name) | +| 0.1.0 | 2023-09-27 | [\#30445](https://github.com/airbytehq/airbyte/pull/30445) | First launch, including shared classes for all connectors. | +| 0.0.2 | 2023-08-21 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Version bump only (no other changes). | +| 0.0.1 | 2023-08-08 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Initial release for testing. | diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java index 74231cbe77a9..b02ebdf131b1 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java @@ -51,7 +51,7 @@ public class FlushWorkers implements AutoCloseable { private static final long SUPERVISOR_INITIAL_DELAY_SECS = 0L; private static final long SUPERVISOR_PERIOD_SECS = 1L; private static final long DEBUG_INITIAL_DELAY_SECS = 0L; - private static final long DEBUG_PERIOD_SECS = 10L; + private static final long DEBUG_PERIOD_SECS = 60L; private final ScheduledExecutorService supervisorThread; private final ExecutorService workerPool; @@ -135,14 +135,14 @@ private void retrieveWork() { } private void printWorkerInfo() { - final var workerInfo = new StringBuilder().append("WORKER INFO").append(System.lineSeparator()); + final var workerInfo = new StringBuilder().append("[ASYNC WORKER INFO] "); final ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) workerPool; final int queueSize = threadPoolExecutor.getQueue().size(); final int activeCount = threadPoolExecutor.getActiveCount(); - workerInfo.append(String.format(" Pool queue size: %d, Active threads: %d", queueSize, activeCount)); + workerInfo.append(String.format("Pool queue size: %d, Active threads: %d", queueSize, activeCount)); log.info(workerInfo.toString()); } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferManager.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferManager.java index 9d8f2104a044..1d824a2b14c0 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferManager.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferManager.java @@ -10,6 +10,7 @@ import io.airbyte.cdk.integrations.destination_async.GlobalMemoryManager; import io.airbyte.cdk.integrations.destination_async.state.GlobalAsyncStateManager; import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.ArrayList; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executors; @@ -33,6 +34,7 @@ public class BufferManager { private final GlobalAsyncStateManager stateManager; private final ScheduledExecutorService debugLoop; + private static final long DEBUG_PERIOD_SECS = 60L; public static final double MEMORY_LIMIT_RATIO = 0.7; @@ -55,7 +57,7 @@ public BufferManager(final long memoryLimit) { bufferEnqueue = new BufferEnqueue(memoryManager, buffers, stateManager); bufferDequeue = new BufferDequeue(memoryManager, buffers, stateManager); debugLoop = Executors.newSingleThreadScheduledExecutor(); - debugLoop.scheduleAtFixedRate(this::printQueueInfo, 0, 10, TimeUnit.SECONDS); + debugLoop.scheduleAtFixedRate(this::printQueueInfo, 0, DEBUG_PERIOD_SECS, TimeUnit.SECONDS); } public GlobalAsyncStateManager getStateManager() { @@ -91,31 +93,27 @@ public void close() throws Exception { } private void printQueueInfo() { - final var queueInfo = new StringBuilder().append("START OF QUEUE INFO").append(System.lineSeparator()) - .append("This represents an estimation of the size of the elements contain in the in memory buffer.") - .append(System.lineSeparator()); + final var queueInfo = new StringBuilder().append("[ASYNC QUEUE INFO] "); + final ArrayList messages = new ArrayList<>(); - queueInfo - .append(String.format(" Global Mem Manager -- max: %s, allocated: %s (%s MB), %% used: %s", + messages + .add(String.format("Global: max: %s, allocated: %s (%s MB), %% used: %s", AirbyteFileUtils.byteCountToDisplaySize(memoryManager.getMaxMemoryBytes()), AirbyteFileUtils.byteCountToDisplaySize(memoryManager.getCurrentMemoryBytes()), (double) memoryManager.getCurrentMemoryBytes() / 1024 / 1024, - (double) memoryManager.getCurrentMemoryBytes() / memoryManager.getMaxMemoryBytes())) - .append(System.lineSeparator()); + (double) memoryManager.getCurrentMemoryBytes() / memoryManager.getMaxMemoryBytes())); for (final var entry : buffers.entrySet()) { final var queue = entry.getValue(); - queueInfo.append( - String.format(" Queue name: %s, num records: %d, num bytes: %s, allocated bytes: %s", + messages.add( + String.format("Queue `%s`, num records: %d, num bytes: %s, allocated bytes: %s", entry.getKey().getName(), queue.size(), AirbyteFileUtils.byteCountToDisplaySize(queue.getCurrentMemoryUsage()), - AirbyteFileUtils.byteCountToDisplaySize(queue.getMaxMemoryUsage()))) - .append(System.lineSeparator()); + AirbyteFileUtils.byteCountToDisplaySize(queue.getMaxMemoryUsage()))); } - queueInfo.append(stateManager.getMemoryUsageMessage()) - .append(System.lineSeparator()); + messages.add(stateManager.getMemoryUsageMessage()); - queueInfo.append("END OF QUEUE INFO"); + queueInfo.append(String.join(" | ", messages)); log.info(queueInfo.toString()); } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties index 3cf6558a4b17..6c39f216d22b 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties @@ -1 +1 @@ -version=0.4.10 +version=0.4.11 diff --git a/airbyte-integrations/connectors/destination-bigquery/build.gradle b/airbyte-integrations/connectors/destination-bigquery/build.gradle index 33dab9b167e4..78277e3e53f9 100644 --- a/airbyte-integrations/connectors/destination-bigquery/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.4.9' + cdkVersionRequired = '0.4.11' features = ['db-destinations', 's3-destinations'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml index 2891145670f8..7cdb95a0bd77 100644 --- a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml +++ b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 - dockerImageTag: 2.3.15 + dockerImageTag: 2.3.16 dockerRepository: airbyte/destination-bigquery githubIssueLabel: destination-bigquery icon: bigquery.svg diff --git a/airbyte-integrations/connectors/destination-snowflake/build.gradle b/airbyte-integrations/connectors/destination-snowflake/build.gradle index ee416b90798a..0ac5603a6f8a 100644 --- a/airbyte-integrations/connectors/destination-snowflake/build.gradle +++ b/airbyte-integrations/connectors/destination-snowflake/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.4.3' + cdkVersionRequired = '0.4.11' features = ['db-destinations', 's3-destinations'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml index 83487bebfd1c..f37244686991 100644 --- a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml +++ b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 424892c4-daac-4491-b35d-c6688ba547ba - dockerImageTag: 3.4.10 + dockerImageTag: 3.4.11 dockerRepository: airbyte/destination-snowflake githubIssueLabel: destination-snowflake icon: snowflake.svg diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index 4ece99595e00..4be09c13e851 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -140,6 +140,7 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 2.3.16 | 2023-11-14 | [\#32526](https://github.com/airbytehq/airbyte/pull/32526) | Clean up memory manager logs. | | 2.3.15 | 2023-11-13 | [\#32468](https://github.com/airbytehq/airbyte/pull/32468) | Further error grouping enhancments | | 2.3.14 | 2023-11-06 | [\#32234](https://github.com/airbytehq/airbyte/pull/32234) | Remove unused config option. | | 2.3.13 | 2023-11-08 | [\#32125](https://github.com/airbytehq/airbyte/pull/32125) | fix compiler warnings | diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index 3216a67bc06b..6680040989e0 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -156,22 +156,22 @@ Navigate to the Airbyte UI to set up Snowflake as a destination. You can authent ### Login and Password -| Field | Description | -|-------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [Host](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html) | The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com). Example: `accountname.us-east-2.aws.snowflakecomputing.com` | -| [Role](https://docs.snowflake.com/en/user-guide/security-access-control-overview.html#roles) | The role you created in Step 1 for Airbyte to access Snowflake. Example: `AIRBYTE_ROLE` | -| [Warehouse](https://docs.snowflake.com/en/user-guide/warehouses-overview.html#overview-of-warehouses) | The warehouse you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_WAREHOUSE` | -| [Database](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl) | The database you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_DATABASE` | -| [Schema](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl) | The default schema used as the target schema for all statements issued from the connection that do not explicitly specify a schema name. | -| Username | The username you created in Step 1 to allow Airbyte to access the database. Example: `AIRBYTE_USER` | -| Password | The password associated with the username. | -| [JDBC URL Params](https://docs.snowflake.com/en/user-guide/jdbc-parameters.html) (Optional) | Additional properties to pass to the JDBC URL string when connecting to the database formatted as `key=value` pairs separated by the symbol `&`. Example: `key1=value1&key2=value2&key3=value3` | -| Disable Final Tables (Optional) | Disables writing final Typed tables See [output schema](#output-schema). WARNING! The data format in _airbyte_data is likely stable but there are no guarantees that other metadata columns will remain the same in future versions | +| Field | Description | +| ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| [Host](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html) | The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com). Example: `accountname.us-east-2.aws.snowflakecomputing.com` | +| [Role](https://docs.snowflake.com/en/user-guide/security-access-control-overview.html#roles) | The role you created in Step 1 for Airbyte to access Snowflake. Example: `AIRBYTE_ROLE` | +| [Warehouse](https://docs.snowflake.com/en/user-guide/warehouses-overview.html#overview-of-warehouses) | The warehouse you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_WAREHOUSE` | +| [Database](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl) | The database you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_DATABASE` | +| [Schema](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl) | The default schema used as the target schema for all statements issued from the connection that do not explicitly specify a schema name. | +| Username | The username you created in Step 1 to allow Airbyte to access the database. Example: `AIRBYTE_USER` | +| Password | The password associated with the username. | +| [JDBC URL Params](https://docs.snowflake.com/en/user-guide/jdbc-parameters.html) (Optional) | Additional properties to pass to the JDBC URL string when connecting to the database formatted as `key=value` pairs separated by the symbol `&`. Example: `key1=value1&key2=value2&key3=value3` | +| Disable Final Tables (Optional) | Disables writing final Typed tables See [output schema](#output-schema). WARNING! The data format in \_airbyte_data is likely stable but there are no guarantees that other metadata columns will remain the same in future versions | ### OAuth 2.0 | Field | Description | -|:------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| :---------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | [Host](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html) | The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com). Example: `accountname.us-east-2.aws.snowflakecomputing.com` | | [Role](https://docs.snowflake.com/en/user-guide/security-access-control-overview.html#roles) | The role you created in Step 1 for Airbyte to access Snowflake. Example: `AIRBYTE_ROLE` | | [Warehouse](https://docs.snowflake.com/en/user-guide/warehouses-overview.html#overview-of-warehouses) | The warehouse you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_WAREHOUSE` | @@ -208,7 +208,7 @@ Navigate to the Airbyte UI to set up Snowflake as a destination. You can authent To use AWS S3 as the cloud storage, enter the information for the S3 bucket you created in Step 2: | Field | Description | -|--------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | S3 Bucket Name | The name of the staging S3 bucket (Example: `airbyte.staging`). Airbyte will write files to this bucket and read them via statements on Snowflake. | | S3 Bucket Region | The S3 staging bucket region used. | | S3 Key Id \* | The Access Key ID granting access to the S3 staging bucket. Airbyte requires Read and Write permissions for the bucket. | @@ -221,7 +221,7 @@ To use AWS S3 as the cloud storage, enter the information for the S3 bucket you To use a Google Cloud Storage bucket, enter the information for the bucket you created in Step 2: | Field | Description | -|--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| ------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | GCP Project ID | The name of the GCP project ID for your credentials. (Example: `my-project`) | | GCP Bucket Name | The name of the staging bucket. Airbyte will write files to this bucket and read them via statements on Snowflake. (Example: `airbyte-staging`) | | Google Application Credentials | The contents of the JSON key file that has read/write permissions to the staging GCS bucket. You will separately need to grant bucket access to your Snowflake GCP service account. See the [Google Cloud docs](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys) for more information on how to generate a JSON key for your service account. | @@ -233,7 +233,7 @@ Airbyte outputs each stream into its own raw table in `airbyte_internal` schema ### Raw Table schema | Airbyte field | Description | Column type | -|------------------------|--------------------------------------------------------------------|--------------------------| +| ---------------------- | ------------------------------------------------------------------ | ------------------------ | | \_airbyte_raw_id | A UUID assigned to each processed event | VARCHAR | | \_airbyte_extracted_at | A timestamp for when the event was pulled from the data source | TIMESTAMP WITH TIME ZONE | | \_airbyte_loaded_at | Timestamp to indicate when the record was loaded into Typed tables | TIMESTAMP WITH TIME ZONE | @@ -276,7 +276,8 @@ Otherwise, make sure to grant the role the required permissions in the desired n ## Changelog | Version | Date | Pull Request | Subject | -|:----------------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| :-------------- | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 3.4.11 | 2023-11-14 | [\#32526](https://github.com/airbytehq/airbyte/pull/32526) | Clean up memory manager logs. | | 3.4.10 | 2023-11-08 | [\#32125](https://github.com/airbytehq/airbyte/pull/32125) | Fix compilation warnings. | | 3.4.9 | 2023-11-06 | [\#32026](https://github.com/airbytehq/airbyte/pull/32026) | Add separate TRY_CAST transaction to reduce compute usage | | 3.4.8 | 2023-11-06 | [\#32190](https://github.com/airbytehq/airbyte/pull/32190) | Further improve error reporting | From c979f704902a35f0acb7cf8d12901f8eacb31421 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Wed, 15 Nov 2023 11:36:58 -0800 Subject: [PATCH 066/124] pass airbytestream instead of configured stream to as_airbyte_message (#32412) --- .../airbyte_cdk/sources/abstract_source.py | 20 +++------- .../manifest_declarative_source.py | 4 -- .../airbyte_cdk/utils/stream_status_utils.py | 6 +-- .../utils/test_stream_status_utils.py | 37 +++++++------------ 4 files changed, 21 insertions(+), 46 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py b/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py index 50d111ff6570..a444f8ab0609 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py @@ -111,14 +111,13 @@ def read( ) try: - self._apply_log_level_to_stream_logger(logger, stream_instance) timer.start_event(f"Syncing stream {configured_stream.stream.name}") stream_is_available, reason = stream_instance.check_availability(logger, self) if not stream_is_available: logger.warning(f"Skipped syncing stream '{stream_instance.name}' because it was unavailable. {reason}") continue logger.info(f"Marking stream {configured_stream.stream.name} as STARTED") - yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.STARTED) + yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.STARTED) yield from self._read_stream( logger=logger, stream_instance=stream_instance, @@ -127,15 +126,15 @@ def read( internal_config=internal_config, ) logger.info(f"Marking stream {configured_stream.stream.name} as STOPPED") - yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.COMPLETE) + yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.COMPLETE) except AirbyteTracedException as e: - yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.INCOMPLETE) + yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.INCOMPLETE) raise e except Exception as e: yield from self._emit_queued_messages() logger.exception(f"Encountered an exception while reading stream {configured_stream.stream.name}") logger.info(f"Marking stream {configured_stream.stream.name} as STOPPED") - yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.INCOMPLETE) + yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.INCOMPLETE) display_message = stream_instance.get_error_display_message(e) if display_message: raise AirbyteTracedException.from_exception(e, message=display_message) from e @@ -197,7 +196,7 @@ def _read_stream( if record_counter == 1: logger.info(f"Marking stream {stream_name} as RUNNING") # If we just read the first record of the stream, emit the transition to the RUNNING state - yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.RUNNING) + yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.RUNNING) yield from self._emit_queued_messages() yield record @@ -259,15 +258,6 @@ def _read_full_refresh( if internal_config.is_limit_reached(total_records_counter): return - @staticmethod - def _apply_log_level_to_stream_logger(logger: logging.Logger, stream_instance: Stream) -> None: - """ - Necessary because we use different loggers at the source and stream levels. We must - apply the source's log level to each stream's logger. - """ - if hasattr(logger, "level"): - stream_instance.logger.setLevel(logger.level) - def _get_message(self, record_data_or_message: Union[StreamData, AirbyteMessage], stream: Stream) -> AirbyteMessage: """ Converts the input to an AirbyteMessage if it is a StreamData. Returns the input as is if it is already an AirbyteMessage diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py index 7ca2986e3b61..b49ad2bb426b 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py @@ -98,10 +98,6 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: ) for stream_config in self._stream_configs(self._source_config) ] - - for stream in source_streams: - # make sure the log level is always applied to the stream's logger - self._apply_log_level_to_stream_logger(self.logger, stream) return source_streams def spec(self, logger: logging.Logger) -> ConnectorSpecification: diff --git a/airbyte-cdk/python/airbyte_cdk/utils/stream_status_utils.py b/airbyte-cdk/python/airbyte_cdk/utils/stream_status_utils.py index cd9e4527688d..bdd36acf9b34 100644 --- a/airbyte-cdk/python/airbyte_cdk/utils/stream_status_utils.py +++ b/airbyte-cdk/python/airbyte_cdk/utils/stream_status_utils.py @@ -7,17 +7,17 @@ from airbyte_cdk.models import ( AirbyteMessage, + AirbyteStream, AirbyteStreamStatus, AirbyteStreamStatusTraceMessage, AirbyteTraceMessage, - ConfiguredAirbyteStream, StreamDescriptor, TraceType, ) from airbyte_cdk.models import Type as MessageType -def as_airbyte_message(stream: ConfiguredAirbyteStream, current_status: AirbyteStreamStatus) -> AirbyteMessage: +def as_airbyte_message(stream: AirbyteStream, current_status: AirbyteStreamStatus) -> AirbyteMessage: """ Builds an AirbyteStreamStatusTraceMessage for the provided stream """ @@ -28,7 +28,7 @@ def as_airbyte_message(stream: ConfiguredAirbyteStream, current_status: AirbyteS type=TraceType.STREAM_STATUS, emitted_at=now_millis, stream_status=AirbyteStreamStatusTraceMessage( - stream_descriptor=StreamDescriptor(name=stream.stream.name, namespace=stream.stream.namespace), + stream_descriptor=StreamDescriptor(name=stream.name, namespace=stream.namespace), status=current_status, ), ) diff --git a/airbyte-cdk/python/unit_tests/utils/test_stream_status_utils.py b/airbyte-cdk/python/unit_tests/utils/test_stream_status_utils.py index 497ebce7ce6b..5d41ab7a5700 100644 --- a/airbyte-cdk/python/unit_tests/utils/test_stream_status_utils.py +++ b/airbyte-cdk/python/unit_tests/utils/test_stream_status_utils.py @@ -2,71 +2,60 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.models import ( - AirbyteMessage, - AirbyteStream, - AirbyteStreamStatus, - ConfiguredAirbyteStream, - DestinationSyncMode, - SyncMode, - TraceType, -) +from airbyte_cdk.models import AirbyteMessage, AirbyteStream, AirbyteStreamStatus, SyncMode, TraceType from airbyte_cdk.models import Type as MessageType from airbyte_cdk.utils.stream_status_utils import as_airbyte_message as stream_status_as_airbyte_message stream = AirbyteStream(name="name", namespace="namespace", json_schema={}, supported_sync_modes=[SyncMode.full_refresh]) -configured_stream = ConfiguredAirbyteStream( - stream=stream, sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.overwrite -) def test_started_as_message(): stream_status = AirbyteStreamStatus.STARTED - airbyte_message = stream_status_as_airbyte_message(configured_stream, stream_status) + airbyte_message = stream_status_as_airbyte_message(stream, stream_status) assert type(airbyte_message) == AirbyteMessage assert airbyte_message.type == MessageType.TRACE assert airbyte_message.trace.type == TraceType.STREAM_STATUS assert airbyte_message.trace.emitted_at > 0 - assert airbyte_message.trace.stream_status.stream_descriptor.name == configured_stream.stream.name - assert airbyte_message.trace.stream_status.stream_descriptor.namespace == configured_stream.stream.namespace + assert airbyte_message.trace.stream_status.stream_descriptor.name == stream.name + assert airbyte_message.trace.stream_status.stream_descriptor.namespace == stream.namespace assert airbyte_message.trace.stream_status.status == stream_status def test_running_as_message(): stream_status = AirbyteStreamStatus.RUNNING - airbyte_message = stream_status_as_airbyte_message(configured_stream, stream_status) + airbyte_message = stream_status_as_airbyte_message(stream, stream_status) assert type(airbyte_message) == AirbyteMessage assert airbyte_message.type == MessageType.TRACE assert airbyte_message.trace.type == TraceType.STREAM_STATUS assert airbyte_message.trace.emitted_at > 0 - assert airbyte_message.trace.stream_status.stream_descriptor.name == configured_stream.stream.name - assert airbyte_message.trace.stream_status.stream_descriptor.namespace == configured_stream.stream.namespace + assert airbyte_message.trace.stream_status.stream_descriptor.name == stream.name + assert airbyte_message.trace.stream_status.stream_descriptor.namespace == stream.namespace assert airbyte_message.trace.stream_status.status == stream_status def test_complete_as_message(): stream_status = AirbyteStreamStatus.COMPLETE - airbyte_message = stream_status_as_airbyte_message(configured_stream, stream_status) + airbyte_message = stream_status_as_airbyte_message(stream, stream_status) assert type(airbyte_message) == AirbyteMessage assert airbyte_message.type == MessageType.TRACE assert airbyte_message.trace.type == TraceType.STREAM_STATUS assert airbyte_message.trace.emitted_at > 0 - assert airbyte_message.trace.stream_status.stream_descriptor.name == configured_stream.stream.name - assert airbyte_message.trace.stream_status.stream_descriptor.namespace == configured_stream.stream.namespace + assert airbyte_message.trace.stream_status.stream_descriptor.name == stream.name + assert airbyte_message.trace.stream_status.stream_descriptor.namespace == stream.namespace assert airbyte_message.trace.stream_status.status == stream_status def test_incomplete_failed_as_message(): stream_status = AirbyteStreamStatus.INCOMPLETE - airbyte_message = stream_status_as_airbyte_message(configured_stream, stream_status) + airbyte_message = stream_status_as_airbyte_message(stream, stream_status) assert type(airbyte_message) == AirbyteMessage assert airbyte_message.type == MessageType.TRACE assert airbyte_message.trace.type == TraceType.STREAM_STATUS assert airbyte_message.trace.emitted_at > 0 - assert airbyte_message.trace.stream_status.stream_descriptor.name == configured_stream.stream.name - assert airbyte_message.trace.stream_status.stream_descriptor.namespace == configured_stream.stream.namespace + assert airbyte_message.trace.stream_status.stream_descriptor.name == stream.name + assert airbyte_message.trace.stream_status.stream_descriptor.namespace == stream.namespace assert airbyte_message.trace.stream_status.status == stream_status From 9a086d068b83be30ffeaa6a8e84cd1b23e56baac Mon Sep 17 00:00:00 2001 From: girarda Date: Wed, 15 Nov 2023 19:43:56 +0000 Subject: [PATCH 067/124] =?UTF-8?q?=F0=9F=A4=96=20Bump=20patch=20version?= =?UTF-8?q?=20of=20Python=20CDK?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-cdk/python/.bumpversion.cfg | 2 +- airbyte-cdk/python/CHANGELOG.md | 3 +++ airbyte-cdk/python/Dockerfile | 4 ++-- airbyte-cdk/python/setup.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/airbyte-cdk/python/.bumpversion.cfg b/airbyte-cdk/python/.bumpversion.cfg index e597dfab9070..08bf1ea465db 100644 --- a/airbyte-cdk/python/.bumpversion.cfg +++ b/airbyte-cdk/python/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.53.6 +current_version = 0.53.7 commit = False [bumpversion:file:setup.py] diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 68398eb18ee1..8778ed30577b 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.53.7 +low-code: fix debug logging when using --debug flag + ## 0.53.6 Increase maximum_attempts_to_acquire to avoid crashing in acquire_call diff --git a/airbyte-cdk/python/Dockerfile b/airbyte-cdk/python/Dockerfile index 4cca0b6e34a5..fdd41a9820ee 100644 --- a/airbyte-cdk/python/Dockerfile +++ b/airbyte-cdk/python/Dockerfile @@ -10,7 +10,7 @@ RUN apk --no-cache upgrade \ && apk --no-cache add tzdata build-base # install airbyte-cdk -RUN pip install --prefix=/install airbyte-cdk==0.53.6 +RUN pip install --prefix=/install airbyte-cdk==0.53.7 # build a clean environment FROM base @@ -32,5 +32,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] # needs to be the same as CDK -LABEL io.airbyte.version=0.53.6 +LABEL io.airbyte.version=0.53.7 LABEL io.airbyte.name=airbyte/source-declarative-manifest diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 14b770867eb8..9949399cfcca 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -36,7 +36,7 @@ name="airbyte-cdk", # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be # updated if our semver format changes such as using release candidate versions. - version="0.53.6", + version="0.53.7", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", From 603bc11709a02064ff272593a401dc1d6e76ecb8 Mon Sep 17 00:00:00 2001 From: Patrick Nilan Date: Wed, 15 Nov 2023 12:48:31 -0800 Subject: [PATCH 068/124] Docs: Updates Stripe Source Connector Docs (Removes inapp page, adds , adds Limitations & Troubleshooting) (#32370) --- docs/integrations/sources/stripe.inapp.md | 35 --------------- docs/integrations/sources/stripe.md | 54 +++++++++++++++++------ 2 files changed, 40 insertions(+), 49 deletions(-) delete mode 100644 docs/integrations/sources/stripe.inapp.md diff --git a/docs/integrations/sources/stripe.inapp.md b/docs/integrations/sources/stripe.inapp.md deleted file mode 100644 index 5988ba81922d..000000000000 --- a/docs/integrations/sources/stripe.inapp.md +++ /dev/null @@ -1,35 +0,0 @@ -## Prerequisites - -- Access to the Stripe account containing the data to replicate - -## Setup Guide - -:::note -To authenticate the Stripe connector, you need an API key with **Read** privileges for the data to replicate. For steps on obtaining and setting permissions for this key, refer to our [full Stripe documentation](https://docs.airbyte.com/integrations/sources/stripe#setup-guide). -::: - -1. For **Source name**, enter a name to help you identify this source. -2. For **Account ID**, enter your Stripe Account ID. This ID begins with `acct_`, and can be found in the top-right corner of your Stripe [account settings page](https://dashboard.stripe.com/settings/account). -3. For **Secret Key**, enter your Stripe API key, which can be found at your Stripe [API keys page](https://dashboard.stripe.com/apikeys). -4. For **Replication Start Date**, use the provided datepicker or enter a UTC date and time programmatically in the format `YYYY-MM-DDTHH:mm:ssZ`. The data added on and after this date will be replicated. -5. (Optional) For **Lookback Window**, you may specify a number of days from the present day to reread data. This allows the connector to retrieve data that might have been updated after its initial creation, and is useful for handling any post-transaction adjustments (such as tips, refunds, chargebacks, etc). - - - Leaving the **Lookback Window** at its default value of 0 means Airbyte will not re-export data after it has been synced. - - Setting the **Lookback Window** to 1 means Airbyte will re-export and capture any data changes within the last day. - - Setting the **Lookback Window** to 7 means Airbyte will re-export and capture any data changes within the last week. - -6. (Optional) For **Data Request Window**, you may specify the time window in days used by the connector when requesting data from the Stripe API. This window defines the span of time covered in each request, with larger values encompassing more days in a single request. Generally speaking, the lack of overhead from making fewer requests means a larger window is faster to sync. However, this also means the state of the sync will persist less frequently. If an issue occurs or the sync is interrupted, a larger window means more data will need to be resynced, potentially causing a delay in the overall process. - - For example, if you are replicating three years worth of data: - - - A **Data Request Window** of 365 days means Airbyte makes 3 requests, each for a year. This is generally faster but risks needing to resync up to a year's data if the sync is interrupted. - - A **Data Request Window** of 30 days means 36 requests, each for a month. This may be slower but minimizes the amount of data that needs to be resynced if an issue occurs. - - If you are unsure of which value to use, we recommend leaving this setting at its default value of 365 days. -7. Click **Set up source** and wait for the tests to complete. - -### Stripe API limitations - -- When syncing `events` data from Stripe, data is only [returned for the last 30 days](https://stripe.com/docs/api/events). Using the Full Refresh (Overwrite) sync from Airbyte will delete the events data older than 30 days from your target destination. Use an Append sync mode to ensure historical data is retained. - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Stripe](https://docs.airbyte.com/integrations/sources/stripe). diff --git a/docs/integrations/sources/stripe.md b/docs/integrations/sources/stripe.md index 30e1967bf371..06bea065cf1f 100644 --- a/docs/integrations/sources/stripe.md +++ b/docs/integrations/sources/stripe.md @@ -1,6 +1,10 @@ # Stripe -This page contains the setup guide and reference information for the Stripe source connector. + + +This page contains the setup guide and reference information for the [Stripe](https://stripe.com/) source connector. + + ## Prerequisites @@ -8,9 +12,11 @@ This page contains the setup guide and reference information for the Stripe sour ## Setup Guide +:::note To authenticate the Stripe connector, you need to use a Stripe API key. Although you may use an existing key, we recommend that you create a new restricted key specifically for Airbyte and grant it **Read** privileges only. We also recommend granting **Read** privileges to all available permissions, and configuring the specific data you would like to replicate within Airbyte. +::: -### Create a Stripe Secret Key +### Step 1: Set up Stripe 1. Log in to your [Stripe account](https://dashboard.stripe.com/login). 2. In the top navigation bar, click **Developers**. @@ -21,9 +27,9 @@ To authenticate the Stripe connector, you need to use a Stripe API key. Although For more information on Stripe API Keys, see the [Stripe documentation](https://stripe.com/docs/keys). -### Set up the Stripe source connector in Airbyte +### Step 2: Set up the Stripe source connector in Airbyte -1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account. +1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) account or your Airbyte Open Source account. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. 3. Find and select **Stripe** from the list of available sources. 4. For **Source name**, enter a name to help you identify this source. @@ -47,6 +53,8 @@ For more information on Stripe API Keys, see the [Stripe documentation](https:// 10. Click **Set up source** and wait for the tests to complete. + + ## Supported sync modes The Stripe source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): @@ -108,11 +116,33 @@ The Stripe source connector supports the following streams: - [Transfer Reversals](https://stripe.com/docs/api/transfer_reversals/list) - [Usage Records](https://stripe.com/docs/api/usage_records/subscription_item_summary_list) + + + + +### Data type mapping + +The [Stripe API](https://stripe.com/docs/api) uses the same [JSON Schema](https://json-schema.org/understanding-json-schema/reference/index.html) types that Airbyte uses internally \(`string`, `date-time`, `object`, `array`, `boolean`, `integer`, and `number`\), so no type conversions are performed for the Stripe connector. + +## Limitations & Troubleshooting + +
+ +Expand to see details about Stripe connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting + +The Stripe connector should not run into Stripe API limitations under normal usage. See Stripe [Rate limits](https://stripe.com/docs/rate-limits) documentation. [Create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. + :::warning **Stripe API Restriction on Events Data**: Access to the events endpoint is [guaranteed only for the last 30 days](https://stripe.com/docs/api/events) by Stripe. If you use the Full Refresh Overwrite sync, be aware that any events data older than 30 days will be **deleted** from your target destination and replaced with the data from the last 30 days only. Use an Append sync mode to ensure historical data is retained. Please be aware: this also means that any change older than 30 days will not be replicated using the incremental sync mode. If you want all your synced data to remain up to date, please set up your sync frequency to no more than 30 days. ::: +### Troubleshooting :::note Since the Stripe API does not allow querying objects which were updated since the last sync, the Stripe connector uses the Events API under the hood to implement incremental syncs and export data based on its update date. However, not all the entities are supported by the Events API, so the Stripe connector uses the `created` field or its analogue to query for new data in your Stripe account. These are the entities synced based on the date of creation: @@ -158,8 +188,7 @@ On the other hand, the following streams use the `updated` field value as a curs - `Top Ups` - `Transactions` - `Transfers` - - ::: +::: :::note The Stripe API also provides a way to implement incremental deletes for a limited number of streams: @@ -177,16 +206,11 @@ The Stripe API also provides a way to implement incremental deletes for a limite - `Subscriptions` Each record is marked with `is_deleted` flag when the appropriate event happens upstream. +::: - ::: - -### Data type mapping - -The [Stripe API](https://stripe.com/docs/api) uses the same [JSON Schema](https://json-schema.org/understanding-json-schema/reference/index.html) types that Airbyte uses internally \(`string`, `date-time`, `object`, `array`, `boolean`, `integer`, and `number`\), so no type conversions are performed for the Stripe connector. - -### Performance considerations +* Check out common troubleshooting issues for the Stripe source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). -The Stripe connector should not run into Stripe API limitations under normal usage. [Create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. +
## Changelog @@ -274,3 +298,5 @@ The Stripe connector should not run into Stripe API limitations under normal usa | 0.1.10 | 2021-05-28 | [3728](https://github.com/airbytehq/airbyte/pull/3728) | Update data types to be number instead of int | | 0.1.9 | 2021-05-13 | [3367](https://github.com/airbytehq/airbyte/pull/3367) | Add acceptance tests for connected accounts | | 0.1.8 | 2021-05-11 | [3566](https://github.com/airbytehq/airbyte/pull/3368) | Bump CDK connectors | + + \ No newline at end of file From 78326802c88e8f35e02d23c9456a59578ed33e48 Mon Sep 17 00:00:00 2001 From: Patrick Nilan Date: Wed, 15 Nov 2023 12:49:52 -0800 Subject: [PATCH 069/124] Docs: Updates Exchange Rates API Source Connector Docs (Removes inapp page, adds , adds Limitations & Troubleshooting) (#32369) --- .../sources/exchange-rates.inapp.md | 25 ------ docs/integrations/sources/exchange-rates.md | 79 ++++++++++++++----- 2 files changed, 61 insertions(+), 43 deletions(-) delete mode 100644 docs/integrations/sources/exchange-rates.inapp.md diff --git a/docs/integrations/sources/exchange-rates.inapp.md b/docs/integrations/sources/exchange-rates.inapp.md deleted file mode 100644 index a883ff6a7054..000000000000 --- a/docs/integrations/sources/exchange-rates.inapp.md +++ /dev/null @@ -1,25 +0,0 @@ -## Prerequisites - -- API Access Key - -In order to get a free `API Access Key` please go to [this](https://manage.exchangeratesapi.io/signup/free) page and enter the required information. After registration and login, you will see your `API Access Key`. You can also locate it [here](https://manage.exchangeratesapi.io/dashboard). - -If you have a `free` subscription plan, you will have two limitations to the plan: - -1. Limit of 1,000 API calls per month -2. You won't be able to specify the `base` parameter, meaning that you will be only be allowed to use the default base value which is EUR. - -## Setup guide -1. Enter a **Name** for your source. -2. Enter your **API key** as the `access_key` from the prerequisites. -3. Enter the **Start Date** in YYYY-MM-DD format. The data added on and after this date will be replicated. -4. (Optional) Enter a **base** currency. For those on the free plan, `EUR` is the only option available. If none are specified, `EUR` will be used. -5. Click **Set up source**. - -### Exchange Rates data output -- The sync will include one stream: `exchange_rates` -- Each record in the stream contains many fields: - - The date of the record - - One field for every supported [currency](https://www.ecb.europa.eu/stats/policy_and_exchange_rates/euro_reference_exchange_rates/html/index.en.html) which contain the value of that currency on that date. - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Exchange Rates](https://docs.airbyte.com/integrations/sources/exchange-rates/). diff --git a/docs/integrations/sources/exchange-rates.md b/docs/integrations/sources/exchange-rates.md index 6f1cc91705d5..635cea691220 100644 --- a/docs/integrations/sources/exchange-rates.md +++ b/docs/integrations/sources/exchange-rates.md @@ -1,25 +1,45 @@ # Exchange Rates API + + +This page contains the setup guide and reference information for the [Exchange Rates API](https://exchangeratesapi.io/) source connector. + + + ## Overview -The exchange rates integration is a toy integration to demonstrate how Airbyte works with a very simple source. +The Exchange Rates API integration is a toy integration to demonstrate how Airbyte works with a very simple source. -It pulls all its data from [https://apilayer.com/marketplace/exchangerates_data-api](https://apilayer.com/marketplace/exchangerates_data-api) +## Prerequisites -#### Output schema +- Exchange Rates API account +- API Access Key -It contains one stream: `exchange_rates` +## Setup Guide -Each record in the stream contains many fields: +### Step 1: Set up Exchange Rates API -- The date of the record -- One field for every supported [currency](https://www.ecb.europa.eu/stats/policy_and_exchange_rates/euro_reference_exchange_rates/html/index.en.html) which contain the value of that currency on that date. +1. Create an account with [Exchange Rates API](https://manage.exchangeratesapi.io/signup/). +2. Navigate to the [Exchange Rates API Dashboard](https://manage.exchangeratesapi.io/dashboard) to find your `API Access Key`. + +:::note +If you have a `free` subscription plan, you will have two limitations to the plan: + +1. Limit of 1,000 API calls per month +2. You won't be able to specify the `base` parameter, meaning that you will be only be allowed to use the default base value which is `EUR`. +::: -#### Data type mapping +### Step 2: Set up the Exchange Rates connector in Airbyte -Currencies are `number` and the date is a `string`. +1. Enter a **Name** for your source. +2. Enter your **API key** as the `access_key` from the prerequisites. +3. Enter the **Start Date** in YYYY-MM-DD format. The data added on and after this date will be replicated. +4. (Optional) Enter a **base** currency. For those on the free plan, `EUR` is the only option available. If none are specified, `EUR` will be used. +5. Click **Set up source**. -#### Features + + +## Supported sync modes | Feature | Supported? | | :------------------------ | :--------- | @@ -27,20 +47,41 @@ Currencies are `number` and the date is a `string`. | Incremental - Append Sync | Yes | | Namespaces | No | -### Getting started +## Supported streams + +It contains one stream: `exchange_rates` -### Requirements +Each record in the stream contains many fields: -- API Access Key +- The date of the record. +- One field for every supported [currency](https://www.ecb.europa.eu/stats/policy_and_exchange_rates/euro_reference_exchange_rates/html/index.en.html) which contain the value of that currency on that date. + +## Data type map + +| Field | Airbyte Type | +| :------------------------ | :----------- | +| Currency | `number` | +| Date | `string` | -### Setup guide +## Limitations & Troubleshooting -In order to get an `API Access Key` please go to [this](https://apilayer.com/signup) page and enter needed info. After registration and login you will see your `API Access Key`, also you may find it [here](https://apilayer.com/account). You will then need to subscribe your account to the [Exchange Rates Data API](https://apilayer.com/marketplace/exchangerates_data-api) +
+ +Expand to see details about Exchange Rates API connector limitations and troubleshooting. + -If you have `free` subscription plan \(you may check it [here](https://apilayer.com/marketplace/exchangerates_data-api)\) this means that you will have 2 limitations: +### Connector limitations -1. 1000 API calls per month. -2. You won't be able to specify the `base` parameter, meaning that you will be dealing only with default base value which is EUR. +#### Rate limiting + +The Exchange Rates API has rate limits that vary per pricing plan. The free plan is subject to rate limiting of 1,000 requests per month. Review the [Exchange Rates API Pricing Plans](https://exchangeratesapi.io/#pricing_plan) for more information. + +### Troubleshooting + +* With the free plan, you won't be able to specify the `base` parameter, meaning that you will be only be allowed to use the default base value which is `EUR`. +* Check out common troubleshooting issues for the Exchange Rates API source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
## Changelog @@ -58,3 +99,5 @@ If you have `free` subscription plan \(you may check it [here](https://apilayer. | 0.2.2 | 2021-05-28 | [3677](https://github.com/airbytehq/airbyte/pull/3677) | Adding clearer error message when a currency isn't supported. access_key field in spec.json was marked as sensitive | | 0.2.0 | 2021-05-26 | [3566](https://github.com/airbytehq/airbyte/pull/3566) | Move from `api.ratesapi.io/` to `api.exchangeratesapi.io/`. Add required field `access_key` to `config.json`. | | 0.1.0 | 2021-04-19 | [2942](https://github.com/airbytehq/airbyte/pull/2942) | Implement Exchange API using the CDK | + + \ No newline at end of file From 3f76180cf0d844b150f003cebdfae5652c997c2e Mon Sep 17 00:00:00 2001 From: Patrick Nilan Date: Wed, 15 Nov 2023 12:50:54 -0800 Subject: [PATCH 070/124] Docs: Updates Zendesk Support Source Connector Docs (Removes inapp page, adds , adds Limitations & Troubleshooting) (#32368) --- .../sources/zendesk-support.inapp.md | 27 ------- docs/integrations/sources/zendesk-support.md | 74 +++++++++++++------ 2 files changed, 50 insertions(+), 51 deletions(-) delete mode 100644 docs/integrations/sources/zendesk-support.inapp.md diff --git a/docs/integrations/sources/zendesk-support.inapp.md b/docs/integrations/sources/zendesk-support.inapp.md deleted file mode 100644 index e62c88b8a182..000000000000 --- a/docs/integrations/sources/zendesk-support.inapp.md +++ /dev/null @@ -1,27 +0,0 @@ -## Prerequisites - -- A Zendesk account with an Administrator role. - -## Setup Guide - - - -For **Airbyte Open Source** users, we recommend using an API token to authenticate your Zendesk account. You can follow the steps in our [full documentation](https://docs.airbyte.com/integrations/sources/zendesk-support#setup-guide) to generate this token. - - - -1. For **Source name**, enter a name to help you identify this source. -2. You can use OAuth or an API token to authenticate your Zendesk account. We recommend using OAuth for Airbyte Cloud and an API token for Airbyte Open Source. - - - - **For Airbyte Cloud:** To authenticate using OAuth, select **OAuth2.0** from the Authentication dropdown, then click **Authenticate your Zendesk Support account** to sign in with Zendesk and authorize your account. - - - - **For Airbyte Open Source**: To authenticate using an API token, select **API Token** from the Authentication dropdown and enter the [token you generated](https://docs.airbyte.com/integrations/sources/zendesk-support#setup-guide), as well as the email address associated with your Zendesk account. - - -3. For **Start Date**, use the provided datepicker or enter a UTC date and time programmatically in the format `YYYY-MM-DDTHH:mm:ssZ`. The data added on and after this date will be replicated. -4. For **Subdomain**, enter your Zendesk subdomain. This is the subdomain found in your account URL. For example, if your account URL is `https://MY_SUBDOMAIN.zendesk.com/`, then `MY_SUBDOMAIN` is your subdomain. -5. Click **Set up source** and wait for the tests to complete. - -For detailed information on supported sync modes, supported streams and performance considerations, refer to the [full documentation for Zendesk Support](https://docs.airbyte.com/integrations/sources/zendesk-support). diff --git a/docs/integrations/sources/zendesk-support.md b/docs/integrations/sources/zendesk-support.md index 5f194f8322e5..cc3f76a90667 100644 --- a/docs/integrations/sources/zendesk-support.md +++ b/docs/integrations/sources/zendesk-support.md @@ -1,6 +1,10 @@ # Zendesk Support -This page contains the setup guide and reference information for the Zendesk Support source connector. + + +This page contains the setup guide and reference information for the [Zendesk Support](https://www.zendesk.com/) source connector. + + ## Prerequisites @@ -14,16 +18,21 @@ The Zendesk Support source connector supports two authentication methods: - API token -For **Airbyte Cloud** users, we highly recommend using OAuth to authenticate your Zendesk Support account, as it simplifies the setup process and allows you to authenticate [directly from the Airbyte UI](#set-up-the-zendesk-support-source-connector). +**For Airbyte Cloud:** + +We highly recommend using OAuth to authenticate your Zendesk Support account, as it simplifies the setup process and allows you to authenticate [directly from the Airbyte UI](#set-up-the-zendesk-support-source-connector). + -For **Airbyte Open Source** users, we recommend using an API token to authenticate your Zendesk Support account. Please follow the steps below to generate this key. +**For Airbyte Open Source:** + +We recommend using an API token to authenticate your Zendesk Support account. Please follow the steps below to generate this key. :::note If you prefer to authenticate with OAuth for **Airbyte Open Source**, you can follow the steps laid out in [this Zendesk article](https://support.zendesk.com/hc/en-us/articles/4408845965210) to obtain your client ID, client secret and access token. Please ensure you set the scope to `read` when generating the access token. ::: -### (Airbyte Open Source) Enable API token access and generate a token +### Generate an API token 1. Log in to your Zendesk account. 2. Click the **Zendesk Products** icon (four squares) in the top-right corner, then select **Admin Center**. @@ -40,22 +49,23 @@ If you prefer to authenticate with OAuth for **Airbyte Open Source**, you can fo ### Set up the Zendesk Support source connector -1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account. +1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. 3. Find and select **Zendesk Support** from the list of available sources. 4. For **Source name**, enter a name to help you identify this source. -5. You can use OAuth or an API token to authenticate your Zendesk Support account. We recommend using OAuth for Airbyte Cloud and an API key for Airbyte Open Source. - - - - **For Airbyte Cloud**: To authenticate using OAuth, select **OAuth 2.0** from the Authentication dropdown, then click **Authenticate your Zendesk Support account** to sign in with Zendesk Support and authorize your account. - - - - **For Airbyte Open Source**: To authenticate using an API key, select **API Token** from the Authentication dropdown and enter the API token you generated, as well as the email address associated with your Zendesk Support account. - - +5. You can use OAuth or an API token to authenticate your Zendesk Support account. + +- **For Airbyte Cloud**: To authenticate using OAuth, select **OAuth 2.0** from the Authentication dropdown, then click **Authenticate your Zendesk Support account** to sign in with Zendesk Support and authorize your account. + + +- **For Airbyte Open Source**: To authenticate using an API key, select **API Token** from the Authentication dropdown and enter the API token you generated, as well as the email address associated with your Zendesk Support account. + 6. For **Subdomain**, enter your Zendesk subdomain. This is the subdomain found in your account URL. For example, if your account URL is `https://MY_SUBDOMAIN.zendesk.com/`, then `MY_SUBDOMAIN` is your subdomain. 7. (Optional) For **Start Date**, use the provided datepicker or enter a UTC date and time programmatically in the format `YYYY-MM-DDTHH:mm:ssZ`. The data added on and after this date will be replicated. If this field is left blank, Airbyte will replicate the data for the last two years by default. 8. Click **Set up source** and wait for the tests to complete. + + + ## Supported sync modes @@ -66,22 +76,22 @@ The Zendesk Support source connector supports the following [sync modes](https:/ - Incremental Sync | Append - Incremental Sync | Deduped History -## Supported streams - :::note There are two types of incremental sync: -1. Incremental (standard server-side, where API returns only the data updated or generated since the last sync) -2. Client-Side Incremental (API returns all available data and connector filters out only new records) +1. Incremental (standard server-side, where API returns only the data updated or generated since the last sync). +2. Client-Side Incremental (API returns all available data and connector filters out only new records). ::: +## Supported streams + The Zendesk Support source connector supports the following streams: - [Account Attributes](https://developer.zendesk.com/api-reference/ticketing/ticket-management/skill_based_routing/#list-account-attributes) -- [Articles](https://developer.zendesk.com/api-reference/help_center/help-center-api/articles/#list-articles) \(Incremental\) -- [Article Votes](https://developer.zendesk.com/api-reference/help_center/help-center-api/votes/#list-votes) \(Incremental\) -- [Article Comments](https://developer.zendesk.com/api-reference/help_center/help-center-api/article_comments/#list-comments) \(Incremental\) -- [Article Comment Votes](https://developer.zendesk.com/api-reference/help_center/help-center-api/votes/#list-votes) \(Incremental\) +- [Articles](https://developer.zendesk.com/api-reference/help_center/help-center-api/articles/#list-articles) \(Incremental\) +- [Article Votes](https://developer.zendesk.com/api-reference/help_center/help-center-api/votes/#list-votes) \(Incremental\) +- [Article Comments](https://developer.zendesk.com/api-reference/help_center/help-center-api/article_comments/#list-comments) \(Incremental\) +- [Article Comment Votes](https://developer.zendesk.com/api-reference/help_center/help-center-api/votes/#list-votes) \(Incremental\) - [Attribute Definitions](https://developer.zendesk.com/api-reference/ticketing/ticket-management/skill_based_routing/#list-routing-attribute-definitions) - [Audit Logs](https://developer.zendesk.com/api-reference/ticketing/account-configuration/audit_logs/#list-audit-logs)\(Incremental\) (Only available for enterprise accounts) - [Brands](https://developer.zendesk.com/api-reference/ticketing/account-configuration/brands/#list-brands) @@ -123,13 +133,27 @@ The Zendesk Support connector fetches deleted records in the following streams: | **Ticket Metric Events** | `deleted` | | **Tickets** | `status`==`deleted` | +## Limitations & Troubleshooting + +
+ +Expand to see details about Zendesk Support connector limitations and troubleshooting. + -## Performance considerations +### Connector limitations + +#### Rate limiting The connector is restricted by normal Zendesk [requests limitation](https://developer.zendesk.com/rest_api/docs/support/usage_limits). The Zendesk connector ideally should not run into Zendesk API limitations under normal usage. [Create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. +### Troubleshooting + +* Check out common troubleshooting issues for the Zendesk Support source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
+ ## Changelog | Version | Date | Pull Request | Subject | @@ -211,4 +235,6 @@ The Zendesk connector ideally should not run into Zendesk API limitations under | `0.1.3` | 2021-10-17 | [7097](https://github.com/airbytehq/airbyte/pull/7097) | Corrected the connector's specification | | `0.1.2` | 2021-10-16 | [6513](https://github.com/airbytehq/airbyte/pull/6513) | Fixed TicketComments stream | | `0.1.1` | 2021-09-02 | [5787](https://github.com/airbytehq/airbyte/pull/5787) | Fixed incremental logic for the ticket_comments stream | -| `0.1.0` | 2021-07-21 | [4861](https://github.com/airbytehq/airbyte/pull/4861) | Created CDK native zendesk connector | \ No newline at end of file +| `0.1.0` | 2021-07-21 | [4861](https://github.com/airbytehq/airbyte/pull/4861) | Created CDK native zendesk connector | + + \ No newline at end of file From 688c255c2437af91873878d0adb623b3097e7935 Mon Sep 17 00:00:00 2001 From: Patrick Nilan Date: Wed, 15 Nov 2023 12:52:12 -0800 Subject: [PATCH 071/124] Docs: Updates Google Analytics v4 Source Connector Docs (Removes inapp page, adds , adds Limitations & Troubleshooting) (#32367) --- .../sources/google-analytics-v4.inapp.md | 101 ------------ .../sources/google-analytics-v4.md | 149 ++++++++++++++---- 2 files changed, 122 insertions(+), 128 deletions(-) delete mode 100644 docs/integrations/sources/google-analytics-v4.inapp.md diff --git a/docs/integrations/sources/google-analytics-v4.inapp.md b/docs/integrations/sources/google-analytics-v4.inapp.md deleted file mode 100644 index 305fd218af5a..000000000000 --- a/docs/integrations/sources/google-analytics-v4.inapp.md +++ /dev/null @@ -1,101 +0,0 @@ -:::caution - -**The Google Analytics (Universal Analytics) connector will be deprecated soon.** - -Google is phasing out Universal Analytics in favor of Google Analytics 4 (GA4). In consequence, we are deprecating the Google Analytics (Universal Analytics) connector and recommend that you migrate to the [Google Analytics 4 (GA4) connector](https://docs.airbyte.com/integrations/sources/google-analytics-data-api) as soon as possible to ensure your syncs are not affected. - -Due to this deprecation, we will not be accepting new contributions for this source. - -For more information, see ["Universal Analytics is going away"](https://support.google.com/analytics/answer/11583528). - -::: - -## Prerequisite - -* Administrator access to a Google Analytics 4 (GA4) property - -## Setup guide - -1. Click **Authenticate your account** by selecting Oauth (recommended). - * If you select Service Account Key Authentication, follow the instructions in our [full documentation](https://docs.airbyte.com/integrations/sources/google-analytics-v4). -2. Log in and Authorize the Google Analytics account. -3. Enter your [Property ID](https://developers.google.com/analytics/devguides/reporting/data/v1/property-id#what_is_my_property_id) -4. Enter the **Start Date** from which to replicate report data in the format YYYY-MM-DD. -5. (Optional) Airbyte generates 8 default reports. To add more reports, you need to add **Custom Reports** as a JSON array describing the custom reports you want to sync from Google Analytics. See below for more information. -6. (Optional) Enter the **Data request time increment in days**. The bigger this value is, the faster the sync will be, but the more likely that sampling will be applied to your data, potentially causing inaccuracies in the returned results. We recommend setting this to 1 unless you have a hard requirement to make the sync faster at the expense of accuracy. The minimum allowed value for this field is 1, and the maximum is 364. - -## (Optional) Custom Reports -Custom Reports allow for flexibility in the reporting dimensions and metrics to meet your specific use case. Use the [GA4 Query Explorer](https://ga-dev-tools.google/ga4/query-explorer/) to help build your report. To ensure your dimensions and metrics are compatible, you can also refer to the [GA4 Dimensions & Metrics Explorer](https://ga-dev-tools.google/ga4/dimensions-metrics-explorer/). - -A custom report is formatted as: `[{"name": "", "dimensions": ["", ...], "metrics": ["", ...]}]` - -Example of a custom report: -``` -[{ - "name" : "page_views_and_users", - "dimensions" :[ - "ga:date", - "ga:pagePath", - "ga:sessionDefaultChannelGrouping" - ], - "metrics" :[ - "ga:screenPageViews", - "ga:totalUsers" - ] -}] -``` -Multiple custom reports should be entered with a comma separator. Each custom report is created as it's own stream. -Example of multiple custom reports: -``` -[ - { - "name" : "page_views_and_users", - "dimensions" :[ - "ga:date", - "ga:pagePath" - ], - "metrics" :[ - "ga:screenPageViews", - "ga:totalUsers" - ] - }, - { - "name" : "sessions_by_region", - "dimensions" :[ - "ga:date", - "ga:region" - ], - "metrics" :[ - "ga:totalUsers", - "ga:sessions" - ] - } -] -``` - -Custom reports can also include segments and filters to pull a subset of your data. The report should be formatted as: `[{"name": "", "dimensions": ["", ...], "metrics": ["", ...], "segments": [""}]` - -* When using segments, make sure you also add the `ga:segment` dimension. - -Example of a custom report with segments and/or filters: -``` -[{ "name" : "page_views_and_users", - "dimensions" :[ - "ga:date", - "ga:pagePath", - "ga:segment" - ], - "metrics" :[ - "ga:sessions", - "ga:totalUsers" - ], - "segments" :[ - "ga:sessionSource!=(direct)" - ], - "filter" :[ - "ga:sessionSource!=(direct);ga:sessionSource!=(not set)" - ] -}] -``` - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Google Analytics 4 (GA4)](https://docs.airbyte.com/integrations/sources/google-analytics-v4). diff --git a/docs/integrations/sources/google-analytics-v4.md b/docs/integrations/sources/google-analytics-v4.md index cf83af464af8..835d1d324df5 100644 --- a/docs/integrations/sources/google-analytics-v4.md +++ b/docs/integrations/sources/google-analytics-v4.md @@ -1,9 +1,13 @@ # Google Analytics (Universal Analytics) + + This page contains the setup guide and reference information for the Google Analytics (Universal Analytics) source connector. This connector supports Universal Analytics properties through the [Reporting API v4](https://developers.google.com/analytics/devguides/reporting/core/v4). + + :::caution **The Google Analytics (Universal Analytics) connector will be deprecated soon.** @@ -34,8 +38,6 @@ A Google Cloud account with [Viewer permissions](https://support.google.com/anal **For Airbyte Cloud:** -To set up Google Analytics as a source in Airbyte Cloud: - 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. 3. On the Set up the source page, select **Google Analytics** from the **Source type** dropdown. @@ -52,19 +54,20 @@ To set up Google Analytics as a source in Airbyte Cloud: **For Airbyte Open Source:** -To set up Google Analytics as a source in Airbyte Open Source: - -1. Go to the Airbyte UI and click **Sources** and then click **+ New source**. -2. On the Set up the source page, select **Google Analytics** from the **Source type** dropdown. -3. Enter a name for the Google Analytics connector. -4. Authenticate your Google account via OAuth or Service Account Key Authentication: +1. Navigate to the Airbyte Open Source dashboard. +2. Go to the Airbyte UI and click **Sources** and then click **+ New source**. +3. On the Set up the source page, select **Google Analytics** from the **Source type** dropdown. +4. Enter a name for the Google Analytics connector. +5. Authenticate your Google account via OAuth or Service Account Key Authentication: - To authenticate your Google account via OAuth, enter your Google application's [client ID, client secret, and refresh token](https://developers.google.com/identity/protocols/oauth2). - To authenticate your Google account via Service Account Key Authentication, enter your [Google Cloud service account key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys) in JSON format. Use the service account email address to [add a user](https://support.google.com/analytics/answer/1009702) to the Google analytics view you want to access via the API and grant [Read and Analyze permissions](https://support.google.com/analytics/answer/2884495). 5. Enter the **Replication Start Date** in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. + 6. Enter the [**View ID**](https://ga-dev-tools.appspot.com/account-explorer/) for the Google Analytics View you want to fetch data from. 7. Optionally, enter a JSON object as a string in the **Custom Reports** field. For details, refer to [Requesting custom reports](#requesting-custom-reports) 8. Leave **Data request time increment in days (Optional)** blank or set to 1. For faster syncs, set this value to more than 1 but that might result in the Google Analytics API returning [sampled data](#sampled-data-in-reports), potentially causing inaccuracies in the returned results. The maximum allowed value is 364. - + + ## Supported sync modes @@ -119,28 +122,90 @@ If you are not on the Google Analytics 360 tier, the Google Analytics API may re In order to minimize the chances of sampling being applied to your data, Airbyte makes data requests to Google in one day increments (the smallest allowed date increment). This reduces the amount of data the Google API processes per request, thus minimizing the chances of sampling being applied. The downside of requesting data in one day increments is that it increases the time it takes to export your Google Analytics data. If sampling is not a concern, you can override this behavior by setting the optional `window_in_day` parameter to specify the number of days to look back and avoid sampling. When sampling occurs, a warning is logged to the sync log. -## Data processing latency - -According to the [Google Analytics API documentation](https://support.google.com/analytics/answer/1070983?hl=en#DataProcessingLatency&zippy=%2Cin-this-article), all report data may continue to be updated 48 hours after it appears in the Google Analytics API. This means if you request the same report twice within 48 hours of that data being sent to Google Analytics, the report data might be different across the two requests. This happens when Google Analytics is still processing all events it received. - -When this occurs, the returned data will set the flag `isDataGolden` to false. As mentioned in the [Google Analytics API docs](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#reportdata), the `isDataGolden` flag indicates if [data] is golden or not. Data is golden when the exact same request [for a report] will not produce any new results if asked at a later point in time. - -To address this issue, the connector adds a lookback window of 2 days to ensure any previously synced non-golden data is re-synced with its potential updates. For example: If your last sync occurred 5 days ago and a sync is initiated today, the connector will attempt to sync data from 7 days ago up to the latest data available. - -To determine whether data is finished processing or not, the `isDataGolden` flag is exposed and should be used. - ## Requesting Custom Reports -To replicate Google Analytics [Custom Reports](https://support.google.com/analytics/answer/1033013?hl=en) using this connector, input a JSON object as a string in the **Custom Reports** field when setting up the connector. The JSON is an array of objects where each object has the following schema: - -```text -{"name": string, "dimensions": [string], "metrics": [string]} +Custom Reports allow for flexibility in the reporting dimensions and metrics to meet your specific use case. Use the [GA4 Query Explorer](https://ga-dev-tools.google/ga4/query-explorer/) to help build your report. To ensure your dimensions and metrics are compatible, you can also refer to the [GA4 Dimensions & Metrics Explorer](https://ga-dev-tools.google/ga4/dimensions-metrics-explorer/). + +A custom report is formatted as: `[{"name": "", "dimensions": ["", ...], "metrics": ["", ...]}]` + +Example of a custom report: +```json +[{ + "name" : "page_views_and_users", + "dimensions" :[ + "ga:date", + "ga:pagePath", + "ga:sessionDefaultChannelGrouping" + ], + "metrics" :[ + "ga:screenPageViews", + "ga:totalUsers" + ] +}] +``` +Multiple custom reports should be entered with a comma separator. Each custom report is created as it's own stream. +Example of multiple custom reports: +```json +[ + { + "name" : "page_views_and_users", + "dimensions" :[ + "ga:date", + "ga:pagePath" + ], + "metrics" :[ + "ga:screenPageViews", + "ga:totalUsers" + ] + }, + { + "name" : "sessions_by_region", + "dimensions" :[ + "ga:date", + "ga:region" + ], + "metrics" :[ + "ga:totalUsers", + "ga:sessions" + ] + } +] ``` -Here is an example input "Custom Reports" field: +Custom reports can also include segments and filters to pull a subset of your data. The report should be formatted as: +```json +[ + { + "name": "", + "dimensions": ["", ...], + "metrics": ["", ...], + "segments": ["", ...], + "filter": "" + } +] +``` -```text -[{"name": "new_users_per_day", "dimensions": ["ga:date","ga:country","ga:region"], "metrics": ["ga:newUsers"]}, {"name": "users_per_city", "dimensions": ["ga:city"], "metrics": ["ga:users"]}] +* When using segments, make sure you also add the `ga:segment` dimension. + +Example of a custom report with segments and/or filters: +```json +[{ "name" : "page_views_and_users", + "dimensions" :[ + "ga:date", + "ga:pagePath", + "ga:segment" + ], + "metrics" :[ + "ga:sessions", + "ga:totalUsers" + ], + "segments" :[ + "ga:sessionSource!=(direct)" + ], + "filter" :[ + "ga:sessionSource!=(direct);ga:sessionSource!=(not set)" + ] +}] ``` To create a list of dimensions, you can use default Google Analytics dimensions (listed below) or custom dimensions if you have some defined. Each report can contain no more than 7 dimensions, and they must all be unique. The default Google Analytics dimensions are: @@ -186,6 +251,34 @@ A custom report can contain no more than 10 unique metrics. The default availabl Incremental sync is supported only if you add `ga:date` dimension to your custom report. +## Limitations & Troubleshooting + +
+ +Expand to see details about Google Analytics v4 connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting + +[Analytics Reporting API v4](https://developers.google.com/analytics/devguides/reporting/core/v4/limits-quotas) + +- Number of requests per day per project: 50,000 +- Number of requests per view (profile) per day: 10,000 (cannot be increased) +- Number of requests per 100 seconds per project: 2,000 +- Number of requests per 100 seconds per user per project: 100 (can be increased in Google API Console to 1,000). + +The Google Analytics connector should not run into the "requests per 100 seconds" limitation under normal usage. [Create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully and try increasing the `window_in_days` value. + +### Troubleshooting + + + +* Check out common troubleshooting issues for the Google Analytics v4 source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
+ ## Changelog | Version | Date | Pull Request | Subject | @@ -227,4 +320,6 @@ Incremental sync is supported only if you add `ga:date` dimension to your custom | 0.1.3 | 2021-09-21 | [6357](https://github.com/airbytehq/airbyte/pull/6357) | Fix OAuth workflow parameters | | 0.1.2 | 2021-09-20 | [6306](https://github.com/airbytehq/airbyte/pull/6306) | Support of Airbyte OAuth initialization flow | | 0.1.1 | 2021-08-25 | [5655](https://github.com/airbytehq/airbyte/pull/5655) | Corrected validation of empty custom report | -| 0.1.0 | 2021-08-10 | [5290](https://github.com/airbytehq/airbyte/pull/5290) | Initial Release | \ No newline at end of file +| 0.1.0 | 2021-08-10 | [5290](https://github.com/airbytehq/airbyte/pull/5290) | Initial Release | + + \ No newline at end of file From 8dce753d803b772e4a9e3900dbd0ee4638c71428 Mon Sep 17 00:00:00 2001 From: Patrick Nilan Date: Wed, 15 Nov 2023 12:52:31 -0800 Subject: [PATCH 072/124] Mailchimp Source: Updated QL to 200 (#32393) --- airbyte-integrations/connectors/source-mailchimp/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-mailchimp/metadata.yaml b/airbyte-integrations/connectors/source-mailchimp/metadata.yaml index 85bb1badfd2e..11ff4b1bbbcc 100644 --- a/airbyte-integrations/connectors/source-mailchimp/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailchimp/metadata.yaml @@ -1,6 +1,6 @@ data: ab_internal: - ql: 400 + ql: 200 sl: 200 allowedHosts: hosts: From 933029ca0e8f54f34283e53966fb9a400e843f64 Mon Sep 17 00:00:00 2001 From: Patrick Nilan Date: Wed, 15 Nov 2023 12:57:23 -0800 Subject: [PATCH 073/124] Docs: Updates GitHub Source Connector Docs (Removes inapp page, adds , adds Limitations & Troubleshooting) (#32362) --- docs/integrations/sources/github.inapp.md | 46 --------------- docs/integrations/sources/github.md | 71 ++++++++++++++++------- 2 files changed, 51 insertions(+), 66 deletions(-) delete mode 100644 docs/integrations/sources/github.inapp.md diff --git a/docs/integrations/sources/github.inapp.md b/docs/integrations/sources/github.inapp.md deleted file mode 100644 index 4cfe94e5190e..000000000000 --- a/docs/integrations/sources/github.inapp.md +++ /dev/null @@ -1,46 +0,0 @@ -## Prerequisites - -- List of GitHub Repositories (and access for them in case they are private) - -## Setup guide - -1. Name your source. -2. Click `Authenticate your GitHub account` or use a [Personal Access Token](https://github.com/settings/tokens) for Authentication. For Personal Access Tokens, refer to the list of required [permissions and scopes](https://docs.airbyte.com/integrations/sources/github#permissions-and-scopes). -3. **GitHub Repositories** - Enter a list of GitHub organizations or repositories. -4. (Optional) **Start date** Enter the date you'd like to replicate data from. - -These streams will only sync records generated on or after the **Start Date**: - -`comments`, `commit_comment_reactions`, `commit_comments`, `commits`, `deployments`, `events`, `issue_comment_reactions`, `issue_events`, `issue_milestones`, `issue_reactions`, `issues`, `project_cards`, `project_columns`, `projects`, `pull_request_comment_reactions`, `pull_requests`, `pull_requeststats`, `releases`, `review_comments`, `reviews`, `stargazers`, `workflow_runs`, `workflows`. - -The **Start Date** does not apply to the streams below and all data will be synced for these streams: - -`assignees`, `branches`, `collaborators`, `issue_labels`, `organizations`, `pull_request_commits`, `pull_request_stats`, `repositories`, `tags`, `teams`, `users` - -Example of a single repository: -``` -airbytehq/airbyte -``` -Example of multiple repositories: -``` -airbytehq/airbyte airbytehq/another-repo -``` -Example of an organization to receive data from all of its repositories: -``` -airbytehq/* -``` -Repositories which have a misspelled name, do not exist, or have the wrong name format will return an error. - -5. (Optional) **Branch** - Enter a list of GitHub repository branches to pull commits for, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled. (e.g. `airbytehq/airbyte/master airbytehq/airbyte/my-branch`). -6. (Optional) **Max requests per hour** - The GitHub API allows for a maximum of 5000 requests per hour (15,000 for Github Enterprise). You can specify a lower value to limit your use of the API quota. - -### Incremental Sync Methods -Incremental sync is offered for most streams, with some differences in sync behavior. - -1. `comments`, `commits`, `issues` and `review comments` only syncs new records. Only new records will be synced. - -2. `workflow_runs` and `worflow_jobs` syncs new records and any records run in the [last 30 days](https://docs.github.com/en/actions/managing-workflow-runs/re-running-workflows-and-jobs) - -3. All other incremental streams sync all historical records and output any updated or new records. - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [GitHub](https://docs.airbyte.com/integrations/sources/github/). diff --git a/docs/integrations/sources/github.md b/docs/integrations/sources/github.md index 1cef7c9764d5..e6175ad7e465 100644 --- a/docs/integrations/sources/github.md +++ b/docs/integrations/sources/github.md @@ -1,6 +1,10 @@ # GitHub -This page contains the setup guide and reference information for the GitHub source connector. + + +This page contains the setup guide and reference information for the [GitHub](https://www.github.com) source connector. + + ## Prerequisites @@ -9,8 +13,8 @@ This page contains the setup guide and reference information for the GitHub sour **For Airbyte Cloud:** -- Personal Access Token (see [Permissions and scopes](https://docs.airbyte.com/integrations/sources/github#permissions-and-scopes)) - OAuth +- Personal Access Token (see [Permissions and scopes](https://docs.airbyte.com/integrations/sources/github#permissions-and-scopes)) @@ -33,28 +37,39 @@ Log into [GitHub](https://github.com) and then generate a [personal access token ### Step 2: Set up the GitHub connector in Airbyte + +**For Airbyte Cloud:** 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. -2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. -3. On the source setup page, select **GitHub** from the Source type dropdown and enter a name for this connector. -4. Click `Authenticate your GitHub account` by selecting Oauth or Personal Access Token for Authentication. +2. In the left navigation bar, click **Sources**. +3. On the source selection page, select **GitHub** from the list of Sources. +4. Add a name for your GitHub connector. 5. To authenticate: - -- **For Airbyte Cloud**: Click **Authenticate your account** to authorize your GitHub account. Airbyte will authenticate the GitHub account you are already logged in to. Please make sure you are logged into the right account. - - -- **For Airbyte Open Source**: Authenticate with **Personal Access Token**. - -6. **GitHub Repositories** - List of GitHub organizations/repositories, e.g. `airbytehq/airbyte` for single repository, `airbytehq/airbyte airbytehq/another-repo` for multiple repositories. If you want to specify the organization to receive data from all its repositories, then you should specify it according to the following example: `airbytehq/*`. + + + - **For Airbyte Cloud:** **Authenticate your GitHub account** to authorize your GitHub account. Airbyte will authenticate the GitHub account you are already logged in to. Please make sure you are logged into the right account. + + + + - **For Airbyte Open Source:** Authenticate with **Personal Access Token**. To generate a personal access token, log into [GitHub](https://github.com) and then generate a [personal access token](https://github.com/settings/tokens). Enter your GitHub personal access token. To load balance your API quota consumption across multiple API tokens, input multiple tokens separated with `,`. + + +6. **GitHub Repositories** - Enter a list of GitHub organizations/repositories, e.g. `airbytehq/airbyte` for single repository, `airbytehq/airbyte airbytehq/another-repo` for multiple repositories. If you want to specify the organization to receive data from all its repositories, then you should specify it according to the following example: `airbytehq/*`. :::caution Repositories with the wrong name or repositories that do not exist or have the wrong name format will be skipped with `WARN` message in the logs. ::: -7. **Start date (Optional)** - The date from which you'd like to replicate data for streams. If the date is not set, all data will be replicated. Using for streams: `Comments`, `Commit comment reactions`, `Commit comments`, `Commits`, `Deployments`, `Events`, `Issue comment reactions`, `Issue events`, `Issue milestones`, `Issue reactions`, `Issues`, `Project cards`, `Project columns`, `Projects`, `Pull request comment reactions`, `Pull requests`, `Pull request stats`, `Releases`, `Review comments`, `Reviews`, `Stargazers`, `Workflow runs`, `Workflows`. -8. **Branch (Optional)** - List of GitHub repository branches to pull commits for, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled. (e.g. `airbytehq/airbyte/master airbytehq/airbyte/my-branch`). -9. **Max requests per hour (Optional)** - The GitHub API allows for a maximum of 5000 requests per hour (15000 for Github Enterprise). You can specify a lower value to limit your use of the API quota. +7. **Start date (Optional)** - The date from which you'd like to replicate data for streams. For streams which support this configuration, only data generated on or after the start date will be replicated. + +- These streams will only sync records generated on or after the **Start Date**: `comments`, `commit_comment_reactions`, `commit_comments`, `commits`, `deployments`, `events`, `issue_comment_reactions`, `issue_events`, `issue_milestones`, `issue_reactions`, `issues`, `project_cards`, `project_columns`, `projects`, `pull_request_comment_reactions`, `pull_requests`, `pull_requeststats`, `releases`, `review_comments`, `reviews`, `stargazers`, `workflow_runs`, `workflows`. +- The **Start Date** does not apply to the streams below and all data will be synced for these streams: `assignees`, `branches`, `collaborators`, `issue_labels`, `organizations`, `pull_request_commits`, `pull_request_stats`, `repositories`, `tags`, `teams`, `users` + +8. **Branch (Optional)** - List of GitHub repository branches to pull commits from, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled. (e.g. `airbytehq/airbyte/master airbytehq/airbyte/my-branch`). +9. **Max requests per hour (Optional)** - The GitHub API allows for a maximum of 5,000 requests per hour (15,000 for Github Enterprise). You can specify a lower value to limit your use of the API quota. Refer to GitHub article [Rate limits for the REST API](https://docs.github.com/en/rest/overview/rate-limits-for-the-rest-api). + + ## Supported sync modes @@ -146,9 +161,21 @@ This connector outputs the following incremental streams: - `teams` - `users` -### Permissions and scopes +## Limitations & Troubleshooting + +
+ +Expand to see details about GitHub connector limitations and troubleshooting. + -If you use OAuth authentication method, the oauth2.0 application requests the next list of [scopes](https://docs.github.com/en/developers/apps/building-oauth-apps/scopes-for-oauth-apps#available-scopes): **repo**, **read:org**, **read:repo_hook**, **read:user**, **read:discussion**, **workflow**. For [personal access token](https://github.com/settings/tokens) you need to manually select needed scopes. +### Connector limitations + +#### Rate limiting +The GitHub connector should not run into GitHub API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. Refer to GitHub article [Rate limits for the REST API](https://docs.github.com/en/rest/overview/rate-limits-for-the-rest-api). + +#### Permissions and scopes + +If you use OAuth authentication method, the OAuth2.0 application requests the next list of [scopes](https://docs.github.com/en/developers/apps/building-oauth-apps/scopes-for-oauth-apps#available-scopes): **repo**, **read:org**, **read:repo_hook**, **read:user**, **read:discussion**, **workflow**. For [personal access token](https://github.com/settings/tokens) you need to manually select needed scopes. Your token should have at least the `repo` scope. Depending on which streams you want to sync, the user generating the token needs more permissions: @@ -156,9 +183,11 @@ Your token should have at least the `repo` scope. Depending on which streams you - Syncing [Teams](https://docs.github.com/en/organizations/organizing-members-into-teams/about-teams) is only available to authenticated members of a team's [organization](https://docs.github.com/en/rest/orgs). [Personal user accounts](https://docs.github.com/en/get-started/learning-about-github/types-of-github-accounts) and repositories belonging to them don't have access to Teams features. In this case no records will be synced. - To sync the Projects stream, the repository must have the Projects feature enabled. -### Performance considerations +### Troubleshooting -The GitHub connector should not run into GitHub API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. +* Check out common troubleshooting issues for the GitHub source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions) + +
## Changelog @@ -269,4 +298,6 @@ The GitHub connector should not run into GitHub API limitations under normal usa | 0.1.3 | 2021-08-03 | [5156](https://github.com/airbytehq/airbyte/pull/5156) | Extended existing schemas with `users` property for certain streams | | 0.1.2 | 2021-07-13 | [4708](https://github.com/airbytehq/airbyte/pull/4708) | Fix bug with IssueEvents stream and add handling for rate limiting | | 0.1.1 | 2021-07-07 | [4590](https://github.com/airbytehq/airbyte/pull/4590) | Fix schema in the `pull_request` stream | -| 0.1.0 | 2021-07-06 | [4174](https://github.com/airbytehq/airbyte/pull/4174) | New Source: GitHub | \ No newline at end of file +| 0.1.0 | 2021-07-06 | [4174](https://github.com/airbytehq/airbyte/pull/4174) | New Source: GitHub | + + \ No newline at end of file From 5dcaf6a00c8159d01af325746b8332b93ae38a4e Mon Sep 17 00:00:00 2001 From: Patrick Nilan Date: Wed, 15 Nov 2023 13:48:16 -0800 Subject: [PATCH 074/124] Docs: Updates Google Search Console Source Connector Docs (Removes inapp page, adds , adds Limitations & Troubleshooting) (#32361) --- .../sources/google-search-console.inapp.md | 28 -------- .../sources/google-search-console.md | 64 +++++++++++++------ 2 files changed, 44 insertions(+), 48 deletions(-) delete mode 100644 docs/integrations/sources/google-search-console.inapp.md diff --git a/docs/integrations/sources/google-search-console.inapp.md b/docs/integrations/sources/google-search-console.inapp.md deleted file mode 100644 index 2224bd12ac60..000000000000 --- a/docs/integrations/sources/google-search-console.inapp.md +++ /dev/null @@ -1,28 +0,0 @@ -## Prerequisite - -- A verified property in Google Search Console - -- Google Search Console API enabled for your project (**Airbyte Open Source** only) - - -## Setup guide - -1. For **Source name**, enter a name to help you identify this source. -2. For **Website URL Property**, enter the specific website property in Google Seach Console with data you want to replicate. -3. For **Start Date**, by default the `2021-01-01` is set, use the provided datepicker or enter a date in the format `YYYY-MM-DD`. Any data created on or after this date will be replicated. -4. To authenticate the connection: - - - - **For Airbyte Cloud**: Select **Oauth** from the Authentication dropdown, then click **Sign in with Google** to authorize your account. More information on authentication methods can be found in our [full Google Search Console documentation](https://docs.airbyte.io/integrations/sources/google-search-console#setup-guide). - - - - (Recommended) To authenticate with a service account, select **Service Account Key Authorization** from the Authentication dropdown, then enter the **Admin Email** and **Service Account JSON Key**. For the key, copy and paste the JSON key you obtained during the service account setup. It should begin with `{"type": "service account", "project_id": YOUR_PROJECT_ID, "private_key_id": YOUR_PRIVATE_KEY, ...}`. - - To authenticate with OAuth, select **Oauth** from the Authentication dropdown, then enter your **Client ID**, **Client Secret**, **Access Token** and **Refresh Token**. More information on authentication methods for Airbyte Open Source can be found in our [full Google Search Console documentation](https://docs.airbyte.io/integrations/sources/google-search-console#setup-guide). - - -5. (Optional) For **End Date**, you may optionally provide a date in the format `YYYY-MM-DD`. Any data created between the defined Start Date and End Date will be replicated. Leaving this field blank will replicate all data created on or after the Start Date to the present. -6. (Optional) For **Custom Reports**, you may optionally provide a custom report representing any additional report you wish to query the API with. Refer to the [Custom reports](https://docs.airbyte.com/integrations/sources/google-search-console#custom-reports) section in our full documentation for more information on formulating these reports. -7. (Optional) For **Data Freshness**, you may choose whether to include "fresh" data that has not been finalized by Google, and may be subject to change. Please note that if you are using Incremental sync mode, we highly recommend leaving this option to its default value of `final`. Refer to the [Data Freshness](https://docs.airbyte.com/integrations/sources/google-search-console#data-freshness) section in our full documentation for more information on this parameter. -8. Click **Set up source** and wait for the tests to complete. - -For detailed information on supported sync modes, supported streams, and performance considerations, refer to the full documentation for [Google Search Console](https://docs.airbyte.com/integrations/sources/google-search-console/). diff --git a/docs/integrations/sources/google-search-console.md b/docs/integrations/sources/google-search-console.md index 843d92ccb850..12d4ba6128b1 100644 --- a/docs/integrations/sources/google-search-console.md +++ b/docs/integrations/sources/google-search-console.md @@ -1,7 +1,11 @@ # Google Search Console + + This page contains the setup guide and reference information for the Google Search Console source connector. + + ## Prerequisites - A verified property in Google Search Console (or the list of the `Site URLs` (Website URL Properties)) @@ -11,13 +15,13 @@ This page contains the setup guide and reference information for the Google Sear ## Setup guide -### Step 1: Set up Google Search Console authentication +### Step 1: Set up Google Search Console To authenticate the Google Search Console connector, you will need to use one of the following methods: -#### I: OAuth (Recommended for Airbyte Cloud) - +#### OAuth (Recommended for Airbyte Cloud) + You can authenticate using your Google Account with OAuth if you are the owner of the Google Search Console property or have view permissions. Follow [Google's instructions](https://support.google.com/webmasters/answer/7687615?sjid=11103698321670173176-NA) to ensure that your account has the necessary permissions (**Owner** or **Full User**) to view the Google Search Console property. This option is recommended for **Airbyte Cloud** users, as it significantly simplifies the setup process and allows you to authenticate the connection [directly from the Airbyte UI](#step-2-set-up-the-google-search-console-connector-in-airbyte). @@ -31,7 +35,7 @@ To authenticate with OAuth in **Airbyte Open Source**, you will need to create a More information on the steps to create an OAuth app to access Google APIs and obtain these credentials can be found [in Google's documentation](https://developers.google.com/identity/protocols/oauth2). -#### II: Google service account with JSON key file (Recommended for Airbyte Open Source) +#### Google service account with JSON key file (Recommended for Airbyte Open Source) You can authenticate the connection using a JSON key file associated with a Google service account. This option is recommended for **Airbyte Open Source** users. Follow the steps below to create a service account and generate the JSON key file: @@ -70,28 +74,32 @@ For more information on this topic, please refer to [this Google article](https: ### Step 2: Set up the Google Search Console connector in Airbyte -1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account. + +**For Airbyte Cloud:** + +1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. 3. Find and select **Google Search Console** from the list of available sources. 4. For **Source name**, enter a name to help you identify this source. 5. For **Website URL Property**, enter the specific website property in Google Seach Console with data you want to replicate. 6. For **Start Date**, by default the `2021-01-01` is set, use the provided datepicker or enter a date in the format `YYYY-MM-DD`. Any data created on or after this date will be replicated. 7. To authenticate the connection: - - - - **For Airbyte Cloud**: Select **Oauth** from the Authentication dropdown, then click **Sign in with Google** to authorize your account. - - - - **For Airbyte Open Source**: - - (Recommended) Select **Service Account Key Authorization** from the Authentication dropdown, then enter the **Admin Email** and **Service Account JSON Key**. For the key, copy and paste the JSON key you obtained during the service account setup. It should begin with `{"type": "service account", "project_id": YOUR_PROJECT_ID, "private_key_id": YOUR_PRIVATE_KEY, ...}` - - Select **Oauth** from the Authentication dropdown, then enter your **Client ID**, **Client Secret**, **Access Token** and **Refresh Token**. - - + +- **For Airbyte Cloud:** + - Select **Oauth** from the Authentication dropdown, then click **Sign in with Google** to authorize your account. + + +- **For Airbyte Open Source:** + - (Recommended) Select **Service Account Key Authorization** from the Authentication dropdown, then enter the **Admin Email** and **Service Account JSON Key**. For the key, copy and paste the JSON key you obtained during the service account setup. It should begin with `{"type": "service account", "project_id": YOUR_PROJECT_ID, "private_key_id": YOUR_PRIVATE_KEY, ...}` + - Select **Oauth** from the Authentication dropdown, then enter your **Client ID**, **Client Secret**, **Access Token** and **Refresh Token**. + 8. (Optional) For **End Date**, you may optionally provide a date in the format `YYYY-MM-DD`. Any data created between the defined Start Date and End Date will be replicated. Leaving this field blank will replicate all data created on or after the Start Date to the present. 9. (Optional) For **Custom Reports**, you may optionally provide an array of JSON objects representing any custom reports you wish to query the API with. Refer to the [Custom reports](#custom-reports) section below for more information on formulating these reports. 10. (Optional) For **Data Freshness**, you may choose whether to include "fresh" data that has not been finalized by Google, and may be subject to change. Please note that if you are using Incremental sync mode, we highly recommend leaving this option to its default value of `final`. Refer to the [Data Freshness](#data-freshness) section below for more information on this parameter. 11. Click **Set up source** and wait for the tests to complete. + + ## Supported sync modes The Google Search Console Source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): @@ -163,10 +171,6 @@ The **Data Freshness** parameter deals with the "freshness", or finality of the When using Incremental Sync mode, we recommend leaving this parameter to its default state of `final`, as the `all` option may cause discrepancies between the data in your destination table and the finalized data in Google Search Console. ::: -## Performance considerations - -This connector attempts to back off gracefully when it hits Reports API's rate limits. To find more information about limits, see [Usage Limits](https://developers.google.com/webmaster-tools/limits) documentation. - ## Data type map | Integration Type | Airbyte Type | Notes | @@ -176,6 +180,24 @@ This connector attempts to back off gracefully when it hits Reports API's rate l | `array` | `array` | | | `object` | `object` | | +## Limitations & Troubleshooting + +
+ +Expand to see details about Google Search Console connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting +This connector attempts to back off gracefully when it hits Reports API's rate limits. To find more information about limits, see [Usage Limits](https://developers.google.com/webmaster-tools/limits) documentation. + +### Troubleshooting + +* Check out common troubleshooting issues for the Google Search Console source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
+ ## Changelog | Version | Date | Pull Request | Subject | @@ -215,4 +237,6 @@ This connector attempts to back off gracefully when it hits Reports API's rate l | `0.1.3` | 2021-09-23 | [6405](https://github.com/airbytehq/airbyte/pull/6405) | Correct Spec File | | `0.1.2` | 2021-09-17 | [6222](https://github.com/airbytehq/airbyte/pull/6222) | Correct Spec File | | `0.1.1` | 2021-09-22 | [6315](https://github.com/airbytehq/airbyte/pull/6315) | Verify access to all sites when performing connection check | -| `0.1.0` | 2021-09-03 | [5350](https://github.com/airbytehq/airbyte/pull/5350) | Initial Release | \ No newline at end of file +| `0.1.0` | 2021-09-03 | [5350](https://github.com/airbytehq/airbyte/pull/5350) | Initial Release | + + \ No newline at end of file From 1f1223235117a8a3045fa23177534c86504f165f Mon Sep 17 00:00:00 2001 From: Patrick Nilan Date: Wed, 15 Nov 2023 13:57:10 -0800 Subject: [PATCH 075/124] Docs: Updates Bing Ads Source Connector Docs (Removes inapp page, adds , adds Limitations & Troubleshooting) (#32372) --- docs/integrations/sources/bing-ads.md | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md index 0986d6250757..1ba6d3570e56 100644 --- a/docs/integrations/sources/bing-ads.md +++ b/docs/integrations/sources/bing-ads.md @@ -1,7 +1,11 @@ # Bing Ads + + This page contains the setup guide and reference information for the Bing Ads source connector. + + ## Prerequisites - Microsoft Advertising account - Microsoft Developer Token @@ -84,6 +88,8 @@ The tenant is used in the authentication URL, for example: `https://login.micros 10. Click **Set up source**. + + ## Supported sync modes The Bing Ads source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): @@ -179,10 +185,25 @@ For example, if you select a report with daily aggregation, the report will cont A report's aggregation window is indicated in its name. For example, `account_performance_report_hourly` is the Account Performance Reported aggregated using an hourly window. -## Performance considerations +## Limitations & Troubleshooting + +
+ +Expand to see details about Bing Ads connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting The Bing Ads API limits the number of requests for all Microsoft Advertising clients. You can find detailed info [here](https://docs.microsoft.com/en-us/advertising/guides/services-protocol?view=bingads-13#throttling). +### Troubleshooting + +* Check out common troubleshooting issues for the Bing Ads source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
+ ## Changelog | Version | Date | Pull Request | Subject | @@ -233,4 +254,6 @@ The Bing Ads API limits the number of requests for all Microsoft Advertising cli | 0.1.3 | 2022-01-14 | [9510](https://github.com/airbytehq/airbyte/pull/9510) | Fixed broken dependency that blocked connector's operations | | 0.1.2 | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update titles and descriptions | | 0.1.1 | 2021-08-31 | [5750](https://github.com/airbytehq/airbyte/pull/5750) | Added reporting streams\) | -| 0.1.0 | 2021-07-22 | [4911](https://github.com/airbytehq/airbyte/pull/4911) | Initial release supported core streams \(Accounts, Campaigns, Ads, AdGroups\) | \ No newline at end of file +| 0.1.0 | 2021-07-22 | [4911](https://github.com/airbytehq/airbyte/pull/4911) | Initial release supported core streams \(Accounts, Campaigns, Ads, AdGroups\) | + + \ No newline at end of file From 94abd2a411d699d87faf320e1dc2795ab10fcf6e Mon Sep 17 00:00:00 2001 From: Patrick Nilan Date: Wed, 15 Nov 2023 13:58:19 -0800 Subject: [PATCH 076/124] Docs: Updates Slack Source Connector Docs (Removes inapp page, adds , adds Limitations & Troubleshooting) (#32366) --- docs/integrations/sources/slack.inapp.md | 58 ------------------------ docs/integrations/sources/slack.md | 31 ++++++++++++- 2 files changed, 30 insertions(+), 59 deletions(-) delete mode 100644 docs/integrations/sources/slack.inapp.md diff --git a/docs/integrations/sources/slack.inapp.md b/docs/integrations/sources/slack.inapp.md deleted file mode 100644 index 37bb4d82f283..000000000000 --- a/docs/integrations/sources/slack.inapp.md +++ /dev/null @@ -1,58 +0,0 @@ -## Prerequisites -- Access to Slack via OAuth or API Token (via Slack App or Legacy API Key) - - -## Setup Guide - -1. Enter a name for your connector -2. Select `Authenticate your Slack account` (preferred) and authorize into the Slack account. To use an API token instead, see the instructions below on creating one. -3. Toggle on **Join all channels** (recommended) to join all channels the user has access to or to sync data only from channels the app (if using API token) is already in. If false, you'll need to manually add all the channels from which you'd like to sync messages. -4. Enter a **Threads Lookback Window (Days)** to set how far back to look for messages in threads from when each sync start. -5. Enter a **Start Date**, enter the date in `YYYY-MM-DDTHH:mm:ssZ` format. Data created on and after this date will be replicated. -8. (Optional) Enter your `Channel name filter` to filter the list of channels Airbyte can access. If none are entered, Airbyte will sync all channels. It can be helpful to only sync required channels to avoid Slack's [requests limits](https://api.slack.com/docs/rate-limits). - - -9. Click **Set up source**. - -### Creating an API token - -You can no longer create "Legacy" API Keys, but if you already have one, you can use it with this source as the API key and skip setting up an application. - -In order to pull data out of your Slack instance, you need to create a Slack App. This may sound daunting, but it is actually pretty straight forward. Slack supplies [documentation](https://api.slack.com/start) on how to build apps. Feel free to follow that if you want to do something fancy. We'll describe the steps we followed to creat the Slack App for this tutorial. - -:::info -This tutorial assumes that you are an administrator on your slack instance. If you are not, you will need to coordinate with your administrator on the steps that require setting permissions for your app. -::: - -1. Go to the [apps page](https://api.slack.com/apps) -2. Click "Create New App" -3. It will request a name and the slack instance you want to create the app for. Make sure you select the instance form which you want to pull data. -4. Completing that form will take you to the "Basic Information" page for your app. -5. Now we need to grant the correct permissions to the app. \(This is the part that requires you to be an administrator\). Go to "Permissions". Then under "Bot Token Scopes" click on "Add an OAuth Scope". We will now need to add the following scopes: - - ```text - channels:history - channels:join - channels:read - files:read - groups:read - links:read - reactions:read - remote_files:read - team:read - usergroups:read - users.profile:read - users:read - ``` - - This may look daunting, but the search functionality in the dropdown should make this part go pretty quick. - -6. Scroll to the top of the page and click "Install to Workspace". This will generate a "Bot User OAuth Access Token". We will need this in a moment. -7. Now go to your slack instance. For any public channel go to info => more => add apps. In the search bar search for the name of your app. \(If using the desktop version of slack, you may need to restart Slack for it to pick up the new app\). Airbyte will only replicate messages from channels that the Slack bot has been added to. - - ![](../../.gitbook/assets/slack-add-apps.png) - -8. In Airbyte, create a Slack source. The "Bot User OAuth Access Token" from the earlier should be used as the token. -9. You can now pull data from your slack instance! - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Slack](https://docs.airbyte.com/integrations/sources/slack). diff --git a/docs/integrations/sources/slack.md b/docs/integrations/sources/slack.md index 84a8e03bb493..6baf8f9953a2 100644 --- a/docs/integrations/sources/slack.md +++ b/docs/integrations/sources/slack.md @@ -1,9 +1,15 @@ # Slack -This page contains the setup guide and reference information for the Slack source connector. + + +This page contains the setup guide and reference information for the [Slack](https://www.slack.com) source connector. + + ## Prerequisites +OAuth or API Token (via Slack App or Legacy API Key) is required for access to Slack. + You can no longer create "Legacy" API Keys, but if you already have one, you can use it with this source. Fill it into the API key section. We recommend creating a restricted, read-only key specifically for Airbyte access. This will allow you to control which resources Airbyte should be able to access. @@ -11,6 +17,7 @@ We recommend creating a restricted, read-only key specifically for Airbyte acces Note that refresh token are entirely optional for Slack and are not required to use Airbyte. You can learn more about refresh tokens [here](https://api.slack.com/authentication/rotation). ## Setup guide + ### Step 1: Set up Slack :::info @@ -97,6 +104,8 @@ We recommend creating a restricted, read-only key specifically for Airbyte acces 8. Click **Set up source**. + + ## Supported sync modes The Slack source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): @@ -133,6 +142,24 @@ It is recommended to sync required channels only, this can be done by specifying | `array` | `array` | | `object` | `object` | +## Limitations & Troubleshooting + +
+ +Expand to see details about Slack connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting +Slack has [rate limit restrictions](https://api.slack.com/docs/rate-limits). + +### Troubleshooting + +* Check out common troubleshooting issues for the Slack source connector on our Airbyte Forum [here](https://github.com/airbytehq/airbyte/discussions). + +
+ ## Changelog | Version | Date | Pull Request | Subject | @@ -164,3 +191,5 @@ It is recommended to sync required channels only, this can be done by specifying | 0.1.9 | 2021-07-20 | [4860](https://github.com/airbytehq/airbyte/pull/4860) | Fix reading threads issue | | 0.1.8 | 2021-07-14 | [4683](https://github.com/airbytehq/airbyte/pull/4683) | Add float\_ts primary key | | 0.1.7 | 2021-06-25 | [3978](https://github.com/airbytehq/airbyte/pull/3978) | Release Slack CDK Connector | + + \ No newline at end of file From 472c807544cbf61b4d90b291349506141bc83739 Mon Sep 17 00:00:00 2001 From: Daryna Ishchenko <80129833+darynaishchenko@users.noreply.github.com> Date: Thu, 16 Nov 2023 00:21:47 +0200 Subject: [PATCH 077/124] :bug: Source Bing Ads: campaigns, accounts, search query report streams: update schemas + date-time convert `rfc3339` (#31995) Co-authored-by: darynaishchenko Co-authored-by: Artem Inzhyyants Co-authored-by: artem1205 Co-authored-by: Artem Inzhyyants <36314070+artem1205@users.noreply.github.com> --- .../acceptance-test-config.yml | 2 + .../integration_tests/expected_records.jsonl | 26 +- .../{state.json => sample_state.json} | 14 +- .../connectors/source-bing-ads/metadata.yaml | 7 +- .../connectors/source-bing-ads/setup.py | 1 + .../source_bing_ads/base_streams.py | 326 ++++ .../source_bing_ads/bulk_streams.py | 190 +++ .../source_bing_ads/report_streams.py | 748 +++++++++ .../source_bing_ads/reports.py | 336 ---- ...account_impression_performance_report.json | 3 +- ..._impression_performance_report_hourly.json | 4 +- .../schemas/account_performance_report.json | 3 +- .../account_performance_report_hourly.json | 122 ++ .../source_bing_ads/schemas/accounts.json | 14 +- ...d_group_impression_performance_report.json | 3 +- ..._impression_performance_report_hourly.json | 4 +- .../schemas/ad_group_labels.json | 4 +- .../schemas/ad_group_performance_report.json | 3 +- .../ad_group_performance_report_hourly.json | 158 ++ .../schemas/ad_performance_report.json | 9 +- .../schemas/ad_performance_report_hourly.json | 152 ++ .../schemas/age_gender_audience_report.json | 3 +- .../age_gender_audience_report_hourly.json | 110 ++ .../schemas/app_install_ad_labels.json | 4 +- .../schemas/app_install_ads.json | 4 +- .../schemas/budget_summary_report.json | 3 +- ...ampaign_impression_performance_report.json | 3 +- ..._impression_performance_report_hourly.json | 4 +- .../schemas/campaign_labels.json | 4 +- .../schemas/campaign_performance_report.json | 3 +- .../campaign_performance_report_hourly.json | 176 +++ .../source_bing_ads/schemas/campaigns.json | 2 +- .../geographic_performance_report.json | 3 +- .../geographic_performance_report_hourly.json | 213 +++ .../schemas/keyword_labels.json | 4 +- .../schemas/keyword_performance_report.json | 15 +- .../keyword_performance_report_daily.json | 190 +++ .../keyword_performance_report_hourly.json | 180 +++ .../source_bing_ads/schemas/keywords.json | 4 +- .../source_bing_ads/schemas/labels.json | 4 +- .../search_query_performance_report.json | 5 +- ...earch_query_performance_report_hourly.json | 182 +++ .../user_location_performance_report.json | 3 +- ...er_location_performance_report_hourly.json | 215 +++ .../source-bing-ads/source_bing_ads/source.py | 15 +- .../source_bing_ads/streams.py | 1358 ----------------- .../source-bing-ads/source_bing_ads/utils.py | 29 + .../source-bing-ads/unit_tests/conftest.py | 62 + .../unit_tests/test_bulk_streams.py | 112 ++ .../unit_tests/test_reports.py | 316 +++- .../source-bing-ads/unit_tests/test_source.py | 300 +--- .../sources/bing-ads-migrations.md | 37 + docs/integrations/sources/bing-ads.md | 99 +- 53 files changed, 3645 insertions(+), 2146 deletions(-) rename airbyte-integrations/connectors/source-bing-ads/integration_tests/{state.json => sample_state.json} (61%) create mode 100644 airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py create mode 100644 airbyte-integrations/connectors/source-bing-ads/source_bing_ads/bulk_streams.py create mode 100644 airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py delete mode 100644 airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py create mode 100644 airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_performance_report_hourly.json create mode 100644 airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_performance_report_hourly.json create mode 100644 airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_performance_report_hourly.json create mode 100644 airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/age_gender_audience_report_hourly.json create mode 100644 airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_performance_report_hourly.json create mode 100644 airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/geographic_performance_report_hourly.json create mode 100644 airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report_daily.json create mode 100644 airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report_hourly.json create mode 100644 airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/search_query_performance_report_hourly.json create mode 100644 airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/user_location_performance_report_hourly.json delete mode 100644 airbyte-integrations/connectors/source-bing-ads/source_bing_ads/streams.py create mode 100644 airbyte-integrations/connectors/source-bing-ads/source_bing_ads/utils.py create mode 100644 airbyte-integrations/connectors/source-bing-ads/unit_tests/conftest.py create mode 100644 airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py diff --git a/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml index 480c522ac5a8..c61a97075ea3 100644 --- a/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml @@ -9,6 +9,8 @@ acceptance_tests: discovery: tests: - config_path: secrets/config.json + backward_compatibility_tests_config: + disable_for_version: "1.4.0" connection: tests: - config_path: secrets/config.json diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl index a4154ed8d18f..62627cf0aeae 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl @@ -1,5 +1,5 @@ {"stream":"ad_groups","data":{"AdRotation":null,"AudienceAdsBidAdjustment":null,"BiddingScheme":{"Type":"InheritFromParent","InheritedBidStrategyType":"EnhancedCpc"},"CpcBid":{"Amount":2.27},"EndDate":null,"FinalUrlSuffix":null,"ForwardCompatibilityMap":null,"Id":1356799861840328,"Language":null,"Name":"keywords","Network":"OwnedAndOperatedAndSyndicatedSearch","PrivacyStatus":null,"Settings":null,"StartDate":{"Day":7,"Month":11,"Year":2023},"Status":"Active","TrackingUrlTemplate":null,"UrlCustomParameters":null,"AdScheduleUseSearcherTimeZone":false,"AdGroupType":"SearchStandard","CpvBid":{"Amount":null},"CpmBid":{"Amount":null},"CampaignId":531016227,"AccountId":180519267,"CustomerId":251186883},"emitted_at":1699913367220} -{"stream":"ads","data":{"AdFormatPreference":"All","DevicePreference":0,"EditorialStatus":"Active","FinalAppUrls":null,"FinalMobileUrls":null,"FinalUrlSuffix":null,"FinalUrls":{"string":["https://airbyte.com"]},"ForwardCompatibilityMap":null,"Id":84800390693061,"Status":"Active","TrackingUrlTemplate":null,"Type":"ResponsiveSearch","UrlCustomParameters":null,"Descriptions":{"AssetLink":[{"Asset":{"Id":10239363892977,"Name":null,"Type":"TextAsset","Text":"Connect, integrate, and sync data seamlessly with Airbyte's 800+ contributors and growing!"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363892976,"Name":null,"Type":"TextAsset","Text":"Move data like a pro with our powerful tool trusted by 40,000+ engineers worldwide!"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null}]},"Domain":"airbyte.com","Headlines":{"AssetLink":[{"Asset":{"Id":10239363892979,"Name":null,"Type":"TextAsset","Text":"Get synced with Airbyte"},"AssetPerformanceLabel":"Good","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363893384,"Name":null,"Type":"TextAsset","Text":"Data management made easy"},"AssetPerformanceLabel":"Low","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363892978,"Name":null,"Type":"TextAsset","Text":"Connectors for every need"},"AssetPerformanceLabel":"Best","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363892980,"Name":null,"Type":"TextAsset","Text":"Industry-leading connectors"},"AssetPerformanceLabel":"Best","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363893383,"Name":null,"Type":"TextAsset","Text":"Try Airbyte now for free"},"AssetPerformanceLabel":"Good","EditorialStatus":"Active","PinnedField":null}]},"Path1":null,"Path2":null,"AdGroupId":1356799861840328,"AccountId":180519267,"CustomerId":251186883},"emitted_at":1699913377498} +{"stream":"ads","data":{"AdFormatPreference":"All","DevicePreference":0,"EditorialStatus":"Active","FinalAppUrls":null,"FinalMobileUrls":null,"FinalUrlSuffix":null,"FinalUrls":{"string":["https://airbyte.com"]},"ForwardCompatibilityMap":null,"Id":84800390693061,"Status":"Active","TrackingUrlTemplate":null,"Type":"ResponsiveSearch","UrlCustomParameters":null,"Descriptions":{"AssetLink":[{"Asset":{"Id":10239363892977,"Name":null,"Type":"TextAsset","Text":"Connect, integrate, and sync data seamlessly with Airbyte's 800+ contributors and growing!"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363892976,"Name":null,"Type":"TextAsset","Text":"Move data like a pro with our powerful tool trusted by 40,000+ engineers worldwide!"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null}]},"Domain":"airbyte.com","Headlines":{"AssetLink":[{"Asset":{"Id":10239363892979,"Name":null,"Type":"TextAsset","Text":"Get synced with Airbyte"},"AssetPerformanceLabel":"Good","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363893384,"Name":null,"Type":"TextAsset","Text":"Data management made easy"},"AssetPerformanceLabel":"Good","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363892978,"Name":null,"Type":"TextAsset","Text":"Connectors for every need"},"AssetPerformanceLabel":"Best","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363892980,"Name":null,"Type":"TextAsset","Text":"Industry-leading connectors"},"AssetPerformanceLabel":"Best","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363893383,"Name":null,"Type":"TextAsset","Text":"Try Airbyte now for free"},"AssetPerformanceLabel":"Good","EditorialStatus":"Active","PinnedField":null}]},"Path1":null,"Path2":null,"AdGroupId":1356799861840328,"AccountId":180519267,"CustomerId":251186883},"emitted_at":1700075716309} {"stream":"campaigns","data":{"AudienceAdsBidAdjustment":0,"BiddingScheme":{"Type":"EnhancedCpc"},"BudgetType":"DailyBudgetStandard","DailyBudget":2.0,"ExperimentId":null,"FinalUrlSuffix":null,"ForwardCompatibilityMap":null,"Id":531016227,"MultimediaAdsBidAdjustment":40,"Name":"Airbyte test","Status":"Active","SubType":null,"TimeZone":"CentralTimeUSCanada","TrackingUrlTemplate":null,"UrlCustomParameters":null,"CampaignType":"Search","Settings":{"Setting":[{"Type":"TargetSetting","Details":{"TargetSettingDetail":[{"CriterionTypeGroup":"Audience","TargetAndBid":false}]}}]},"BudgetId":null,"Languages":{"string":["English"]},"AdScheduleUseSearcherTimeZone":false,"AccountId":180519267,"CustomerId":251186883},"emitted_at":1699913381852} {"stream":"accounts","data":{"BillToCustomerId":251186883,"CurrencyCode":"USD","AccountFinancialStatus":"ClearFinancialStatus","Id":180535609,"Language":"English","LastModifiedByUserId":0,"LastModifiedTime":"2023-08-11T08:24:26.603000","Name":"DEMO-ACCOUNT","Number":"F149W3B6","ParentCustomerId":251186883,"PaymentMethodId":null,"PaymentMethodType":null,"PrimaryUserId":138225488,"AccountLifeCycleStatus":"Pause","TimeStamp":"AAAAAH10c1A=","TimeZone":"Santiago","PauseReason":2,"ForwardCompatibilityMap":null,"LinkedAgencies":null,"SalesHouseCustomerId":null,"TaxInformation":null,"BackUpPaymentInstrumentId":null,"BillingThresholdAmount":null,"BusinessAddress":{"City":"San Francisco","CountryCode":"US","Id":149694999,"Line1":"350 29th avenue","Line2":null,"Line3":null,"Line4":null,"PostalCode":"94121","StateOrProvince":"CA","TimeStamp":null,"BusinessName":"Daxtarity Inc."},"AutoTagType":"Inactive","SoldToPaymentInstrumentId":null,"AccountMode":"Expert"},"emitted_at":1699913384475} {"stream":"account_performance_report_daily","data":{"AccountId":180519267,"TimePeriod":"2023-11-07","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Syndicated search partners","DeliveredMatchType":"Phrase","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","AccountNumber":"F149MJ18","PhoneImpressions":0,"PhoneCalls":0,"Clicks":1,"Ctr":16.67,"Spend":0.33,"Impressions":6,"CostPerConversion":null,"Ptr":null,"Assists":0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"AverageCpc":0.33,"AveragePosition":0.0,"AverageCpm":55.0,"Conversions":0.0,"ConversionRate":0.0,"ConversionsQualified":0.0,"LowQualityClicks":0,"LowQualityClicksPercent":0.0,"LowQualityImpressions":0,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1699953356703} @@ -8,15 +8,15 @@ {"stream":"ad_group_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"TimePeriod":"2023-11-05","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Syndicated search partners","DeliveredMatchType":"Exact","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","Language":"English","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","AdGroupName":"keywords","AdGroupType":"Standard","Impressions":3,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"CostPerAssist":null,"CustomParameters":null,"FinalUrlSuffix":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"HistoricalQualityScore":5.0,"HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":1.0},"emitted_at":1699953500072} {"stream":"ad_group_impression_performance_report_daily","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-11-07","Status":"Active","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":1,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"DeviceType":"Computer","Language":"German","ImpressionSharePercent":null,"ImpressionLostToBudgetPercent":null,"ImpressionLostToRankAggPercent":null,"QualityScore":5,"ExpectedCtr":2.0,"AdRelevance":3,"LandingPageExperience":1,"HistoricalQualityScore":null,"HistoricalExpectedCtr":null,"HistoricalAdRelevance":null,"HistoricalLandingPageExperience":null,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","CampaignStatus":"Active","AdGroupLabels":null,"ExactMatchImpressionSharePercent":null,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":null,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":null,"TopImpressionShareLostToBudgetPercent":null,"AbsoluteTopImpressionShareLostToRankPercent":null,"AbsoluteTopImpressionShareLostToBudgetPercent":null,"TopImpressionSharePercent":null,"AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"BaseCampaignId":531016227,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AdGroupType":"Standard","AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1699955374204} {"stream":"ad_group_impression_performance_report_weekly","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-11-05","Status":"Active","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":3,"Clicks":1,"Ctr":33.33,"AverageCpc":0.08,"Spend":0.08,"AveragePosition":0.0,"Conversions":0,"ConversionRate":0.0,"CostPerConversion":null,"DeviceType":"Computer","Language":"Danish","ImpressionSharePercent":11.11,"ImpressionLostToBudgetPercent":0.0,"ImpressionLostToRankAggPercent":88.89,"QualityScore":5,"ExpectedCtr":2.0,"AdRelevance":3,"LandingPageExperience":1,"HistoricalQualityScore":5,"HistoricalExpectedCtr":2,"HistoricalAdRelevance":3,"HistoricalLandingPageExperience":1,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Microsoft sites and select traffic","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","CampaignStatus":"Active","AdGroupLabels":null,"ExactMatchImpressionSharePercent":null,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":null,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":null,"TopImpressionShareLostToBudgetPercent":null,"AbsoluteTopImpressionShareLostToRankPercent":null,"AbsoluteTopImpressionShareLostToBudgetPercent":null,"TopImpressionSharePercent":null,"AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"BaseCampaignId":531016227,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":0.0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AdGroupType":"Standard","AverageCpm":26.67,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1699955407065} -{"stream":"ad_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"AdId":84800390693061,"TimePeriod":"2023-11-07","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Language":"German","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","DeliveredMatchType":"Phrase","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","AdGroupName":"keywords","Impressions":2,"Clicks":1,"Ctr":50.0,"Spend":0.79,"CostPerConversion":null,"DestinationUrl":null,"Assists":0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"AdDescription":null,"AdDescription2":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"Conversions":0.0,"ConversionRate":0.0,"ConversionsQualified":0.0,"AverageCpc":0.79,"AveragePosition":0.0,"AverageCpm":395.0,"AllConversions":0,"AllConversionRate":0.0,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1699953597299} -{"stream":"ad_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"AdId":84800390693061,"TimePeriod":"2023-11-05","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Language":"Danish","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","DeliveredMatchType":"Phrase","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","AdGroupName":"keywords","Impressions":3,"Clicks":1,"Ctr":33.33,"Spend":0.08,"CostPerConversion":null,"DestinationUrl":null,"Assists":0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"AdDescription":null,"AdDescription2":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"Conversions":0.0,"ConversionRate":0.0,"ConversionsQualified":0.0,"AverageCpc":0.08,"AveragePosition":0.0,"AverageCpm":26.67,"AllConversions":0,"AllConversionRate":0.0,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1699953623795} -{"stream":"budget_summary_report","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"CampaignName":"Airbyte test","CampaignId":531016227,"Date":"11/7/2023","MonthlyBudget":48.0,"DailySpend":3.12,"MonthToDateSpend":3.12},"emitted_at":1699953978176} +{"stream":"ad_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"AdId":84800390693061,"TimePeriod":"2023-11-07","AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":100.0,"CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Language":"German","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","DeliveredMatchType":"Phrase","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","AdGroupName":"keywords","Impressions":2,"Clicks":1,"Ctr":50.0,"Spend":0.79,"CostPerConversion":null,"DestinationUrl":null,"Assists":0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"AdDescription":null,"AdDescription2":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"Conversions":0.0,"ConversionRate":0.0,"ConversionsQualified":0.0,"AverageCpc":0.79,"AveragePosition":0.0,"AverageCpm":395.0,"AllConversions":0,"AllConversionRate":0.0,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1700076965521} +{"stream":"ad_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"AdId":84800390693061,"TimePeriod":"2023-11-05","AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Language":"Danish","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","DeliveredMatchType":"Phrase","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","AdGroupName":"keywords","Impressions":3,"Clicks":1,"Ctr":33.33,"Spend":0.08,"CostPerConversion":null,"DestinationUrl":null,"Assists":0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"AdDescription":null,"AdDescription2":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"Conversions":0.0,"ConversionRate":0.0,"ConversionsQualified":0.0,"AverageCpc":0.08,"AveragePosition":0.0,"AverageCpm":26.67,"AllConversions":0,"AllConversionRate":0.0,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1700077221065} +{"stream":"budget_summary_report","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"CampaignName":"Airbyte test","CampaignId":531016227,"Date":"2023-11-07","MonthlyBudget":48.0,"DailySpend":3.12,"MonthToDateSpend":3.12},"emitted_at":1699953978176} {"stream":"campaign_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"TimePeriod":"2023-11-07","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Syndicated search partners","DeliveredMatchType":"Phrase","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Other","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","CampaignStatus":"Active","CampaignLabels":null,"Impressions":0,"Clicks":0,"Ctr":null,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"AdRelevance":3.0,"LandingPageExperience":1.0,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":1,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"BudgetName":null,"BudgetStatus":null,"BudgetAssociationStatus":"Current","HistoricalQualityScore":null,"HistoricalExpectedCtr":null,"HistoricalAdRelevance":null,"HistoricalLandingPageExperience":null},"emitted_at":1699954051067} {"stream":"campaign_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"TimePeriod":"2023-11-05","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Syndicated search partners","DeliveredMatchType":"Exact","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","CampaignStatus":"Active","CampaignLabels":null,"Impressions":9,"Clicks":1,"Ctr":11.11,"Spend":0.03,"CostPerConversion":null,"QualityScore":5.0,"AdRelevance":3.0,"LandingPageExperience":1.0,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"CustomParameters":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.03,"AveragePosition":0.0,"AverageCpm":3.33,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"LowQualityClicks":0,"LowQualityClicksPercent":0.0,"LowQualityImpressions":0,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"BudgetName":null,"BudgetStatus":null,"BudgetAssociationStatus":"Current","HistoricalQualityScore":5.0,"HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":1.0},"emitted_at":1699954081143} {"stream":"campaign_impression_performance_report_daily","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-11-07","CampaignStatus":"Active","CampaignName":"Airbyte test","CampaignId":531016227,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":10,"Clicks":1,"Ctr":10.0,"AverageCpc":0.33,"Spend":0.33,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"LowQualityClicks":0,"LowQualityClicksPercent":0.0,"LowQualityImpressions":9,"LowQualityImpressionsPercent":47.37,"LowQualityConversions":0,"LowQualityConversionRate":null,"DeviceType":"Computer","ImpressionSharePercent":3.37,"ImpressionLostToBudgetPercent":85.19,"ImpressionLostToRankAggPercent":11.45,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"HistoricalQualityScore":null,"HistoricalExpectedCtr":null,"HistoricalAdRelevance":null,"HistoricalLandingPageExperience":null,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":0,"CampaignLabels":null,"ExactMatchImpressionSharePercent":null,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":6.02,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":14.63,"TopImpressionShareLostToBudgetPercent":77.24,"AbsoluteTopImpressionShareLostToRankPercent":15.66,"AbsoluteTopImpressionShareLostToBudgetPercent":78.31,"TopImpressionSharePercent":8.13,"AbsoluteTopImpressionRatePercent":50.0,"TopImpressionRatePercent":100.0,"BaseCampaignId":531016227,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AverageCpm":33.0,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1699954182626} {"stream":"campaign_impression_performance_report_weekly","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-11-05","CampaignStatus":"Active","CampaignName":"Airbyte test","CampaignId":531016227,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":10,"Clicks":1,"Ctr":10.0,"AverageCpc":0.33,"Spend":0.33,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"LowQualityClicks":0,"LowQualityClicksPercent":0.0,"LowQualityImpressions":9,"LowQualityImpressionsPercent":47.37,"LowQualityConversions":0,"LowQualityConversionRate":null,"DeviceType":"Computer","ImpressionSharePercent":10.87,"ImpressionLostToBudgetPercent":17.05,"ImpressionLostToRankAggPercent":72.08,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"HistoricalQualityScore":null,"HistoricalExpectedCtr":null,"HistoricalAdRelevance":null,"HistoricalLandingPageExperience":null,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":0,"CampaignLabels":null,"ExactMatchImpressionSharePercent":29.07,"ClickSharePercent":2.89,"AbsoluteTopImpressionSharePercent":8.88,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":76.51,"TopImpressionShareLostToBudgetPercent":9.99,"AbsoluteTopImpressionShareLostToRankPercent":81.99,"AbsoluteTopImpressionShareLostToBudgetPercent":9.13,"TopImpressionSharePercent":13.5,"AbsoluteTopImpressionRatePercent":50.0,"TopImpressionRatePercent":100.0,"BaseCampaignId":531016227,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AverageCpm":33.0,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1699954211223} -{"stream":"keyword_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"KeywordId":84801135055365,"AdId":84800390693061,"TimePeriod":"2023-11-07","CurrencyCode":"USD","DeliveredMatchType":"Phrase","AdDistribution":"Search","DeviceType":"Computer","Language":"German","Network":"Syndicated search partners","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","Keyword":"connector","KeywordStatus":"Active","Impressions":1,"Clicks":0,"Ctr":0.0,"CurrentMaxCpc":2.27,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"QualityImpact":0.0,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"Mainline1Bid":null,"MainlineBid":1.2,"FirstPageBid":0.56,"FinalUrlSuffix":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"HistoricalQualityScore":null,"HistoricalExpectedCtr":null,"HistoricalAdRelevance":null,"HistoricalLandingPageExperience":null},"emitted_at":1699954312976} -{"stream":"keyword_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"KeywordId":84801135055365,"AdId":84800390693061,"TimePeriod":"2023-11-05","CurrencyCode":"USD","DeliveredMatchType":"Exact","AdDistribution":"Search","DeviceType":"Computer","Language":"English","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","Keyword":"connector","KeywordStatus":"Active","Impressions":2,"Clicks":0,"Ctr":0.0,"CurrentMaxCpc":2.27,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"QualityImpact":0.0,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"Mainline1Bid":null,"MainlineBid":1.2,"FirstPageBid":0.56,"FinalUrlSuffix":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1699954348243} +{"stream":"keyword_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"KeywordId":84801135055365,"Keyword":"connector","AdId":84800390693061,"TimePeriod":"2023-11-07","CurrencyCode":"USD","DeliveredMatchType":"Phrase","AdDistribution":"Search","DeviceType":"Computer","Language":"German","Network":"Syndicated search partners","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","KeywordStatus":"Active","HistoricalExpectedCtr":null,"HistoricalAdRelevance":null,"HistoricalLandingPageExperience":null,"HistoricalQualityScore":null,"Impressions":1,"Clicks":0,"Ctr":0.0,"CurrentMaxCpc":2.27,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"QualityImpact":0.0,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"Mainline1Bid":null,"MainlineBid":1.18,"FirstPageBid":0.51,"FinalUrlSuffix":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1700078149400} +{"stream":"keyword_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"KeywordId":84801135055365,"Keyword":"connector","AdId":84800390693061,"TimePeriod":"2023-11-05","CurrencyCode":"USD","DeliveredMatchType":"Exact","AdDistribution":"Search","DeviceType":"Computer","Language":"English","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","KeywordStatus":"Active","Impressions":2,"Clicks":0,"Ctr":0.0,"CurrentMaxCpc":2.27,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"QualityImpact":0.0,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"Mainline1Bid":null,"MainlineBid":1.18,"FirstPageBid":0.51,"FinalUrlSuffix":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1700078299436} {"stream":"geographic_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"TimePeriod":"2023-11-07","Country":"Australia","CurrencyCode":"USD","DeliveredMatchType":"Broad","AdDistribution":"Search","DeviceType":"Computer","Language":"English","Network":"Syndicated search partners","DeviceOS":"Windows","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","MetroArea":null,"State":"New South Wales","City":null,"AdGroupName":"keywords","Ctr":0.0,"ProximityTargetLocation":null,"Radius":"0","Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"LocationType":"Physical location","MostSpecificLocation":"2000","AccountStatus":"Active","CampaignStatus":"Active","AdGroupStatus":"Active","County":null,"PostalCode":"2000","LocationId":"122395","BaseCampaignId":"531016227","AllCostPerConversion":null,"AllReturnOnAdSpend":null,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":"100.00","AllConversionsQualified":"0.00","ViewThroughConversionsQualified":null,"Neighborhood":null,"ViewThroughRevenue":"0.00","CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null,"AssetGroupStatus":null,"Clicks":0,"Spend":0.0,"Impressions":1,"CostPerConversion":null,"AccountName":"Airbyte","AccountNumber":"F149MJ18","CampaignName":"Airbyte test","Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1699956863587} {"stream":"geographic_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"TimePeriod":"2023-11-05","Country":"Argentina","CurrencyCode":"USD","DeliveredMatchType":"Exact","AdDistribution":"Search","DeviceType":"Computer","Language":"Spanish","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","MetroArea":null,"State":"Buenos Aires Province","City":null,"AdGroupName":"keywords","Ctr":0.0,"ProximityTargetLocation":null,"Radius":"0","Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"LocationType":"Physical location","MostSpecificLocation":"Buenos Aires Province","AccountStatus":"Active","CampaignStatus":"Active","AdGroupStatus":"Active","County":null,"PostalCode":null,"LocationId":"141965","BaseCampaignId":"531016227","AllCostPerConversion":null,"AllReturnOnAdSpend":null,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":"100.00","AllConversionsQualified":"0.00","ViewThroughConversionsQualified":null,"Neighborhood":null,"ViewThroughRevenue":"0.00","CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null,"AssetGroupStatus":null,"Clicks":0,"Spend":0.0,"Impressions":1,"CostPerConversion":null,"AccountName":"Airbyte","AccountNumber":"F149MJ18","CampaignName":"Airbyte test","Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1699953673210} {"stream":"age_gender_audience_report_daily","data":{"AccountId":180519267,"AgeGroup":"Unknown","Gender":"Unknown","TimePeriod":"2023-11-07","AllConversions":0,"AccountName":"Airbyte","AccountNumber":"F149MJ18","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"AdDistribution":"Search","Impressions":3,"Clicks":1,"Conversions":0.0,"Spend":0.79,"Revenue":0.0,"ExtendedCost":0.0,"Assists":0,"Language":"German","AccountStatus":"Active","CampaignStatus":"Active","AdGroupStatus":"Active","BaseCampaignId":"531016227","AllRevenue":0.0,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":33.33,"TopImpressionRatePercent":100.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0},"emitted_at":1699954406862} @@ -27,10 +27,10 @@ {"stream":"user_location_performance_report_weekly","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-11-05","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"Country":"Argentina","State":null,"MetroArea":null,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":1,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"ProximityTargetLocation":null,"Radius":0,"Language":"Spanish","City":null,"QueryIntentCountry":"Argentina","QueryIntentState":null,"QueryIntentCity":null,"QueryIntentDMA":null,"BidMatchType":"Broad","DeliveredMatchType":"Phrase","Network":"Syndicated search partners","TopVsOther":"Syndicated search partners - Top","DeviceType":"Computer","DeviceOS":"Unknown","Assists":0,"Conversions":0,"ConversionRate":null,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerConversion":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"County":null,"PostalCode":null,"QueryIntentCounty":null,"QueryIntentPostalCode":null,"LocationId":8,"QueryIntentLocationId":8,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"Neighborhood":null,"QueryIntentNeighborhood":null,"ViewThroughRevenue":0.0,"CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null},"emitted_at":1699954842196} {"stream":"account_impression_performance_report_daily","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-11-07","CurrencyCode":"USD","AdDistribution":"Search","Impressions":10,"Clicks":1,"Ctr":10.0,"AverageCpc":0.33,"Spend":0.33,"AveragePosition":0.0,"Conversions":0,"ConversionRate":0.0,"CostPerConversion":null,"LowQualityClicks":0,"LowQualityClicksPercent":0.0,"LowQualityImpressions":9,"LowQualityImpressionsPercent":47.37,"LowQualityConversions":0,"LowQualityConversionRate":null,"DeviceType":"Computer","ImpressionSharePercent":3.37,"ImpressionLostToBudgetPercent":85.19,"ImpressionLostToRankAggPercent":11.45,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":0,"ExactMatchImpressionSharePercent":null,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":6.02,"TopImpressionShareLostToRankPercent":14.63,"TopImpressionShareLostToBudgetPercent":77.24,"AbsoluteTopImpressionShareLostToRankPercent":15.66,"AbsoluteTopImpressionShareLostToBudgetPercent":78.31,"TopImpressionSharePercent":8.13,"AbsoluteTopImpressionRatePercent":50.0,"TopImpressionRatePercent":100.0,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":0.0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"AverageCpm":33.0,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1699955144374} {"stream":"account_impression_performance_report_weekly","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-11-05","CurrencyCode":"USD","AdDistribution":"Search","Impressions":457,"Clicks":5,"Ctr":1.09,"AverageCpc":0.38,"Spend":1.88,"AveragePosition":0.0,"Conversions":0,"ConversionRate":0.0,"CostPerConversion":null,"LowQualityClicks":6,"LowQualityClicksPercent":54.55,"LowQualityImpressions":183,"LowQualityImpressionsPercent":28.59,"LowQualityConversions":0,"LowQualityConversionRate":0.0,"DeviceType":"Computer","ImpressionSharePercent":10.87,"ImpressionLostToBudgetPercent":17.05,"ImpressionLostToRankAggPercent":72.08,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","LowQualityGeneralClicks":4,"LowQualitySophisticatedClicks":2,"ExactMatchImpressionSharePercent":29.07,"ClickSharePercent":2.89,"AbsoluteTopImpressionSharePercent":8.88,"TopImpressionShareLostToRankPercent":76.51,"TopImpressionShareLostToBudgetPercent":9.99,"AbsoluteTopImpressionShareLostToRankPercent":81.99,"AbsoluteTopImpressionShareLostToBudgetPercent":9.13,"TopImpressionSharePercent":13.5,"AbsoluteTopImpressionRatePercent":39.61,"TopImpressionRatePercent":81.62,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":0.0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"AverageCpm":4.11,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1699955205307} -{"stream":"labels","data":{"Status":"Active","Id":10239203506495,"Client Id":null,"Modified Time":"04/27/2023 17:16:49.170","Description":null,"Label":"what a new label","Color":"#B0B20F","Account Id":180278106},"emitted_at":1698687185070} -{"stream":"campaign_labels","data":{"Status":"Active","Id":10239203506494,"Parent Id":413444833,"Campaign":null,"Client Id":null,"Modified Time":"04/27/2023 17:57:38.110","Account Id":180278106},"emitted_at":1698687214960} -{"stream":"keyword_labels","data":{"Status":"Active","Id":10239203506495,"Parent Id":84868925026027,"Client Id":null,"Modified Time":"04/27/2023 17:22:52.733","Account Id":180278106},"emitted_at":1698687224964} -{"stream":"ad_group_labels","data":{"Status":"Active","Id":10239203506494,"Parent Id":1350201453189474,"Campaign":null,"Ad Group":null,"Client Id":null,"Modified Time":"04/27/2023 18:00:14.970","Account Id":180278106},"emitted_at":1698751388918} -{"stream": "keywords", "data": {"Status": "Active", "Id": 84525593559627, "Parent Id": "1352400325389092", "Campaign": "Test 2", "Ad Group": "Airbyte", "Client Id": null, "Modified Time": "08/13/2021 02:33:08.560", "Tracking Template": null, "Final Url Suffix": null, "Custom Parameter": null, "Final Url": null, "Mobile Final Url": null, "Bid Strategy Type": "InheritFromParent", "Inherited Bid Strategy Type": "MaxClicks", "Destination Url": null, "Editorial Status": "Active", "Editorial Location": null, "Editorial Term": null, "Editorial Reason Code": null, "Editorial Appeal Status": null, "Keyword": "big data integration tools", "Match Type": "Broad", "Bid": "0.11", "Param1": null, "Param2": null, "Param3": null, "Publisher Countries": null, "Quality Score": null, "Keyword Relevance": null, "Landing Page Relevance": null, "Landing Page User Experience": null, "Account Id": 180278106}, "emitted_at": 1698768109942} -{"stream": "keywords", "data": {"Status": "Active", "Id": 84525593559628, "Parent Id": "1352400325389092", "Campaign": "Test 2", "Ad Group": "Airbyte", "Client Id": null, "Modified Time": "08/13/2021 02:33:14.270", "Tracking Template": null, "Final Url Suffix": null, "Custom Parameter": null, "Final Url": null, "Mobile Final Url": null, "Bid Strategy Type": "InheritFromParent", "Inherited Bid Strategy Type": "MaxClicks", "Destination Url": null, "Editorial Status": "Active", "Editorial Location": null, "Editorial Term": null, "Editorial Reason Code": null, "Editorial Appeal Status": null, "Keyword": "best data integration tool", "Match Type": "Broad", "Bid": "0.11", "Param1": null, "Param2": null, "Param3": null, "Publisher Countries": null, "Quality Score": null, "Keyword Relevance": null, "Landing Page Relevance": null, "Landing Page User Experience": null, "Account Id": 180278106}, "emitted_at": 1698768109942} -{"stream": "keywords", "data": {"Status": "Active", "Id": 84525593559629, "Parent Id": "1352400325389092", "Campaign": "Test 2", "Ad Group": "Airbyte", "Client Id": null, "Modified Time": "08/13/2021 02:33:19.053", "Tracking Template": null, "Final Url Suffix": null, "Custom Parameter": null, "Final Url": null, "Mobile Final Url": null, "Bid Strategy Type": "InheritFromParent", "Inherited Bid Strategy Type": "MaxClicks", "Destination Url": null, "Editorial Status": "Active", "Editorial Location": null, "Editorial Term": null, "Editorial Reason Code": null, "Editorial Appeal Status": null, "Keyword": "data integration tools", "Match Type": "Broad", "Bid": "0.11", "Param1": null, "Param2": null, "Param3": null, "Publisher Countries": null, "Quality Score": null, "Keyword Relevance": null, "Landing Page Relevance": null, "Landing Page User Experience": null, "Account Id": 180278106}, "emitted_at": 1698768109943} +{"stream":"labels","data":{"Status":"Active","Id":10239203506495,"Client Id":null,"Modified Time":"2023-04-27T17:16:49.170+00:00","Description":null,"Label":"what a new label","Color":"#B0B20F","Account Id":180278106},"emitted_at":1698687185070} +{"stream":"campaign_labels","data":{"Status":"Active","Id":10239203506494,"Parent Id":413444833,"Campaign":null,"Client Id":null,"Modified Time":"2023-04-27T17:57:38.110+00:00","Account Id":180278106},"emitted_at":1698687214960} +{"stream":"keyword_labels","data":{"Status":"Active","Id":10239203506495,"Parent Id":84868925026027,"Client Id":null,"Modified Time":"2023-04-27T17:22:52.733+00:00","Account Id":180278106},"emitted_at":1698687224964} +{"stream":"ad_group_labels","data":{"Status":"Active","Id":10239203506494,"Parent Id":1350201453189474,"Campaign":null,"Ad Group":null,"Client Id":null,"Modified Time":"2023-04-27T18:00:14.970+00:00","Account Id":180278106},"emitted_at":1698751388918} +{"stream": "keywords", "data": {"Status": "Active", "Id": 84525593559627, "Parent Id": "1352400325389092", "Campaign": "Test 2", "Ad Group": "Airbyte", "Client Id": null, "Modified Time": "2021-08-13T02:33:08.560+00:00", "Tracking Template": null, "Final Url Suffix": null, "Custom Parameter": null, "Final Url": null, "Mobile Final Url": null, "Bid Strategy Type": "InheritFromParent", "Inherited Bid Strategy Type": "MaxClicks", "Destination Url": null, "Editorial Status": "Active", "Editorial Location": null, "Editorial Term": null, "Editorial Reason Code": null, "Editorial Appeal Status": null, "Keyword": "big data integration tools", "Match Type": "Broad", "Bid": "0.11", "Param1": null, "Param2": null, "Param3": null, "Publisher Countries": null, "Quality Score": null, "Keyword Relevance": null, "Landing Page Relevance": null, "Landing Page User Experience": null, "Account Id": 180278106}, "emitted_at": 1698768109942} +{"stream": "keywords", "data": {"Status": "Active", "Id": 84525593559628, "Parent Id": "1352400325389092", "Campaign": "Test 2", "Ad Group": "Airbyte", "Client Id": null, "Modified Time": "2021-08-13T02:33:14.270+00:00", "Tracking Template": null, "Final Url Suffix": null, "Custom Parameter": null, "Final Url": null, "Mobile Final Url": null, "Bid Strategy Type": "InheritFromParent", "Inherited Bid Strategy Type": "MaxClicks", "Destination Url": null, "Editorial Status": "Active", "Editorial Location": null, "Editorial Term": null, "Editorial Reason Code": null, "Editorial Appeal Status": null, "Keyword": "best data integration tool", "Match Type": "Broad", "Bid": "0.11", "Param1": null, "Param2": null, "Param3": null, "Publisher Countries": null, "Quality Score": null, "Keyword Relevance": null, "Landing Page Relevance": null, "Landing Page User Experience": null, "Account Id": 180278106}, "emitted_at": 1698768109942} +{"stream": "keywords", "data": {"Status": "Active", "Id": 84525593559629, "Parent Id": "1352400325389092", "Campaign": "Test 2", "Ad Group": "Airbyte", "Client Id": null, "Modified Time": "2021-08-13T02:33:19.053+00:00", "Tracking Template": null, "Final Url Suffix": null, "Custom Parameter": null, "Final Url": null, "Mobile Final Url": null, "Bid Strategy Type": "InheritFromParent", "Inherited Bid Strategy Type": "MaxClicks", "Destination Url": null, "Editorial Status": "Active", "Editorial Location": null, "Editorial Term": null, "Editorial Reason Code": null, "Editorial Appeal Status": null, "Keyword": "data integration tools", "Match Type": "Broad", "Bid": "0.11", "Param1": null, "Param2": null, "Param3": null, "Publisher Countries": null, "Quality Score": null, "Keyword Relevance": null, "Landing Page Relevance": null, "Landing Page User Experience": null, "Account Id": 180278106}, "emitted_at": 1698768109943} diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json b/airbyte-integrations/connectors/source-bing-ads/integration_tests/sample_state.json similarity index 61% rename from airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json rename to airbyte-integrations/connectors/source-bing-ads/integration_tests/sample_state.json index ef99548a8bb2..8b6447f78e3e 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/sample_state.json @@ -4,7 +4,7 @@ "stream": { "stream_state": { "180278106": { - "TimePeriod": 1627820152 + "TimePeriod": "2021-08-01T14:00:00+00:00" } }, "stream_descriptor": { "name": "keyword_performance_report_hourly" } @@ -15,10 +15,10 @@ "stream": { "stream_state": { "180278106": { - "Date": 1627800152 + "Date": "2021-08-01" } }, - "stream_descriptor": { "name": "budget_summary_report_hourly" } + "stream_descriptor": { "name": "budget_summary_report" } } }, { @@ -26,10 +26,10 @@ "stream": { "stream_state": { "180278106": { - "TimePeriod": 1627795152 + "TimePeriod": "2021-08-01" } }, - "stream_descriptor": { "name": "ad_performance_report_hourly" } + "stream_descriptor": { "name": "ad_performance_report_weekly" } } }, { @@ -37,10 +37,10 @@ "stream": { "stream_state": { "180278106": { - "TimePeriod": 1727810152 + "TimePeriod": "2023-04-27T15:16:49.170+00:00" } }, - "stream_descriptor": { "name": "campaign_performance_report_hourly" } + "stream_descriptor": { "name": "labels" } } } ] diff --git a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml index 2fa6e2d659d9..63f2a336f97f 100644 --- a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml @@ -16,7 +16,7 @@ data: connectorSubtype: api connectorType: source definitionId: 47f25999-dd5e-4636-8c39-e7cea2453331 - dockerImageTag: 1.13.0 + dockerImageTag: 2.0.0 dockerRepository: airbyte/source-bing-ads documentationUrl: https://docs.airbyte.com/integrations/sources/bing-ads githubIssueLabel: source-bing-ads @@ -39,6 +39,11 @@ data: primary keys. A data reset and schema refresh of all the affected streams is required for the changes to take effect. upgradeDeadline: "2023-10-25" + 2.0.0: + message: > + Version 2.0.0 updates schemas for all hourly reports (end in report_hourly), and the following streams: Accounts, Campaigns, Search Query Performance Report, AppInstallAds, AppInstallAdLabels, Labels, Campaign Labels, Keyword Labels, Ad Group Labels, Keywords, and Budget Summary Report. + Users will need to refresh the source schema and reset affected streams after upgrading. + upgradeDeadline: "2023-11-29" suggestedStreams: streams: - campaigns diff --git a/airbyte-integrations/connectors/source-bing-ads/setup.py b/airbyte-integrations/connectors/source-bing-ads/setup.py index 29de9b3aded4..5c326b2d9aad 100644 --- a/airbyte-integrations/connectors/source-bing-ads/setup.py +++ b/airbyte-integrations/connectors/source-bing-ads/setup.py @@ -8,6 +8,7 @@ MAIN_REQUIREMENTS = ["airbyte-cdk", "bingads~=13.0.17", "urllib3<2.0", "pandas"] TEST_REQUIREMENTS = [ + "freezegun", "requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1", diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py new file mode 100644 index 000000000000..39f1eec1957e --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py @@ -0,0 +1,326 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import ssl +import time +from abc import ABC, abstractmethod +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union +from urllib.error import URLError + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams import Stream +from bingads.service_client import ServiceClient +from bingads.v13.reporting.reporting_service_manager import ReportingServiceManager +from source_bing_ads.client import Client +from suds import sudsobject + + +class BingAdsBaseStream(Stream, ABC): + primary_key: Optional[Union[str, List[str], List[List[str]]]] = None + + def __init__(self, client: Client, config: Mapping[str, Any]) -> None: + super().__init__() + self.client = client + self.config = config + + +class BingAdsStream(BingAdsBaseStream, ABC): + @property + @abstractmethod + def operation_name(self) -> str: + """ + Specifies operation name to use for a current stream + """ + + @property + @abstractmethod + def service_name(self) -> str: + """ + Specifies bing ads service name for a current stream + """ + + @property + def parent_key_to_foreign_key_map(self) -> MutableMapping[str, str]: + """ + Specifies dict with field in record as kay and slice key as value to be inserted in record in transform method. + """ + return {} + + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + foreign_keys = {key: stream_slice.get(value) for key, value in self.parent_key_to_foreign_key_map.items()} + return record | foreign_keys + + @property + def _service(self) -> Union[ServiceClient, ReportingServiceManager]: + return self.client.get_service(service_name=self.service_name) + + @property + def _user_id(self) -> int: + return self._get_user_id() + + # TODO remove once Microsoft support confirm their SSL certificates are always valid... + def _get_user_id(self, number_of_retries=10): + """""" + try: + return self._service.GetUser().User.Id + except URLError as error: + if isinstance(error.reason, ssl.SSLError): + self.logger.warning("SSL certificate error, retrying...") + if number_of_retries > 0: + time.sleep(1) + return self._get_user_id(number_of_retries - 1) + else: + raise error + + def next_page_token(self, response: sudsobject.Object, **kwargs: Mapping[str, Any]) -> Optional[Mapping[str, Any]]: + """ + Default method for streams that don't support pagination + """ + return None + + def send_request(self, params: Mapping[str, Any], customer_id: str, account_id: str = None) -> Mapping[str, Any]: + request_kwargs = { + "service_name": self.service_name, + "customer_id": customer_id, + "account_id": account_id, + "operation_name": self.operation_name, + "params": params, + } + request = self.client.request(**request_kwargs) + return request + + def read_records( + self, + sync_mode: SyncMode, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + **kwargs: Mapping[str, Any], + ) -> Iterable[Mapping[str, Any]]: + stream_state = stream_state or {} + next_page_token = None + account_id = str(stream_slice.get("account_id")) if stream_slice else None + customer_id = str(stream_slice.get("customer_id")) if stream_slice else None + + while True: + params = self.request_params( + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + account_id=account_id, + ) + response = self.send_request(params, customer_id=customer_id, account_id=account_id) + for record in self.parse_response(response): + yield self.transform(record, stream_slice) + + next_page_token = self.next_page_token(response, current_page_token=next_page_token) + if not next_page_token: + break + + def parse_response(self, response: sudsobject.Object, **kwargs) -> Iterable[Mapping]: + if response is not None and hasattr(response, self.data_field): + yield from self.client.asdict(response)[self.data_field] + + +class BingAdsCampaignManagementStream(BingAdsStream, ABC): + service_name: str = "CampaignManagement" + + @property + @abstractmethod + def data_field(self) -> str: + """ + Specifies root object name in a stream response + """ + + @property + @abstractmethod + def additional_fields(self) -> Optional[str]: + """ + Specifies which additional fields to fetch for a current stream. + Expected format: field names separated by space + """ + + def parse_response(self, response: sudsobject.Object, **kwargs) -> Iterable[Mapping]: + if response is not None and hasattr(response, self.data_field): + yield from self.client.asdict(response)[self.data_field] + + +class Accounts(BingAdsStream): + """ + Searches for accounts that the current authenticated user can access. + API doc: https://docs.microsoft.com/en-us/advertising/customer-management-service/searchaccounts?view=bingads-13 + Account schema: https://docs.microsoft.com/en-us/advertising/customer-management-service/advertiseraccount?view=bingads-13 + Stream caches incoming responses to be able to reuse this data in Campaigns stream + """ + + primary_key = "Id" + # Stream caches incoming responses to avoid duplicated http requests + use_cache: bool = True + data_field: str = "AdvertiserAccount" + service_name: str = "CustomerManagementService" + operation_name: str = "SearchAccounts" + additional_fields: str = "TaxCertificate AccountMode" + # maximum page size + page_size_limit: int = 1000 + + def next_page_token(self, response: sudsobject.Object, current_page_token: Optional[int]) -> Optional[Mapping[str, Any]]: + current_page_token = current_page_token or 0 + if response is not None and hasattr(response, self.data_field): + return None if self.page_size_limit > len(response[self.data_field]) else current_page_token + 1 + else: + return None + + def request_params( + self, + next_page_token: Mapping[str, Any] = None, + **kwargs: Mapping[str, Any], + ) -> MutableMapping[str, Any]: + predicates = { + "Predicate": [ + { + "Field": "UserId", + "Operator": "Equals", + "Value": self._user_id, + } + ] + } + + paging = self._service.factory.create("ns5:Paging") + paging.Index = next_page_token or 0 + paging.Size = self.page_size_limit + return { + "PageInfo": paging, + "Predicates": predicates, + "ReturnAdditionalFields": self.additional_fields, + } + + +class Campaigns(BingAdsCampaignManagementStream): + """ + Gets the campaigns for all provided accounts. + API doc: https://docs.microsoft.com/en-us/advertising/campaign-management-service/getcampaignsbyaccountid?view=bingads-13 + Campaign schema: https://docs.microsoft.com/en-us/advertising/campaign-management-service/campaign?view=bingads-13 + Stream caches incoming responses to be able to reuse this data in AdGroups stream + """ + + primary_key = "Id" + # Stream caches incoming responses to avoid duplicated http requests + use_cache: bool = True + data_field: str = "Campaign" + operation_name: str = "GetCampaignsByAccountId" + additional_fields: Iterable[str] = [ + "AdScheduleUseSearcherTimeZone", + "BidStrategyId", + "CpvCpmBiddingScheme", + "DynamicDescriptionSetting", + "DynamicFeedSetting", + "MaxConversionValueBiddingScheme", + "MultimediaAdsBidAdjustment", + "TargetImpressionShareBiddingScheme", + "TargetSetting", + "VerifiedTrackingSetting", + ] + campaign_types: Iterable[str] = ["Audience", "DynamicSearchAds", "Search", "Shopping"] + + parent_key_to_foreign_key_map = { + "AccountId": "account_id", + "CustomerId": "customer_id", + } + + def request_params( + self, + stream_slice: Mapping[str, Any] = None, + **kwargs: Mapping[str, Any], + ) -> MutableMapping[str, Any]: + return { + "AccountId": stream_slice["account_id"], + "CampaignType": " ".join(self.campaign_types), + "ReturnAdditionalFields": " ".join(self.additional_fields), + } + + def stream_slices( + self, + **kwargs: Mapping[str, Any], + ) -> Iterable[Optional[Mapping[str, Any]]]: + for account in Accounts(self.client, self.config).read_records(SyncMode.full_refresh): + yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} + + +class AdGroups(BingAdsCampaignManagementStream): + """ + Gets the ad groups for all provided accounts. + API doc: https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadgroupsbycampaignid?view=bingads-13 + AdGroup schema: https://docs.microsoft.com/en-us/advertising/campaign-management-service/adgroup?view=bingads-13 + Stream caches incoming responses to be able to reuse this data in Ads stream + """ + + primary_key = "Id" + # Stream caches incoming responses to avoid duplicated http requests + use_cache: bool = True + data_field: str = "AdGroup" + operation_name: str = "GetAdGroupsByCampaignId" + additional_fields: str = "AdGroupType AdScheduleUseSearcherTimeZone CpmBid CpvBid MultimediaAdsBidAdjustment" + + parent_key_to_foreign_key_map = {"CampaignId": "campaign_id", "AccountId": "account_id", "CustomerId": "customer_id"} + + def request_params( + self, + stream_slice: Mapping[str, Any] = None, + **kwargs: Mapping[str, Any], + ) -> MutableMapping[str, Any]: + return {"CampaignId": stream_slice["campaign_id"], "ReturnAdditionalFields": self.additional_fields} + + def stream_slices( + self, + **kwargs: Mapping[str, Any], + ) -> Iterable[Optional[Mapping[str, Any]]]: + campaigns = Campaigns(self.client, self.config) + for account in Accounts(self.client, self.config).read_records(SyncMode.full_refresh): + for campaign in campaigns.read_records( + sync_mode=SyncMode.full_refresh, stream_slice={"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} + ): + yield {"campaign_id": campaign["Id"], "account_id": account["Id"], "customer_id": account["ParentCustomerId"]} + + +class Ads(BingAdsCampaignManagementStream): + """ + Retrieves the ads for all provided accounts. + API doc: https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadsbyadgroupid?view=bingads-13 + Ad schema: https://docs.microsoft.com/en-us/advertising/campaign-management-service/ad?view=bingads-13 + """ + + primary_key = "Id" + data_field: str = "Ad" + operation_name: str = "GetAdsByAdGroupId" + additional_fields: str = "ImpressionTrackingUrls Videos LongHeadlines" + ad_types: Iterable[str] = [ + "Text", + "Image", + "Product", + "AppInstall", + "ExpandedText", + "DynamicSearch", + "ResponsiveAd", + "ResponsiveSearch", + ] + + parent_key_to_foreign_key_map = {"AdGroupId": "ad_group_id", "AccountId": "account_id", "CustomerId": "customer_id"} + + def request_params( + self, + stream_slice: Mapping[str, Any] = None, + **kwargs: Mapping[str, Any], + ) -> MutableMapping[str, Any]: + return { + "AdGroupId": stream_slice["ad_group_id"], + "AdTypes": {"AdType": self.ad_types}, + "ReturnAdditionalFields": self.additional_fields, + } + + def stream_slices( + self, + **kwargs: Mapping[str, Any], + ) -> Iterable[Optional[Mapping[str, Any]]]: + ad_groups = AdGroups(self.client, self.config) + for slice in ad_groups.stream_slices(sync_mode=SyncMode.full_refresh): + for ad_group in ad_groups.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice): + yield {"ad_group_id": ad_group["Id"], "account_id": slice["account_id"], "customer_id": slice["customer_id"]} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/bulk_streams.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/bulk_streams.py new file mode 100644 index 000000000000..34d13b2627ca --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/bulk_streams.py @@ -0,0 +1,190 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import os +from abc import ABC, abstractmethod +from datetime import timezone +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import pandas as pd +import pendulum +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams import IncrementalMixin +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from numpy import nan +from source_bing_ads.base_streams import Accounts, BingAdsBaseStream +from source_bing_ads.utils import transform_bulk_datetime_format_to_rfc_3339 + + +class BingAdsBulkStream(BingAdsBaseStream, IncrementalMixin, ABC): + + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization) + cursor_field = "Modified Time" + primary_key = "Id" + _state = {} + + @staticmethod + @transformer.registerCustomTransform + def custom_transform_date_rfc3339(original_value, field_schema): + if original_value and "format" in field_schema and field_schema["format"] == "date-time": + transformed_value = transform_bulk_datetime_format_to_rfc_3339(original_value) + return transformed_value + return original_value + + @property + @abstractmethod + def data_scope(self) -> List[str]: + """ + Defines scopes or types of data to download. Docs: https://learn.microsoft.com/en-us/advertising/bulk-service/datascope?view=bingads-13 + """ + + @property + @abstractmethod + def download_entities(self) -> List[str]: + """ + Defines the entities that should be downloaded. Docs: https://learn.microsoft.com/en-us/advertising/bulk-service/downloadentity?view=bingads-13 + """ + + def stream_slices( + self, + **kwargs: Mapping[str, Any], + ) -> Iterable[Optional[Mapping[str, Any]]]: + for account in Accounts(self.client, self.config).read_records(SyncMode.full_refresh): + yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} + + @property + def state(self) -> Mapping[str, Any]: + return self._state + + @state.setter + def state(self, value: Mapping[str, Any]): + current_state_value = self._state.get(str(value["Account Id"]), {}).get(self.cursor_field, "") + if value[self.cursor_field]: + record_state_value = transform_bulk_datetime_format_to_rfc_3339(value[self.cursor_field]) + new_state_value = max(current_state_value, record_state_value) + self._state.update({str(value["Account Id"]): {self.cursor_field: new_state_value}}) + + def get_start_date(self, stream_state: Mapping[str, Any] = None, account_id: str = None) -> Optional[pendulum.DateTime]: + """ + The start_date in the query can only be specified if it is within a period of up to 30 days from today. + """ + min_available_date = pendulum.now().subtract(days=30).astimezone(tz=timezone.utc) + start_date = self.client.reports_start_date + if stream_state.get(account_id, {}).get(self.cursor_field): + start_date = pendulum.parse(stream_state[account_id][self.cursor_field]) + return start_date if start_date and start_date > min_available_date else None + + def read_records( + self, + sync_mode: SyncMode, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + **kwargs: Mapping[str, Any], + ) -> Iterable[Mapping[str, Any]]: + stream_state = stream_state or {} + account_id = str(stream_slice.get("account_id")) if stream_slice else None + customer_id = str(stream_slice.get("customer_id")) if stream_slice else None + + report_file_path = self.client.get_bulk_entity( + data_scope=self.data_scope, + download_entities=self.download_entities, + customer_id=customer_id, + account_id=account_id, + start_date=self.get_start_date(stream_state, account_id), + ) + for record in self.read_with_chunks(report_file_path): + record = self.transform(record, stream_slice) + yield record + self.state = record + + def read_with_chunks(self, path: str, chunk_size: int = 1024) -> Iterable[Tuple[int, Mapping[str, Any]]]: + try: + with open(path, "r") as data: + chunks = pd.read_csv(data, chunksize=chunk_size, iterator=True, dialect="unix", dtype=object) + for chunk in chunks: + chunk = chunk.replace({nan: None}).to_dict(orient="records") + for row in chunk: + if row.get("Type") not in ("Format Version", "Account"): + yield row + except pd.errors.EmptyDataError as e: + self.logger.info(f"Empty data received. {e}") + except IOError as ioe: + self.logger.fatal( + f"The IO/Error occurred while reading tmp data. Called: {path}. Stream: {self.name}", + ) + raise ioe + finally: + # remove binary tmp file, after data is read + os.remove(path) + + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + """ + Bing Ads Bulk API returns all available properties for all entities. + This method filter out only available properties. + """ + actual_record = {key: value for key, value in record.items() if key in self.get_json_schema()["properties"].keys()} + actual_record["Account Id"] = stream_slice.get("account_id") + return actual_record + + +class AppInstallAds(BingAdsBulkStream): + """ + https://learn.microsoft.com/en-us/advertising/bulk-service/app-install-ad?view=bingads-13 + """ + + data_scope = ["EntityData"] + download_entities = ["AppInstallAds"] + + +class AppInstallAdLabels(BingAdsBulkStream): + """ + https://learn.microsoft.com/en-us/advertising/bulk-service/app-install-ad-label?view=bingads-13 + """ + + data_scope = ["EntityData"] + download_entities = ["AppInstallAdLabels"] + + +class Labels(BingAdsBulkStream): + """ + https://learn.microsoft.com/en-us/advertising/bulk-service/label?view=bingads-13 + """ + + data_scope = ["EntityData"] + download_entities = ["Labels"] + + +class KeywordLabels(BingAdsBulkStream): + """ + https://learn.microsoft.com/en-us/advertising/bulk-service/keyword-label?view=bingads-13 + """ + + data_scope = ["EntityData"] + download_entities = ["KeywordLabels"] + + +class Keywords(BingAdsBulkStream): + """ + https://learn.microsoft.com/en-us/advertising/bulk-service/keyword?view=bingads-13 + """ + + data_scope = ["EntityData"] + download_entities = ["Keywords"] + + +class CampaignLabels(BingAdsBulkStream): + """ + https://learn.microsoft.com/en-us/advertising/bulk-service/campaign-label?view=bingads-13 + """ + + data_scope = ["EntityData"] + download_entities = ["CampaignLabels"] + + +class AdGroupLabels(BingAdsBulkStream): + """ + https://learn.microsoft.com/en-us/advertising/bulk-service/ad-group-label?view=bingads-13 + """ + + data_scope = ["EntityData"] + download_entities = ["AdGroupLabels"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py new file mode 100644 index 000000000000..2be5dd890834 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py @@ -0,0 +1,748 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import re +import xml.etree.ElementTree as ET +from abc import ABC, abstractmethod +from datetime import datetime +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union +from urllib.parse import urlparse + +import _csv +import pendulum +from airbyte_cdk.sources.streams.core import package_name_from_class +from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from airbyte_protocol.models import SyncMode +from bingads import ServiceClient +from bingads.v13.internal.reporting.row_report import _RowReport +from bingads.v13.internal.reporting.row_report_iterator import _RowReportRecord +from bingads.v13.reporting import ReportingDownloadParameters +from source_bing_ads.base_streams import Accounts, BingAdsStream +from source_bing_ads.utils import transform_date_format_to_rfc_3339, transform_report_hourly_datetime_format_to_rfc_3339 +from suds import WebFault, sudsobject + + +class HourlyReportTransformerMixin: + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization) + + @staticmethod + @transformer.registerCustomTransform + def custom_transform_datetime_rfc3339(original_value, field_schema): + if original_value and "format" in field_schema and field_schema["format"] == "date-time": + print(original_value) + transformed_value = transform_report_hourly_datetime_format_to_rfc_3339(original_value) + return transformed_value + return original_value + + +class BingAdsReportingServiceStream(BingAdsStream, ABC): + # The directory where the file with report will be downloaded. + file_directory: str = "/tmp" + # timeout for reporting download operations in milliseconds + timeout: int = 300000 + report_file_format: str = "Csv" + + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + primary_key: List[str] = ["TimePeriod", "Network", "DeviceType"] + + cursor_field = "TimePeriod" + service_name: str = "ReportingService" + operation_name: str = "download_report" + + def get_json_schema(self) -> Mapping[str, Any]: + return ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema(self.report_schema_name) + + @property + @abstractmethod + def report_name(self) -> str: + """ + Specifies bing ads report naming + """ + + @property + @abstractmethod + def report_aggregation(self) -> Optional[str]: + """ + Specifies bing ads report aggregation type + Supported types: Hourly, Daily, Weekly, Monthly + """ + + @property + @abstractmethod + def report_schema_name(self) -> str: + """ + Specifies file name with schema + """ + + @property + def default_time_periods(self): + # used when reports start date is not provided + return ["LastYear", "ThisYear"] if self.report_aggregation not in ("DayOfWeek", "HourOfDay") else ["ThisYear"] + + @property + def report_columns(self) -> Iterable[str]: + return list(self.get_json_schema().get("properties", {}).keys()) + + def parse_response(self, response: sudsobject.Object, **kwargs: Mapping[str, Any]) -> Iterable[Mapping]: + if response is not None: + try: + for row in response.report_records: + yield {column: self.get_column_value(row, column) for column in self.report_columns} + except _csv.Error as e: + self.logger.warning(f"CSV report file for stream `{self.name}` is broken or cannot be read correctly: {e}, skipping ...") + + def get_column_value(self, row: _RowReportRecord, column: str) -> Union[str, None, int, float]: + """ + Reads field value from row and transforms: + 1. empty values to logical None + 2. Percent values to numeric string e.g. "12.25%" -> "12.25" + """ + value = row.value(column) + if not value or value == "--": + return None + if "%" in value: + value = value.replace("%", "") + if value and set(self.get_json_schema()["properties"].get(column, {}).get("type")) & {"integer", "number"}: + value = value.replace(",", "") + return value + + def get_request_date(self, reporting_service: ServiceClient, date: datetime) -> sudsobject.Object: + """ + Creates XML Date object based on datetime. + https://docs.microsoft.com/en-us/advertising/reporting-service/date?view=bingads-13 + The [suds.client.Factory-class.html factory] namespace provides a factory that may be used + to create instances of objects and types defined in the WSDL. + """ + request_date = reporting_service.factory.create("Date") + request_date.Day = date.day + request_date.Month = date.month + request_date.Year = date.year + return request_date + + def request_params( + self, stream_state: Mapping[str, Any] = None, account_id: str = None, **kwargs: Mapping[str, Any] + ) -> Mapping[str, Any]: + stream_slice = kwargs["stream_slice"] + start_date = self.get_start_date(stream_state, account_id) + + reporting_service = self.client.get_service("ReportingService") + request_time_zone = reporting_service.factory.create("ReportTimeZone") + + report_time = reporting_service.factory.create("ReportTime") + report_time.ReportTimeZone = request_time_zone.GreenwichMeanTimeDublinEdinburghLisbonLondon + if start_date: + report_time.CustomDateRangeStart = self.get_request_date(reporting_service, start_date) + report_time.CustomDateRangeEnd = self.get_request_date(reporting_service, datetime.utcnow()) + report_time.PredefinedTime = None + else: + report_time.CustomDateRangeStart = None + report_time.CustomDateRangeEnd = None + report_time.PredefinedTime = stream_slice["time_period"] + + report_request = self.get_report_request(account_id, False, False, False, self.report_file_format, False, report_time) + + return { + "report_request": report_request, + "result_file_directory": self.file_directory, + "result_file_name": self.report_name, + "overwrite_result_file": True, + "timeout_in_milliseconds": self.timeout, + } + + def get_start_date(self, stream_state: Mapping[str, Any] = None, account_id: str = None): + if stream_state and account_id: + if stream_state.get(account_id, {}).get(self.cursor_field): + return pendulum.parse(self.get_report_record_timestamp(stream_state[account_id][self.cursor_field])) + + return self.client.reports_start_date + + def get_updated_state( + self, + current_stream_state: MutableMapping[str, Any], + latest_record: Mapping[str, Any], + ) -> Mapping[str, Any]: + account_id = str(latest_record["AccountId"]) + current_stream_state[account_id] = current_stream_state.get(account_id, {}) + current_stream_state[account_id][self.cursor_field] = max( + self.get_report_record_timestamp(latest_record[self.cursor_field]), + current_stream_state.get(account_id, {}).get(self.cursor_field, ""), + ) + return current_stream_state + + def send_request(self, params: Mapping[str, Any], customer_id: str, account_id: str) -> _RowReport: + request_kwargs = { + "service_name": None, + "customer_id": customer_id, + "account_id": account_id, + "operation_name": self.operation_name, + "is_report_service": True, + "params": {"download_parameters": ReportingDownloadParameters(**params)}, + } + return self.client.request(**request_kwargs) + + def get_report_request( + self, + account_id: str, + exclude_column_headers: bool, + exclude_report_footer: bool, + exclude_report_header: bool, + report_file_format: str, + return_only_complete_data: bool, + time: sudsobject.Object, + ) -> sudsobject.Object: + reporting_service = self.client.get_service(self.service_name) + report_request = reporting_service.factory.create(f"{self.report_name}Request") + if self.report_aggregation: + report_request.Aggregation = self.report_aggregation + + report_request.ExcludeColumnHeaders = exclude_column_headers + report_request.ExcludeReportFooter = exclude_report_footer + report_request.ExcludeReportHeader = exclude_report_header + report_request.Format = report_file_format + report_request.FormatVersion = "2.0" + report_request.ReturnOnlyCompleteData = return_only_complete_data + report_request.Time = time + report_request.ReportName = self.report_name + # Defines the set of accounts and campaigns to include in the report. + scope = reporting_service.factory.create("AccountThroughCampaignReportScope") + scope.AccountIds = {"long": [account_id]} + scope.Campaigns = None + report_request.Scope = scope + + columns = reporting_service.factory.create(f"ArrayOf{self.report_name}Column") + getattr(columns, f"{self.report_name}Column").append(self.report_columns) + report_request.Columns = columns + return report_request + + def get_report_record_timestamp(self, datestring: str) -> str: + """ + Parse report date field based on aggregation type + """ + return ( + self.transformer._custom_normalizer(datestring, self.get_json_schema()["properties"][self.cursor_field]) + if self.transformer._custom_normalizer + else datestring + ) + + def stream_slices( + self, + **kwargs: Mapping[str, Any], + ) -> Iterable[Optional[Mapping[str, Any]]]: + for account in Accounts(self.client, self.config).read_records(SyncMode.full_refresh): + for period in self.default_time_periods: + yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"], "time_period": period} + + +class BingAdsReportingServicePerformanceStream(BingAdsReportingServiceStream, ABC): + def get_start_date(self, stream_state: Mapping[str, Any] = None, account_id: str = None): + start_date = super().get_start_date(stream_state, account_id) + + if self.config.get("lookback_window") and start_date: + # Datetime subtract won't work with days = 0 + # it'll output an AirbyteError + return start_date.subtract(days=self.config["lookback_window"]) + else: + return start_date + + +class BudgetSummaryReport(BingAdsReportingServiceStream): + report_name: str = "BudgetSummaryReport" + report_aggregation = None + cursor_field = "Date" + report_schema_name = "budget_summary_report" + primary_key = "Date" + + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization) + + @staticmethod + @transformer.registerCustomTransform + def custom_transform_date_rfc3339(original_value, field_schema): + if original_value and "format" in field_schema and field_schema["format"] == "date": + transformed_value = transform_date_format_to_rfc_3339(original_value) + return transformed_value + return original_value + + +class CampaignPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + report_name: str = "CampaignPerformanceReport" + + report_schema_name = "campaign_performance_report" + primary_key = [ + "AccountId", + "CampaignId", + "TimePeriod", + "CurrencyCode", + "AdDistribution", + "DeviceType", + "Network", + "DeliveredMatchType", + "DeviceOS", + "TopVsOther", + "BidMatchType", + ] + + +class CampaignPerformanceReportHourly(HourlyReportTransformerMixin, CampaignPerformanceReport): + report_aggregation = "Hourly" + + report_schema_name = "campaign_performance_report_hourly" + + +class CampaignPerformanceReportDaily(CampaignPerformanceReport): + report_aggregation = "Daily" + + +class CampaignPerformanceReportWeekly(CampaignPerformanceReport): + report_aggregation = "Weekly" + + +class CampaignPerformanceReportMonthly(CampaignPerformanceReport): + report_aggregation = "Monthly" + + +class CampaignImpressionPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + """ + https://learn.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13 + Primary key cannot be set: due to included `Impression Share Performance Statistics` some fields should be removed, + see https://learn.microsoft.com/en-us/advertising/guides/reports?view=bingads-13#columnrestrictions for more info. + """ + + report_name: str = "CampaignPerformanceReport" + + report_schema_name = "campaign_impression_performance_report" + + primary_key = None + + +class CampaignImpressionPerformanceReportHourly(HourlyReportTransformerMixin, CampaignImpressionPerformanceReport): + report_aggregation = "Hourly" + + report_schema_name = "campaign_impression_performance_report_hourly" + + +class CampaignImpressionPerformanceReportDaily(CampaignImpressionPerformanceReport): + report_aggregation = "Daily" + + +class CampaignImpressionPerformanceReportWeekly(CampaignImpressionPerformanceReport): + report_aggregation = "Weekly" + + +class CampaignImpressionPerformanceReportMonthly(CampaignImpressionPerformanceReport): + report_aggregation = "Monthly" + + +class AdPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + report_name: str = "AdPerformanceReport" + + report_schema_name = "ad_performance_report" + primary_key = [ + "AccountId", + "CampaignId", + "AdGroupId", + "AdId", + "TimePeriod", + "CurrencyCode", + "AdDistribution", + "DeviceType", + "Language", + "Network", + "DeviceOS", + "TopVsOther", + "BidMatchType", + "DeliveredMatchType", + ] + + +class AdPerformanceReportHourly(HourlyReportTransformerMixin, AdPerformanceReport): + report_aggregation = "Hourly" + report_schema_name = "ad_performance_report_hourly" + + +class AdPerformanceReportDaily(AdPerformanceReport): + report_aggregation = "Daily" + + +class AdPerformanceReportWeekly(AdPerformanceReport): + report_aggregation = "Weekly" + + +class AdPerformanceReportMonthly(AdPerformanceReport): + report_aggregation = "Monthly" + + +class AdGroupPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + report_name: str = "AdGroupPerformanceReport" + report_schema_name = "ad_group_performance_report" + + primary_key = [ + "AccountId", + "CampaignId", + "AdGroupId", + "TimePeriod", + "CurrencyCode", + "AdDistribution", + "DeviceType", + "Network", + "DeliveredMatchType", + "DeviceOS", + "TopVsOther", + "BidMatchType", + "Language", + ] + + +class AdGroupPerformanceReportHourly(HourlyReportTransformerMixin, AdGroupPerformanceReport): + report_aggregation = "Hourly" + report_schema_name = "ad_group_performance_report_hourly" + + +class AdGroupPerformanceReportDaily(AdGroupPerformanceReport): + report_aggregation = "Daily" + + +class AdGroupPerformanceReportWeekly(AdGroupPerformanceReport): + report_aggregation = "Weekly" + + +class AdGroupPerformanceReportMonthly(AdGroupPerformanceReport): + report_aggregation = "Monthly" + + +class AdGroupImpressionPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + """ + https://learn.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13 + Primary key cannot be set: due to included `Impression Share Performance Statistics` some fields should be removed, + see https://learn.microsoft.com/en-us/advertising/guides/reports?view=bingads-13#columnrestrictions for more info. + """ + + report_name: str = "AdGroupPerformanceReport" + report_schema_name = "ad_group_impression_performance_report" + + +class AdGroupImpressionPerformanceReportHourly(HourlyReportTransformerMixin, AdGroupImpressionPerformanceReport): + report_aggregation = "Hourly" + report_schema_name = "ad_group_impression_performance_report_hourly" + + +class AdGroupImpressionPerformanceReportDaily(AdGroupImpressionPerformanceReport): + report_aggregation = "Daily" + + +class AdGroupImpressionPerformanceReportWeekly(AdGroupImpressionPerformanceReport): + report_aggregation = "Weekly" + + +class AdGroupImpressionPerformanceReportMonthly(AdGroupImpressionPerformanceReport): + report_aggregation = "Monthly" + + +class KeywordPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + report_name: str = "KeywordPerformanceReport" + report_schema_name = "keyword_performance_report" + primary_key = [ + "AccountId", + "CampaignId", + "AdGroupId", + "KeywordId", + "AdId", + "TimePeriod", + "CurrencyCode", + "DeliveredMatchType", + "AdDistribution", + "DeviceType", + "Language", + "Network", + "DeviceOS", + "TopVsOther", + "BidMatchType", + ] + + +class KeywordPerformanceReportHourly(HourlyReportTransformerMixin, KeywordPerformanceReport): + report_aggregation = "Hourly" + report_schema_name = "keyword_performance_report_hourly" + + +class KeywordPerformanceReportDaily(KeywordPerformanceReport): + report_aggregation = "Daily" + report_schema_name = "keyword_performance_report_daily" + + +class KeywordPerformanceReportWeekly(KeywordPerformanceReport): + report_aggregation = "Weekly" + + +class KeywordPerformanceReportMonthly(KeywordPerformanceReport): + report_aggregation = "Monthly" + + +class GeographicPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + report_name: str = "GeographicPerformanceReport" + report_schema_name = "geographic_performance_report" + + # Need to override the primary key here because the one inherited from the PerformanceReportsMixin + # is incorrect for the geographic performance reports + primary_key = None + + +class GeographicPerformanceReportHourly(HourlyReportTransformerMixin, GeographicPerformanceReport): + report_aggregation = "Hourly" + report_schema_name = "geographic_performance_report_hourly" + + +class GeographicPerformanceReportDaily(GeographicPerformanceReport): + report_aggregation = "Daily" + + +class GeographicPerformanceReportWeekly(GeographicPerformanceReport): + report_aggregation = "Weekly" + + +class GeographicPerformanceReportMonthly(GeographicPerformanceReport): + report_aggregation = "Monthly" + + +class AccountPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + report_name: str = "AccountPerformanceReport" + report_schema_name = "account_performance_report" + primary_key = [ + "AccountId", + "TimePeriod", + "CurrencyCode", + "AdDistribution", + "DeviceType", + "Network", + "DeliveredMatchType", + "DeviceOS", + "TopVsOther", + "BidMatchType", + ] + + +class AccountPerformanceReportHourly(HourlyReportTransformerMixin, AccountPerformanceReport): + report_aggregation = "Hourly" + report_schema_name = "account_performance_report_hourly" + + +class AccountPerformanceReportDaily(AccountPerformanceReport): + report_aggregation = "Daily" + + +class AccountPerformanceReportWeekly(AccountPerformanceReport): + report_aggregation = "Weekly" + + +class AccountPerformanceReportMonthly(AccountPerformanceReport): + report_aggregation = "Monthly" + + +class AccountImpressionPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + """ + Report source: https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13 + Primary key cannot be set: due to included `Impression Share Performance Statistics` some fields should be removed, + see https://learn.microsoft.com/en-us/advertising/guides/reports?view=bingads-13#columnrestrictions for more info. + """ + + report_name: str = "AccountPerformanceReport" + report_schema_name = "account_impression_performance_report" + primary_key = None + + +class AccountImpressionPerformanceReportHourly(HourlyReportTransformerMixin, AccountImpressionPerformanceReport): + report_aggregation = "Hourly" + report_schema_name = "account_impression_performance_report_hourly" + + +class AccountImpressionPerformanceReportDaily(AccountImpressionPerformanceReport): + report_aggregation = "Daily" + + +class AccountImpressionPerformanceReportWeekly(AccountImpressionPerformanceReport): + report_aggregation = "Weekly" + + +class AccountImpressionPerformanceReportMonthly(AccountImpressionPerformanceReport): + report_aggregation = "Monthly" + + +class AgeGenderAudienceReport(BingAdsReportingServicePerformanceStream, ABC): + report_name: str = "AgeGenderAudienceReport" + + report_schema_name = "age_gender_audience_report" + primary_key = ["AgeGroup", "Gender", "TimePeriod", "AccountId", "CampaignId", "Language", "AdDistribution"] + + +class AgeGenderAudienceReportHourly(HourlyReportTransformerMixin, AgeGenderAudienceReport): + report_aggregation = "Hourly" + report_schema_name = "age_gender_audience_report_hourly" + + +class AgeGenderAudienceReportDaily(AgeGenderAudienceReport): + report_aggregation = "Daily" + + +class AgeGenderAudienceReportWeekly(AgeGenderAudienceReport): + report_aggregation = "Weekly" + + +class AgeGenderAudienceReportMonthly(AgeGenderAudienceReport): + report_aggregation = "Monthly" + + +class SearchQueryPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + report_name: str = "SearchQueryPerformanceReport" + report_schema_name = "search_query_performance_report" + + primary_key = [ + "SearchQuery", + "Keyword", + "TimePeriod", + "AccountId", + "CampaignId", + "Language", + "DeliveredMatchType", + "DeviceType", + "DeviceOS", + "TopVsOther", + ] + + +class SearchQueryPerformanceReportHourly(HourlyReportTransformerMixin, SearchQueryPerformanceReport): + report_aggregation = "Hourly" + report_schema_name = "search_query_performance_report_hourly" + + +class SearchQueryPerformanceReportDaily(SearchQueryPerformanceReport): + report_aggregation = "Daily" + + +class SearchQueryPerformanceReportWeekly(SearchQueryPerformanceReport): + report_aggregation = "Weekly" + + +class SearchQueryPerformanceReportMonthly(SearchQueryPerformanceReport): + report_aggregation = "Monthly" + + +class UserLocationPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + report_name: str = "UserLocationPerformanceReport" + report_schema_name = "user_location_performance_report" + primary_key = [ + "AccountId", + "AdGroupId", + "CampaignId", + "DeliveredMatchType", + "DeviceOS", + "DeviceType", + "Language", + "LocationId", + "QueryIntentLocationId", + "TimePeriod", + "TopVsOther", + ] + + +class UserLocationPerformanceReportHourly(HourlyReportTransformerMixin, UserLocationPerformanceReport): + report_aggregation = "Hourly" + report_schema_name = "user_location_performance_report_hourly" + + +class UserLocationPerformanceReportDaily(UserLocationPerformanceReport): + report_aggregation = "Daily" + + +class UserLocationPerformanceReportWeekly(UserLocationPerformanceReport): + report_aggregation = "Weekly" + + +class UserLocationPerformanceReportMonthly(UserLocationPerformanceReport): + report_aggregation = "Monthly" + + +class CustomReport(BingAdsReportingServicePerformanceStream, ABC): + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + custom_report_columns = [] + report_schema_name = None + primary_key = None + + @property + def cursor_field(self) -> Union[str, List[str]]: + # Summary aggregation doesn't include TimePeriod field + if self.report_aggregation not in ("Summary", "DayOfWeek", "HourOfDay"): + return "TimePeriod" + + @property + def report_columns(self): + # adding common and default columns + if "AccountId" not in self.custom_report_columns: + self.custom_report_columns.append("AccountId") + if self.cursor_field and self.cursor_field not in self.custom_report_columns: + self.custom_report_columns.append(self.cursor_field) + return list(frozenset(self.custom_report_columns)) + + def get_json_schema(self) -> Mapping[str, Any]: + columns_schema = {col: {"type": ["null", "string"]} for col in self.report_columns} + schema: Mapping[str, Any] = { + "$schema": "https://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": True, + "properties": columns_schema, + } + return schema + + def validate_report_configuration(self) -> Tuple[bool, str]: + # gets /bingads/v13/proxies/production/reporting_service.xml + reporting_service_file = self.client.get_service(self.service_name)._get_service_info_dict(self.client.api_version)[ + ("reporting", self.client.environment) + ] + tree = ET.parse(urlparse(reporting_service_file).path) + request_object = tree.find(f".//{{*}}complexType[@name='{self.report_name}Request']") + + report_object_columns = self._get_object_columns(request_object, tree) + is_custom_cols_in_report_object_cols = all(x in report_object_columns for x in self.custom_report_columns) + + if not is_custom_cols_in_report_object_cols: + return False, ( + f"Reporting Columns are invalid. Columns that you provided don't belong to Reporting Data Object Columns:" + f" {self.custom_report_columns}. Please ensure it is correct in Bing Ads Docs." + ) + + return True, "" + + def _clear_namespace(self, type: str) -> str: + return re.sub(r"^[a-z]+:", "", type) + + def _get_object_columns(self, request_el: ET.Element, tree: ET.ElementTree) -> List[str]: + column_el = request_el.find(".//{*}element[@name='Columns']") + array_of_columns_name = self._clear_namespace(column_el.get("type")) + + array_of_columns_elements = tree.find(f".//{{*}}complexType[@name='{array_of_columns_name}']") + inner_array_of_columns_elements = array_of_columns_elements.find(".//{*}element") + column_el_name = self._clear_namespace(inner_array_of_columns_elements.get("type")) + + column_el = tree.find(f".//{{*}}simpleType[@name='{column_el_name}']") + column_enum_items = column_el.findall(".//{*}enumeration") + column_enum_items_values = [el.get("value") for el in column_enum_items] + return column_enum_items_values + + def get_report_record_timestamp(self, datestring: str) -> str: + """ + Parse report date field based on aggregation type + """ + if not self.report_aggregation or self.report_aggregation == "Summary": + datestring = transform_date_format_to_rfc_3339(datestring) + elif self.report_aggregation == "Hourly": + datestring = transform_report_hourly_datetime_format_to_rfc_3339(datestring) + return datestring + + def send_request(self, params: Mapping[str, Any], customer_id: str, account_id: str) -> _RowReport: + try: + return super().send_request(params, customer_id, account_id) + except WebFault as e: + self.logger.error( + f"Could not sync custom report {self.name}: Please validate your column and aggregation configuration. " + f"Error form server: [{e.fault.faultstring}]" + ) diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py deleted file mode 100644 index 731954bad986..000000000000 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py +++ /dev/null @@ -1,336 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from abc import ABC, abstractmethod -from datetime import datetime -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional - -import pendulum -import source_bing_ads.source -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.core import package_name_from_class -from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader -from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from bingads.service_client import ServiceClient -from bingads.v13.internal.reporting.row_report import _RowReport -from bingads.v13.reporting import ReportingDownloadParameters -from suds import sudsobject - -AVERAGE_FIELD_TYPES = { - "AverageCpc": "number", - "AveragePosition": "number", - "AverageCpm": "number", -} -AVERAGE_FIELDS = list(AVERAGE_FIELD_TYPES.keys()) - -CONVERSION_FIELD_TYPES = { - "Conversions": "number", - "ConversionRate": "number", - "ConversionsQualified": "number", -} -CONVERSION_FIELDS = list(CONVERSION_FIELD_TYPES.keys()) - -ALL_CONVERSION_FIELD_TYPES = { - "AllConversions": "integer", - "AllConversionRate": "number", -} -ALL_CONVERSION_FIELDS = list(ALL_CONVERSION_FIELD_TYPES.keys()) - -LOW_QUALITY_FIELD_TYPES = { - "LowQualityClicks": "integer", - "LowQualityClicksPercent": "number", - "LowQualityImpressions": "integer", - "LowQualitySophisticatedClicks": "integer", - "LowQualityConversions": "integer", - "LowQualityConversionRate": "number", -} -LOW_QUALITY_FIELDS = list(LOW_QUALITY_FIELD_TYPES.keys()) - -REVENUE_FIELD_TYPES = { - "Revenue": "number", - "RevenuePerConversion": "number", - "RevenuePerAssist": "number", -} -REVENUE_FIELDS = list(REVENUE_FIELD_TYPES.keys()) - -ALL_REVENUE_FIELD_TYPES = { - "AllRevenue": "number", - "AllRevenuePerConversion": "number", -} -ALL_REVENUE_FIELDS = list(ALL_REVENUE_FIELD_TYPES.keys()) - -IMPRESSION_FIELD_TYPES = { - "ImpressionSharePercent": "number", - "ImpressionLostToBudgetPercent": "number", - "ImpressionLostToRankAggPercent": "number", -} -IMPRESSION_FIELDS = list(IMPRESSION_FIELD_TYPES.keys()) - - -HISTORICAL_FIELD_TYPES = { - "HistoricalQualityScore": "number", - "HistoricalExpectedCtr": "number", - "HistoricalAdRelevance": "number", - "HistoricalLandingPageExperience": "number", -} -HISTORICAL_FIELDS = list(HISTORICAL_FIELD_TYPES.keys()) - -BUDGET_FIELD_TYPES = { - "BudgetName": "string", - "BudgetStatus": "string", - "BudgetAssociationStatus": "string", -} -BUDGET_FIELDS = list(BUDGET_FIELD_TYPES.keys()) - -REPORT_FIELD_TYPES = { - "AccountId": "integer", - "AdId": "integer", - "AdGroupCriterionId": "integer", - "AdGroupId": "integer", - "AdRelevance": "number", - "Assists": "integer", - "AllCostPerConversion": "number", - "AllReturnOnAdSpend": "number", - "BusinessCategoryId": "integer", - "BusinessListingId": "integer", - "CampaignId": "integer", - "ClickCalls": "integer", - "Clicks": "integer", - "CostPerAssist": "number", - "CostPerConversion": "number", - "Ctr": "number", - "CurrentMaxCpc": "number", - "EstimatedClickPercent": "number", - "EstimatedClicks": "integer", - "EstimatedConversionRate": "number", - "EstimatedConversions": "integer", - "EstimatedCtr": "number", - "EstimatedImpressionPercent": "number", - "EstimatedImpressions": "integer", - "ExactMatchImpressionSharePercent": "number", - "Impressions": "integer", - "ImpressionSharePercent": "number", - "KeywordId": "integer", - "LandingPageExperience": "number", - "PhoneCalls": "integer", - "PhoneImpressions": "integer", - "Ptr": "number", - "QualityImpact": "number", - "QualityScore": "number", - "ReturnOnAdSpend": "number", - "SidebarBid": "number", - "Spend": "number", - "MonthlyBudget": "number", - "DailySpend": "number", - "MonthToDateSpend": "number", - "AbsoluteTopImpressionRatePercent": "number", - "ViewThroughConversions": "integer", - "ViewThroughConversionsQualified": "number", - "MainlineBid": "number", - "Mainline1Bid": "number", - "FirstPageBid": "number", - **AVERAGE_FIELD_TYPES, - **CONVERSION_FIELD_TYPES, - **ALL_CONVERSION_FIELD_TYPES, - **LOW_QUALITY_FIELD_TYPES, - **REVENUE_FIELD_TYPES, - **ALL_REVENUE_FIELD_TYPES, - **IMPRESSION_FIELD_TYPES, - **HISTORICAL_FIELD_TYPES, - **BUDGET_FIELD_TYPES, -} - - -class ReportsMixin(ABC): - # The directory where the file with report will be downloaded. - file_directory: str = "/tmp" - # timeout for reporting download operations in milliseconds - timeout: int = 300000 - report_file_format: str = "Csv" - # used when reports start date is not provided - time_periods = ["LastYear", "ThisYear"] - - transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - primary_key: List[str] = ["TimePeriod", "Network", "DeviceType"] - - @property - @abstractmethod - def report_name(self) -> str: - """ - Specifies bing ads report naming - """ - pass - - @property - @abstractmethod - def report_columns(self) -> Iterable[str]: - """ - Specifies bing ads report naming - TODO: refactor to use list(self.get_json_schema().get("properties", {}).keys()), see AgeGenderAudienceReport - """ - pass - - @property - @abstractmethod - def report_aggregation(self) -> Optional[str]: - """ - Specifies bing ads report aggregation type - Supported types: Hourly, Daily, Weekly, Monthly - """ - pass - - @property - @abstractmethod - def report_schema_name(self) -> str: - """ - Specifies file name with schema - """ - pass - - def get_json_schema(self) -> Mapping[str, Any]: - return ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema(self.report_schema_name) - - def get_request_date(self, reporting_service: ServiceClient, date: datetime) -> sudsobject.Object: - """ - Creates XML Date object based on datetime. - https://docs.microsoft.com/en-us/advertising/reporting-service/date?view=bingads-13 - The [suds.client.Factory-class.html factory] namespace provides a factory that may be used - to create instances of objects and types defined in the WSDL. - """ - request_date = reporting_service.factory.create("Date") - request_date.Day = date.day - request_date.Month = date.month - request_date.Year = date.year - return request_date - - def request_params( - self, stream_state: Mapping[str, Any] = None, account_id: str = None, **kwargs: Mapping[str, Any] - ) -> Mapping[str, Any]: - stream_slice = kwargs["stream_slice"] - start_date = self.get_start_date(stream_state, account_id) - - reporting_service = self.client.get_service("ReportingService") - request_time_zone = reporting_service.factory.create("ReportTimeZone") - - report_time = reporting_service.factory.create("ReportTime") - report_time.ReportTimeZone = request_time_zone.GreenwichMeanTimeDublinEdinburghLisbonLondon - if start_date: - report_time.CustomDateRangeStart = self.get_request_date(reporting_service, start_date) - report_time.CustomDateRangeEnd = self.get_request_date(reporting_service, datetime.utcnow()) - report_time.PredefinedTime = None - else: - report_time.CustomDateRangeStart = None - report_time.CustomDateRangeEnd = None - report_time.PredefinedTime = stream_slice["time_period"] - - report_request = self.get_report_request(account_id, False, False, False, self.report_file_format, False, report_time) - - return { - "report_request": report_request, - "result_file_directory": self.file_directory, - "result_file_name": self.report_name, - "overwrite_result_file": True, - "timeout_in_milliseconds": self.timeout, - } - - def get_start_date(self, stream_state: Mapping[str, Any] = None, account_id: str = None): - if stream_state and account_id: - if stream_state.get(account_id, {}).get(self.cursor_field): - return pendulum.from_timestamp(stream_state[account_id][self.cursor_field]) - - return self.client.reports_start_date - - def get_updated_state( - self, - current_stream_state: MutableMapping[str, Any], - latest_record: Mapping[str, Any], - ) -> Mapping[str, Any]: - account_id = str(latest_record["AccountId"]) - current_stream_state[account_id] = current_stream_state.get(account_id, {}) - current_stream_state[account_id][self.cursor_field] = max( - self.get_report_record_timestamp(latest_record[self.cursor_field]), - current_stream_state.get(account_id, {}).get(self.cursor_field, 1), - ) - return current_stream_state - - def send_request(self, params: Mapping[str, Any], customer_id: str, account_id: str) -> _RowReport: - request_kwargs = { - "service_name": None, - "customer_id": customer_id, - "account_id": account_id, - "operation_name": self.operation_name, - "is_report_service": True, - "params": {"download_parameters": ReportingDownloadParameters(**params)}, - } - return self.client.request(**request_kwargs) - - def get_report_request( - self, - account_id: str, - exclude_column_headers: bool, - exclude_report_footer: bool, - exclude_report_header: bool, - report_file_format: str, - return_only_complete_data: bool, - time: sudsobject.Object, - ) -> sudsobject.Object: - reporting_service = self.client.get_service(self.service_name) - report_request = reporting_service.factory.create(f"{self.report_name}Request") - if self.report_aggregation: - report_request.Aggregation = self.report_aggregation - - report_request.ExcludeColumnHeaders = exclude_column_headers - report_request.ExcludeReportFooter = exclude_report_footer - report_request.ExcludeReportHeader = exclude_report_header - report_request.Format = report_file_format - report_request.FormatVersion = "2.0" - report_request.ReturnOnlyCompleteData = return_only_complete_data - report_request.Time = time - report_request.ReportName = self.report_name - # Defines the set of accounts and campaigns to include in the report. - scope = reporting_service.factory.create("AccountThroughCampaignReportScope") - scope.AccountIds = {"long": [account_id]} - scope.Campaigns = None - report_request.Scope = scope - - columns = reporting_service.factory.create(f"ArrayOf{self.report_name}Column") - getattr(columns, f"{self.report_name}Column").append(self.report_columns) - report_request.Columns = columns - return report_request - - def get_report_record_timestamp(self, datestring: str) -> int: - """ - Parse report date field based on aggregation type - """ - if not self.report_aggregation: - date = pendulum.from_format(datestring, "M/D/YYYY") - else: - if self.report_aggregation == "Hourly": - date = pendulum.from_format(datestring, "YYYY-MM-DD|H") - else: - date = pendulum.parse(datestring) - - return date.int_timestamp - - def stream_slices( - self, - **kwargs: Mapping[str, Any], - ) -> Iterable[Optional[Mapping[str, Any]]]: - for account in source_bing_ads.source.Accounts(self.client, self.config).read_records(SyncMode.full_refresh): - for period in self.time_periods: - yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"], "time_period": period} - - yield from [] - - -class PerformanceReportsMixin(ReportsMixin): - def get_start_date(self, stream_state: Mapping[str, Any] = None, account_id: str = None): - start_date = super().get_start_date(stream_state, account_id) - - if self.config.get("lookback_window") and start_date: - # Datetime subtract won't work with days = 0 - # it'll output an AirbyteError - return start_date.subtract(days=self.config["lookback_window"]) - else: - return start_date diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_impression_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_impression_performance_report.json index 530308e88e98..f2ce126eff0d 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_impression_performance_report.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_impression_performance_report.json @@ -12,7 +12,8 @@ "type": ["null", "integer"] }, "TimePeriod": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "CurrencyCode": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_impression_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_impression_performance_report_hourly.json index 253549bbbdf4..7c0cff82f779 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_impression_performance_report_hourly.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_impression_performance_report_hourly.json @@ -12,7 +12,9 @@ "type": ["null", "integer"] }, "TimePeriod": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "CurrencyCode": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_performance_report.json index 27f13fb2f71c..ab5c273877ab 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_performance_report.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_performance_report.json @@ -6,7 +6,8 @@ "type": ["null", "integer"] }, "TimePeriod": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "CurrencyCode": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_performance_report_hourly.json new file mode 100644 index 000000000000..4013dd037402 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_performance_report_hourly.json @@ -0,0 +1,122 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "Network": { + "type": ["null", "string"] + }, + "DeliveredMatchType": { + "type": ["null", "string"] + }, + "DeviceOS": { + "type": ["null", "string"] + }, + "TopVsOther": { + "type": ["null", "string"] + }, + "BidMatchType": { + "type": ["null", "string"] + }, + "AccountName": { + "type": ["null", "string"] + }, + "AccountNumber": { + "type": ["null", "string"] + }, + "PhoneImpressions": { + "type": ["null", "integer"] + }, + "PhoneCalls": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "Ptr": { + "type": ["null", "number"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "Conversions": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "LowQualityClicks": { + "type": ["null", "integer"] + }, + "LowQualityClicksPercent": { + "type": ["null", "number"] + }, + "LowQualityImpressions": { + "type": ["null", "integer"] + }, + "LowQualitySophisticatedClicks": { + "type": ["null", "integer"] + }, + "LowQualityConversions": { + "type": ["null", "integer"] + }, + "LowQualityConversionRate": { + "type": ["null", "number"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/accounts.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/accounts.json index 858ecfe2d9bc..63bf3d699add 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/accounts.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/accounts.json @@ -27,7 +27,15 @@ "type": ["null", "string"] }, "LinkedAgencies": { - "type": ["null", "string"] + "type": ["null", "object"], + "properties": { + "Id": { + "type": ["null", "integer"] + }, + "Name": { + "type": ["null", "string"] + } + } }, "TaxInformation": { "type": ["null", "string"] @@ -89,7 +97,9 @@ "type": ["null", "number"] }, "LastModifiedTime": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, "Name": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_impression_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_impression_performance_report.json index 7db370e256d2..beee977875e8 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_impression_performance_report.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_impression_performance_report.json @@ -12,7 +12,8 @@ "type": ["null", "integer"] }, "TimePeriod": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "Status": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_impression_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_impression_performance_report_hourly.json index 272a62bd655d..9ebbe2b60f2f 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_impression_performance_report_hourly.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_impression_performance_report_hourly.json @@ -12,7 +12,9 @@ "type": ["null", "integer"] }, "TimePeriod": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "Status": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_labels.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_labels.json index 6cb4d85c3353..5daab1862dbd 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_labels.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_labels.json @@ -21,7 +21,9 @@ "type": ["null", "integer"] }, "Modified Time": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "Status": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_performance_report.json index b0e952bb0aec..6b8d28e98867 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_performance_report.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_performance_report.json @@ -12,7 +12,8 @@ "type": ["null", "integer"] }, "TimePeriod": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "CurrencyCode": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_performance_report_hourly.json new file mode 100644 index 000000000000..a1300b327c69 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_performance_report_hourly.json @@ -0,0 +1,158 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountId": { + "type": ["null", "integer"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "Network": { + "type": ["null", "string"] + }, + "DeliveredMatchType": { + "type": ["null", "string"] + }, + "DeviceOS": { + "type": ["null", "string"] + }, + "TopVsOther": { + "type": ["null", "string"] + }, + "BidMatchType": { + "type": ["null", "string"] + }, + "Language": { + "type": ["null", "string"] + }, + "AccountName": { + "type": ["null", "string"] + }, + "CampaignName": { + "type": ["null", "string"] + }, + "CampaignType": { + "type": ["null", "string"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "AdGroupType": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "QualityScore": { + "type": ["null", "number"] + }, + "ExpectedCtr": { + "type": ["null", "string"] + }, + "AdRelevance": { + "type": ["null", "number"] + }, + "LandingPageExperience": { + "type": ["null", "number"] + }, + "PhoneImpressions": { + "type": ["null", "integer"] + }, + "PhoneCalls": { + "type": ["null", "integer"] + }, + "Ptr": { + "type": ["null", "number"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "CustomParameters": { + "type": ["null", "string"] + }, + "FinalUrlSuffix": { + "type": ["null", "string"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "Conversions": { + "type": ["null", "number"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_performance_report.json index 44945b40758e..c884c8e5ffb3 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_performance_report.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_performance_report.json @@ -15,7 +15,14 @@ "type": ["null", "integer"] }, "TimePeriod": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" + }, + "AbsoluteTopImpressionRatePercent": { + "type": ["null", "number"] + }, + "TopImpressionRatePercent": { + "type": ["null", "number"] }, "CurrencyCode": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_performance_report_hourly.json new file mode 100644 index 000000000000..93a690f08cd3 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_performance_report_hourly.json @@ -0,0 +1,152 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountId": { + "type": ["null", "integer"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "AdId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "Language": { + "type": ["null", "string"] + }, + "Network": { + "type": ["null", "string"] + }, + "DeviceOS": { + "type": ["null", "string"] + }, + "TopVsOther": { + "type": ["null", "string"] + }, + "BidMatchType": { + "type": ["null", "string"] + }, + "DeliveredMatchType": { + "type": ["null", "string"] + }, + "AccountName": { + "type": ["null", "string"] + }, + "CampaignName": { + "type": ["null", "string"] + }, + "CampaignType": { + "type": ["null", "string"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "DestinationUrl": { + "type": ["null", "string"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "CustomParameters": { + "type": ["null", "string"] + }, + "FinalAppUrl": { + "type": ["null", "string"] + }, + "AdDescription": { + "type": ["null", "string"] + }, + "AdDescription2": { + "type": ["null", "string"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "ViewThroughConversionsQualified": { + "type": ["null", "number"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "Conversions": { + "type": ["null", "number"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/age_gender_audience_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/age_gender_audience_report.json index 29b9b94cb932..36285f85a5e2 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/age_gender_audience_report.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/age_gender_audience_report.json @@ -12,7 +12,8 @@ "type": ["null", "string"] }, "TimePeriod": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "AllConversions": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/age_gender_audience_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/age_gender_audience_report_hourly.json new file mode 100644 index 000000000000..544559e884d5 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/age_gender_audience_report_hourly.json @@ -0,0 +1,110 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountId": { + "type": ["null", "integer"] + }, + "AgeGroup": { + "type": ["null", "string"] + }, + "Gender": { + "type": ["null", "string"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AccountName": { + "type": ["null", "string"] + }, + "AccountNumber": { + "type": ["null", "string"] + }, + "CampaignName": { + "type": ["null", "string"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Conversions": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "ExtendedCost": { + "type": ["null", "number"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "Language": { + "type": ["null", "string"] + }, + "AccountStatus": { + "type": ["null", "string"] + }, + "CampaignStatus": { + "type": ["null", "string"] + }, + "AdGroupStatus": { + "type": ["null", "string"] + }, + "BaseCampaignId": { + "type": ["null", "string"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "Goal": { + "type": ["null", "string"] + }, + "GoalType": { + "type": ["null", "string"] + }, + "AbsoluteTopImpressionRatePercent": { + "type": ["null", "number"] + }, + "TopImpressionRatePercent": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AllConversionsQualified": { + "type": ["null", "number"] + }, + "ViewThroughConversionsQualified": { + "type": ["null", "number"] + }, + "ViewThroughRevenue": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/app_install_ad_labels.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/app_install_ad_labels.json index 37bd0ad224a4..74ebe7d23dfe 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/app_install_ad_labels.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/app_install_ad_labels.json @@ -15,7 +15,9 @@ "type": ["null", "integer"] }, "Modified Time": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "Status": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/app_install_ads.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/app_install_ads.json index b409f06d62cd..4db96b254020 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/app_install_ads.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/app_install_ads.json @@ -45,7 +45,9 @@ "type": ["null", "integer"] }, "Modified Time": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "Parent Id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/budget_summary_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/budget_summary_report.json index f8366861a8c4..6c4cf7f5a939 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/budget_summary_report.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/budget_summary_report.json @@ -18,7 +18,8 @@ "type": ["null", "string"] }, "Date": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "MonthlyBudget": { "type": ["null", "number"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_impression_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_impression_performance_report.json index 9390d3de2967..4b2983bd3258 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_impression_performance_report.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_impression_performance_report.json @@ -12,7 +12,8 @@ "type": ["null", "integer"] }, "TimePeriod": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "CampaignStatus": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_impression_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_impression_performance_report_hourly.json index 90a90d070b7f..a5e48498383f 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_impression_performance_report_hourly.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_impression_performance_report_hourly.json @@ -12,7 +12,9 @@ "type": ["null", "integer"] }, "TimePeriod": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "CampaignStatus": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_labels.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_labels.json index e5fdf2701bef..7db5d82599fa 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_labels.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_labels.json @@ -18,7 +18,9 @@ "type": ["null", "integer"] }, "Modified Time": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "Status": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_performance_report.json index eff183e144e9..6ac3320854d6 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_performance_report.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_performance_report.json @@ -9,7 +9,8 @@ "type": ["null", "integer"] }, "TimePeriod": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "CurrencyCode": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_performance_report_hourly.json new file mode 100644 index 000000000000..bda17310348f --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_performance_report_hourly.json @@ -0,0 +1,176 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountId": { + "type": ["null", "integer"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "Network": { + "type": ["null", "string"] + }, + "DeliveredMatchType": { + "type": ["null", "string"] + }, + "DeviceOS": { + "type": ["null", "string"] + }, + "TopVsOther": { + "type": ["null", "string"] + }, + "BidMatchType": { + "type": ["null", "string"] + }, + "AccountName": { + "type": ["null", "string"] + }, + "CampaignName": { + "type": ["null", "string"] + }, + "CampaignType": { + "type": ["null", "string"] + }, + "CampaignStatus": { + "type": ["null", "string"] + }, + "CampaignLabels": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "QualityScore": { + "type": ["null", "number"] + }, + "AdRelevance": { + "type": ["null", "number"] + }, + "LandingPageExperience": { + "type": ["null", "number"] + }, + "PhoneImpressions": { + "type": ["null", "integer"] + }, + "PhoneCalls": { + "type": ["null", "integer"] + }, + "Ptr": { + "type": ["null", "number"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "CustomParameters": { + "type": ["null", "string"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "Conversions": { + "type": ["null", "number"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "LowQualityClicks": { + "type": ["null", "integer"] + }, + "LowQualityClicksPercent": { + "type": ["null", "number"] + }, + "LowQualityImpressions": { + "type": ["null", "integer"] + }, + "LowQualitySophisticatedClicks": { + "type": ["null", "integer"] + }, + "LowQualityConversions": { + "type": ["null", "integer"] + }, + "LowQualityConversionRate": { + "type": ["null", "number"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + }, + "BudgetName": { + "type": ["null", "string"] + }, + "BudgetStatus": { + "type": ["null", "string"] + }, + "BudgetAssociationStatus": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaigns.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaigns.json index 6721b299b263..c5790cd8928f 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaigns.json @@ -21,7 +21,7 @@ "type": ["null", "object"], "properties": { "Amount": { - "type": ["null", "string"] + "type": ["null", "number"] } } } diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/geographic_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/geographic_performance_report.json index f2c65f08ff39..8e55c0a61121 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/geographic_performance_report.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/geographic_performance_report.json @@ -13,7 +13,8 @@ "type": ["null", "integer"] }, "TimePeriod": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "AccountNumber": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/geographic_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/geographic_performance_report_hourly.json new file mode 100644 index 000000000000..9b79a66cfe24 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/geographic_performance_report_hourly.json @@ -0,0 +1,213 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "AccountId": { + "type": ["null", "integer"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "AccountNumber": { + "type": ["null", "string"] + }, + "Country": { + "type": ["null", "string"] + }, + "State": { + "type": ["null", "string"] + }, + "MetroArea": { + "type": ["null", "string"] + }, + "City": { + "type": ["null", "string"] + }, + "ProximityTargetLocation": { + "type": ["null", "string"] + }, + "Radius": { + "type": ["null", "string"] + }, + "LocationType": { + "type": ["null", "string"] + }, + "MostSpecificLocation": { + "type": ["null", "string"] + }, + "AccountStatus": { + "type": ["null", "string"] + }, + "CampaignStatus": { + "type": ["null", "string"] + }, + "AdGroupStatus": { + "type": ["null", "string"] + }, + "County": { + "type": ["null", "string"] + }, + "PostalCode": { + "type": ["null", "string"] + }, + "LocationId": { + "type": ["null", "string"] + }, + "BaseCampaignId": { + "type": ["null", "string"] + }, + "Goal": { + "type": ["null", "string"] + }, + "GoalType": { + "type": ["null", "string"] + }, + "AbsoluteTopImpressionRatePercent": { + "type": ["null", "number"] + }, + "TopImpressionRatePercent": { + "type": ["null", "string"] + }, + "AllConversionsQualified": { + "type": ["null", "string"] + }, + "Neighborhood": { + "type": ["null", "string"] + }, + "ViewThroughRevenue": { + "type": ["null", "string"] + }, + "CampaignType": { + "type": ["null", "string"] + }, + "AssetGroupId": { + "type": ["null", "string"] + }, + "AssetGroupName": { + "type": ["null", "string"] + }, + "AssetGroupStatus": { + "type": ["null", "string"] + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "DeliveredMatchType": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "Language": { + "type": ["null", "string"] + }, + "Network": { + "type": ["null", "string"] + }, + "DeviceOS": { + "type": ["null", "string"] + }, + "TopVsOther": { + "type": ["null", "string"] + }, + "BidMatchType": { + "type": ["null", "string"] + }, + "AccountName": { + "type": ["null", "string"] + }, + "CampaignName": { + "type": ["null", "string"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "ViewThroughConversionsQualified": { + "type": ["null", "number"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "Conversions": { + "type": ["null", "number"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_labels.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_labels.json index 37bd0ad224a4..74ebe7d23dfe 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_labels.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_labels.json @@ -15,7 +15,9 @@ "type": ["null", "integer"] }, "Modified Time": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "Status": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report.json index c6c6279fbde0..70ccc68a0b93 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report.json @@ -22,7 +22,8 @@ "type": ["null", "integer"] }, "TimePeriod": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "CurrencyCode": { "type": ["null", "string"] @@ -63,18 +64,6 @@ "KeywordStatus": { "type": ["null", "string"] }, - "HistoricalExpectedCtr": { - "type": ["null", "number"] - }, - "HistoricalAdRelevance": { - "type": ["null", "number"] - }, - "HistoricalLandingPageExperience": { - "type": ["null", "number"] - }, - "HistoricalQualityScore": { - "type": ["null", "number"] - }, "Impressions": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report_daily.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report_daily.json new file mode 100644 index 000000000000..48e35d9f3ce9 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report_daily.json @@ -0,0 +1,190 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountId": { + "type": ["null", "integer"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "KeywordId": { + "type": ["null", "integer"] + }, + "Keyword": { + "type": ["null", "string"] + }, + "AdId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date" + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "DeliveredMatchType": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "Language": { + "type": ["null", "string"] + }, + "Network": { + "type": ["null", "string"] + }, + "DeviceOS": { + "type": ["null", "string"] + }, + "TopVsOther": { + "type": ["null", "string"] + }, + "BidMatchType": { + "type": ["null", "string"] + }, + "AccountName": { + "type": ["null", "string"] + }, + "CampaignName": { + "type": ["null", "string"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "KeywordStatus": { + "type": ["null", "string"] + }, + "HistoricalExpectedCtr": { + "type": ["null", "number"] + }, + "HistoricalAdRelevance": { + "type": ["null", "number"] + }, + "HistoricalLandingPageExperience": { + "type": ["null", "number"] + }, + "HistoricalQualityScore": { + "type": ["null", "number"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "CurrentMaxCpc": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "QualityScore": { + "type": ["null", "number"] + }, + "ExpectedCtr": { + "type": ["null", "string"] + }, + "AdRelevance": { + "type": ["null", "number"] + }, + "LandingPageExperience": { + "type": ["null", "number"] + }, + "QualityImpact": { + "type": ["null", "number"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "CustomParameters": { + "type": ["null", "string"] + }, + "FinalAppUrl": { + "type": ["null", "string"] + }, + "Mainline1Bid": { + "type": ["null", "number"] + }, + "MainlineBid": { + "type": ["null", "number"] + }, + "FirstPageBid": { + "type": ["null", "number"] + }, + "FinalUrlSuffix": { + "type": ["null", "string"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "ViewThroughConversionsQualified": { + "type": ["null", "number"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "Conversions": { + "type": ["null", "number"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report_hourly.json new file mode 100644 index 000000000000..831c389d24a1 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report_hourly.json @@ -0,0 +1,180 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "AccountId": { + "type": ["null", "integer"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "KeywordId": { + "type": ["null", "integer"] + }, + "Keyword": { + "type": ["null", "string"] + }, + "AdId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "DeliveredMatchType": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "Language": { + "type": ["null", "string"] + }, + "Network": { + "type": ["null", "string"] + }, + "DeviceOS": { + "type": ["null", "string"] + }, + "TopVsOther": { + "type": ["null", "string"] + }, + "BidMatchType": { + "type": ["null", "string"] + }, + "AccountName": { + "type": ["null", "string"] + }, + "CampaignName": { + "type": ["null", "string"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "KeywordStatus": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "CurrentMaxCpc": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "QualityScore": { + "type": ["null", "number"] + }, + "ExpectedCtr": { + "type": ["null", "string"] + }, + "AdRelevance": { + "type": ["null", "number"] + }, + "LandingPageExperience": { + "type": ["null", "number"] + }, + "QualityImpact": { + "type": ["null", "number"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "CustomParameters": { + "type": ["null", "string"] + }, + "FinalAppUrl": { + "type": ["null", "string"] + }, + "Mainline1Bid": { + "type": ["null", "number"] + }, + "MainlineBid": { + "type": ["null", "number"] + }, + "FirstPageBid": { + "type": ["null", "number"] + }, + "FinalUrlSuffix": { + "type": ["null", "string"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "ViewThroughConversionsQualified": { + "type": ["null", "number"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "Conversions": { + "type": ["null", "number"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keywords.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keywords.json index 9987fe828be2..4f25c1378753 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keywords.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keywords.json @@ -30,7 +30,9 @@ "type": ["null", "string"] }, "Modified Time": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "Editorial Appeal Status": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/labels.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/labels.json index a23858825318..40845b0eb035 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/labels.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/labels.json @@ -21,7 +21,9 @@ "type": ["null", "string"] }, "Modified Time": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "Status": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/search_query_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/search_query_performance_report.json index ca9dcb71d9c6..57c50442e395 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/search_query_performance_report.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/search_query_performance_report.json @@ -12,7 +12,8 @@ "type": ["null", "integer"] }, "TimePeriod": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "CampaignName": { "type": ["null", "string"] @@ -81,7 +82,7 @@ "type": ["null", "number"] }, "CostPerConversion": { - "type": ["null", "integer"] + "type": ["null", "number"] }, "Language": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/search_query_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/search_query_performance_report_hourly.json new file mode 100644 index 000000000000..27e35b2fc670 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/search_query_performance_report_hourly.json @@ -0,0 +1,182 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountName": { + "type": ["null", "string"] + }, + "AccountNumber": { + "type": ["null", "string"] + }, + "AccountId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "CampaignName": { + "type": ["null", "string"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "AdId": { + "type": ["null", "integer"] + }, + "AdType": { + "type": ["null", "string"] + }, + "DestinationUrl": { + "type": ["null", "string"] + }, + "BidMatchType": { + "type": ["null", "string"] + }, + "DeliveredMatchType": { + "type": ["null", "string"] + }, + "CampaignStatus": { + "type": ["null", "string"] + }, + "AdStatus": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "SearchQuery": { + "type": ["null", "string"] + }, + "Keyword": { + "type": ["null", "string"] + }, + "AdGroupCriterionId": { + "type": ["null", "string"] + }, + "Conversions": { + "type": ["null", "integer"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "Language": { + "type": ["null", "string"] + }, + "KeywordId": { + "type": ["null", "integer"] + }, + "Network": { + "type": ["null", "string"] + }, + "TopVsOther": { + "type": ["null", "string"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "DeviceOS": { + "type": ["null", "string"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + }, + "AccountStatus": { + "type": ["null", "string"] + }, + "AdGroupStatus": { + "type": ["null", "string"] + }, + "KeywordStatus": { + "type": ["null", "string"] + }, + "CampaignType": { + "type": ["null", "string"] + }, + "CustomerId": { + "type": ["null", "integer"] + }, + "CustomerName": { + "type": ["null", "string"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "Goal": { + "type": ["null", "string"] + }, + "GoalType": { + "type": ["null", "string"] + }, + "AbsoluteTopImpressionRatePercent": { + "type": ["null", "number"] + }, + "TopImpressionRatePercent": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AllConversionsQualified": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/user_location_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/user_location_performance_report.json index acee84403de1..8edbd095b605 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/user_location_performance_report.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/user_location_performance_report.json @@ -12,7 +12,8 @@ "type": ["null", "integer"] }, "TimePeriod": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "CampaignName": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/user_location_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/user_location_performance_report_hourly.json new file mode 100644 index 000000000000..1bd42e6b8087 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/user_location_performance_report_hourly.json @@ -0,0 +1,215 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountName": { + "type": ["null", "string"] + }, + "AccountNumber": { + "type": ["null", "string"] + }, + "AccountId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "CampaignName": { + "type": ["null", "string"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "Country": { + "type": ["null", "string"] + }, + "State": { + "type": ["null", "string"] + }, + "MetroArea": { + "type": ["null", "string"] + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "ProximityTargetLocation": { + "type": ["null", "string"] + }, + "Radius": { + "type": ["null", "integer"] + }, + "Language": { + "type": ["null", "string"] + }, + "City": { + "type": ["null", "string"] + }, + "QueryIntentCountry": { + "type": ["null", "string"] + }, + "QueryIntentState": { + "type": ["null", "string"] + }, + "QueryIntentCity": { + "type": ["null", "string"] + }, + "QueryIntentDMA": { + "type": ["null", "string"] + }, + "BidMatchType": { + "type": ["null", "string"] + }, + "DeliveredMatchType": { + "type": ["null", "string"] + }, + "Network": { + "type": ["null", "string"] + }, + "TopVsOther": { + "type": ["null", "string"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "DeviceOS": { + "type": ["null", "string"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "Conversions": { + "type": ["null", "integer"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + }, + "County": { + "type": ["null", "string"] + }, + "PostalCode": { + "type": ["null", "string"] + }, + "QueryIntentCounty": { + "type": ["null", "string"] + }, + "QueryIntentPostalCode": { + "type": ["null", "string"] + }, + "LocationId": { + "type": ["null", "integer"] + }, + "QueryIntentLocationId": { + "type": ["null", "integer"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "Goal": { + "type": ["null", "string"] + }, + "GoalType": { + "type": ["null", "string"] + }, + "AbsoluteTopImpressionRatePercent": { + "type": ["null", "number"] + }, + "TopImpressionRatePercent": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AllConversionsQualified": { + "type": ["null", "number"] + }, + "ViewThroughConversionsQualified": { + "type": ["null", "number"] + }, + "Neighborhood": { + "type": ["null", "string"] + }, + "QueryIntentNeighborhood": { + "type": ["null", "string"] + }, + "ViewThroughRevenue": { + "type": ["null", "number"] + }, + "CampaignType": { + "type": ["null", "string"] + }, + "AssetGroupId": { + "type": ["null", "integer"] + }, + "AssetGroupName": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py index 9ff7a581479c..c18fcf5a913f 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py @@ -9,8 +9,10 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.utils import AirbyteTracedException +from source_bing_ads.base_streams import Accounts, AdGroups, Ads, Campaigns +from source_bing_ads.bulk_streams import AdGroupLabels, AppInstallAdLabels, AppInstallAds, CampaignLabels, KeywordLabels, Keywords, Labels from source_bing_ads.client import Client -from source_bing_ads.streams import ( # noqa: F401 +from source_bing_ads.report_streams import ( # noqa: F401 AccountImpressionPerformanceReportDaily, AccountImpressionPerformanceReportHourly, AccountImpressionPerformanceReportMonthly, @@ -19,52 +21,41 @@ AccountPerformanceReportHourly, AccountPerformanceReportMonthly, AccountPerformanceReportWeekly, - Accounts, AdGroupImpressionPerformanceReportDaily, AdGroupImpressionPerformanceReportHourly, AdGroupImpressionPerformanceReportMonthly, AdGroupImpressionPerformanceReportWeekly, - AdGroupLabels, AdGroupPerformanceReportDaily, AdGroupPerformanceReportHourly, AdGroupPerformanceReportMonthly, AdGroupPerformanceReportWeekly, - AdGroups, AdPerformanceReportDaily, AdPerformanceReportHourly, AdPerformanceReportMonthly, AdPerformanceReportWeekly, - Ads, AgeGenderAudienceReportDaily, AgeGenderAudienceReportHourly, AgeGenderAudienceReportMonthly, AgeGenderAudienceReportWeekly, - AppInstallAdLabels, - AppInstallAds, BingAdsReportingServiceStream, BudgetSummaryReport, CampaignImpressionPerformanceReportDaily, CampaignImpressionPerformanceReportHourly, CampaignImpressionPerformanceReportMonthly, CampaignImpressionPerformanceReportWeekly, - CampaignLabels, CampaignPerformanceReportDaily, CampaignPerformanceReportHourly, CampaignPerformanceReportMonthly, CampaignPerformanceReportWeekly, - Campaigns, CustomReport, GeographicPerformanceReportDaily, GeographicPerformanceReportHourly, GeographicPerformanceReportMonthly, GeographicPerformanceReportWeekly, - KeywordLabels, KeywordPerformanceReportDaily, KeywordPerformanceReportHourly, KeywordPerformanceReportMonthly, KeywordPerformanceReportWeekly, - Keywords, - Labels, SearchQueryPerformanceReportDaily, SearchQueryPerformanceReportHourly, SearchQueryPerformanceReportMonthly, diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/streams.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/streams.py deleted file mode 100644 index ecbb4cb450f8..000000000000 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/streams.py +++ /dev/null @@ -1,1358 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# -import os -import re -import ssl -import time -import xml.etree.ElementTree as ET -from abc import ABC, abstractmethod -from datetime import timezone -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union -from urllib.error import URLError -from urllib.parse import urlparse - -import _csv -import pandas as pd -import pendulum -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams import IncrementalMixin, Stream -from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from bingads.service_client import ServiceClient -from bingads.v13.internal.reporting.row_report import _RowReport -from bingads.v13.internal.reporting.row_report_iterator import _RowReportRecord -from bingads.v13.reporting.reporting_service_manager import ReportingServiceManager -from numpy import nan -from source_bing_ads.client import Client -from source_bing_ads.reports import ( - ALL_CONVERSION_FIELDS, - ALL_REVENUE_FIELDS, - AVERAGE_FIELDS, - BUDGET_FIELDS, - CONVERSION_FIELDS, - HISTORICAL_FIELDS, - LOW_QUALITY_FIELDS, - REVENUE_FIELDS, - PerformanceReportsMixin, - ReportsMixin, -) -from suds import WebFault, sudsobject - - -class BingAdsBaseStream(Stream, ABC): - primary_key: Optional[Union[str, List[str], List[List[str]]]] = None - - def __init__(self, client: Client, config: Mapping[str, Any]) -> None: - super().__init__() - self.client = client - self.config = config - - -class BingAdsStream(BingAdsBaseStream, ABC): - @property - @abstractmethod - def operation_name(self) -> str: - """ - Specifies operation name to use for a current stream - """ - pass - - @property - @abstractmethod - def service_name(self) -> str: - """ - Specifies bing ads service name for a current stream - """ - pass - - @property - def parent_key_to_foreign_key_map(self) -> MutableMapping[str, str]: - """ - Specifies dict with field in record as kay and slice key as value to be inserted in record in transform method. - """ - return {} - - def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - foreign_keys = {key: stream_slice.get(value) for key, value in self.parent_key_to_foreign_key_map.items()} - return record | foreign_keys - - @property - def _service(self) -> Union[ServiceClient, ReportingServiceManager]: - return self.client.get_service(service_name=self.service_name) - - @property - def _user_id(self) -> int: - return self._get_user_id() - - # TODO remove once Microsoft support confirm their SSL certificates are always valid... - def _get_user_id(self, number_of_retries=10): - """""" - try: - return self._service.GetUser().User.Id - except URLError as error: - if isinstance(error.reason, ssl.SSLError): - self.logger.warning("SSL certificate error, retrying...") - if number_of_retries > 0: - time.sleep(1) - return self._get_user_id(number_of_retries - 1) - else: - raise error - - def next_page_token(self, response: sudsobject.Object, **kwargs: Mapping[str, Any]) -> Optional[Mapping[str, Any]]: - """ - Default method for streams that don't support pagination - """ - return None - - def send_request(self, params: Mapping[str, Any], customer_id: str, account_id: str = None) -> Mapping[str, Any]: - request_kwargs = { - "service_name": self.service_name, - "customer_id": customer_id, - "account_id": account_id, - "operation_name": self.operation_name, - "params": params, - } - request = self.client.request(**request_kwargs) - return request - - def read_records( - self, - sync_mode: SyncMode, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - **kwargs: Mapping[str, Any], - ) -> Iterable[Mapping[str, Any]]: - stream_state = stream_state or {} - next_page_token = None - account_id = str(stream_slice.get("account_id")) if stream_slice else None - customer_id = str(stream_slice.get("customer_id")) if stream_slice else None - - while True: - params = self.request_params( - stream_state=stream_state, - stream_slice=stream_slice, - next_page_token=next_page_token, - account_id=account_id, - ) - response = self.send_request(params, customer_id=customer_id, account_id=account_id) - for record in self.parse_response(response): - yield self.transform(record, stream_slice) - - next_page_token = self.next_page_token(response, current_page_token=next_page_token) - if not next_page_token: - break - - yield from [] - - def parse_response(self, response: sudsobject.Object, **kwargs) -> Iterable[Mapping]: - if response is not None and hasattr(response, self.data_field): - yield from self.client.asdict(response)[self.data_field] - - yield from [] - - -class BingAdsCampaignManagementStream(BingAdsStream, ABC): - service_name: str = "CampaignManagement" - - @property - @abstractmethod - def data_field(self) -> str: - """ - Specifies root object name in a stream response - """ - pass - - @property - @abstractmethod - def additional_fields(self) -> Optional[str]: - """ - Specifies which additional fields to fetch for a current stream. - Expected format: field names separated by space - """ - pass - - def parse_response(self, response: sudsobject.Object, **kwargs) -> Iterable[Mapping]: - if response is not None and hasattr(response, self.data_field): - yield from self.client.asdict(response)[self.data_field] - - yield from [] - - -class BingAdsReportingServiceStream(BingAdsStream, ABC): - - cursor_field = "TimePeriod" - service_name: str = "ReportingService" - operation_name: str = "download_report" - - def parse_response(self, response: sudsobject.Object, **kwargs: Mapping[str, Any]) -> Iterable[Mapping]: - if response is not None: - try: - for row in response.report_records: - yield {column: self.get_column_value(row, column) for column in self.report_columns} - except _csv.Error as e: - self.logger.warning(f"CSV report file for stream `{self.name}` is broken or cannot be read correctly: {e}, skipping ...") - - yield from [] - - def get_column_value(self, row: _RowReportRecord, column: str) -> Union[str, None, int, float]: - """ - Reads field value from row and transforms: - 1. empty values to logical None - 2. Percent values to numeric string e.g. "12.25%" -> "12.25" - """ - value = row.value(column) - if not value or value == "--": - return None - if "%" in value: - value = value.replace("%", "") - if value and set(self.get_json_schema()["properties"].get(column, {}).get("type")) & {"integer", "number"}: - value = value.replace(",", "") - return value - - -class BingAdsBulkStream(BingAdsBaseStream, IncrementalMixin, ABC): - - transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - cursor_field = "Modified Time" - _state = {} - - @property - @abstractmethod - def data_scope(self) -> List[str]: - """ - Defines scopes or types of data to download. Docs: https://learn.microsoft.com/en-us/advertising/bulk-service/datascope?view=bingads-13 - """ - - @property - @abstractmethod - def download_entities(self) -> List[str]: - """ - Defines the entities that should be downloaded. Docs: https://learn.microsoft.com/en-us/advertising/bulk-service/downloadentity?view=bingads-13 - """ - - def stream_slices( - self, - **kwargs: Mapping[str, Any], - ) -> Iterable[Optional[Mapping[str, Any]]]: - for account in Accounts(self.client, self.config).read_records(SyncMode.full_refresh): - yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} - - yield from [] - - @property - def state(self) -> Mapping[str, Any]: - return self._state - - @state.setter - def state(self, value: Mapping[str, Any]): - self._state.update({str(value["Account Id"]): {self.cursor_field: value[self.cursor_field]}}) - - def get_start_date(self, stream_state: Mapping[str, Any] = None, account_id: str = None): - """ - Start_date in request can be provided only if it is sooner than 30 days from now - """ - min_available_date = pendulum.now().subtract(days=30).astimezone(tz=timezone.utc) - start_date = self.client.reports_start_date - if stream_state.get(account_id, {}).get(self.cursor_field): - start_date = pendulum.from_format(stream_state[account_id][self.cursor_field], "MM/DD/YYYY HH:mm:ss.SSS") - return None if start_date < min_available_date else min_available_date - - def read_records( - self, - sync_mode: SyncMode, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - **kwargs: Mapping[str, Any], - ) -> Iterable[Mapping[str, Any]]: - stream_state = stream_state or {} - account_id = str(stream_slice.get("account_id")) if stream_slice else None - customer_id = str(stream_slice.get("customer_id")) if stream_slice else None - - report_file_path = self.client.get_bulk_entity( - data_scope=self.data_scope, - download_entities=self.download_entities, - customer_id=customer_id, - account_id=account_id, - start_date=self.get_start_date(stream_state, account_id), - ) - for record in self.read_with_chunks(report_file_path): - record = self.transform(record, stream_slice) - yield record - self.state = record - - yield from [] - - def read_with_chunks(self, path: str, chunk_size: int = 1024) -> Iterable[Tuple[int, Mapping[str, Any]]]: - try: - with open(path, "r") as data: - chunks = pd.read_csv(data, chunksize=chunk_size, iterator=True, dialect="unix", dtype=object) - for chunk in chunks: - chunk = chunk.replace({nan: None}).to_dict(orient="records") - for row in chunk: - if row.get("Type") not in ("Format Version", "Account"): - yield row - except pd.errors.EmptyDataError as e: - self.logger.info(f"Empty data received. {e}") - yield from [] - except IOError as ioe: - self.logger.fatal( - f"The IO/Error occurred while reading tmp data. Called: {path}. Stream: {self.name}", - ) - raise ioe - finally: - # remove binary tmp file, after data is read - os.remove(path) - - def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - """ - Bing Ads Bulk API returns all available properties for all entities. - This method filter out only available properties. - """ - actual_record = {key: value for key, value in record.items() if key in self.get_json_schema()["properties"].keys()} - actual_record["Account Id"] = stream_slice.get("account_id") - return actual_record - - -class AppInstallAds(BingAdsBulkStream): - """ - https://learn.microsoft.com/en-us/advertising/bulk-service/app-install-ad?view=bingads-13 - """ - - data_scope = ["EntityData"] - download_entities = ["AppInstallAds"] - - primary_key = "Id" - - -class AppInstallAdLabels(BingAdsBulkStream): - """ - https://learn.microsoft.com/en-us/advertising/bulk-service/app-install-ad-label?view=bingads-13 - """ - - data_scope = ["EntityData"] - download_entities = ["AppInstallAdLabels"] - - primary_key = "Id" - - -class Labels(BingAdsBulkStream): - """ - https://learn.microsoft.com/en-us/advertising/bulk-service/label?view=bingads-13 - """ - - data_scope = ["EntityData"] - download_entities = ["Labels"] - - primary_key = "Id" - - -class KeywordLabels(BingAdsBulkStream): - """ - https://learn.microsoft.com/en-us/advertising/bulk-service/keyword-label?view=bingads-13 - """ - - data_scope = ["EntityData"] - download_entities = ["KeywordLabels"] - - primary_key = "Id" - - -class Keywords(BingAdsBulkStream): - """ - https://learn.microsoft.com/en-us/advertising/bulk-service/keyword?view=bingads-13 - """ - - data_scope = ["EntityData"] - download_entities = ["Keywords"] - - primary_key = "Id" - - -class CampaignLabels(BingAdsBulkStream): - """ - https://learn.microsoft.com/en-us/advertising/bulk-service/campaign-label?view=bingads-13 - """ - - data_scope = ["EntityData"] - download_entities = ["CampaignLabels"] - - primary_key = "Id" - - -class AdGroupLabels(BingAdsBulkStream): - """ - https://learn.microsoft.com/en-us/advertising/bulk-service/ad-group-label?view=bingads-13 - """ - - data_scope = ["EntityData"] - download_entities = ["AdGroupLabels"] - - primary_key = "Id" - - -class Accounts(BingAdsStream): - """ - Searches for accounts that the current authenticated user can access. - API doc: https://docs.microsoft.com/en-us/advertising/customer-management-service/searchaccounts?view=bingads-13 - Account schema: https://docs.microsoft.com/en-us/advertising/customer-management-service/advertiseraccount?view=bingads-13 - Stream caches incoming responses to be able to reuse this data in Campaigns stream - """ - - primary_key = "Id" - # Stream caches incoming responses to avoid duplicated http requests - use_cache: bool = True - data_field: str = "AdvertiserAccount" - service_name: str = "CustomerManagementService" - operation_name: str = "SearchAccounts" - additional_fields: str = "TaxCertificate AccountMode" - # maximum page size - page_size_limit: int = 1000 - - def next_page_token(self, response: sudsobject.Object, current_page_token: Optional[int]) -> Optional[Mapping[str, Any]]: - current_page_token = current_page_token or 0 - if response is not None and hasattr(response, self.data_field): - return None if self.page_size_limit > len(response[self.data_field]) else current_page_token + 1 - else: - return None - - def request_params( - self, - next_page_token: Mapping[str, Any] = None, - **kwargs: Mapping[str, Any], - ) -> MutableMapping[str, Any]: - predicates = { - "Predicate": [ - { - "Field": "UserId", - "Operator": "Equals", - "Value": self._user_id, - } - ] - } - - paging = self._service.factory.create("ns5:Paging") - paging.Index = next_page_token or 0 - paging.Size = self.page_size_limit - return { - "PageInfo": paging, - "Predicates": predicates, - "ReturnAdditionalFields": self.additional_fields, - } - - -class Campaigns(BingAdsCampaignManagementStream): - """ - Gets the campaigns for all provided accounts. - API doc: https://docs.microsoft.com/en-us/advertising/campaign-management-service/getcampaignsbyaccountid?view=bingads-13 - Campaign schema: https://docs.microsoft.com/en-us/advertising/campaign-management-service/campaign?view=bingads-13 - Stream caches incoming responses to be able to reuse this data in AdGroups stream - """ - - primary_key = "Id" - # Stream caches incoming responses to avoid duplicated http requests - use_cache: bool = True - data_field: str = "Campaign" - operation_name: str = "GetCampaignsByAccountId" - additional_fields: Iterable[str] = [ - "AdScheduleUseSearcherTimeZone", - "BidStrategyId", - "CpvCpmBiddingScheme", - "DynamicDescriptionSetting", - "DynamicFeedSetting", - "MaxConversionValueBiddingScheme", - "MultimediaAdsBidAdjustment", - "TargetImpressionShareBiddingScheme", - "TargetSetting", - "VerifiedTrackingSetting", - ] - campaign_types: Iterable[str] = ["Audience", "DynamicSearchAds", "Search", "Shopping"] - - parent_key_to_foreign_key_map = { - "AccountId": "account_id", - "CustomerId": "customer_id", - } - - def request_params( - self, - stream_slice: Mapping[str, Any] = None, - **kwargs: Mapping[str, Any], - ) -> MutableMapping[str, Any]: - return { - "AccountId": stream_slice["account_id"], - "CampaignType": " ".join(self.campaign_types), - "ReturnAdditionalFields": " ".join(self.additional_fields), - } - - def stream_slices( - self, - **kwargs: Mapping[str, Any], - ) -> Iterable[Optional[Mapping[str, Any]]]: - for account in Accounts(self.client, self.config).read_records(SyncMode.full_refresh): - yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} - - yield from [] - - -class AdGroups(BingAdsCampaignManagementStream): - """ - Gets the ad groups for all provided accounts. - API doc: https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadgroupsbycampaignid?view=bingads-13 - AdGroup schema: https://docs.microsoft.com/en-us/advertising/campaign-management-service/adgroup?view=bingads-13 - Stream caches incoming responses to be able to reuse this data in Ads stream - """ - - primary_key = "Id" - # Stream caches incoming responses to avoid duplicated http requests - use_cache: bool = True - data_field: str = "AdGroup" - operation_name: str = "GetAdGroupsByCampaignId" - additional_fields: str = "AdGroupType AdScheduleUseSearcherTimeZone CpmBid CpvBid MultimediaAdsBidAdjustment" - - parent_key_to_foreign_key_map = {"CampaignId": "campaign_id", "AccountId": "account_id", "CustomerId": "customer_id"} - - def request_params( - self, - stream_slice: Mapping[str, Any] = None, - **kwargs: Mapping[str, Any], - ) -> MutableMapping[str, Any]: - return {"CampaignId": stream_slice["campaign_id"], "ReturnAdditionalFields": self.additional_fields} - - def stream_slices( - self, - **kwargs: Mapping[str, Any], - ) -> Iterable[Optional[Mapping[str, Any]]]: - campaigns = Campaigns(self.client, self.config) - for account in Accounts(self.client, self.config).read_records(SyncMode.full_refresh): - for campaign in campaigns.read_records( - sync_mode=SyncMode.full_refresh, stream_slice={"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} - ): - yield {"campaign_id": campaign["Id"], "account_id": account["Id"], "customer_id": account["ParentCustomerId"]} - - yield from [] - - -class Ads(BingAdsCampaignManagementStream): - """ - Retrieves the ads for all provided accounts. - API doc: https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadsbyadgroupid?view=bingads-13 - Ad schema: https://docs.microsoft.com/en-us/advertising/campaign-management-service/ad?view=bingads-13 - """ - - primary_key = "Id" - data_field: str = "Ad" - operation_name: str = "GetAdsByAdGroupId" - additional_fields: str = "ImpressionTrackingUrls Videos LongHeadlines" - ad_types: Iterable[str] = [ - "Text", - "Image", - "Product", - "AppInstall", - "ExpandedText", - "DynamicSearch", - "ResponsiveAd", - "ResponsiveSearch", - ] - - parent_key_to_foreign_key_map = {"AdGroupId": "ad_group_id", "AccountId": "account_id", "CustomerId": "customer_id"} - - def request_params( - self, - stream_slice: Mapping[str, Any] = None, - **kwargs: Mapping[str, Any], - ) -> MutableMapping[str, Any]: - return { - "AdGroupId": stream_slice["ad_group_id"], - "AdTypes": {"AdType": self.ad_types}, - "ReturnAdditionalFields": self.additional_fields, - } - - def stream_slices( - self, - **kwargs: Mapping[str, Any], - ) -> Iterable[Optional[Mapping[str, Any]]]: - ad_groups = AdGroups(self.client, self.config) - for slice in ad_groups.stream_slices(sync_mode=SyncMode.full_refresh): - for ad_group in ad_groups.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice): - yield {"ad_group_id": ad_group["Id"], "account_id": slice["account_id"], "customer_id": slice["customer_id"]} - yield from [] - - -class BudgetSummaryReport(ReportsMixin, BingAdsReportingServiceStream): - report_name: str = "BudgetSummaryReport" - report_aggregation = None - cursor_field = "Date" - report_schema_name = "budget_summary_report" - primary_key = "Date" - - report_columns = [ - "AccountName", - "AccountNumber", - "AccountId", - "CampaignName", - "CampaignId", - "Date", - "MonthlyBudget", - "DailySpend", - "MonthToDateSpend", - ] - - -class CampaignPerformanceReport(PerformanceReportsMixin, BingAdsReportingServiceStream): - report_name: str = "CampaignPerformanceReport" - - report_schema_name = "campaign_performance_report" - primary_key = [ - "AccountId", - "CampaignId", - "TimePeriod", - "CurrencyCode", - "AdDistribution", - "DeviceType", - "Network", - "DeliveredMatchType", - "DeviceOS", - "TopVsOther", - "BidMatchType", - ] - - report_columns = [ - *primary_key, - "AccountName", - "CampaignName", - "CampaignType", - "CampaignStatus", - "CampaignLabels", - "Impressions", - "Clicks", - "Ctr", - "Spend", - "CostPerConversion", - "QualityScore", - "AdRelevance", - "LandingPageExperience", - "PhoneImpressions", - "PhoneCalls", - "Ptr", - "Assists", - "ReturnOnAdSpend", - "CostPerAssist", - "CustomParameters", - "ViewThroughConversions", - "AllCostPerConversion", - "AllReturnOnAdSpend", - *ALL_CONVERSION_FIELDS, - *ALL_REVENUE_FIELDS, - *AVERAGE_FIELDS, - *CONVERSION_FIELDS, - *LOW_QUALITY_FIELDS, - *REVENUE_FIELDS, - *BUDGET_FIELDS, - ] - - -class CampaignPerformanceReportHourly(CampaignPerformanceReport): - report_aggregation = "Hourly" - - -class CampaignPerformanceReportDaily(CampaignPerformanceReport): - report_aggregation = "Daily" - report_columns = [ - *CampaignPerformanceReport.report_columns, - *HISTORICAL_FIELDS, - ] - - -class CampaignPerformanceReportWeekly(CampaignPerformanceReport): - report_aggregation = "Weekly" - report_columns = [ - *CampaignPerformanceReport.report_columns, - *HISTORICAL_FIELDS, - ] - - -class CampaignPerformanceReportMonthly(CampaignPerformanceReport): - report_aggregation = "Monthly" - report_columns = [ - *CampaignPerformanceReport.report_columns, - *HISTORICAL_FIELDS, - ] - - -class CampaignImpressionPerformanceReport(PerformanceReportsMixin, BingAdsReportingServiceStream, ABC): - """ - https://learn.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13 - Primary key cannot be set: due to included `Impression Share Performance Statistics` some fields should be removed, - see https://learn.microsoft.com/en-us/advertising/guides/reports?view=bingads-13#columnrestrictions for more info. - """ - - report_name: str = "CampaignPerformanceReport" - - report_schema_name = "campaign_impression_performance_report" - - primary_key = None - - @property - def report_columns(self) -> Iterable[str]: - return list(self.get_json_schema().get("properties", {}).keys()) - - -class CampaignImpressionPerformanceReportHourly(CampaignImpressionPerformanceReport): - report_aggregation = "Hourly" - - report_schema_name = "campaign_impression_performance_report_hourly" - - -class CampaignImpressionPerformanceReportDaily(CampaignImpressionPerformanceReport): - report_aggregation = "Daily" - - -class CampaignImpressionPerformanceReportWeekly(CampaignImpressionPerformanceReport): - report_aggregation = "Weekly" - - -class CampaignImpressionPerformanceReportMonthly(CampaignImpressionPerformanceReport): - report_aggregation = "Monthly" - - -class AdPerformanceReport(PerformanceReportsMixin, BingAdsReportingServiceStream): - report_name: str = "AdPerformanceReport" - - report_schema_name = "ad_performance_report" - primary_key = [ - "AccountId", - "CampaignId", - "AdGroupId", - "AdId", - "TimePeriod", - "CurrencyCode", - "AdDistribution", - "DeviceType", - "Language", - "Network", - "DeviceOS", - "TopVsOther", - "BidMatchType", - "DeliveredMatchType", - ] - - report_columns = [ - *primary_key, - "AccountName", - "CampaignName", - "CampaignType", - "AdGroupName", - "Impressions", - "Clicks", - "Ctr", - "Spend", - "CostPerConversion", - "DestinationUrl", - "Assists", - "ReturnOnAdSpend", - "CostPerAssist", - "CustomParameters", - "FinalAppUrl", - "AdDescription", - "AdDescription2", - "ViewThroughConversions", - "ViewThroughConversionsQualified", - "AllCostPerConversion", - "AllReturnOnAdSpend", - *CONVERSION_FIELDS, - *AVERAGE_FIELDS, - *ALL_CONVERSION_FIELDS, - *ALL_REVENUE_FIELDS, - *REVENUE_FIELDS, - ] - - -class AdPerformanceReportHourly(AdPerformanceReport): - report_aggregation = "Hourly" - - -class AdPerformanceReportDaily(AdPerformanceReport): - report_aggregation = "Daily" - - -class AdPerformanceReportWeekly(AdPerformanceReport): - report_aggregation = "Weekly" - - -class AdPerformanceReportMonthly(AdPerformanceReport): - report_aggregation = "Monthly" - - -class AdGroupPerformanceReport(PerformanceReportsMixin, BingAdsReportingServiceStream, ABC): - report_name: str = "AdGroupPerformanceReport" - - report_schema_name = "ad_group_performance_report" - - primary_key = [ - "AccountId", - "CampaignId", - "AdGroupId", - "TimePeriod", - "CurrencyCode", - "AdDistribution", - "DeviceType", - "Network", - "DeliveredMatchType", - "DeviceOS", - "TopVsOther", - "BidMatchType", - "Language", - ] - - report_columns = [ - *primary_key, - "AccountName", - "CampaignName", - "CampaignType", - "AdGroupName", - "AdGroupType", - "Impressions", - "Clicks", - "Ctr", - "Spend", - "CostPerConversion", - "QualityScore", - "ExpectedCtr", - "AdRelevance", - "LandingPageExperience", - "PhoneImpressions", - "PhoneCalls", - "Ptr", - "Assists", - "CostPerAssist", - "CustomParameters", - "FinalUrlSuffix", - "ViewThroughConversions", - "AllCostPerConversion", - "AllReturnOnAdSpend", - *ALL_CONVERSION_FIELDS, - *ALL_REVENUE_FIELDS, - *AVERAGE_FIELDS, - *CONVERSION_FIELDS, - *REVENUE_FIELDS, - ] - - -class AdGroupPerformanceReportHourly(AdGroupPerformanceReport): - report_aggregation = "Hourly" - - -class AdGroupPerformanceReportDaily(AdGroupPerformanceReport): - report_aggregation = "Daily" - report_columns = [ - *AdGroupPerformanceReport.report_columns, - *HISTORICAL_FIELDS, - ] - - -class AdGroupPerformanceReportWeekly(AdGroupPerformanceReport): - report_aggregation = "Weekly" - report_columns = [ - *AdGroupPerformanceReport.report_columns, - *HISTORICAL_FIELDS, - ] - - -class AdGroupPerformanceReportMonthly(AdGroupPerformanceReport): - report_aggregation = "Monthly" - report_columns = [ - *AdGroupPerformanceReport.report_columns, - *HISTORICAL_FIELDS, - ] - - -class AdGroupImpressionPerformanceReport(PerformanceReportsMixin, BingAdsReportingServiceStream, ABC): - """ - https://learn.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13 - Primary key cannot be set: due to included `Impression Share Performance Statistics` some fields should be removed, - see https://learn.microsoft.com/en-us/advertising/guides/reports?view=bingads-13#columnrestrictions for more info. - """ - - report_name: str = "AdGroupPerformanceReport" - - report_schema_name = "ad_group_impression_performance_report" - - @property - def report_columns(self) -> Iterable[str]: - return list(self.get_json_schema().get("properties", {}).keys()) - - -class AdGroupImpressionPerformanceReportHourly(AdGroupImpressionPerformanceReport): - report_aggregation = "Hourly" - report_schema_name = "ad_group_impression_performance_report_hourly" - - -class AdGroupImpressionPerformanceReportDaily(AdGroupImpressionPerformanceReport): - report_aggregation = "Daily" - - -class AdGroupImpressionPerformanceReportWeekly(AdGroupImpressionPerformanceReport): - report_aggregation = "Weekly" - - -class AdGroupImpressionPerformanceReportMonthly(AdGroupImpressionPerformanceReport): - report_aggregation = "Monthly" - - -class KeywordPerformanceReport(PerformanceReportsMixin, BingAdsReportingServiceStream, ABC): - - report_name: str = "KeywordPerformanceReport" - - report_schema_name = "keyword_performance_report" - primary_key = [ - "AccountId", - "CampaignId", - "AdGroupId", - "KeywordId", - "AdId", - "TimePeriod", - "CurrencyCode", - "DeliveredMatchType", - "AdDistribution", - "DeviceType", - "Language", - "Network", - "DeviceOS", - "TopVsOther", - "BidMatchType", - ] - - report_columns = [ - *primary_key, - "AccountName", - "CampaignName", - "AdGroupName", - "Keyword", - "KeywordStatus", - "Impressions", - "Clicks", - "Ctr", - "CurrentMaxCpc", - "Spend", - "CostPerConversion", - "QualityScore", - "ExpectedCtr", - "AdRelevance", - "LandingPageExperience", - "QualityImpact", - "Assists", - "ReturnOnAdSpend", - "CostPerAssist", - "CustomParameters", - "FinalAppUrl", - "Mainline1Bid", - "MainlineBid", - "FirstPageBid", - "FinalUrlSuffix", - "ViewThroughConversions", - "ViewThroughConversionsQualified", - "AllCostPerConversion", - "AllReturnOnAdSpend", - *CONVERSION_FIELDS, - *AVERAGE_FIELDS, - *ALL_CONVERSION_FIELDS, - *ALL_REVENUE_FIELDS, - *REVENUE_FIELDS, - ] - - -class KeywordPerformanceReportHourly(KeywordPerformanceReport): - report_aggregation = "Hourly" - - -class KeywordPerformanceReportDaily(KeywordPerformanceReport): - report_aggregation = "Daily" - report_columns = [ - *KeywordPerformanceReport.report_columns, - *HISTORICAL_FIELDS, - ] - - -class KeywordPerformanceReportWeekly(KeywordPerformanceReport): - report_aggregation = "Weekly" - - -class KeywordPerformanceReportMonthly(KeywordPerformanceReport): - report_aggregation = "Monthly" - - -class GeographicPerformanceReport(PerformanceReportsMixin, BingAdsReportingServiceStream, ABC): - - report_name: str = "GeographicPerformanceReport" - - report_schema_name = "geographic_performance_report" - - # Need to override the primary key here because the one inherited from the PerformanceReportsMixin - # is incorrect for the geographic performance reports - primary_key = None - - report_columns = [ - "AccountId", - "CampaignId", - "AdGroupId", - "TimePeriod", - "Country", - "CurrencyCode", - "DeliveredMatchType", - "AdDistribution", - "DeviceType", - "Language", - "Network", - "DeviceOS", - "TopVsOther", - "BidMatchType", - "MetroArea", - "State", - "City", - "AdGroupName", - "Ctr", - "ProximityTargetLocation", - "Radius", - "Assists", - "ReturnOnAdSpend", - "CostPerAssist", - "LocationType", - "MostSpecificLocation", - "AccountStatus", - "CampaignStatus", - "AdGroupStatus", - "County", - "PostalCode", - "LocationId", - "BaseCampaignId", - "AllCostPerConversion", - "AllReturnOnAdSpend", - "ViewThroughConversions", - "Goal", - "GoalType", - "AbsoluteTopImpressionRatePercent", - "TopImpressionRatePercent", - "AllConversionsQualified", - "ViewThroughConversionsQualified", - "Neighborhood", - "ViewThroughRevenue", - "CampaignType", - "AssetGroupId", - "AssetGroupName", - "AssetGroupStatus", - "Clicks", - "Spend", - "Impressions", - "CostPerConversion", - "AccountName", - "AccountNumber", - "CampaignName", - *CONVERSION_FIELDS, - *AVERAGE_FIELDS, - *ALL_CONVERSION_FIELDS, - *ALL_REVENUE_FIELDS, - *REVENUE_FIELDS, - ] - - -class GeographicPerformanceReportHourly(GeographicPerformanceReport): - report_aggregation = "Hourly" - - -class GeographicPerformanceReportDaily(GeographicPerformanceReport): - report_aggregation = "Daily" - - -class GeographicPerformanceReportWeekly(GeographicPerformanceReport): - report_aggregation = "Weekly" - - -class GeographicPerformanceReportMonthly(GeographicPerformanceReport): - report_aggregation = "Monthly" - - -class AccountPerformanceReport(PerformanceReportsMixin, BingAdsReportingServiceStream): - - report_name: str = "AccountPerformanceReport" - - report_schema_name = "account_performance_report" - primary_key = [ - "AccountId", - "TimePeriod", - "CurrencyCode", - "AdDistribution", - "DeviceType", - "Network", - "DeliveredMatchType", - "DeviceOS", - "TopVsOther", - "BidMatchType", - ] - - report_columns = [ - *primary_key, - "AccountName", - "AccountNumber", - "PhoneImpressions", - "PhoneCalls", - "Clicks", - "Ctr", - "Spend", - "Impressions", - "CostPerConversion", - "Ptr", - "Assists", - "ReturnOnAdSpend", - "CostPerAssist", - *AVERAGE_FIELDS, - *CONVERSION_FIELDS, - *LOW_QUALITY_FIELDS, - *REVENUE_FIELDS, - ] - - -class AccountPerformanceReportHourly(AccountPerformanceReport): - report_aggregation = "Hourly" - - -class AccountPerformanceReportDaily(AccountPerformanceReport): - report_aggregation = "Daily" - - -class AccountPerformanceReportWeekly(AccountPerformanceReport): - report_aggregation = "Weekly" - - -class AccountPerformanceReportMonthly(AccountPerformanceReport): - report_aggregation = "Monthly" - - -class AccountImpressionPerformanceReport(PerformanceReportsMixin, BingAdsReportingServiceStream, ABC): - """ - Report source: https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13 - Primary key cannot be set: due to included `Impression Share Performance Statistics` some fields should be removed, - see https://learn.microsoft.com/en-us/advertising/guides/reports?view=bingads-13#columnrestrictions for more info. - """ - - report_name: str = "AccountPerformanceReport" - report_schema_name = "account_impression_performance_report" - primary_key = None - - @property - def report_columns(self): - return list(self.get_json_schema().get("properties", {}).keys()) - - -class AccountImpressionPerformanceReportHourly(AccountImpressionPerformanceReport): - report_aggregation = "Hourly" - - report_schema_name = "account_impression_performance_report_hourly" - - -class AccountImpressionPerformanceReportDaily(AccountImpressionPerformanceReport): - report_aggregation = "Daily" - - -class AccountImpressionPerformanceReportWeekly(AccountImpressionPerformanceReport): - report_aggregation = "Weekly" - - -class AccountImpressionPerformanceReportMonthly(AccountImpressionPerformanceReport): - report_aggregation = "Monthly" - - -class AgeGenderAudienceReport(PerformanceReportsMixin, BingAdsReportingServiceStream, ABC): - - report_name: str = "AgeGenderAudienceReport" - - report_schema_name = "age_gender_audience_report" - primary_key = ["AgeGroup", "Gender", "TimePeriod", "AccountId", "CampaignId", "Language", "AdDistribution"] - - @property - def report_columns(self): - return list(self.get_json_schema().get("properties", {}).keys()) - - -class AgeGenderAudienceReportHourly(AgeGenderAudienceReport): - report_aggregation = "Hourly" - - -class AgeGenderAudienceReportDaily(AgeGenderAudienceReport): - report_aggregation = "Daily" - - -class AgeGenderAudienceReportWeekly(AgeGenderAudienceReport): - report_aggregation = "Weekly" - - -class AgeGenderAudienceReportMonthly(AgeGenderAudienceReport): - report_aggregation = "Monthly" - - -class SearchQueryPerformanceReport(PerformanceReportsMixin, BingAdsReportingServiceStream, ABC): - - report_name: str = "SearchQueryPerformanceReport" - - report_schema_name = "search_query_performance_report" - primary_key = [ - "SearchQuery", - "Keyword", - "TimePeriod", - "AccountId", - "CampaignId", - "Language", - "DeliveredMatchType", - "DeviceType", - "DeviceOS", - "TopVsOther", - ] - - @property - def report_columns(self) -> List[str]: - return list(self.get_json_schema().get("properties", {}).keys()) - - -class SearchQueryPerformanceReportHourly(SearchQueryPerformanceReport): - report_aggregation = "Hourly" - - -class SearchQueryPerformanceReportDaily(SearchQueryPerformanceReport): - report_aggregation = "Daily" - - -class SearchQueryPerformanceReportWeekly(SearchQueryPerformanceReport): - report_aggregation = "Weekly" - - -class SearchQueryPerformanceReportMonthly(SearchQueryPerformanceReport): - report_aggregation = "Monthly" - - -class UserLocationPerformanceReport(PerformanceReportsMixin, BingAdsReportingServiceStream, ABC): - - report_name: str = "UserLocationPerformanceReport" - report_schema_name = "user_location_performance_report" - primary_key = [ - "AccountId", - "AdGroupId", - "CampaignId", - "DeliveredMatchType", - "DeviceOS", - "DeviceType", - "Language", - "LocationId", - "QueryIntentLocationId", - "TimePeriod", - "TopVsOther", - ] - - @property - def report_columns(self) -> List[str]: - return list(self.get_json_schema().get("properties", {}).keys()) - - -class UserLocationPerformanceReportHourly(UserLocationPerformanceReport): - report_aggregation = "Hourly" - - -class UserLocationPerformanceReportDaily(UserLocationPerformanceReport): - report_aggregation = "Daily" - - -class UserLocationPerformanceReportWeekly(UserLocationPerformanceReport): - report_aggregation = "Weekly" - - -class UserLocationPerformanceReportMonthly(UserLocationPerformanceReport): - report_aggregation = "Monthly" - - -class CustomReport(PerformanceReportsMixin, BingAdsReportingServiceStream, ABC): - transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - custom_report_columns = [] - report_schema_name = None - primary_key = None - - @property - def cursor_field(self) -> Union[str, List[str]]: - # Summary aggregation doesn't include TimePeriod field - if self.report_aggregation != "Summary": - return "TimePeriod" - - @property - def report_columns(self): - # adding common and default columns - if "AccountId" not in self.custom_report_columns: - self.custom_report_columns.append("AccountId") - if self.cursor_field and self.cursor_field not in self.custom_report_columns: - self.custom_report_columns.append(self.cursor_field) - return list(frozenset(self.custom_report_columns)) - - def get_json_schema(self) -> Mapping[str, Any]: - columns_schema = {col: {"type": ["null", "string"]} for col in self.report_columns} - schema: Mapping[str, Any] = { - "$schema": "https://json-schema.org/draft-07/schema#", - "type": ["null", "object"], - "additionalProperties": True, - "properties": columns_schema, - } - return schema - - def validate_report_configuration(self) -> Tuple[bool, str]: - # gets /bingads/v13/proxies/production/reporting_service.xml - reporting_service_file = self.client.get_service(self.service_name)._get_service_info_dict(self.client.api_version)[ - ("reporting", self.client.environment) - ] - tree = ET.parse(urlparse(reporting_service_file).path) - request_object = tree.find(f".//{{*}}complexType[@name='{self.report_name}Request']") - - report_object_columns = self._get_object_columns(request_object, tree) - is_custom_cols_in_report_object_cols = all(x in report_object_columns for x in self.custom_report_columns) - - if not is_custom_cols_in_report_object_cols: - return False, ( - f"Reporting Columns are invalid. Columns that you provided don't belong to Reporting Data Object Columns:" - f" {self.custom_report_columns}. Please ensure it is correct in Bing Ads Docs." - ) - - return True, "" - - def _clear_namespace(self, type: str) -> str: - return re.sub(r"^[a-z]+:", "", type) - - def _get_object_columns(self, request_el: ET.Element, tree: ET.ElementTree) -> List[str]: - column_el = request_el.find(".//{*}element[@name='Columns']") - array_of_columns_name = self._clear_namespace(column_el.get("type")) - - array_of_columns_elements = tree.find(f".//{{*}}complexType[@name='{array_of_columns_name}']") - inner_array_of_columns_elements = array_of_columns_elements.find(".//{*}element") - column_el_name = self._clear_namespace(inner_array_of_columns_elements.get("type")) - - column_el = tree.find(f".//{{*}}simpleType[@name='{column_el_name}']") - column_enum_items = column_el.findall(".//{*}enumeration") - column_enum_items_values = [el.get("value") for el in column_enum_items] - return column_enum_items_values - - def get_report_record_timestamp(self, datestring: str) -> int: - """ - Parse report date field based on aggregation type - """ - if not self.report_aggregation: - date = pendulum.from_format(datestring, "M/D/YYYY") - else: - if self.report_aggregation in ["DayOfWeek", "HourOfDay"]: - return int(datestring) - if self.report_aggregation == "Hourly": - date = pendulum.from_format(datestring, "YYYY-MM-DD|H") - else: - date = pendulum.parse(datestring) - - return date.int_timestamp - - def send_request(self, params: Mapping[str, Any], customer_id: str, account_id: str) -> _RowReport: - try: - return super().send_request(params, customer_id, account_id) - except WebFault as e: - self.logger.error( - f"Could not sync custom report {self.name}: Please validate your column and aggregation configuration. " - f"Error form server: [{e.fault.faultstring}]" - ) diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/utils.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/utils.py new file mode 100644 index 000000000000..785b1237452a --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/utils.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from datetime import datetime, timezone + + +def transform_bulk_datetime_format_to_rfc_3339(original_value: str) -> str: + """ + Bing Ads Bulk API provides datetime fields in custom format with milliseconds: "04/27/2023 18:00:14.970" + Return datetime in RFC3339 format: "2023-04-27T18:00:14.970+00:00" + """ + return datetime.strptime(original_value, "%m/%d/%Y %H:%M:%S.%f").replace(tzinfo=timezone.utc).isoformat(timespec="milliseconds") + + +def transform_date_format_to_rfc_3339(original_value: str) -> str: + """ + Bing Ads API provides date fields in custom format: "04/27/2023" + Return date in RFC3339 format: "2023-04-27" + """ + return datetime.strptime(original_value, "%m/%d/%Y").replace(tzinfo=timezone.utc).strftime("%Y-%m-%d") + + +def transform_report_hourly_datetime_format_to_rfc_3339(original_value: str) -> str: + """ + Bing Ads API reports with hourly aggregation provides date fields in custom format: "2023-11-04|11" + Return date in RFC3339 format: "2023-11-04T11:00:00+00:00" + """ + return datetime.strptime(original_value, "%Y-%m-%d|%H").replace(tzinfo=timezone.utc).isoformat(timespec="seconds") diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/conftest.py new file mode 100644 index 000000000000..fec097978fff --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/conftest.py @@ -0,0 +1,62 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from unittest.mock import patch + +import pytest + + +@pytest.fixture(name="config") +def config_fixture(): + """Generates streams settings from a config file""" + return { + "tenant_id": "common", + "developer_token": "fake_developer_token", + "refresh_token": "fake_refresh_token", + "client_id": "fake_client_id", + "reports_start_date": "2020-01-01", + "lookback_window": 0, + } + + +@pytest.fixture(name="config_with_custom_reports") +def config_with_custom_reports_fixture(): + """Generates streams settings with custom reports from a config file""" + return { + "tenant_id": "common", + "developer_token": "fake_developer_token", + "refresh_token": "fake_refresh_token", + "client_id": "fake_client_id", + "reports_start_date": "2020-01-01", + "lookback_window": 0, + "custom_reports": [ + { + "name": "my test custom report", + "reporting_object": "DSAAutoTargetPerformanceReport", + "report_columns": [ + "AbsoluteTopImpressionRatePercent", + "AccountId", + "AccountName", + "AccountNumber", + "AccountStatus", + "AdDistribution", + "AdGroupId", + "AdGroupName", + "AdGroupStatus", + "AdId", + "AllConversionRate", + "AllConversions", + "AllConversionsQualified", + "AllCostPerConversion", + "AllReturnOnAdSpend", + "AllRevenue", + ], + "report_aggregation": "Weekly", + } + ], + } + + +@pytest.fixture(name="logger_mock") +def logger_mock_fixture(): + return patch("source_bing_ads.source.AirbyteLogger") diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py new file mode 100644 index 000000000000..23f7efb7bb8e --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py @@ -0,0 +1,112 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from pathlib import Path +from unittest.mock import patch + +import pendulum +import pytest +import source_bing_ads +from freezegun import freeze_time +from pendulum import UTC, DateTime +from source_bing_ads.base_streams import Accounts +from source_bing_ads.bulk_streams import AppInstallAds + + +@patch.object(source_bing_ads.source, "Client") +def test_bulk_stream_stream_slices(mocked_client, config): + slices = AppInstallAds(mocked_client, config).stream_slices() + assert list(slices) == [] + + app_install_ads = AppInstallAds(mocked_client, config) + accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) + with patch.object(Accounts, "read_records", return_value=accounts_read_records): + slices = app_install_ads.stream_slices() + assert list(slices) == [{"account_id": 180519267, "customer_id": 100}, {"account_id": 180278106, "customer_id": 200}] + + +@patch.object(source_bing_ads.source, "Client") +def test_bulk_stream_transform(mocked_client, config): + record = {"Ad Group": "Ad Group", "App Id": "App Id", "Campaign": "Campaign", "Custom Parameter": "Custom Parameter"} + transformed_record = AppInstallAds(mocked_client, config).transform( + record=record, stream_slice={"account_id": 180519267, "customer_id": 100} + ) + assert transformed_record == { + "Account Id": 180519267, + "Ad Group": "Ad Group", + "App Id": "App Id", + "Campaign": "Campaign", + "Custom Parameter": "Custom Parameter", + } + + +@patch.object(source_bing_ads.source, "Client") +def test_bulk_stream_read_with_chunks(mocked_client, config): + path_to_file = Path(__file__).parent / "app_install_ads.csv" + path_to_file_base = Path(__file__).parent / "app_install_ads_base.csv" + with open(path_to_file_base, "r") as f1, open(path_to_file, "a") as f2: + for line in f1: + f2.write(line) + + app_install_ads = AppInstallAds(mocked_client, config) + result = app_install_ads.read_with_chunks(path=path_to_file) + assert next(result) == { + "Ad Group": "AdGroupNameGoesHere", + "App Id": "AppStoreIdGoesHere", + "App Platform": "Android", + "Campaign": "ParentCampaignNameGoesHere", + "Client Id": "ClientIdGoesHere", + "Custom Parameter": "{_promoCode}=PROMO1; {_season}=summer", + "Destination Url": None, + "Device Preference": "All", + "Display Url": None, + "Final Url": "FinalUrlGoesHere", + "Final Url Suffix": None, + "Id": None, + "Mobile Final Url": None, + "Modified Time": None, + "Name": None, + "Parent Id": "-1111", + "Promotion": None, + "Status": "Active", + "Text": "Find New Customers & Increase Sales!", + "Title": "Contoso Quick Setup", + "Tracking Template": None, + "Type": "App Install Ad", + } + + +@patch.object(source_bing_ads.source, "Client") +@freeze_time("2023-11-01T12:00:00.000+00:00") +@pytest.mark.parametrize( + "stream_state, config_start_date, expected_start_date", + [ + ({"some_account_id": {"Modified Time": "2023-10-15T12:00:00.000+00:00"}}, "2020-01-01", DateTime(2023, 10, 15, 12, 0, 0, tzinfo=UTC)), + ({"another_account_id": {"Modified Time": "2023-10-15T12:00:00.000+00:00"}}, "2020-01-01", None), + ({}, "2020-01-01", None), + ({}, "2023-10-21", DateTime(2023, 10, 21, 0, 0, 0, tzinfo=UTC)), + ], + ids=["state_within_30_days", "state_within_30_days_another_account_id", "empty_state", "empty_state_start_date_within_30"] +) +def test_bulk_stream_start_date(mocked_client, config, stream_state, config_start_date, expected_start_date): + mocked_client.reports_start_date = pendulum.parse(config_start_date) if config_start_date else None + stream = AppInstallAds(mocked_client, config) + assert expected_start_date == stream.get_start_date(stream_state, 'some_account_id') + + +@patch.object(source_bing_ads.source, "Client") +def test_bulk_stream_stream_state(mocked_client, config): + stream = AppInstallAds(mocked_client, config) + stream.state = {"Account Id": "some_account_id", "Modified Time": "04/27/2023 18:00:14.970"} + assert stream.state == {"some_account_id": {"Modified Time": "2023-04-27T18:00:14.970+00:00"}} + stream.state = {"Account Id": "some_account_id", "Modified Time": "05/27/2023 18:00:14.970"} + assert stream.state == {"some_account_id": {"Modified Time": "2023-05-27T18:00:14.970+00:00"}} + stream.state = {"Account Id": "some_account_id", "Modified Time": "05/25/2023 18:00:14.970"} + assert stream.state == {"some_account_id": {"Modified Time": "2023-05-27T18:00:14.970+00:00"}} + + +@patch.object(source_bing_ads.source, "Client") +def test_bulk_stream_custom_transform_date_rfc3339(mocked_client, config): + stream = AppInstallAds(mocked_client, config) + assert "2023-04-27T18:00:14.970+00:00" == stream.custom_transform_date_rfc3339("04/27/2023 18:00:14.970", stream.get_json_schema()["properties"][stream.cursor_field]) diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py index 3c7ecc853430..44c5329bd83a 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py @@ -3,27 +3,65 @@ # import copy -from unittest.mock import Mock +import xml.etree.ElementTree as ET +from unittest.mock import MagicMock, Mock, patch +from urllib.parse import urlparse +import _csv import pendulum import pytest +import source_bing_ads +from bingads.service_info import SERVICE_INFO_DICT_V13 from bingads.v13.internal.reporting.row_report_iterator import _RowReportRecord, _RowValues -from source_bing_ads.reports import PerformanceReportsMixin, ReportsMixin -from source_bing_ads.source import SourceBingAds -from source_bing_ads.streams import ( +from source_bing_ads.base_streams import Accounts +from source_bing_ads.report_streams import ( + AccountImpressionPerformanceReportDaily, + AccountImpressionPerformanceReportHourly, + AccountPerformanceReportDaily, + AccountPerformanceReportHourly, + AccountPerformanceReportMonthly, + AdGroupImpressionPerformanceReportDaily, + AdGroupImpressionPerformanceReportHourly, + AdGroupPerformanceReportDaily, + AdGroupPerformanceReportHourly, + AdPerformanceReportDaily, + AdPerformanceReportHourly, + AgeGenderAudienceReportDaily, + AgeGenderAudienceReportHourly, + BingAdsReportingServicePerformanceStream, BingAdsReportingServiceStream, + BudgetSummaryReport, + CampaignImpressionPerformanceReportDaily, + CampaignImpressionPerformanceReportHourly, + CampaignPerformanceReportDaily, + CampaignPerformanceReportHourly, GeographicPerformanceReportDaily, GeographicPerformanceReportHourly, GeographicPerformanceReportMonthly, GeographicPerformanceReportWeekly, + KeywordPerformanceReportDaily, + KeywordPerformanceReportHourly, + SearchQueryPerformanceReportDaily, + SearchQueryPerformanceReportHourly, + UserLocationPerformanceReportDaily, + UserLocationPerformanceReportHourly, ) +from source_bing_ads.source import SourceBingAds +from suds import WebFault + +TEST_CONFIG = { + "developer_token": "developer_token", + "client_id": "client_id", + "refresh_token": "refresh_token", + "reports_start_date": "2020-01-01T00:00:00Z", +} class TestClient: pass -class TestReport(ReportsMixin, BingAdsReportingServiceStream, SourceBingAds): +class TestReport(BingAdsReportingServiceStream, SourceBingAds): date_format, report_columns, report_name, cursor_field = "YYYY-MM-DD", None, None, "Time" report_aggregation = "Monthly" report_schema_name = "campaign_performance_report" @@ -32,7 +70,7 @@ def __init__(self) -> None: self.client = TestClient() -class TestPerformanceReport(PerformanceReportsMixin, BingAdsReportingServiceStream, SourceBingAds): +class TestPerformanceReport(BingAdsReportingServicePerformanceStream, SourceBingAds): date_format, report_columns, report_name, cursor_field = "YYYY-MM-DD", None, None, "Time" report_aggregation = "Monthly" report_schema_name = "campaign_performance_report" @@ -65,25 +103,38 @@ def test_get_column_value(): assert test_report.get_column_value(record, "Assists") == "123456789" +@patch.object(source_bing_ads.source, "Client") +def test_AccountPerformanceReportMonthly_request_params(mocked_client, config): + accountperformancereportmonthly = AccountPerformanceReportMonthly(mocked_client, config) + request_params = accountperformancereportmonthly.request_params(account_id=180278106, stream_slice={"time_period": "ThisYear"}) + del request_params["report_request"] + assert request_params == { + "overwrite_result_file": True, + "result_file_directory": "/tmp", + "result_file_name": "AccountPerformanceReport", + "timeout_in_milliseconds": 300000, + } + + def test_get_updated_state_init_state(): test_report = TestReport() stream_state = {} latest_record = {"AccountId": 123, "Time": "2020-01-02"} new_state = test_report.get_updated_state(stream_state, latest_record) - assert new_state["123"]["Time"] == (pendulum.parse("2020-01-02")).timestamp() + assert new_state["123"]["Time"] == "2020-01-02" def test_get_updated_state_new_state(): test_report = TestReport() - stream_state = {"123": {"Time": pendulum.parse("2020-01-01").timestamp()}} + stream_state = {"123": {"Time": "2020-01-01"}} latest_record = {"AccountId": 123, "Time": "2020-01-02"} new_state = test_report.get_updated_state(stream_state, latest_record) - assert new_state["123"]["Time"] == pendulum.parse("2020-01-02").timestamp() + assert new_state["123"]["Time"] == "2020-01-02" def test_get_updated_state_state_unchanged(): test_report = TestReport() - stream_state = {"123": {"Time": pendulum.parse("2020-01-03").timestamp()}} + stream_state = {"123": {"Time": "2020-01-03"}} latest_record = {"AccountId": 123, "Time": "2020-01-02"} new_state = test_report.get_updated_state(copy.deepcopy(stream_state), latest_record) assert stream_state == new_state @@ -91,29 +142,60 @@ def test_get_updated_state_state_unchanged(): def test_get_updated_state_state_new_account(): test_report = TestReport() - stream_state = {"123": {"Time": pendulum.parse("2020-01-03").timestamp()}} + stream_state = {"123": {"Time": "2020-01-03"}} latest_record = {"AccountId": 234, "Time": "2020-01-02"} new_state = test_report.get_updated_state(stream_state, latest_record) assert "234" in new_state and "123" in new_state - assert new_state["234"]["Time"] == pendulum.parse("2020-01-02").timestamp() + assert new_state["234"]["Time"] == "2020-01-02" -def test_get_report_record_timestamp_daily(): - test_report = TestReport() - test_report.report_aggregation = "Daily" - assert pendulum.parse("2020-01-01").timestamp() == test_report.get_report_record_timestamp("2020-01-01") +@pytest.mark.parametrize( + "stream_report_daily_cls", + ( + AccountImpressionPerformanceReportDaily, + AccountPerformanceReportDaily, + AdGroupImpressionPerformanceReportDaily, + AdGroupPerformanceReportDaily, + AgeGenderAudienceReportDaily, + AdPerformanceReportDaily, + CampaignImpressionPerformanceReportDaily, + CampaignPerformanceReportDaily, + KeywordPerformanceReportDaily, + SearchQueryPerformanceReportDaily, + UserLocationPerformanceReportDaily, + GeographicPerformanceReportDaily, + ), +) +def test_get_report_record_timestamp_daily(stream_report_daily_cls): + stream_report = stream_report_daily_cls(client=Mock(), config=TEST_CONFIG) + assert "2020-01-01" == stream_report.get_report_record_timestamp("2020-01-01") def test_get_report_record_timestamp_without_aggregation(): - test_report = TestReport() - test_report.report_aggregation = None - assert pendulum.parse("2020-07-20").timestamp() == test_report.get_report_record_timestamp("7/20/2020") + stream_report = BudgetSummaryReport(client=Mock(), config=TEST_CONFIG) + assert "2020-07-20" == stream_report.get_report_record_timestamp("7/20/2020") -def test_get_report_record_timestamp_hourly(): - test_report = TestReport() - test_report.report_aggregation = "Hourly" - assert pendulum.parse("2020-01-01T15:00:00").timestamp() == test_report.get_report_record_timestamp("2020-01-01|15") +@pytest.mark.parametrize( + "stream_report_hourly_cls", + ( + AccountImpressionPerformanceReportHourly, + AccountPerformanceReportHourly, + AdGroupImpressionPerformanceReportHourly, + AdGroupPerformanceReportHourly, + AgeGenderAudienceReportHourly, + AdPerformanceReportHourly, + CampaignImpressionPerformanceReportHourly, + CampaignPerformanceReportHourly, + KeywordPerformanceReportHourly, + SearchQueryPerformanceReportHourly, + UserLocationPerformanceReportHourly, + GeographicPerformanceReportHourly, + ), +) +def test_get_report_record_timestamp_hourly(stream_report_hourly_cls): + stream_report = stream_report_hourly_cls(client=Mock(), config=TEST_CONFIG) + assert "2020-01-01T15:00:00+00:00" == stream_report.get_report_record_timestamp("2020-01-01|15") def test_report_get_start_date_wo_stream_state(): @@ -130,7 +212,7 @@ def test_report_get_start_date_with_stream_state(): test_report = TestReport() test_report.cursor_field = "cursor_field" test_report.client.reports_start_date = "2020-01-01" - stream_state = {"123": {"cursor_field": 1681766997}} + stream_state = {"123": {"cursor_field": "2023-04-17T21:29:57+00:00"}} account_id = "123" assert expected_start_date == test_report.get_start_date(stream_state, account_id) @@ -140,7 +222,7 @@ def test_report_get_start_date_performance_report_with_stream_state(): test_report = TestPerformanceReport() test_report.cursor_field = "cursor_field" test_report.config = {"lookback_window": 10} - stream_state = {"123": {"cursor_field": 1681766997}} + stream_state = {"123": {"cursor_field": "2023-04-17T21:29:57+00:00"}} account_id = "123" assert expected_start_date == test_report.get_start_date(stream_state, account_id) @@ -167,11 +249,181 @@ def test_report_get_start_date_performance_report_wo_stream_state(): ), ) def test_geographic_performance_report_pk(performance_report_cls): - config = { - "developer_token": "developer_token", - "client_id": "client_id", - "refresh_token": "refresh_token", - "reports_start_date": "2020-01-01T00:00:00Z", - } - stream = performance_report_cls(client=Mock(), config=config) + stream = performance_report_cls(client=Mock(), config=TEST_CONFIG) assert stream.primary_key is None + + +def test_report_parse_response_csv_error(caplog): + stream_report = AccountPerformanceReportHourly(client=Mock(), config=TEST_CONFIG) + fake_response = MagicMock() + fake_response.report_records.__iter__ = MagicMock(side_effect=_csv.Error) + list(stream_report.parse_response(fake_response)) + assert "CSV report file for stream `account_performance_report_hourly` is broken or cannot be read correctly: , skipping ..." in caplog.messages + + +@patch.object(source_bing_ads.source, "Client") +def test_custom_report_clear_namespace(mocked_client, config_with_custom_reports, logger_mock): + custom_report = SourceBingAds().get_custom_reports(config_with_custom_reports, mocked_client)[0] + assert custom_report._clear_namespace("tns:ReportAggregation") == "ReportAggregation" + + +@patch.object(source_bing_ads.source, "Client") +def test_custom_report_get_object_columns(mocked_client, config_with_custom_reports, logger_mock): + reporting_service_mock = MagicMock() + reporting_service_mock._get_service_info_dict.return_value = SERVICE_INFO_DICT_V13 + mocked_client.get_service.return_value = reporting_service_mock + mocked_client.environment = "production" + + custom_report = SourceBingAds().get_custom_reports(config_with_custom_reports, mocked_client)[0] + + tree = ET.parse(urlparse(SERVICE_INFO_DICT_V13[("reporting", mocked_client.environment)]).path) + request_object = tree.find(f".//{{*}}complexType[@name='{custom_report.report_name}Request']") + + assert custom_report._get_object_columns(request_object, tree) == [ + "TimePeriod", + "AccountId", + "AccountName", + "AccountNumber", + "AccountStatus", + "CampaignId", + "CampaignName", + "CampaignStatus", + "AdGroupId", + "AdGroupName", + "AdGroupStatus", + "AdDistribution", + "Language", + "Network", + "TopVsOther", + "DeviceType", + "DeviceOS", + "BidStrategyType", + "TrackingTemplate", + "CustomParameters", + "DynamicAdTargetId", + "DynamicAdTarget", + "DynamicAdTargetStatus", + "WebsiteCoverage", + "Impressions", + "Clicks", + "Ctr", + "AverageCpc", + "Spend", + "AveragePosition", + "Conversions", + "ConversionRate", + "CostPerConversion", + "Assists", + "Revenue", + "ReturnOnAdSpend", + "CostPerAssist", + "RevenuePerConversion", + "RevenuePerAssist", + "AllConversions", + "AllRevenue", + "AllConversionRate", + "AllCostPerConversion", + "AllReturnOnAdSpend", + "AllRevenuePerConversion", + "ViewThroughConversions", + "Goal", + "GoalType", + "AbsoluteTopImpressionRatePercent", + "TopImpressionRatePercent", + "AverageCpm", + "ConversionsQualified", + "AllConversionsQualified", + "ViewThroughConversionsQualified", + "AdId", + "ViewThroughRevenue", + ] + + +@patch.object(source_bing_ads.source, "Client") +def test_custom_report_send_request(mocked_client, config_with_custom_reports, logger_mock, caplog): + class Fault: + faultstring = "Invalid Client Data" + + custom_report = SourceBingAds().get_custom_reports(config_with_custom_reports, mocked_client)[0] + with patch.object(BingAdsReportingServiceStream, "send_request", side_effect=WebFault(fault=Fault(), document=None)): + custom_report.send_request(params={}, customer_id="13131313", account_id="800800808") + assert ( + "Could not sync custom report my test custom report: Please validate your column and aggregation configuration. " + "Error form server: [Invalid Client Data]" + ) in caplog.text + + +@pytest.mark.parametrize( + "aggregation, datastring, expected", + [ + ( + "Summary", + "11/13/2023", + "2023-11-13", + ), + ( + "Hourly", + "2022-11-13|10", + "2022-11-13T10:00:00+00:00", + ), + ( + "Daily", + "2022-11-13", + "2022-11-13", + ), + ( + "Weekly", + "2022-11-13", + "2022-11-13", + ), + ( + "Monthly", + "2022-11-13", + "2022-11-13", + ), + ( + "WeeklyStartingMonday", + "2022-11-13", + "2022-11-13", + ), + ], +) +@patch.object(source_bing_ads.source, "Client") +def test_custom_report_get_report_record_timestamp(mocked_client, config_with_custom_reports, aggregation, datastring, expected): + custom_report = SourceBingAds().get_custom_reports(config_with_custom_reports, mocked_client)[0] + custom_report.report_aggregation = aggregation + assert custom_report.get_report_record_timestamp(datastring) == expected + + +@patch.object(source_bing_ads.source, "Client") +def test_account_performance_report_monthly_stream_slices(mocked_client, config): + account_performance_report_monthly = AccountPerformanceReportMonthly(mocked_client, config) + accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) + with patch.object(Accounts, "read_records", return_value=accounts_read_records): + stream_slice = list(account_performance_report_monthly.stream_slices()) + assert stream_slice == [ + {'account_id': 180519267, 'customer_id': 100, 'time_period': 'LastYear'}, + {'account_id': 180519267, 'customer_id': 100, 'time_period': 'ThisYear'}, + {'account_id': 180278106, 'customer_id': 200, 'time_period': 'LastYear'}, + {'account_id': 180278106, 'customer_id': 200, 'time_period': 'ThisYear'} + ] + + +@pytest.mark.parametrize( + "aggregation", + [ + "DayOfWeek", + "HourOfDay", + ], +) +@patch.object(source_bing_ads.source, "Client") +def test_custom_performance_report_no_last_year_stream_slices(mocked_client, config_with_custom_reports, aggregation): + custom_report = SourceBingAds().get_custom_reports(config_with_custom_reports, mocked_client)[0] + custom_report.report_aggregation = aggregation + accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) + with patch.object(Accounts, "read_records", return_value=accounts_read_records): + stream_slice = list(custom_report.stream_slices()) + assert stream_slice == [ + {"account_id": 180519267, "customer_id": 100, "time_period": "ThisYear"}, + {"account_id": 180278106, "customer_id": 200, "time_period": "ThisYear"}, + ] diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py index 379eb5afacb1..7d89ff112b1a 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py @@ -2,76 +2,15 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import xml.etree.ElementTree as ET -from pathlib import Path from unittest.mock import MagicMock, patch -from urllib.parse import urlparse import pytest import source_bing_ads from airbyte_cdk.models import SyncMode from airbyte_cdk.utils import AirbyteTracedException from bingads.service_info import SERVICE_INFO_DICT_V13 -from source_bing_ads.reports import ReportsMixin +from source_bing_ads.base_streams import Accounts, AdGroups, Ads, Campaigns from source_bing_ads.source import SourceBingAds -from source_bing_ads.streams import AccountPerformanceReportMonthly, Accounts, AdGroups, Ads, AppInstallAds, Campaigns -from suds import WebFault - - -@pytest.fixture(name="config") -def config_fixture(): - """Generates streams settings from a config file""" - return { - "tenant_id": "common", - "developer_token": "fake_developer_token", - "refresh_token": "fake_refresh_token", - "client_id": "fake_client_id", - "reports_start_date": "2020-01-01", - "lookback_window": 0, - } - - -@pytest.fixture(name="config_with_custom_reports") -def config_with_custom_reports_fixture(): - """Generates streams settings with custom reports from a config file""" - return { - "tenant_id": "common", - "developer_token": "fake_developer_token", - "refresh_token": "fake_refresh_token", - "client_id": "fake_client_id", - "reports_start_date": "2020-01-01", - "lookback_window": 0, - "custom_reports": [ - { - "name": "my test custom report", - "reporting_object": "DSAAutoTargetPerformanceReport", - "report_columns": [ - "AbsoluteTopImpressionRatePercent", - "AccountId", - "AccountName", - "AccountNumber", - "AccountStatus", - "AdDistribution", - "AdGroupId", - "AdGroupName", - "AdGroupStatus", - "AdId", - "AllConversionRate", - "AllConversions", - "AllConversionsQualified", - "AllCostPerConversion", - "AllReturnOnAdSpend", - "AllRevenue", - ], - "report_aggregation": "Weekly", - } - ], - } - - -@pytest.fixture(name="logger_mock") -def logger_mock_fixture(): - return patch("source_bing_ads.source.AirbyteLogger") @patch.object(source_bing_ads.source, "Client") @@ -234,36 +173,6 @@ def test_ads_stream_slices(mocked_client, config): ] -@patch.object(source_bing_ads.source, "Client") -def test_AccountPerformanceReportMonthly_request_params(mocked_client, config): - - accountperformancereportmonthly = AccountPerformanceReportMonthly(mocked_client, config) - request_params = accountperformancereportmonthly.request_params(account_id=180278106, stream_slice={"time_period": "ThisYear"}) - del request_params["report_request"] - assert request_params == { - "overwrite_result_file": True, - # 'report_request': , - "result_file_directory": "/tmp", - "result_file_name": "AccountPerformanceReport", - "timeout_in_milliseconds": 300000, - } - - -@patch.object(source_bing_ads.source, "Client") -def test_AccountPerformanceReportMonthly_stream_slices(mocked_client, config): - - accountperformancereportmonthly = AccountPerformanceReportMonthly(mocked_client, config) - accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) - with patch.object(Accounts, "read_records", return_value=accounts_read_records): - stream_slice = list(accountperformancereportmonthly.stream_slices()) - assert stream_slice == [ - {"account_id": 180519267, "customer_id": 100, "time_period": "LastYear"}, - {"account_id": 180519267, "customer_id": 100, "time_period": "ThisYear"}, - {"account_id": 180278106, "customer_id": 200, "time_period": "LastYear"}, - {"account_id": 180278106, "customer_id": 200, "time_period": "ThisYear"}, - ] - - @pytest.mark.parametrize( ("stream", "stream_slice"), ( @@ -280,69 +189,6 @@ def test_streams_full_refresh(mocked_client, config, stream, stream_slice): mocked_client.request.assert_called_once() -@patch.object(source_bing_ads.source, "Client") -def test_bulk_stream_stream_slices(mocked_client, config): - slices = AppInstallAds(mocked_client, config).stream_slices() - assert list(slices) == [] - - app_install_ads = AppInstallAds(mocked_client, config) - accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) - with patch.object(Accounts, "read_records", return_value=accounts_read_records): - slices = app_install_ads.stream_slices() - assert list(slices) == [{"account_id": 180519267, "customer_id": 100}, {"account_id": 180278106, "customer_id": 200}] - - -@patch.object(source_bing_ads.source, "Client") -def test_bulk_stream_transfrom(mocked_client, config): - record = {"Ad Group": "Ad Group", "App Id": "App Id", "Campaign": "Campaign", "Custom Parameter": "Custom Parameter"} - transformed_record = AppInstallAds(mocked_client, config).transform( - record=record, stream_slice={"account_id": 180519267, "customer_id": 100} - ) - assert transformed_record == { - "Account Id": 180519267, - "Ad Group": "Ad Group", - "App Id": "App Id", - "Campaign": "Campaign", - "Custom Parameter": "Custom Parameter", - } - - -@patch.object(source_bing_ads.source, "Client") -def test_bulk_stream_read_with_chunks(mocked_client, config): - path_to_file = Path(__file__).parent / "app_install_ads.csv" - path_to_file_base = Path(__file__).parent / "app_install_ads_base.csv" - with open(path_to_file_base, "r") as f1, open(path_to_file, "a") as f2: - for line in f1: - f2.write(line) - - app_install_ads = AppInstallAds(mocked_client, config) - result = app_install_ads.read_with_chunks(path=path_to_file) - assert next(result) == { - "Ad Group": "AdGroupNameGoesHere", - "App Id": "AppStoreIdGoesHere", - "App Platform": "Android", - "Campaign": "ParentCampaignNameGoesHere", - "Client Id": "ClientIdGoesHere", - "Custom Parameter": "{_promoCode}=PROMO1; {_season}=summer", - "Destination Url": None, - "Device Preference": "All", - "Display Url": None, - "Final Url": "FinalUrlGoesHere", - "Final Url Suffix": None, - "Id": None, - "Mobile Final Url": None, - "Modified Time": None, - "Name": None, - "Parent Id": "-1111", - "Promotion": None, - "Status": "Active", - "Text": "Find New Customers & Increase Sales!", - "Title": "Contoso Quick Setup", - "Tracking Template": None, - "Type": "App Install Ad", - } - - @patch.object(source_bing_ads.source, "Client") def test_transform(mocked_client, config): record = {"AdFormatPreference": "All", "DevicePreference": 0, "EditorialStatus": "ActiveLimited", "FinalAppUrls": None} @@ -358,147 +204,3 @@ def test_transform(mocked_client, config): "EditorialStatus": "ActiveLimited", "FinalAppUrls": None, } - - -@patch.object(source_bing_ads.source, "Client") -def test_custom_report_clear_namespace(mocked_client, config_with_custom_reports, logger_mock): - custom_report = SourceBingAds().get_custom_reports(config_with_custom_reports, mocked_client)[0] - assert custom_report._clear_namespace("tns:ReportAggregation") == "ReportAggregation" - - -@patch.object(source_bing_ads.source, "Client") -def test_custom_report_get_object_columns(mocked_client, config_with_custom_reports, logger_mock): - reporting_service_mock = MagicMock() - reporting_service_mock._get_service_info_dict.return_value = SERVICE_INFO_DICT_V13 - mocked_client.get_service.return_value = reporting_service_mock - mocked_client.environment = "production" - - custom_report = SourceBingAds().get_custom_reports(config_with_custom_reports, mocked_client)[0] - - tree = ET.parse(urlparse(SERVICE_INFO_DICT_V13[("reporting", mocked_client.environment)]).path) - request_object = tree.find(f".//{{*}}complexType[@name='{custom_report.report_name}Request']") - - assert custom_report._get_object_columns(request_object, tree) == [ - "TimePeriod", - "AccountId", - "AccountName", - "AccountNumber", - "AccountStatus", - "CampaignId", - "CampaignName", - "CampaignStatus", - "AdGroupId", - "AdGroupName", - "AdGroupStatus", - "AdDistribution", - "Language", - "Network", - "TopVsOther", - "DeviceType", - "DeviceOS", - "BidStrategyType", - "TrackingTemplate", - "CustomParameters", - "DynamicAdTargetId", - "DynamicAdTarget", - "DynamicAdTargetStatus", - "WebsiteCoverage", - "Impressions", - "Clicks", - "Ctr", - "AverageCpc", - "Spend", - "AveragePosition", - "Conversions", - "ConversionRate", - "CostPerConversion", - "Assists", - "Revenue", - "ReturnOnAdSpend", - "CostPerAssist", - "RevenuePerConversion", - "RevenuePerAssist", - "AllConversions", - "AllRevenue", - "AllConversionRate", - "AllCostPerConversion", - "AllReturnOnAdSpend", - "AllRevenuePerConversion", - "ViewThroughConversions", - "Goal", - "GoalType", - "AbsoluteTopImpressionRatePercent", - "TopImpressionRatePercent", - "AverageCpm", - "ConversionsQualified", - "AllConversionsQualified", - "ViewThroughConversionsQualified", - "AdId", - "ViewThroughRevenue", - ] - - -@patch.object(source_bing_ads.source, "Client") -def test_custom_report_send_request(mocked_client, config_with_custom_reports, logger_mock, caplog): - class Fault: - faultstring = "Invalid Client Data" - - custom_report = SourceBingAds().get_custom_reports(config_with_custom_reports, mocked_client)[0] - with patch.object(ReportsMixin, "send_request", side_effect=WebFault(fault=Fault(), document=None)): - custom_report.send_request(params={}, customer_id="13131313", account_id="800800808") - assert ( - "Could not sync custom report my test custom report: Please validate your column and aggregation configuration. " - "Error form server: [Invalid Client Data]" - ) in caplog.text - - -@pytest.mark.parametrize( - "aggregation,datastring,expected", - ( - ( - "DayOfWeek", - "1", - 1, - ), - ( - "HourOfDay", - "20", - 20, - ), - ( - "Hourly", - "2022-11-13|10", - 1668333600, - ), - ( - "Hourly", - "2022-11-13|10", - 1668333600, - ), - ( - "Daily", - "2022-11-13", - 1668297600, - ), - ( - "Weekly", - "2022-11-13", - 1668297600, - ), - ( - "Monthly", - "2022-11-13", - 1668297600, - ), - ( - "WeeklyStartingMonday", - "2022-11-13", - 1668297600, - ), - ), -) -@patch.object(source_bing_ads.source, "Client") -def test_custom_report_get_report_record_timestamp(mocked_client, config_with_custom_reports, aggregation, datastring, expected): - custom_report = SourceBingAds().get_custom_reports(config_with_custom_reports, mocked_client)[0] - custom_report.report_aggregation = aggregation - assert custom_report.get_report_record_timestamp(datastring) == expected diff --git a/docs/integrations/sources/bing-ads-migrations.md b/docs/integrations/sources/bing-ads-migrations.md index 3d378d2517ad..c078d1d0cb56 100644 --- a/docs/integrations/sources/bing-ads-migrations.md +++ b/docs/integrations/sources/bing-ads-migrations.md @@ -1,5 +1,42 @@ # Bing Ads Migration Guide +## Upgrading to 2.0.0 + +This version update affects all hourly reports (end in report_hourly) and the following streams: + +- Accounts +- Campaigns +- Search Query Performance Report +- AppInstallAds +- AppInstallAdLabels +- Labels +- Campaign Labels +- Keyword Labels +- Ad Group Labels +- Keywords +- Budget Summary Report + +All `date` and `date-time` fields will be converted to standard `RFC3339`. Stream state format will be updated as well. + +For the changes to take effect, please refresh the source schema and reset affected streams after you have applied the upgrade. + +| Stream field | Current Airbyte Type | New Airbyte Type | +|-----------------------------|----------------------|-------------------| +| LinkedAgencies | string | object | +| BiddingScheme.MaxCpc.Amount | string | number | +| CostPerConversion | integer | number | +| Modified Time | string | timestamp with tz | +| Date | string | date | +| TimePeriod | string | timestamp with tz | + +Detailed date-time field change examples: + +| Affected streams | Field_name | Old type | New type (`RFC3339`) | +|----------------------------------------------------------------------------------------------------------------------|-----------------|---------------------------|---------------------------------| +| `AppInstallAds`, `AppInstallAdLabels`, `Labels`, `Campaign Labels`, `Keyword Labels`, `Ad Group Labels`, `Keywords` | `Modified Time` | `04/27/2023 18:00:14.970` | `2023-04-27T16:00:14.970+00:00` | +| `Budget Summary Report` | `Date` | `6/10/2021` | `2021-06-10` | +| `* Report Hourly` | `TimePeriod` | `2023-11-04\|11` | `2023-11-04T11:00:00+00:00` | + ## Upgrading to 1.0.0 This version update only affects the geographic performance reports streams. diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md index 1ba6d3570e56..cab52dc73a5b 100644 --- a/docs/integrations/sources/bing-ads.md +++ b/docs/integrations/sources/bing-ads.md @@ -206,54 +206,55 @@ The Bing Ads API limits the number of requests for all Microsoft Advertising cli ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------| -| 1.13.0 | 2023-11-13 | [32306](https://github.com/airbytehq/airbyte/pull/32306) | Add Custom reports and decrease backoff max tries number | -| 1.12.1 | 2023-11-10 | [32422](https://github.com/airbytehq/airbyte/pull/32422) | Normalize numeric values in reports | -| 1.12.0 | 2023-11-09 | [32340](https://github.com/airbytehq/airbyte/pull/32340) | Remove default start date in favor of Time Period - Last Year and This Year, if start date is not provided | -| 1.11.0 | 2023-11-06 | [32201](https://github.com/airbytehq/airbyte/pull/32201) | Skip broken CSV report files | -| 1.10.0 | 2023-11-06 | [32148](https://github.com/airbytehq/airbyte/pull/32148) | Add new fields to stream Ads: "BusinessName", "CallToAction", "Headline", "Images", "Videos", "Text" | -| 1.9.0 | 2023-11-03 | [32131](https://github.com/airbytehq/airbyte/pull/32131) | Add "CampaignId", "AccountId", "CustomerId" fields to Ad Groups, Ads and Campaigns streams. | -| 1.8.0 | 2023-11-02 | [32059](https://github.com/airbytehq/airbyte/pull/32059) | Add new streams `CampaignImpressionPerformanceReport` (daily, hourly, weekly, monthly) | -| 1.7.1 | 2023-11-02 | [32088](https://github.com/airbytehq/airbyte/pull/32088) | Raise config error when user does not have accounts | -| 1.7.0 | 2023-11-01 | [32027](https://github.com/airbytehq/airbyte/pull/32027) | Add new streams `AdGroupImpressionPerformanceReport` | -| 1.6.0 | 2023-10-31 | [32008](https://github.com/airbytehq/airbyte/pull/32008) | Add new streams `Keywords` | -| 1.5.0 | 2023-10-30 | [31952](https://github.com/airbytehq/airbyte/pull/31952) | Add new streams `Labels`, `App install ads`, `Keyword Labels`, `Campaign Labels`, `App Install Ad Labels`, `Ad Group Labels` | -| 1.4.0 | 2023-10-27 | [31885](https://github.com/airbytehq/airbyte/pull/31885) | Add new stream: `AccountImpressionPerformanceReport` (daily, hourly, weekly, monthly) | -| 1.3.0 | 2023-10-26 | [31837](https://github.com/airbytehq/airbyte/pull/31837) | Add new stream: `UserLocationPerformanceReport` (daily, hourly, weekly, monthly) | -| 1.2.0 | 2023-10-24 | [31783](https://github.com/airbytehq/airbyte/pull/31783) | Add new stream: `SearchQueryPerformanceReport` (daily, hourly, weekly, monthly) | -| 1.1.0 | 2023-10-24 | [31712](https://github.com/airbytehq/airbyte/pull/31712) | Add new stream: `AgeGenderAudienceReport` (daily, hourly, weekly, monthly) | -| 1.0.2 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | -| 1.0.1 | 2023-10-16 | [31432](https://github.com/airbytehq/airbyte/pull/31432) | Remove primary keys from the geographic performance reports - complete what was missed in version 1.0.0 | -| 1.0.0 | 2023-10-11 | [31277](https://github.com/airbytehq/airbyte/pull/31277) | Remove primary keys from the geographic performance reports. | -| 0.2.3 | 2023-09-28 | [30834](https://github.com/airbytehq/airbyte/pull/30834) | Wrap auth error with the config error. | -| 0.2.2 | 2023-09-27 | [30791](https://github.com/airbytehq/airbyte/pull/30791) | Fix missing fields for geographic performance reports. | -| 0.2.1 | 2023-09-04 | [30128](https://github.com/airbytehq/airbyte/pull/30128) | Add increasing download timeout if ReportingDownloadException occurs | -| 0.2.0 | 2023-08-17 | [27619](https://github.com/airbytehq/airbyte/pull/27619) | Add Geographic Performance Report | -| 0.1.24 | 2023-06-22 | [27619](https://github.com/airbytehq/airbyte/pull/27619) | Retry request after facing temporary name resolution error. | -| 0.1.23 | 2023-05-11 | [25996](https://github.com/airbytehq/airbyte/pull/25996) | Implement a retry logic if SSL certificate validation fails. | -| 0.1.22 | 2023-05-08 | [24223](https://github.com/airbytehq/airbyte/pull/24223) | Add CampaignLabels report column in campaign performance report | -| 0.1.21 | 2023-04-28 | [25668](https://github.com/airbytehq/airbyte/pull/25668) | Add undeclared fields to accounts, campaigns, campaign_performance_report, keyword_performance_report and account_performance_report streams | -| 0.1.20 | 2023-03-09 | [23663](https://github.com/airbytehq/airbyte/pull/23663) | Add lookback window for performance reports in incremental mode | -| 0.1.19 | 2023-03-08 | [23868](https://github.com/airbytehq/airbyte/pull/23868) | Add dimensional-type columns for reports. | -| 0.1.18 | 2023-01-30 | [22073](https://github.com/airbytehq/airbyte/pull/22073) | Fix null values in the `Keyword` column of `keyword_performance_report` streams | -| 0.1.17 | 2022-12-10 | [20005](https://github.com/airbytehq/airbyte/pull/20005) | Add `Keyword` to `keyword_performance_report` stream | -| 0.1.16 | 2022-10-12 | [17873](https://github.com/airbytehq/airbyte/pull/17873) | Fix: added missing campaign types in (Audience, Shopping and DynamicSearchAds) in campaigns stream | -| 0.1.15 | 2022-10-03 | [17505](https://github.com/airbytehq/airbyte/pull/17505) | Fix: limit cache size for ServiceClient instances | -| 0.1.14 | 2022-09-29 | [17403](https://github.com/airbytehq/airbyte/pull/17403) | Fix: limit cache size for ReportingServiceManager instances | -| 0.1.13 | 2022-09-29 | [17386](https://github.com/airbytehq/airbyte/pull/17386) | Migrate to per-stream states. | -| 0.1.12 | 2022-09-05 | [16335](https://github.com/airbytehq/airbyte/pull/16335) | Added backoff for socket.timeout | -| 0.1.11 | 2022-08-25 | [15684](https://github.com/airbytehq/airbyte/pull/15684) (published in [15987](https://github.com/airbytehq/airbyte/pull/15987)) | Fixed log messages being unreadable | -| 0.1.10 | 2022-08-12 | [15602](https://github.com/airbytehq/airbyte/pull/15602) | Fixed bug caused Hourly Reports to crash due to invalid fields set | -| 0.1.9 | 2022-08-02 | [14862](https://github.com/airbytehq/airbyte/pull/14862) | Added missing columns | -| 0.1.8 | 2022-06-15 | [13801](https://github.com/airbytehq/airbyte/pull/13801) | All reports `hourly/daily/weekly/monthly` will be generated by default, these options are removed from input configuration | -| 0.1.7 | 2022-05-17 | [12937](https://github.com/airbytehq/airbyte/pull/12937) | Added OAuth2.0 authentication method, removed `redirect_uri` from input configuration | -| 0.1.6 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | -| 0.1.5 | 2022-01-01 | [11652](https://github.com/airbytehq/airbyte/pull/11652) | Rebump attempt after DockerHub failure at registring the 0.1.4 | -| 0.1.4 | 2022-03-22 | [11311](https://github.com/airbytehq/airbyte/pull/11311) | Added optional Redirect URI & Tenant ID to spec | -| 0.1.3 | 2022-01-14 | [9510](https://github.com/airbytehq/airbyte/pull/9510) | Fixed broken dependency that blocked connector's operations | -| 0.1.2 | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update titles and descriptions | -| 0.1.1 | 2021-08-31 | [5750](https://github.com/airbytehq/airbyte/pull/5750) | Added reporting streams\) | -| 0.1.0 | 2021-07-22 | [4911](https://github.com/airbytehq/airbyte/pull/4911) | Initial release supported core streams \(Accounts, Campaigns, Ads, AdGroups\) | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.0.0 | 2023-11-07 | [31995](https://github.com/airbytehq/airbyte/pull/31995) | Schema update for Accounts, Campaigns and Search Query Performance Report streams. Convert `date` and `date-time` fields to standard `RFC3339` | +| 1.13.0 | 2023-11-13 | [32306](https://github.com/airbytehq/airbyte/pull/32306) | Add Custom reports and decrease backoff max tries number | +| 1.12.1 | 2023-11-10 | [32422](https://github.com/airbytehq/airbyte/pull/32422) | Normalize numeric values in reports | +| 1.12.0 | 2023-11-09 | [32340](https://github.com/airbytehq/airbyte/pull/32340) | Remove default start date in favor of Time Period - Last Year and This Year, if start date is not provided | +| 1.11.0 | 2023-11-06 | [32201](https://github.com/airbytehq/airbyte/pull/32201) | Skip broken CSV report files | +| 1.10.0 | 2023-11-06 | [32148](https://github.com/airbytehq/airbyte/pull/32148) | Add new fields to stream Ads: "BusinessName", "CallToAction", "Headline", "Images", "Videos", "Text" | +| 1.9.0 | 2023-11-03 | [32131](https://github.com/airbytehq/airbyte/pull/32131) | Add "CampaignId", "AccountId", "CustomerId" fields to Ad Groups, Ads and Campaigns streams. | +| 1.8.0 | 2023-11-02 | [32059](https://github.com/airbytehq/airbyte/pull/32059) | Add new streams `CampaignImpressionPerformanceReport` (daily, hourly, weekly, monthly) | +| 1.7.1 | 2023-11-02 | [32088](https://github.com/airbytehq/airbyte/pull/32088) | Raise config error when user does not have accounts | +| 1.7.0 | 2023-11-01 | [32027](https://github.com/airbytehq/airbyte/pull/32027) | Add new streams `AdGroupImpressionPerformanceReport` | +| 1.6.0 | 2023-10-31 | [32008](https://github.com/airbytehq/airbyte/pull/32008) | Add new streams `Keywords` | +| 1.5.0 | 2023-10-30 | [31952](https://github.com/airbytehq/airbyte/pull/31952) | Add new streams `Labels`, `App install ads`, `Keyword Labels`, `Campaign Labels`, `App Install Ad Labels`, `Ad Group Labels` | +| 1.4.0 | 2023-10-27 | [31885](https://github.com/airbytehq/airbyte/pull/31885) | Add new stream: `AccountImpressionPerformanceReport` (daily, hourly, weekly, monthly) | +| 1.3.0 | 2023-10-26 | [31837](https://github.com/airbytehq/airbyte/pull/31837) | Add new stream: `UserLocationPerformanceReport` (daily, hourly, weekly, monthly) | +| 1.2.0 | 2023-10-24 | [31783](https://github.com/airbytehq/airbyte/pull/31783) | Add new stream: `SearchQueryPerformanceReport` (daily, hourly, weekly, monthly) | +| 1.1.0 | 2023-10-24 | [31712](https://github.com/airbytehq/airbyte/pull/31712) | Add new stream: `AgeGenderAudienceReport` (daily, hourly, weekly, monthly) | +| 1.0.2 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 1.0.1 | 2023-10-16 | [31432](https://github.com/airbytehq/airbyte/pull/31432) | Remove primary keys from the geographic performance reports - complete what was missed in version 1.0.0 | +| 1.0.0 | 2023-10-11 | [31277](https://github.com/airbytehq/airbyte/pull/31277) | Remove primary keys from the geographic performance reports. | +| 0.2.3 | 2023-09-28 | [30834](https://github.com/airbytehq/airbyte/pull/30834) | Wrap auth error with the config error. | +| 0.2.2 | 2023-09-27 | [30791](https://github.com/airbytehq/airbyte/pull/30791) | Fix missing fields for geographic performance reports. | +| 0.2.1 | 2023-09-04 | [30128](https://github.com/airbytehq/airbyte/pull/30128) | Add increasing download timeout if ReportingDownloadException occurs | +| 0.2.0 | 2023-08-17 | [27619](https://github.com/airbytehq/airbyte/pull/27619) | Add Geographic Performance Report | +| 0.1.24 | 2023-06-22 | [27619](https://github.com/airbytehq/airbyte/pull/27619) | Retry request after facing temporary name resolution error. | +| 0.1.23 | 2023-05-11 | [25996](https://github.com/airbytehq/airbyte/pull/25996) | Implement a retry logic if SSL certificate validation fails. | +| 0.1.22 | 2023-05-08 | [24223](https://github.com/airbytehq/airbyte/pull/24223) | Add CampaignLabels report column in campaign performance report | +| 0.1.21 | 2023-04-28 | [25668](https://github.com/airbytehq/airbyte/pull/25668) | Add undeclared fields to accounts, campaigns, campaign_performance_report, keyword_performance_report and account_performance_report streams | +| 0.1.20 | 2023-03-09 | [23663](https://github.com/airbytehq/airbyte/pull/23663) | Add lookback window for performance reports in incremental mode | +| 0.1.19 | 2023-03-08 | [23868](https://github.com/airbytehq/airbyte/pull/23868) | Add dimensional-type columns for reports. | +| 0.1.18 | 2023-01-30 | [22073](https://github.com/airbytehq/airbyte/pull/22073) | Fix null values in the `Keyword` column of `keyword_performance_report` streams | +| 0.1.17 | 2022-12-10 | [20005](https://github.com/airbytehq/airbyte/pull/20005) | Add `Keyword` to `keyword_performance_report` stream | +| 0.1.16 | 2022-10-12 | [17873](https://github.com/airbytehq/airbyte/pull/17873) | Fix: added missing campaign types in (Audience, Shopping and DynamicSearchAds) in campaigns stream | +| 0.1.15 | 2022-10-03 | [17505](https://github.com/airbytehq/airbyte/pull/17505) | Fix: limit cache size for ServiceClient instances | +| 0.1.14 | 2022-09-29 | [17403](https://github.com/airbytehq/airbyte/pull/17403) | Fix: limit cache size for ReportingServiceManager instances | +| 0.1.13 | 2022-09-29 | [17386](https://github.com/airbytehq/airbyte/pull/17386) | Migrate to per-stream states. | +| 0.1.12 | 2022-09-05 | [16335](https://github.com/airbytehq/airbyte/pull/16335) | Added backoff for socket.timeout | +| 0.1.11 | 2022-08-25 | [15684](https://github.com/airbytehq/airbyte/pull/15684) (published in [15987](https://github.com/airbytehq/airbyte/pull/15987)) | Fixed log messages being unreadable | +| 0.1.10 | 2022-08-12 | [15602](https://github.com/airbytehq/airbyte/pull/15602) | Fixed bug caused Hourly Reports to crash due to invalid fields set | +| 0.1.9 | 2022-08-02 | [14862](https://github.com/airbytehq/airbyte/pull/14862) | Added missing columns | +| 0.1.8 | 2022-06-15 | [13801](https://github.com/airbytehq/airbyte/pull/13801) | All reports `hourly/daily/weekly/monthly` will be generated by default, these options are removed from input configuration | +| 0.1.7 | 2022-05-17 | [12937](https://github.com/airbytehq/airbyte/pull/12937) | Added OAuth2.0 authentication method, removed `redirect_uri` from input configuration | +| 0.1.6 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | +| 0.1.5 | 2022-01-01 | [11652](https://github.com/airbytehq/airbyte/pull/11652) | Rebump attempt after DockerHub failure at registring the 0.1.4 | +| 0.1.4 | 2022-03-22 | [11311](https://github.com/airbytehq/airbyte/pull/11311) | Added optional Redirect URI & Tenant ID to spec | +| 0.1.3 | 2022-01-14 | [9510](https://github.com/airbytehq/airbyte/pull/9510) | Fixed broken dependency that blocked connector's operations | +| 0.1.2 | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update titles and descriptions | +| 0.1.1 | 2021-08-31 | [5750](https://github.com/airbytehq/airbyte/pull/5750) | Added reporting streams\) | +| 0.1.0 | 2021-07-22 | [4911](https://github.com/airbytehq/airbyte/pull/4911) | Initial release supported core streams \(Accounts, Campaigns, Ads, AdGroups\) | \ No newline at end of file From 6253c3f365fcf83a38d649b2aa487c3563d6a482 Mon Sep 17 00:00:00 2001 From: Augustin Date: Thu, 16 Nov 2023 11:04:30 +0100 Subject: [PATCH 078/124] airbyte-ci: use new self hosted runners (#32537) --- .github/workflows/airbyte-ci-tests.yml | 2 +- .github/workflows/cat-tests.yml | 2 +- .github/workflows/connectors_nightly_build.yml | 8 ++++---- .github/workflows/connectors_tests.yml | 4 ++-- .github/workflows/connectors_weekly_build.yml | 8 ++++---- .github/workflows/format_check.yml | 2 +- .github/workflows/format_fix.yml | 2 +- .../metadata_service_deploy_orchestrator_dagger.yml | 2 +- .github/workflows/publish_connectors.yml | 4 ++-- .github/workflows/publish_pypi.yml | 4 ++-- 10 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/airbyte-ci-tests.yml b/.github/workflows/airbyte-ci-tests.yml index 6069ea917a70..f11f6947b469 100644 --- a/.github/workflows/airbyte-ci-tests.yml +++ b/.github/workflows/airbyte-ci-tests.yml @@ -14,7 +14,7 @@ on: jobs: run-airbyte-ci-tests: name: Run Airbyte CI tests - runs-on: "conn-prod-xlarge-runner" + runs-on: "ci-runner-connector-test-large-dagger-0-6-4" steps: - name: Checkout Airbyte uses: actions/checkout@v3 diff --git a/.github/workflows/cat-tests.yml b/.github/workflows/cat-tests.yml index 97de1c1fda89..e341407dd8f1 100644 --- a/.github/workflows/cat-tests.yml +++ b/.github/workflows/cat-tests.yml @@ -16,7 +16,7 @@ on: jobs: run-cat-unit-tests: name: Run CAT unit tests - runs-on: "conn-prod-xlarge-runner" + runs-on: "ci-runner-connector-test-large-dagger-0-6-4" steps: - name: Checkout Airbyte uses: actions/checkout@v3 diff --git a/.github/workflows/connectors_nightly_build.yml b/.github/workflows/connectors_nightly_build.yml index c7f7eb7dddf5..6b9d5d6ce5fa 100644 --- a/.github/workflows/connectors_nightly_build.yml +++ b/.github/workflows/connectors_nightly_build.yml @@ -8,19 +8,19 @@ on: inputs: runs-on: type: string - default: conn-nightly-xlarge-runner + default: ci-runner-connector-nightly-xlarge-dagger-0-6-4 required: true test-connectors-options: default: --concurrency=5 --support-level=certified required: true -run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}" +run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }}" jobs: test_connectors: - name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}" + name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }}" timeout-minutes: 720 # 12 hours - runs-on: ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }} + runs-on: ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }} steps: - name: Checkout Airbyte uses: actions/checkout@v3 diff --git a/.github/workflows/connectors_tests.yml b/.github/workflows/connectors_tests.yml index 610e4fc94ad1..c371735c35bb 100644 --- a/.github/workflows/connectors_tests.yml +++ b/.github/workflows/connectors_tests.yml @@ -19,7 +19,7 @@ on: default: "--modified" runner: description: "The runner to use for this job" - default: "conn-prod-xlarge-runner" + default: "ci-runner-connector-test-large-dagger-0-6-4" pull_request: types: - opened @@ -29,7 +29,7 @@ jobs: connectors_ci: name: Connectors CI timeout-minutes: 1440 # 24 hours - runs-on: ${{ inputs.runner || 'conn-prod-xlarge-runner'}} + runs-on: ${{ inputs.runner || 'ci-runner-connector-test-large-dagger-0-6-4'}} steps: - name: Checkout Airbyte uses: actions/checkout@v3 diff --git a/.github/workflows/connectors_weekly_build.yml b/.github/workflows/connectors_weekly_build.yml index ccf1f0b52199..aa96a832b9b8 100644 --- a/.github/workflows/connectors_weekly_build.yml +++ b/.github/workflows/connectors_weekly_build.yml @@ -8,19 +8,19 @@ on: inputs: runs-on: type: string - default: conn-nightly-xlarge-runner + default: ci-runner-connector-nightly-xlarge-dagger-0-6-4 required: true test-connectors-options: default: --concurrency=3 --support-level=community required: true -run-name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}" +run-name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }}" jobs: test_connectors: - name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}" + name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }}" timeout-minutes: 8640 # 6 days - runs-on: ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }} + runs-on: ${{ inputs.runs-on || 'ci-runner-connector-nightly-xlarge-dagger-0-6-4' }} steps: - name: Checkout Airbyte uses: actions/checkout@v3 diff --git a/.github/workflows/format_check.yml b/.github/workflows/format_check.yml index 290eb5662fd9..97530ad424e7 100644 --- a/.github/workflows/format_check.yml +++ b/.github/workflows/format_check.yml @@ -10,7 +10,7 @@ on: - master jobs: format-check: - runs-on: "conn-prod-xlarge-runner" + runs-on: "ci-runner-connector-format-medium-dagger-0-6-4" name: "Check for formatting errors on ${{ github.head_ref }}" timeout-minutes: 40 steps: diff --git a/.github/workflows/format_fix.yml b/.github/workflows/format_fix.yml index b1b2c31562aa..d689a8891063 100644 --- a/.github/workflows/format_fix.yml +++ b/.github/workflows/format_fix.yml @@ -10,7 +10,7 @@ on: pull_request: jobs: format-fix: - runs-on: "conn-prod-xlarge-runner" + runs-on: "ci-runner-connector-format-medium-dagger-0-6-4" name: "Apply All Formatting Rules" timeout-minutes: 40 steps: diff --git a/.github/workflows/metadata_service_deploy_orchestrator_dagger.yml b/.github/workflows/metadata_service_deploy_orchestrator_dagger.yml index 0da841726893..9550a5b1635c 100644 --- a/.github/workflows/metadata_service_deploy_orchestrator_dagger.yml +++ b/.github/workflows/metadata_service_deploy_orchestrator_dagger.yml @@ -10,7 +10,7 @@ on: jobs: connector_metadata_service_deploy_orchestrator: name: Connector metadata service deploy orchestrator - runs-on: medium-runner + runs-on: ci-runner-connector-test-large-dagger-0-6-4 steps: - name: Checkout Airbyte uses: actions/checkout@v2 diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml index 5fdc8dfcde60..ded1f9fb11ae 100644 --- a/.github/workflows/publish_connectors.yml +++ b/.github/workflows/publish_connectors.yml @@ -16,12 +16,12 @@ on: default: "--pre-release" runs-on: type: string - default: conn-prod-xlarge-runner + default: ci-runner-connector-publish-large-dagger-0-6-4 required: true jobs: publish_connectors: name: Publish connectors - runs-on: ${{ inputs.runs-on || 'conn-prod-xlarge-runner' }} + runs-on: ${{ inputs.runs-on || 'ci-runner-connector-publish-large-dagger-0-6-4' }} steps: - name: Checkout Airbyte uses: actions/checkout@v3 diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 477db6af6d6a..05deefcb7845 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -5,12 +5,12 @@ on: inputs: runs-on: type: string - default: conn-prod-xlarge-runner + default: ci-runner-connector-publish-large-dagger-0-6-4 required: true jobs: no-op: name: No-op - runs-on: ${{ inputs.runs-on || 'conn-prod-xlarge-runner' }} + runs-on: ${{ inputs.runs-on || 'ci-runner-connector-publish-large-dagger-0-6-4' }} steps: - run: echo 'hi!' From 316069e66d358cff18b8cc993f8a8c4a16cd4a13 Mon Sep 17 00:00:00 2001 From: Augustin Date: Thu, 16 Nov 2023 12:28:04 +0100 Subject: [PATCH 079/124] airbyte-ci bin release: trigger on all PRs (#32590) --- .github/workflows/airbyte-ci-release.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/airbyte-ci-release.yml b/.github/workflows/airbyte-ci-release.yml index 1d42e96e7d3b..aad8f7629559 100644 --- a/.github/workflows/airbyte-ci-release.yml +++ b/.github/workflows/airbyte-ci-release.yml @@ -6,8 +6,6 @@ concurrency: on: push: - branches: - - master paths: - "airbyte-ci/connectors/pipelines/**" workflow_dispatch: From fc1b08c65b7c43359ae6f89de6432cad6bea229b Mon Sep 17 00:00:00 2001 From: Artem Inzhyyants <36314070+artem1205@users.noreply.github.com> Date: Thu, 16 Nov 2023 14:35:24 +0100 Subject: [PATCH 080/124] =?UTF-8?q?=F0=9F=9A=A8=F0=9F=9A=A8=20Source=20Ama?= =?UTF-8?q?zon=20Seller=20Partner:=20remove=20brand=20analytics=20reports?= =?UTF-8?q?=20from=20cloud=20(#32355)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: artem1205 Co-authored-by: Anton Karpets --- .../acceptance-test-config.yml | 10 ----- ...ba_fulfillment_current_inventory_data.json | 28 -------------- ...ulfillment_inventory_adjustments_data.json | 31 ---------------- ...a_fulfillment_inventory_receipts_data.json | 27 -------------- ...ba_fulfillment_inventory_summary_data.json | 28 -------------- ...ba_fulfillment_monthly_inventory_data.json | 29 --------------- .../integration_tests/sample_state.json | 15 -------- .../metadata.yaml | 7 +++- ...BA_FULFILLMENT_CURRENT_INVENTORY_DATA.json | 16 -------- ...ULFILLMENT_INVENTORY_ADJUSTMENTS_DATA.json | 19 ---------- ...A_FULFILLMENT_INVENTORY_RECEIPTS_DATA.json | 15 -------- ...BA_FULFILLMENT_INVENTORY_SUMMARY_DATA.json | 16 -------- ...BA_FULFILLMENT_MONTHLY_INVENTORY_DATA.json | 17 --------- .../source_amazon_seller_partner/source.py | 37 +++++++++---------- .../source_amazon_seller_partner/streams.py | 20 ---------- .../amazon-seller-partner-migrations.md | 21 +++++++++++ .../sources/amazon-seller-partner.md | 19 +++++----- 17 files changed, 53 insertions(+), 302 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_current_inventory_data.json delete mode 100644 airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_inventory_adjustments_data.json delete mode 100644 airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_inventory_receipts_data.json delete mode 100644 airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_inventory_summary_data.json delete mode 100644 airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_monthly_inventory_data.json delete mode 100644 airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_CURRENT_INVENTORY_DATA.json delete mode 100644 airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_INVENTORY_ADJUSTMENTS_DATA.json delete mode 100644 airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_INVENTORY_RECEIPTS_DATA.json delete mode 100644 airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_INVENTORY_SUMMARY_DATA.json delete mode 100644 airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_MONTHLY_INVENTORY_DATA.json create mode 100644 docs/integrations/sources/amazon-seller-partner-migrations.md diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml index 1ea5e6f7eb52..47cd75672448 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml @@ -55,8 +55,6 @@ acceptance_tests: bypass_reason: "no records" - name: GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE bypass_reason: "no records" - - name: GET_FBA_FULFILLMENT_MONTHLY_INVENTORY_DATA - bypass_reason: "no records" - name: GET_VENDOR_SALES_REPORT bypass_reason: "no records" - name: GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT @@ -67,8 +65,6 @@ acceptance_tests: bypass_reason: "no records" - name: GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE bypass_reason: "no records" - - name: GET_FBA_FULFILLMENT_INVENTORY_SUMMARY_DATA - bypass_reason: "no records" - name: GET_BRAND_ANALYTICS_ITEM_COMPARISON_REPORT bypass_reason: "no records" - name: GET_AFN_INVENTORY_DATA @@ -83,8 +79,6 @@ acceptance_tests: bypass_reason: "no records" - name: GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT bypass_reason: "no records" - - name: GET_FBA_FULFILLMENT_INVENTORY_ADJUSTMENTS_DATA - bypass_reason: "no records" - name: GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT bypass_reason: "no records" - name: GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT @@ -97,8 +91,6 @@ acceptance_tests: bypass_reason: "no records" - name: GET_FBA_SNS_PERFORMANCE_DATA bypass_reason: "no records" - - name: GET_FBA_FULFILLMENT_CURRENT_INVENTORY_DATA - bypass_reason: "no records" - name: GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA bypass_reason: "no records" - name: GET_FBA_INVENTORY_PLANNING_DATA @@ -113,8 +105,6 @@ acceptance_tests: bypass_reason: "no records" - name: GET_STRANDED_INVENTORY_UI_DATA bypass_reason: "no records" - - name: GET_FBA_FULFILLMENT_INVENTORY_RECEIPTS_DATA - bypass_reason: "no records" - name: GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL bypass_reason: "no records" - name: ListFinancialEvents diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_current_inventory_data.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_current_inventory_data.json deleted file mode 100644 index 376c90214355..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_current_inventory_data.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "GET_FBA_FULFILLMENT_CURRENT_INVENTORY_DATA", - "json_schema": { - "title": "FBA Daily Inventory History Report", - "description": "", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "snapshot-date": { "type": ["null", "string"] }, - "fnsku": { "type": ["null", "string"] }, - "sku": { "type": ["null", "string"] }, - "product-name": { "type": ["null", "string"] }, - "quantity": { "type": ["null", "string"] }, - "fulfillment-center-id": { "type": ["null", "string"] }, - "detailed-disposition": { "type": ["null", "string"] }, - "country": { "type": ["null", "string"] } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_inventory_adjustments_data.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_inventory_adjustments_data.json deleted file mode 100644 index eb241d9e7e3e..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_inventory_adjustments_data.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "GET_FBA_FULFILLMENT_INVENTORY_ADJUSTMENTS_DATA", - "json_schema": { - "title": "FBA Inventory Adjustments Report", - "description": "", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "adjusted-date": { "type": ["null", "string"] }, - "transaction-item-id": { "type": ["null", "string"] }, - "fnsku": { "type": ["null", "string"] }, - "sku": { "type": ["null", "string"] }, - "product-name": { "type": ["null", "string"] }, - "fulfillment-center-id": { "type": ["null", "string"] }, - "quantity": { "type": ["null", "string"] }, - "reason": { "type": ["null", "string"] }, - "disposition": { "type": ["null", "string"] }, - "reconciled": { "type": ["null", "string"] }, - "unreconciled": { "type": ["null", "string"] } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_inventory_receipts_data.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_inventory_receipts_data.json deleted file mode 100644 index 92575cfb052e..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_inventory_receipts_data.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "GET_FBA_FULFILLMENT_INVENTORY_RECEIPTS_DATA", - "json_schema": { - "title": "FBA Received Inventory Report", - "description": "", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "received-date": { "type": ["null", "string"] }, - "fnsku": { "type": ["null", "string"] }, - "sku": { "type": ["null", "string"] }, - "product-name": { "type": ["null", "string"] }, - "quantity": { "type": ["null", "string"] }, - "fba-shipment-id": { "type": ["null", "string"] }, - "fulfillment-center-id": { "type": ["null", "string"] } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_inventory_summary_data.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_inventory_summary_data.json deleted file mode 100644 index b38e9b2849ef..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_inventory_summary_data.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "GET_FBA_FULFILLMENT_INVENTORY_SUMMARY_DATA", - "json_schema": { - "title": "FBA Inventory Event Detail Report", - "description": "", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "snapshot-date": { "type": ["null", "string"] }, - "transaction-type": { "type": ["null", "string"] }, - "fnsku": { "type": ["null", "string"] }, - "sku": { "type": ["null", "string"] }, - "product-name": { "type": ["null", "string"] }, - "fulfillment-center-id": { "type": ["null", "string"] }, - "quantity": { "type": ["null", "string"] }, - "disposition": { "type": ["null", "string"] } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_monthly_inventory_data.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_monthly_inventory_data.json deleted file mode 100644 index c695f887c910..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_monthly_inventory_data.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "GET_FBA_FULFILLMENT_MONTHLY_INVENTORY_DATA", - "json_schema": { - "title": "FBA Monthly Inventory History Report", - "description": "", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "month": { "type": ["null", "string"] }, - "fnsku": { "type": ["null", "string"] }, - "sku": { "type": ["null", "string"] }, - "product-name": { "type": ["null", "string"] }, - "average-quantity": { "type": ["null", "string"] }, - "end-quantity": { "type": ["null", "string"] }, - "fulfillment-center-id": { "type": ["null", "string"] }, - "detailed-disposition": { "type": ["null", "string"] }, - "country": { "type": ["null", "string"] } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/sample_state.json index bebd0fa00a49..a58a834e17d3 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/sample_state.json @@ -41,24 +41,9 @@ "GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA": { "createdTime": "2021-07-01T00:00:00Z" }, - "GET_FBA_FULFILLMENT_CURRENT_INVENTORY_DATA": { - "createdTime": "2021-07-01T00:00:00Z" - }, "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA": { "createdTime": "2021-07-01T00:00:00Z" }, - "GET_FBA_FULFILLMENT_INVENTORY_ADJUSTMENTS_DATA": { - "createdTime": "2021-07-01T00:00:00Z" - }, - "GET_FBA_FULFILLMENT_INVENTORY_RECEIPTS_DATA": { - "createdTime": "2021-07-01T00:00:00Z" - }, - "GET_FBA_FULFILLMENT_INVENTORY_SUMMARY_DATA": { - "createdTime": "2021-07-01T00:00:00Z" - }, - "GET_FBA_FULFILLMENT_MONTHLY_INVENTORY_DATA": { - "createdTime": "2021-07-01T00:00:00Z" - }, "GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA": { "createdTime": "2021-07-01T00:00:00Z" }, diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml index 83c764ab41f1..2bad20c6b106 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: api connectorType: source definitionId: e55879a8-0ef8-4557-abcf-ab34c53ec460 - dockerImageTag: 1.6.2 + dockerImageTag: 2.0.0 dockerRepository: airbyte/source-amazon-seller-partner documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-seller-partner githubIssueLabel: source-amazon-seller-partner @@ -20,6 +20,11 @@ data: oss: enabled: true releaseStage: alpha + releases: + breakingChanges: + 2.0.0: + message: "Deprecated FBA reports will be removed permanently from Cloud and Brand Analytics Reports will be removed temporarily. Updates on Brand Analytics Reports can be tracked here: [#32353](https://github.com/airbytehq/airbyte/issues/32353)" + upgradeDeadline: "2023-11-29" supportLevel: community tags: - language:python diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_CURRENT_INVENTORY_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_CURRENT_INVENTORY_DATA.json deleted file mode 100644 index 401cbf484380..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_CURRENT_INVENTORY_DATA.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "title": "FBA Daily Inventory History Report", - "description": "", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "snapshot-date": { "type": ["null", "string"] }, - "fnsku": { "type": ["null", "string"] }, - "sku": { "type": ["null", "string"] }, - "product-name": { "type": ["null", "string"] }, - "quantity": { "type": ["null", "string"] }, - "fulfillment-center-id": { "type": ["null", "string"] }, - "detailed-disposition": { "type": ["null", "string"] }, - "country": { "type": ["null", "string"] } - } -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_INVENTORY_ADJUSTMENTS_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_INVENTORY_ADJUSTMENTS_DATA.json deleted file mode 100644 index 916f932cc057..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_INVENTORY_ADJUSTMENTS_DATA.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "title": "FBA Inventory Adjustments Report", - "description": "", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "adjusted-date": { "type": ["null", "string"] }, - "transaction-item-id": { "type": ["null", "string"] }, - "fnsku": { "type": ["null", "string"] }, - "sku": { "type": ["null", "string"] }, - "product-name": { "type": ["null", "string"] }, - "fulfillment-center-id": { "type": ["null", "string"] }, - "quantity": { "type": ["null", "string"] }, - "reason": { "type": ["null", "string"] }, - "disposition": { "type": ["null", "string"] }, - "reconciled": { "type": ["null", "string"] }, - "unreconciled": { "type": ["null", "string"] } - } -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_INVENTORY_RECEIPTS_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_INVENTORY_RECEIPTS_DATA.json deleted file mode 100644 index 3d23369d51e1..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_INVENTORY_RECEIPTS_DATA.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "title": "FBA Received Inventory Report", - "description": "", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "received-date": { "type": ["null", "string"] }, - "fnsku": { "type": ["null", "string"] }, - "sku": { "type": ["null", "string"] }, - "product-name": { "type": ["null", "string"] }, - "quantity": { "type": ["null", "string"] }, - "fba-shipment-id": { "type": ["null", "string"] }, - "fulfillment-center-id": { "type": ["null", "string"] } - } -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_INVENTORY_SUMMARY_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_INVENTORY_SUMMARY_DATA.json deleted file mode 100644 index 1ddf4fceca59..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_INVENTORY_SUMMARY_DATA.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "title": "FBA Inventory Event Detail Report", - "description": "", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "snapshot-date": { "type": ["null", "string"] }, - "transaction-type": { "type": ["null", "string"] }, - "fnsku": { "type": ["null", "string"] }, - "sku": { "type": ["null", "string"] }, - "product-name": { "type": ["null", "string"] }, - "fulfillment-center-id": { "type": ["null", "string"] }, - "quantity": { "type": ["null", "string"] }, - "disposition": { "type": ["null", "string"] } - } -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_MONTHLY_INVENTORY_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_MONTHLY_INVENTORY_DATA.json deleted file mode 100644 index 796985c5210e..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_MONTHLY_INVENTORY_DATA.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "title": "FBA Monthly Inventory History Report", - "description": "", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "month": { "type": ["null", "string"] }, - "fnsku": { "type": ["null", "string"] }, - "sku": { "type": ["null", "string"] }, - "product-name": { "type": ["null", "string"] }, - "average-quantity": { "type": ["null", "string"] }, - "end-quantity": { "type": ["null", "string"] }, - "fulfillment-center-id": { "type": ["null", "string"] }, - "detailed-disposition": { "type": ["null", "string"] }, - "country": { "type": ["null", "string"] } - } -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py index 2ca326c5f47f..5a565c21ad71 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py @@ -1,7 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +from os import getenv from typing import Any, List, Mapping, Tuple from airbyte_cdk.logger import AirbyteLogger @@ -20,12 +20,7 @@ FbaAfnInventoryReports, FbaCustomerReturnsReports, FbaEstimatedFbaFeesTxtReport, - FbaFulfillmentCurrentInventoryReport, FbaFulfillmentCustomerShipmentPromotionReport, - FbaFulfillmentInventoryAdjustReport, - FbaFulfillmentInventoryReceiptsReport, - FbaFulfillmentInventorySummaryReport, - FbaFulfillmentMonthlyInventoryReport, FbaInventoryPlaningReport, FbaMyiUnsuppressedInventoryReport, FbaOrdersReports, @@ -127,7 +122,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: :param config: A Mapping of the user input configuration as defined in the connector spec. """ stream_kwargs = self._get_stream_kwargs(config) - return [ + streams = [ FbaCustomerReturnsReports(**stream_kwargs), FbaAfnInventoryReports(**stream_kwargs), FbaAfnInventoryByCountryReports(**stream_kwargs), @@ -144,28 +139,16 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: FulfilledShipmentsReports(**stream_kwargs), MerchantListingsReports(**stream_kwargs), VendorDirectFulfillmentShipping(**stream_kwargs), - VendorInventoryReports(**stream_kwargs), - VendorSalesReports(**stream_kwargs), Orders(**stream_kwargs), OrderItems(**stream_kwargs), OrderReportDataShipping(**stream_kwargs), - SellerAnalyticsSalesAndTrafficReports(**stream_kwargs), SellerFeedbackReports(**stream_kwargs), - BrandAnalyticsMarketBasketReports(**stream_kwargs), - BrandAnalyticsSearchTermsReports(**stream_kwargs), - BrandAnalyticsRepeatPurchaseReports(**stream_kwargs), - BrandAnalyticsAlternatePurchaseReports(**stream_kwargs), - BrandAnalyticsItemComparisonReports(**stream_kwargs), GetXmlBrowseTreeData(**stream_kwargs), ListFinancialEventGroups(**stream_kwargs), ListFinancialEvents(**stream_kwargs), LedgerDetailedViewReports(**stream_kwargs), FbaEstimatedFbaFeesTxtReport(**stream_kwargs), - FbaFulfillmentCurrentInventoryReport(**stream_kwargs), FbaFulfillmentCustomerShipmentPromotionReport(**stream_kwargs), - FbaFulfillmentInventoryAdjustReport(**stream_kwargs), - FbaFulfillmentInventoryReceiptsReport(**stream_kwargs), - FbaFulfillmentInventorySummaryReport(**stream_kwargs), FbaMyiUnsuppressedInventoryReport(**stream_kwargs), MerchantCancelledListingsReport(**stream_kwargs), MerchantListingsReport(**stream_kwargs), @@ -173,7 +156,6 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: MerchantListingsInactiveData(**stream_kwargs), StrandedInventoryUiReport(**stream_kwargs), XmlAllOrdersDataByOrderDataGeneral(**stream_kwargs), - FbaFulfillmentMonthlyInventoryReport(**stream_kwargs), MerchantListingsFypReport(**stream_kwargs), FbaSnsForecastReport(**stream_kwargs), FbaSnsPerformanceReport(**stream_kwargs), @@ -183,3 +165,18 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: LedgerSummaryViewReport(**stream_kwargs), FbaReimbursementsReports(**stream_kwargs), ] + # TODO: Remove after Brand Analytics will be enabled in CLOUD: + # https://github.com/airbytehq/airbyte/issues/32353 + if getenv("DEPLOYMENT_MODE", "").upper() != "CLOUD": + brand_analytics_reports = [ + BrandAnalyticsMarketBasketReports(**stream_kwargs), + BrandAnalyticsSearchTermsReports(**stream_kwargs), + BrandAnalyticsRepeatPurchaseReports(**stream_kwargs), + BrandAnalyticsAlternatePurchaseReports(**stream_kwargs), + BrandAnalyticsItemComparisonReports(**stream_kwargs), + SellerAnalyticsSalesAndTrafficReports(**stream_kwargs), + VendorSalesReports(**stream_kwargs), + VendorInventoryReports(**stream_kwargs), + ] + streams += brand_analytics_reports + return streams diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py index 38304231378d..401332714252 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py @@ -474,26 +474,10 @@ class FbaEstimatedFbaFeesTxtReport(ReportsAmazonSPStream): name = "GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA" -class FbaFulfillmentCurrentInventoryReport(ReportsAmazonSPStream): - name = "GET_FBA_FULFILLMENT_CURRENT_INVENTORY_DATA" - - class FbaFulfillmentCustomerShipmentPromotionReport(ReportsAmazonSPStream): name = "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA" -class FbaFulfillmentInventoryAdjustReport(ReportsAmazonSPStream): - name = "GET_FBA_FULFILLMENT_INVENTORY_ADJUSTMENTS_DATA" - - -class FbaFulfillmentInventoryReceiptsReport(ReportsAmazonSPStream): - name = "GET_FBA_FULFILLMENT_INVENTORY_RECEIPTS_DATA" - - -class FbaFulfillmentInventorySummaryReport(ReportsAmazonSPStream): - name = "GET_FBA_FULFILLMENT_INVENTORY_SUMMARY_DATA" - - class FbaMyiUnsuppressedInventoryReport(ReportsAmazonSPStream): name = "GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA" @@ -532,10 +516,6 @@ class MerchantCancelledListingsReport(ReportsAmazonSPStream): name = "GET_MERCHANT_CANCELLED_LISTINGS_DATA" -class FbaFulfillmentMonthlyInventoryReport(ReportsAmazonSPStream): - name = "GET_FBA_FULFILLMENT_MONTHLY_INVENTORY_DATA" - - class MerchantListingsFypReport(ReportsAmazonSPStream): name = "GET_MERCHANTS_LISTINGS_FYP_REPORT" diff --git a/docs/integrations/sources/amazon-seller-partner-migrations.md b/docs/integrations/sources/amazon-seller-partner-migrations.md new file mode 100644 index 000000000000..4f51ba68b60c --- /dev/null +++ b/docs/integrations/sources/amazon-seller-partner-migrations.md @@ -0,0 +1,21 @@ +# Amazon Seller Partner Migration Guide + +## Upgrading to 2.0.0 + +This change removes Brand Analytics and permanently removes deprecated FBA reports (from Airbyte Cloud). +Customers who have those streams must refresh their schema OR disable the following streams: +* GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT +* GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT +* GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT +* GET_BRAND_ANALYTICS_ALTERNATE_PURCHASE_REPORT +* GET_BRAND_ANALYTICS_ITEM_COMPARISON_REPORT +* GET_SALES_AND_TRAFFIC_REPORT +* GET_VENDOR_SALES_REPORT +* GET_VENDOR_INVENTORY_REPORT + +Customers, who have the following streams, will have to disable them: +* GET_FBA_FULFILLMENT_INVENTORY_ADJUSTMENTS_DATA +* GET_FBA_FULFILLMENT_CURRENT_INVENTORY_DATA +* GET_FBA_FULFILLMENT_INVENTORY_RECEIPTS_DATA +* GET_FBA_FULFILLMENT_INVENTORY_SUMMARY_DATA +* GET_FBA_FULFILLMENT_MONTHLY_INVENTORY_DATA diff --git a/docs/integrations/sources/amazon-seller-partner.md b/docs/integrations/sources/amazon-seller-partner.md index 3841fa3e416a..c5da26fc4d3d 100644 --- a/docs/integrations/sources/amazon-seller-partner.md +++ b/docs/integrations/sources/amazon-seller-partner.md @@ -70,21 +70,16 @@ This source is capable of syncing the following tables and their data: - [Orders](https://developer-docs.amazon.com/sp-api/docs/orders-api-v0-reference) \(incremental\) - [Orders Items](https://developer-docs.amazon.com/sp-api/docs/orders-api-v0-reference#getorderitems) \(incremental\) - [Seller Feedback Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) \(incremental\) -- [Brand Analytics Alternate Purchase Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports) -- [Brand Analytics Item Comparison Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports) -- [Brand Analytics Market Basket Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports) -- [Brand Analytics Repeat Purchase Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports) -- [Brand Analytics Search Terms Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports) +- [Brand Analytics Alternate Purchase Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports) \(only available in OSS\) +- [Brand Analytics Item Comparison Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports) \(only available in OSS\) +- [Brand Analytics Market Basket Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports) \(only available in OSS\) +- [Brand Analytics Repeat Purchase Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports) \(only available in OSS\) +- [Brand Analytics Search Terms Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports) \(only available in OSS\) - [Browse tree report](https://github.com/amzn/selling-partner-api-docs/blob/main/references/reports-api/reporttype-values.md#browse-tree-report) - [Financial Event Groups](https://developer-docs.amazon.com/sp-api/docs/finances-api-reference#get-financesv0financialeventgroups) - [Financial Events](https://developer-docs.amazon.com/sp-api/docs/finances-api-reference#get-financesv0financialevents) - [FBA Fee Preview Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [FBA Daily Inventory History Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) - [FBA Promotions Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [FBA Inventory Adjustments Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [FBA Received Inventory Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [FBA Inventory Event Detail Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [FBA Monthly Inventory History Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) - [FBA Manage Inventory](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) - [Subscribe and Save Forecast Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) - [Subscribe and Save Performance Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) @@ -102,6 +97,9 @@ This source is capable of syncing the following tables and their data: - [Inventory Ledger Report - Summary View](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) - [FBA Reimbursements Report](https://sellercentral.amazon.com/help/hub/reference/G200732720) - [Order Data Shipping Report](https://developer-docs.amazon.com/sp-api/docs/order-reports-attributes#get_order_report_data_shipping) +- [Sales and Traffic Business Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) \(only available in OSS\) +- [Vendor Sales Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) \(only available in OSS\) +- [Vendor Inventory Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) \(only available in OSS\) ## Report options @@ -126,6 +124,7 @@ So, for any value that exceeds the limit, the `period_in_days` will be automatic | Version | Date | Pull Request | Subject | |:---------|:-----------|:--------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `2.0.0` | 2023-11-23 | [\#32355](https://github.com/airbytehq/airbyte/pull/32355) | Remove Brand Analytics from Airbyte Cloud, permanently remove deprecated FBA reports | | `1.6.2` | 2023-11-14 | [\#32508](https://github.com/airbytehq/airbyte/pull/32508) | Do not use AWS signature as it is no longer required by the Amazon API | | `1.6.1` | 2023-11-13 | [\#32457](https://github.com/airbytehq/airbyte/pull/32457) | Fix report decompression | | `1.6.0` | 2023-11-09 | [\#32259](https://github.com/airbytehq/airbyte/pull/32259) | mark "aws_secret_key" and "aws_access_key" as required in specification; update schema for stream `Orders` | From a38eca14ec22dbad6e376851185d5e28815133c3 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Thu, 16 Nov 2023 17:02:31 +0200 Subject: [PATCH 081/124] Enable client-side rate limiting on source-stripe #31512 (#32284) Co-authored-by: Eugene Kulak Co-authored-by: keu --- .../connectors/source-stripe/metadata.yaml | 2 +- .../connectors/source-stripe/setup.py | 2 +- .../source-stripe/source_stripe/source.py | 59 +++++++++++++++++-- .../source-stripe/source_stripe/spec.yaml | 16 +++-- .../source-stripe/unit_tests/test_source.py | 58 +++++++++++++++++- docs/integrations/sources/stripe.md | 3 +- 6 files changed, 128 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/source-stripe/metadata.yaml b/airbyte-integrations/connectors/source-stripe/metadata.yaml index 0c34e7b7c27d..1aaaa9ee97c4 100644 --- a/airbyte-integrations/connectors/source-stripe/metadata.yaml +++ b/airbyte-integrations/connectors/source-stripe/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: e094cb9a-26de-4645-8761-65c0c425d1de - dockerImageTag: 4.5.3 + dockerImageTag: 4.5.4 dockerRepository: airbyte/source-stripe documentationUrl: https://docs.airbyte.com/integrations/sources/stripe githubIssueLabel: source-stripe diff --git a/airbyte-integrations/connectors/source-stripe/setup.py b/airbyte-integrations/connectors/source-stripe/setup.py index 8ce3d6936bdd..55bb256393b6 100644 --- a/airbyte-integrations/connectors/source-stripe/setup.py +++ b/airbyte-integrations/connectors/source-stripe/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk==0.52.8", "stripe==2.56.0", "pendulum==2.1.2"] +MAIN_REQUIREMENTS = ["airbyte-cdk==0.53.6", "stripe==2.56.0", "pendulum==2.1.2"] TEST_REQUIREMENTS = ["pytest-mock~=3.6.1", "pytest~=6.1", "requests-mock", "requests_mock~=1.8", "freezegun==1.2.2"] diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py index e72e4dd5398b..5683c875497e 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py @@ -2,7 +2,9 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging import os +from datetime import timedelta from typing import Any, List, Mapping, MutableMapping, Optional, Tuple import pendulum @@ -13,6 +15,7 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.message.repository import InMemoryMessageRepository from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.call_rate import AbstractAPIBudget, HttpAPIBudget, HttpRequestMatcher, MovingWindowCallRatePolicy, Rate from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade from airbyte_cdk.sources.streams.concurrent.cursor import NoopCursor from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator @@ -33,9 +36,12 @@ UpdatedCursorIncrementalStripeStream, ) -_MAX_CONCURRENCY = 3 +logger = logging.getLogger("airbyte") + +_MAX_CONCURRENCY = 20 _CACHE_DISABLED = os.environ.get("CACHE_DISABLED") USE_CACHE = not _CACHE_DISABLED +STRIPE_TEST_ACCOUNT_PREFIX = "sk_test_" class SourceStripe(AbstractSource): @@ -114,6 +120,52 @@ def customers(**args): **args, ) + @staticmethod + def is_test_account(config: Mapping[str, Any]) -> bool: + """Check if configuration uses Stripe test account (https://stripe.com/docs/keys#obtain-api-keys) + + :param config: + :return: True if configured to use a test account, False - otherwise + """ + + return str(config["client_secret"]).startswith(STRIPE_TEST_ACCOUNT_PREFIX) + + def get_api_call_budget(self, config: Mapping[str, Any]) -> AbstractAPIBudget: + """Get API call budget which connector is allowed to use. + + :param config: + :return: + """ + + max_call_rate = 25 if self.is_test_account(config) else 100 + if config.get("call_rate_limit"): + call_limit = config["call_rate_limit"] + if call_limit > max_call_rate: + logger.warning( + "call_rate_limit is larger than maximum allowed %s, fallback to default %s.", + max_call_rate, + max_call_rate, + ) + call_limit = max_call_rate + else: + call_limit = max_call_rate + + policies = [ + MovingWindowCallRatePolicy( + rates=[Rate(limit=20, interval=timedelta(seconds=1))], + matchers=[ + HttpRequestMatcher(url="https://api.stripe.com/v1/files"), + HttpRequestMatcher(url="https://api.stripe.com/v1/file_links"), + ], + ), + MovingWindowCallRatePolicy( + rates=[Rate(limit=call_limit, interval=timedelta(seconds=1))], + matchers=[], + ), + ] + + return HttpAPIBudget(policies=policies) + def streams(self, config: Mapping[str, Any]) -> List[Stream]: config = self.validate_and_fill_with_defaults(config) authenticator = TokenAuthenticator(config["client_secret"]) @@ -122,6 +174,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: "account_id": config["account_id"], "start_date": config["start_date"], "slice_range": config["slice_range"], + "api_budget": self.get_api_call_budget(config), } incremental_args = {**args, "lookback_window_days": config["lookback_window_days"]} subscriptions = IncrementalStripeStream( @@ -441,9 +494,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: ), ] - # We cap the number of workers to avoid hitting the Stripe rate limit - # The limit can be removed or increased once we have proper rate limiting - concurrency_level = min(config.get("num_workers", 2), _MAX_CONCURRENCY) + concurrency_level = min(config.get("num_workers", 10), _MAX_CONCURRENCY) streams[0].logger.info(f"Using concurrent cdk with concurrency level {concurrency_level}") return [ diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml b/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml index f65886c41298..5a31b610cd27 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml @@ -61,10 +61,18 @@ connectionSpecification: type: integer title: Number of concurrent workers minimum: 1 - maximum: 3 - default: 2 + maximum: 20 + default: 10 examples: [1, 2, 3] description: >- - The number of worker thread to use for the sync. The bigger the value is, the faster the sync will be. - Be careful as rate limiting is not implemented. + The number of worker thread to use for the sync. + The performance upper boundary depends on call_rate_limit setting and type of account. order: 5 + call_rate_limit: + type: integer + title: Max number of API calls per second + examples: [25, 100] + description: >- + The number of API calls per second that you allow connector to make. This value can not be bigger than real + API call rate limit (https://stripe.com/docs/rate-limits). If not specified the default maximum is 25 and 100 + calls per second for test and production tokens respectively. diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py index 61b226b5da83..476dbd38a689 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py @@ -1,7 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +import datetime import logging from contextlib import nullcontext as does_not_raise from unittest.mock import patch @@ -10,7 +10,9 @@ import source_stripe import stripe from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode +from airbyte_cdk.sources.streams.call_rate import CachedLimiterSession, LimiterSession, Rate from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade +from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.utils import AirbyteTracedException from source_stripe import SourceStripe @@ -92,3 +94,57 @@ def test_when_streams_return_full_refresh_as_concurrent(): ).streams(_a_valid_config()) assert len(list(filter(lambda stream: isinstance(stream, StreamFacade), streams))) == 1 + + +@pytest.mark.parametrize( + "input_config, default_call_limit", + ( + ({"account_id": 1, "client_secret": "secret"}, 100), + ({"account_id": 1, "client_secret": "secret", "call_rate_limit": 10}, 10), + ({"account_id": 1, "client_secret": "secret", "call_rate_limit": 110}, 100), + ({"account_id": 1, "client_secret": "sk_test_some_secret"}, 25), + ({"account_id": 1, "client_secret": "sk_test_some_secret", "call_rate_limit": 10}, 10), + ({"account_id": 1, "client_secret": "sk_test_some_secret", "call_rate_limit": 30}, 25), + ), +) +def test_call_budget_creation(mocker, input_config, default_call_limit): + """Test that call_budget was created with specific config i.e., that first policy has specific matchers.""" + + policy_mock = mocker.patch("source_stripe.source.MovingWindowCallRatePolicy") + matcher_mock = mocker.patch("source_stripe.source.HttpRequestMatcher") + source = SourceStripe(catalog=None) + + source.get_api_call_budget(input_config) + + policy_mock.assert_has_calls( + calls=[ + mocker.call(matchers=[mocker.ANY, mocker.ANY], rates=[Rate(limit=20, interval=datetime.timedelta(seconds=1))]), + mocker.call(matchers=[], rates=[Rate(limit=default_call_limit, interval=datetime.timedelta(seconds=1))]), + ], + ) + + matcher_mock.assert_has_calls( + calls=[ + mocker.call(url="https://api.stripe.com/v1/files"), + mocker.call(url="https://api.stripe.com/v1/file_links"), + ] + ) + + +def test_call_budget_passed_to_every_stream(mocker): + """Test that each stream has call_budget passed and creates a proper session""" + + prod_config = {"account_id": 1, "client_secret": "secret"} + source = SourceStripe(catalog=None) + get_api_call_budget_mock = mocker.patch.object(source, "get_api_call_budget") + + streams = source.streams(prod_config) + + assert streams + get_api_call_budget_mock.assert_called_once() + + for stream in streams: + assert isinstance(stream, HttpStream) + session = stream.request_session() + assert isinstance(session, (CachedLimiterSession, LimiterSession)) + assert session._api_budget == get_api_call_budget_mock.return_value diff --git a/docs/integrations/sources/stripe.md b/docs/integrations/sources/stripe.md index 06bea065cf1f..bde9f9d6d0e6 100644 --- a/docs/integrations/sources/stripe.md +++ b/docs/integrations/sources/stripe.md @@ -216,6 +216,7 @@ Each record is marked with `is_deleted` flag when the appropriate event happens | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| 4.5.4 | 2023-11-16 | [32284](https://github.com/airbytehq/airbyte/pull/32284/) | Enable client-side rate limiting | | 4.5.3 | 2023-11-14 | [32473](https://github.com/airbytehq/airbyte/pull/32473/) | Have all full_refresh stream syncs be concurrent | | 4.5.2 | 2023-11-03 | [32146](https://github.com/airbytehq/airbyte/pull/32146/) | Fix multiple BankAccount issues | | 4.5.1 | 2023-11-01 | [32056](https://github.com/airbytehq/airbyte/pull/32056/) | Use CDK version 0.52.8 | @@ -299,4 +300,4 @@ Each record is marked with `is_deleted` flag when the appropriate event happens | 0.1.9 | 2021-05-13 | [3367](https://github.com/airbytehq/airbyte/pull/3367) | Add acceptance tests for connected accounts | | 0.1.8 | 2021-05-11 | [3566](https://github.com/airbytehq/airbyte/pull/3368) | Bump CDK connectors | - \ No newline at end of file + From 509afc9858a9c6444ea9f4e21146de218d4cc821 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 16 Nov 2023 16:23:16 +0100 Subject: [PATCH 082/124] File CDK: Improve unstructured parser (#32554) Co-authored-by: flash1293 --- .../sources/file_based/file_types/unstructured_parser.py | 6 +++--- airbyte-cdk/python/setup.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/unstructured_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/unstructured_parser.py index 54d0ab9d7c73..b91be567f9b3 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/unstructured_parser.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/unstructured_parser.py @@ -115,9 +115,9 @@ def _read_file(self, file_handle: IOBase, remote_file: RemoteFile, format: Unstr if filetype == FileType.PDF: # for PDF, read the file into a BytesIO object because some code paths in pdf parsing are doing an instance check on the file object and don't work with file-like objects file_handle.seek(0) - file = BytesIO(file_handle.read()) - file_handle.seek(0) - elements = unstructured_partition_pdf(file=file) + with BytesIO(file_handle.read()) as file: + file_handle.seek(0) + elements = unstructured_partition_pdf(file=file) elif filetype == FileType.DOCX: elements = unstructured_partition_docx(file=file) elif filetype == FileType.PPTX: diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 9949399cfcca..f31184728351 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -23,8 +23,8 @@ tiktoken_dependency = "tiktoken==0.4.0" unstructured_dependencies = [ - "unstructured==0.10.19", - "unstructured[docx,pptx]==0.10.19", + "unstructured==0.10.27", # can't be bumped higher due to transitive dependencies we can't provide + "unstructured[docx,pptx]==0.10.27", "pdf2image==1.16.3", "pdfminer.six==20221105", "unstructured.pytesseract>=0.3.12", From aa111d2bead8a9bca8c12ae85fb26292b688c47b Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 16 Nov 2023 16:32:15 +0100 Subject: [PATCH 083/124] Vector DB CDK: Delete cdc records (#32496) --- .../vector_db_based/document_processor.py | 23 +- .../document_processor_test.py | 232 ++++++++++++++++-- 2 files changed, 223 insertions(+), 32 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/document_processor.py b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/document_processor.py index 7d7d174baee9..3ed3e3511dd1 100644 --- a/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/document_processor.py +++ b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/document_processor.py @@ -19,6 +19,8 @@ METADATA_STREAM_FIELD = "_ab_stream" METADATA_RECORD_ID_FIELD = "_ab_record_id" +CDC_DELETED_FIELD = "_ab_cdc_deleted_at" + @dataclass class Chunk: @@ -103,6 +105,8 @@ def process(self, record: AirbyteRecordMessage) -> Tuple[List[Chunk], Optional[s :param records: List of AirbyteRecordMessages :return: Tuple of (List of document chunks, record id to delete if a stream is in dedup mode to avoid stale documents in the vector store) """ + if CDC_DELETED_FIELD in record.data and record.data[CDC_DELETED_FIELD]: + return [], self._extract_primary_key(record) doc = self._generate_document(record) if doc is None: text_fields = ", ".join(self.text_fields) if self.text_fields else "all fields" @@ -139,22 +143,27 @@ def _extract_relevant_fields(self, record: AirbyteRecordMessage, fields: Optiona def _extract_metadata(self, record: AirbyteRecordMessage) -> Dict[str, Any]: metadata = self._extract_relevant_fields(record, self.metadata_fields) + metadata[METADATA_STREAM_FIELD] = create_stream_identifier(record) + primary_key = self._extract_primary_key(record) + if primary_key: + metadata[METADATA_RECORD_ID_FIELD] = primary_key + return metadata + + def _extract_primary_key(self, record: AirbyteRecordMessage) -> Optional[str]: stream_identifier = create_stream_identifier(record) current_stream: ConfiguredAirbyteStream = self.streams[stream_identifier] - metadata[METADATA_STREAM_FIELD] = stream_identifier # if the sync mode is deduping, use the primary key to upsert existing records instead of appending new ones - if current_stream.primary_key and current_stream.destination_sync_mode == DestinationSyncMode.append_dedup: - metadata[METADATA_RECORD_ID_FIELD] = f"{stream_identifier}_{self._extract_primary_key(record, current_stream)}" - return metadata + if not current_stream.primary_key or current_stream.destination_sync_mode != DestinationSyncMode.append_dedup: + return None - def _extract_primary_key(self, record: AirbyteRecordMessage, stream: ConfiguredAirbyteStream) -> str: primary_key = [] - for key in stream.primary_key: + for key in current_stream.primary_key: try: primary_key.append(str(dpath.util.get(record.data, key))) except KeyError: primary_key.append("__not_found__") - return "_".join(primary_key) + stringified_primary_key = "_".join(primary_key) + return f"{stream_identifier}_{stringified_primary_key}" def _split_document(self, doc: Document) -> List[Document]: chunks: List[Document] = self.splitter.split_documents([doc]) diff --git a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/document_processor_test.py b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/document_processor_test.py index 59f5f8c011bb..2660ee791512 100644 --- a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/document_processor_test.py +++ b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/document_processor_test.py @@ -23,13 +23,22 @@ def initialize_processor(config=ProcessingConfigModel(chunk_size=48, chunk_overl catalog = ConfiguredAirbyteCatalog( streams=[ ConfiguredAirbyteStream( - stream=AirbyteStream(name="stream1", json_schema={}, namespace="namespace1", supported_sync_modes=[SyncMode.full_refresh]), + stream=AirbyteStream( + name="stream1", + json_schema={}, + namespace="namespace1", + supported_sync_modes=[SyncMode.full_refresh], + ), sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.overwrite, primary_key=[["id"]], ), ConfiguredAirbyteStream( - stream=AirbyteStream(name="stream2", json_schema={}, supported_sync_modes=[SyncMode.full_refresh]), + stream=AirbyteStream( + name="stream2", + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh], + ), sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.overwrite, ), @@ -53,8 +62,14 @@ def initialize_processor(config=ProcessingConfigModel(chunk_size=48, chunk_overl ), (["id"], {"_ab_stream": "namespace1_stream1", "id": 1}), (["id", "non_existing"], {"_ab_stream": "namespace1_stream1", "id": 1}), - (["id", "complex.test"], {"_ab_stream": "namespace1_stream1", "id": 1, "complex.test": "abc"}), - (["id", "arr.*.test"], {"_ab_stream": "namespace1_stream1", "id": 1, "arr.*.test": ["abc", "def"]}), + ( + ["id", "complex.test"], + {"_ab_stream": "namespace1_stream1", "id": 1, "complex.test": "abc"}, + ), + ( + ["id", "arr.*.test"], + {"_ab_stream": "namespace1_stream1", "id": 1, "arr.*.test": ["abc", "def"]}, + ), ], ) def test_process_single_chunk_with_metadata(metadata_fields, expected_metadata): @@ -82,7 +97,7 @@ def test_process_single_chunk_with_metadata(metadata_fields, expected_metadata): assert id_to_delete is None -def test_process_single_chunk_limit4ed_metadata(): +def test_process_single_chunk_limited_metadata(): processor = initialize_processor() record = AirbyteRecordMessage( @@ -112,7 +127,11 @@ def test_process_single_chunk_without_namespace(): catalog = ConfiguredAirbyteCatalog( streams=[ ConfiguredAirbyteStream( - stream=AirbyteStream(name="stream1", json_schema={}, supported_sync_modes=[SyncMode.full_refresh]), + stream=AirbyteStream( + name="stream1", + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh], + ), sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.overwrite, ), @@ -155,7 +174,12 @@ def test_complex_text_fields(): emitted_at=1234, ) - processor.text_fields = ["nested.texts.*.text", "text", "other_nested.non_text", "non.*.existing"] + processor.text_fields = [ + "nested.texts.*.text", + "text", + "other_nested.non_text", + "non.*.existing", + ] processor.metadata_fields = ["non_text", "non_text_2", "id"] chunks, _ = processor.process(record) @@ -169,7 +193,12 @@ def test_complex_text_fields(): other_nested.non_text: \na: xyz b: abc""" ) - assert chunks[0].metadata == {"id": 1, "non_text": "a", "non_text_2": 1, "_ab_stream": "namespace1_stream1"} + assert chunks[0].metadata == { + "id": 1, + "non_text": "a", + "non_text_2": 1, + "_ab_stream": "namespace1_stream1", + } def test_no_text_fields(): @@ -228,7 +257,11 @@ def test_process_multiple_chunks_with_relevant_fields(): 10, 0, None, - ["text: By default, splits are done", "on multi newlines,", "then single newlines, then spaces"], + [ + "text: By default, splits are done", + "on multi newlines,", + "then single newlines, then spaces", + ], ), ( "Overlap splitting", @@ -346,7 +379,11 @@ def test_process_multiple_chunks_with_relevant_fields(): def test_text_splitters(label, text, chunk_size, chunk_overlap, splitter_config, expected_chunks): processor = initialize_processor( ProcessingConfigModel( - chunk_size=chunk_size, chunk_overlap=chunk_overlap, text_fields=["text"], metadata_fields=None, text_splitter=splitter_config + chunk_size=chunk_size, + chunk_overlap=chunk_overlap, + text_fields=["text"], + metadata_fields=None, + text_splitter=splitter_config, ) ) @@ -378,16 +415,42 @@ def test_text_splitters(label, text, chunk_size, chunk_overlap, splitter_config, @pytest.mark.parametrize( "label, split_config, has_error_message", [ - ("Invalid separator", SeparatorSplitterConfigModel(mode="separator", separators=['"xxx']), True), - ("Missing quotes", SeparatorSplitterConfigModel(mode="separator", separators=["xxx"]), True), - ("Non-string separator", SeparatorSplitterConfigModel(mode="separator", separators=["123"]), True), - ("Object separator", SeparatorSplitterConfigModel(mode="separator", separators=["{}"]), True), - ("Proper separator", SeparatorSplitterConfigModel(mode="separator", separators=['"xxx"', '"\\n\\n"']), False), + ( + "Invalid separator", + SeparatorSplitterConfigModel(mode="separator", separators=['"xxx']), + True, + ), + ( + "Missing quotes", + SeparatorSplitterConfigModel(mode="separator", separators=["xxx"]), + True, + ), + ( + "Non-string separator", + SeparatorSplitterConfigModel(mode="separator", separators=["123"]), + True, + ), + ( + "Object separator", + SeparatorSplitterConfigModel(mode="separator", separators=["{}"]), + True, + ), + ( + "Proper separator", + SeparatorSplitterConfigModel(mode="separator", separators=['"xxx"', '"\\n\\n"']), + False, + ), ], ) def test_text_splitter_check(label, split_config, has_error_message): error = DocumentProcessor.check_config( - ProcessingConfigModel(chunk_size=48, chunk_overlap=0, text_fields=None, metadata_fields=None, text_splitter=split_config) + ProcessingConfigModel( + chunk_size=48, + chunk_overlap=0, + text_fields=None, + metadata_fields=None, + text_splitter=split_config, + ) ) if has_error_message: assert error is not None @@ -400,12 +463,22 @@ def test_text_splitter_check(label, split_config, has_error_message): [ (None, {"abc": "def", "xyz": 123}, {"abc": "def", "xyz": 123}), ([], {"abc": "def", "xyz": 123}, {"abc": "def", "xyz": 123}), - ([FieldNameMappingConfigModel(from_field="abc", to_field="AAA")], {"abc": "def", "xyz": 123}, {"AAA": "def", "xyz": 123}), - ([FieldNameMappingConfigModel(from_field="non_existing", to_field="AAA")], {"abc": "def", "xyz": 123}, {"abc": "def", "xyz": 123}), + ( + [FieldNameMappingConfigModel(from_field="abc", to_field="AAA")], + {"abc": "def", "xyz": 123}, + {"AAA": "def", "xyz": 123}, + ), + ( + [FieldNameMappingConfigModel(from_field="non_existing", to_field="AAA")], + {"abc": "def", "xyz": 123}, + {"abc": "def", "xyz": 123}, + ), ], ) def test_rename_metadata_fields( - mappings: Optional[List[FieldNameMappingConfigModel]], fields: Mapping[str, Any], expected_chunk_metadata: Mapping[str, Any] + mappings: Optional[List[FieldNameMappingConfigModel]], + fields: Mapping[str, Any], + expected_chunk_metadata: Mapping[str, Any], ): processor = initialize_processor() @@ -422,21 +495,43 @@ def test_rename_metadata_fields( chunks, id_to_delete = processor.process(record) assert len(chunks) == 1 - assert chunks[0].metadata == {**expected_chunk_metadata, "_ab_stream": "namespace1_stream1", "text": "abc"} + assert chunks[0].metadata == { + **expected_chunk_metadata, + "_ab_stream": "namespace1_stream1", + "text": "abc", + } @pytest.mark.parametrize( "primary_key_value, stringified_primary_key, primary_key", [ ({"id": 99}, "namespace1_stream1_99", [["id"]]), - ({"id": 99, "name": "John Doe"}, "namespace1_stream1_99_John Doe", [["id"], ["name"]]), - ({"id": 99, "name": "John Doe", "age": 25}, "namespace1_stream1_99_John Doe_25", [["id"], ["name"], ["age"]]), - ({"nested": {"id": "abc"}, "name": "John Doe"}, "namespace1_stream1_abc_John Doe", [["nested", "id"], ["name"]]), - ({"nested": {"id": "abc"}}, "namespace1_stream1_abc___not_found__", [["nested", "id"], ["name"]]), + ( + {"id": 99, "name": "John Doe"}, + "namespace1_stream1_99_John Doe", + [["id"], ["name"]], + ), + ( + {"id": 99, "name": "John Doe", "age": 25}, + "namespace1_stream1_99_John Doe_25", + [["id"], ["name"], ["age"]], + ), + ( + {"nested": {"id": "abc"}, "name": "John Doe"}, + "namespace1_stream1_abc_John Doe", + [["nested", "id"], ["name"]], + ), + ( + {"nested": {"id": "abc"}}, + "namespace1_stream1_abc___not_found__", + [["nested", "id"], ["name"]], + ), ], ) def test_process_multiple_chunks_with_dedupe_mode( - primary_key_value: Mapping[str, Any], stringified_primary_key: str, primary_key: List[List[str]] + primary_key_value: Mapping[str, Any], + stringified_primary_key: str, + primary_key: List[List[str]], ): processor = initialize_processor() @@ -462,3 +557,90 @@ def test_process_multiple_chunks_with_dedupe_mode( for chunk in chunks: assert chunk.metadata["_ab_record_id"] == stringified_primary_key assert id_to_delete == stringified_primary_key + + +@pytest.mark.parametrize( + "record, sync_mode, has_chunks, raises, expected_id_to_delete", + [ + pytest.param( + AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={"text": "This is the text", "id": "1"}, + emitted_at=1234, + ), + DestinationSyncMode.append_dedup, + True, + False, + "namespace1_stream1_1", + id="update", + ), + pytest.param( + AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={"text": "This is the text", "id": "1"}, + emitted_at=1234, + ), + DestinationSyncMode.append, + True, + False, + None, + id="append", + ), + pytest.param( + AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={"text": "This is the text", "id": "1", "_ab_cdc_deleted_at": 1234}, + emitted_at=1234, + ), + DestinationSyncMode.append_dedup, + False, + False, + "namespace1_stream1_1", + id="cdc_delete", + ), + pytest.param( + AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={"id": "1", "_ab_cdc_deleted_at": 1234}, + emitted_at=1234, + ), + DestinationSyncMode.append_dedup, + False, + False, + "namespace1_stream1_1", + id="cdc_delete_without_text", + ), + pytest.param( + AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={"id": "1"}, + emitted_at=1234, + ), + DestinationSyncMode.append_dedup, + False, + True, + "namespace1_stream1_1", + id="update_without_text", + ), + ], +) +def test_process_cdc_records(record, sync_mode, has_chunks, raises, expected_id_to_delete): + processor = initialize_processor() + + processor.text_fields = ["text"] + + processor.streams["namespace1_stream1"].destination_sync_mode = sync_mode + + if raises: + with pytest.raises(AirbyteTracedException): + processor.process(record) + else: + chunks, id_to_delete = processor.process(record) + if has_chunks: + assert len(chunks) > 0 + assert id_to_delete == expected_id_to_delete From 261a6fd976e92fe01be130d4327bbbae55008047 Mon Sep 17 00:00:00 2001 From: flash1293 Date: Thu, 16 Nov 2023 15:43:52 +0000 Subject: [PATCH 084/124] =?UTF-8?q?=F0=9F=A4=96=20Bump=20patch=20version?= =?UTF-8?q?=20of=20Python=20CDK?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-cdk/python/.bumpversion.cfg | 2 +- airbyte-cdk/python/CHANGELOG.md | 3 +++ airbyte-cdk/python/Dockerfile | 4 ++-- airbyte-cdk/python/setup.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/airbyte-cdk/python/.bumpversion.cfg b/airbyte-cdk/python/.bumpversion.cfg index 08bf1ea465db..d93cc7ac0f15 100644 --- a/airbyte-cdk/python/.bumpversion.cfg +++ b/airbyte-cdk/python/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.53.7 +current_version = 0.53.8 commit = False [bumpversion:file:setup.py] diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 8778ed30577b..fc944faa64c8 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.53.8 +Vector DB CDK: Remove CDC records, File CDK: Update unstructured parser + ## 0.53.7 low-code: fix debug logging when using --debug flag diff --git a/airbyte-cdk/python/Dockerfile b/airbyte-cdk/python/Dockerfile index fdd41a9820ee..21618bcb99ff 100644 --- a/airbyte-cdk/python/Dockerfile +++ b/airbyte-cdk/python/Dockerfile @@ -10,7 +10,7 @@ RUN apk --no-cache upgrade \ && apk --no-cache add tzdata build-base # install airbyte-cdk -RUN pip install --prefix=/install airbyte-cdk==0.53.7 +RUN pip install --prefix=/install airbyte-cdk==0.53.8 # build a clean environment FROM base @@ -32,5 +32,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] # needs to be the same as CDK -LABEL io.airbyte.version=0.53.7 +LABEL io.airbyte.version=0.53.8 LABEL io.airbyte.name=airbyte/source-declarative-manifest diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index f31184728351..8c2f8921ae71 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -36,7 +36,7 @@ name="airbyte-cdk", # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be # updated if our semver format changes such as using release candidate versions. - version="0.53.7", + version="0.53.8", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", From 7dc53c27e6f3a1169c946a8eff2940e2d0604a40 Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Thu, 16 Nov 2023 07:45:48 -0800 Subject: [PATCH 085/124] enable parallelism on bigquery tests (#32580) --- .../connectors/destination-bigquery/gradle.properties | 1 + 1 file changed, 1 insertion(+) create mode 100644 airbyte-integrations/connectors/destination-bigquery/gradle.properties diff --git a/airbyte-integrations/connectors/destination-bigquery/gradle.properties b/airbyte-integrations/connectors/destination-bigquery/gradle.properties new file mode 100644 index 000000000000..4dbe8b8729df --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/gradle.properties @@ -0,0 +1 @@ +testExecutionConcurrency=-1 From f157e14346cfc3d8f8aa27703b4370d3f924647c Mon Sep 17 00:00:00 2001 From: Maxime Carbonneau-Leclerc Date: Thu, 16 Nov 2023 11:45:52 -0500 Subject: [PATCH 086/124] =?UTF-8?q?=F0=9F=90=9B=20Source=20Bing=20Ads:=20D?= =?UTF-8?q?isable=20upgrade=20to=202.0.0=20(#32604)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-integrations/connectors/source-bing-ads/metadata.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml index 63f2a336f97f..599a7e5c7206 100644 --- a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml @@ -25,9 +25,11 @@ data: name: Bing Ads registries: cloud: + dockerImageTag: 1.13.0 #https://airbytehq-team.slack.com/archives/C0662JB7XPU enabled: true oss: enabled: true + dockerImageTag: 1.13.0 #https://airbytehq-team.slack.com/archives/C0662JB7XPU releaseStage: generally_available releases: breakingChanges: From 136535d546ded551035b5251b7d3169fb21bb127 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 16 Nov 2023 17:48:23 +0100 Subject: [PATCH 087/124] Google Drive: Improve folder URL input (#32547) Co-authored-by: flash1293 --- .../integration_tests/spec.json | 2 ++ .../source-google-drive/metadata.yaml | 2 +- .../connectors/source-google-drive/setup.py | 2 +- .../source_google_drive/spec.py | 2 ++ .../source_google_drive/stream_reader.py | 18 ++---------- .../source_google_drive/utils.py | 21 ++++++++++++++ .../unit_tests/test_utils.py | 28 +++++++++++++++++++ docs/integrations/sources/google-drive.md | 1 + 8 files changed, 58 insertions(+), 18 deletions(-) create mode 100644 airbyte-integrations/connectors/source-google-drive/source_google_drive/utils.py create mode 100644 airbyte-integrations/connectors/source-google-drive/unit_tests/test_utils.py diff --git a/airbyte-integrations/connectors/source-google-drive/integration_tests/spec.json b/airbyte-integrations/connectors/source-google-drive/integration_tests/spec.json index 0b148260c015..709efd036a5b 100644 --- a/airbyte-integrations/connectors/source-google-drive/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-google-drive/integration_tests/spec.json @@ -304,6 +304,8 @@ "https://drive.google.com/drive/folders/1Xaz0vXXXX2enKnNYU5qSt9NS70gvMyYn" ], "order": 0, + "pattern": "^https://drive.google.com/.+", + "pattern_descriptor": "https://drive.google.com/drive/folders/MY-FOLDER-ID", "type": "string" }, "credentials": { diff --git a/airbyte-integrations/connectors/source-google-drive/metadata.yaml b/airbyte-integrations/connectors/source-google-drive/metadata.yaml index f7eb6998beff..e561f16c6a0d 100644 --- a/airbyte-integrations/connectors/source-google-drive/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-drive/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: file connectorType: source definitionId: 9f8dda77-1048-4368-815b-269bf54ee9b8 - dockerImageTag: 0.0.2 + dockerImageTag: 0.0.3 dockerRepository: airbyte/source-google-drive githubIssueLabel: source-google-drive icon: google-drive.svg diff --git a/airbyte-integrations/connectors/source-google-drive/setup.py b/airbyte-integrations/connectors/source-google-drive/setup.py index af0e32a9949e..ed7492559cd9 100644 --- a/airbyte-integrations/connectors/source-google-drive/setup.py +++ b/airbyte-integrations/connectors/source-google-drive/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk[file-based]>=0.53.5", + "airbyte-cdk[file-based]>=0.53.8", "google-api-python-client==2.104.0", "google-auth-httplib2==0.1.1", "google-auth-oauthlib==1.1.0", diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py index 7c2a60b27b82..00a360e0640b 100644 --- a/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py @@ -55,6 +55,8 @@ class Config: description="URL for the folder you want to sync. Using individual streams and glob patterns, it's possible to only sync a subset of all files located in the folder.", examples=["https://drive.google.com/drive/folders/1Xaz0vXXXX2enKnNYU5qSt9NS70gvMyYn"], order=0, + pattern="^https://drive.google.com/.+", + pattern_descriptor="https://drive.google.com/drive/folders/MY-FOLDER-ID", ) credentials: Union[OAuthCredentials, ServiceAccountCredentials] = Field( diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py index 2b6fa5f9cccc..dd786360f7a3 100644 --- a/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py @@ -16,6 +16,7 @@ from google.oauth2 import credentials, service_account from googleapiclient.discovery import build from googleapiclient.http import MediaIoBaseDownload +from source_google_drive.utils import get_folder_id from .spec import SourceGoogleDriveSpec @@ -86,7 +87,7 @@ def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: lo Get all files matching the specified glob patterns. """ service = self.google_drive_service - root_folder_id = self._get_folder_id(self.config.folder_url) + root_folder_id = get_folder_id(self.config.folder_url) # ignore prefix argument as it's legacy only and this is a new connector prefixes = self.get_prefixes_from_globs(globs) @@ -141,21 +142,6 @@ def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: lo if request is None: break - def _get_folder_id(self, url): - # Regular expression pattern to check the URL structure and extract the ID - pattern = r"^https://drive\.google\.com/drive/folders/([a-zA-Z0-9_-]+)$" - - # Find the pattern in the URL - match = re.search(pattern, url) - - if match: - # The matched group is the ID - drive_id = match.group(1) - return drive_id - else: - # If no match is found - raise ValueError(f"Could not extract folder ID from {url}") - def _is_exportable_document(self, mime_type: str): """ Returns true if the given file is a Google App document that can be exported. diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/utils.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/utils.py new file mode 100644 index 000000000000..c0994802358b --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/utils.py @@ -0,0 +1,21 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from urllib.parse import urlparse + + +def get_folder_id(url_string: str) -> str: + """ + Extract the folder ID from a Google Drive folder URL. + + Takes the last path segment of the URL, which is the folder ID (ignoring trailing slashes and query parameters). + """ + try: + parsed_url = urlparse(url_string) + if parsed_url.scheme != "https" or parsed_url.netloc != "drive.google.com": + raise ValueError("Folder URL has to be of the form https://drive.google.com/drive/folders/") + path_segments = list(filter(None, parsed_url.path.split("/"))) + if path_segments[-2] != "folders" or len(path_segments) < 3: + raise ValueError("Folder URL has to be of the form https://drive.google.com/drive/folders/") + return path_segments[-1] + except Exception: + raise ValueError("Folder URL is invalid") diff --git a/airbyte-integrations/connectors/source-google-drive/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-google-drive/unit_tests/test_utils.py new file mode 100644 index 000000000000..8dcb7e52e223 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/unit_tests/test_utils.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + + +import pytest +from source_google_drive.utils import get_folder_id + + +@pytest.mark.parametrize( + "input, output, raises", + [ + ("https://drive.google.com/drive/folders/1q2w3e4r5t6y7u8i9o0p", "1q2w3e4r5t6y7u8i9o0p", False), + ("https://drive.google.com/drive/folders/1q2w3e4r5t6y7u8i9o0p/", "1q2w3e4r5t6y7u8i9o0p", False), + ("https://drive.google.com/drive/folders/1q2w3e4r5t6y7u8i9o0p?usp=link_sharing", "1q2w3e4r5t6y7u8i9o0p", False), + ("https://drive.google.com/drive/u/0/folders/1q2w3e4r5t6y7u8i9o0p/", "1q2w3e4r5t6y7u8i9o0p", False), + ("https://drive.google.com/drive/u/0/folders/1q2w3e4r5t6y7u8i9o0p?usp=link_sharing", "1q2w3e4r5t6y7u8i9o0p", False), + ("https://drive.google.com/drive/u/0/folders/1q2w3e4r5t6y7u8i9o0p#abc", "1q2w3e4r5t6y7u8i9o0p", False), + ("https://docs.google.com/document/d/fsgfjdsh", None, True), + ("https://drive.google.com/drive/my-drive", None, True), + ("http://drive.google.com/drive/u/0/folders/1q2w3e4r5t6y7u8i9o0p/", None, True), + ("https://drive.google.com/", None, True), + ] +) +def test_get_folder_id(input, output, raises): + if raises: + with pytest.raises(ValueError): + get_folder_id(input) + else: + assert get_folder_id(input) == output \ No newline at end of file diff --git a/docs/integrations/sources/google-drive.md b/docs/integrations/sources/google-drive.md index cff121b4e696..df8aa03e2bba 100644 --- a/docs/integrations/sources/google-drive.md +++ b/docs/integrations/sources/google-drive.md @@ -247,6 +247,7 @@ Before parsing each document, the connector exports Google Document files to Doc | Version | Date | Pull Request | Subject | |---------|------------|----------------------------------------------------------|-----------------------------------------------------------------------------------| +| 0.0.3 | 2023-11-16 | [31458](https://github.com/airbytehq/airbyte/pull/31458) | Improve folder id input and update document file type parser | | 0.0.2 | 2023-11-02 | [31458](https://github.com/airbytehq/airbyte/pull/31458) | Allow syncs on shared drives | | 0.0.1 | 2023-11-02 | [31458](https://github.com/airbytehq/airbyte/pull/31458) | Initial Google Drive source | From f5cbe29622c5fe847353a48a276d25ec99095846 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Thu, 16 Nov 2023 19:15:23 +0200 Subject: [PATCH 088/124] :rotating_light: :rotating_light: Source Stripe: fix multiple issues regarding Refunds, CheckoutSessions and CheckoutSessionsLineItems + fix stream schemas (#32286) Co-authored-by: davydov-d --- .../source-stripe/acceptance-test-config.yml | 2 + .../integration_tests/abnormal_state.json | 4 +- .../integration_tests/configured_catalog.json | 8 +- .../integration_tests/expected_records.jsonl | 16 +- .../connectors/source-stripe/metadata.yaml | 7 +- .../schemas/checkout_sessions_line_items.json | 2 + .../source_stripe/schemas/invoices.json | 5 +- .../schemas/subscription_schedule.json | 6 +- .../source_stripe/schemas/subscriptions.json | 5 +- .../source-stripe/source_stripe/source.py | 72 ++- .../source-stripe/source_stripe/spec.yaml | 4 +- .../source-stripe/source_stripe/streams.py | 238 +++---- .../source-stripe/unit_tests/test_streams.py | 582 +++++++++++++++--- .../integrations/sources/stripe-migrations.md | 15 + docs/integrations/sources/stripe.md | 197 +++--- 15 files changed, 788 insertions(+), 375 deletions(-) diff --git a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml index 0dda354b4266..dfdddbb6ca31 100644 --- a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml @@ -13,6 +13,8 @@ acceptance_tests: discovery: tests: - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: 4.4.2 basic_read: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json index e34da831b7be..97d865ec3c49 100644 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json @@ -163,14 +163,14 @@ { "type": "STREAM", "stream": { - "stream_state": { "expires_at": 10000000000 }, + "stream_state": { "updated": 10000000000 }, "stream_descriptor": { "name": "checkout_sessions" } } }, { "type": "STREAM", "stream": { - "stream_state": { "checkout_session_expires_at": 10000000000 }, + "stream_state": { "checkout_session_updated": 10000000000 }, "stream_descriptor": { "name": "checkout_sessions_line_items" } } }, diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-stripe/integration_tests/configured_catalog.json index fc3ccb073b53..281642987467 100644 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/configured_catalog.json @@ -143,11 +143,11 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["checkout_session_expires_at"], + "default_cursor_field": ["checkout_session_updated"], "source_defined_primary_key": [["id"]] }, "primary_key": [["id"]], - "cursor_field": ["checkout_session_expires_at"], + "cursor_field": ["checkout_session_updated"], "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, @@ -459,11 +459,11 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["updated"], + "default_cursor_field": ["created"], "source_defined_primary_key": [["id"]] }, "primary_key": [["id"]], - "cursor_field": ["updated"], + "cursor_field": ["created"], "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl index a3732127059f..a84e541f0ecf 100644 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl @@ -1,12 +1,12 @@ -{"stream": "checkout_sessions_line_items", "data": {"id": "li_1O2XZ1EcXtiJtvvh26q22omU", "object": "item", "amount_discount": 0, "amount_subtotal": 3400, "amount_tax": 0, "amount_total": 3400, "currency": "usd", "description": "Test Product 1", "discounts": [], "price": {"id": "price_1MX364EcXtiJtvvh6jKcimNL", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1675345504, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NHcKselSHfKdfc", "recurring": null, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 1700, "unit_amount_decimal": "1700"}, "quantity": 2, "taxes": [], "checkout_session_id": "cs_test_a1uSLwxkrTLjGhRXgzJweMwh09uvSZcWIkGLcIqDXzYADowSPwkAmJUrAN", "checkout_session_expires_at": 1697713523}, "emitted_at": 1697627220862} +{"stream": "checkout_sessions_line_items", "data": {"checkout_session_id": "cs_test_a1uSLwxkrTLjGhRXgzJweMwh09uvSZcWIkGLcIqDXzYADowSPwkAmJUrAN", "checkout_session_expires_at": 1697713523, "checkout_session_created": 1697627124, "checkout_session_updated": 1697627124, "id": "li_1O2XZ1EcXtiJtvvh26q22omU", "object": "item", "amount_discount": 0, "amount_subtotal": 3400, "amount_tax": 0, "amount_total": 3400, "currency": "usd", "description": "Test Product 1", "discounts": [], "price": {"id": "price_1MX364EcXtiJtvvh6jKcimNL", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1675345504, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NHcKselSHfKdfc", "recurring": null, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 1700, "unit_amount_decimal": "1700"}, "quantity": 2, "taxes": []}, "emitted_at": 1699376426293} {"stream": "customer_balance_transactions", "data": {"id": "cbtxn_1MX2zPEcXtiJtvvhr4L2D3Q1", "object": "customer_balance_transaction", "amount": -50000.0, "created": 1675345091, "credit_note": null, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "description": null, "ending_balance": 0.0, "invoice": "in_1MX2yFEcXtiJtvvhMXhUCgKx", "livemode": false, "metadata": {}, "type": "applied_to_invoice"}, "emitted_at": 1697627222916} {"stream": "customer_balance_transactions", "data": {"id": "cbtxn_1MWIPLEcXtiJtvvhLnQYjVCj", "object": "customer_balance_transaction", "amount": 50000.0, "created": 1675166031, "credit_note": null, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "description": "Test credit balance", "ending_balance": 50000.0, "invoice": null, "livemode": false, "metadata": {}, "type": "adjustment"}, "emitted_at": 1697627222918} {"stream": "setup_attempts", "data": {"id": "setatt_1KnfIjEcXtiJtvvhqDfSlpM4", "object": "setup_attempt", "application": null, "created": 1649752937, "customer": null, "flow_directions": null, "livemode": false, "on_behalf_of": null, "payment_method": "pm_1KnfIj2eZvKYlo2CAlv2Vhqc", "payment_method_details": {"acss_debit": {}, "type": "acss_debit"}, "setup_error": null, "setup_intent": "seti_1KnfIjEcXtiJtvvhPw5znVKY", "status": "succeeded", "usage": "off_session"}, "emitted_at": 1697627241471} {"stream": "setup_attempts", "data": {"id": "setatt_1KnfIdEcXtiJtvvhpDrYVlRP", "object": "setup_attempt", "application": null, "created": 1649752931, "customer": null, "flow_directions": null, "livemode": false, "on_behalf_of": null, "payment_method": "pm_1KnfIc2eZvKYlo2Civ7snSPy", "payment_method_details": {"acss_debit": {}, "type": "acss_debit"}, "setup_error": null, "setup_intent": "seti_1KnfIcEcXtiJtvvh61qlCaDf", "status": "succeeded", "usage": "off_session"}, "emitted_at": 1697627242509} {"stream": "setup_attempts", "data": {"id": "setatt_1KnfIVEcXtiJtvvhqouWGuhD", "object": "setup_attempt", "application": null, "created": 1649752923, "customer": null, "flow_directions": null, "livemode": false, "on_behalf_of": null, "payment_method": "pm_1KnfIV2eZvKYlo2CaOLGBF00", "payment_method_details": {"acss_debit": {}, "type": "acss_debit"}, "setup_error": null, "setup_intent": "seti_1KnfIVEcXtiJtvvhWiIbMkpH", "status": "succeeded", "usage": "off_session"}, "emitted_at": 1697627243547} -{"stream": "accounts", "data": {"id": "acct_1NGp6SD04fX0Aizk", "object": "account", "capabilities": {"acss_debit_payments": "active", "affirm_payments": "active", "afterpay_clearpay_payments": "active", "bancontact_payments": "active", "card_payments": "active", "cartes_bancaires_payments": "pending", "cashapp_payments": "active", "eps_payments": "active", "giropay_payments": "active", "ideal_payments": "active", "klarna_payments": "active", "link_payments": "active", "p24_payments": "active", "sepa_debit_payments": "active", "sofort_payments": "active", "transfers": "active", "us_bank_account_ach_payments": "active"}, "charges_enabled": true, "country": "US", "default_currency": "usd", "details_submitted": true, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "payouts_enabled": true, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "settings": {"bacs_debit_payments": {}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"statement_descriptor_prefix": "AIRBYTE", "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": "Airbyte", "timezone": "Asia/Tbilisi"}, "payments": {"statement_descriptor": "WWW.AIRBYTE.COM", "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "sepa_debit_payments": {}}, "type": "standard"}, "emitted_at": 1697627267880} -{"stream": "accounts", "data": {"id": "acct_1MwD6tIyVv44cUB4", "object": "account", "business_profile": {"mcc": null, "name": null, "product_description": null, "support_address": null, "support_email": null, "support_phone": null, "support_url": null, "url": null}, "business_type": null, "capabilities": {"card_payments": "inactive", "transfers": "inactive"}, "charges_enabled": false, "country": "US", "created": 1681342196, "default_currency": "usd", "details_submitted": false, "email": "jenny.rosen@example.com", "external_accounts": {"object": "list", "data": [], "has_more": false, "total_count": 0, "url": "/v1/accounts/acct_1MwD6tIyVv44cUB4/external_accounts"}, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "metadata": {}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "pending_verification": []}, "settings": {"bacs_debit_payments": {}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"decline_on": {"avs_failure": false, "cvc_failure": false}, "statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "Etc/UTC"}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "payouts": {"debit_negative_balances": false, "schedule": {"delay_days": 2, "interval": "daily"}, "statement_descriptor": null}, "sepa_debit_payments": {}}, "tos_acceptance": {"date": null, "ip": null, "user_agent": null}, "type": "custom"}, "emitted_at": 1697627267882} -{"stream": "accounts", "data": {"id": "acct_1Jx8unEYmRTj5on1", "object": "account", "business_profile": {"mcc": null, "name": "Airbyte", "support_address": null, "support_email": null, "support_phone": null, "support_url": null, "url": null}, "capabilities": {}, "charges_enabled": false, "controller": {"type": "account"}, "country": "US", "default_currency": "usd", "details_submitted": false, "email": null, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "metadata": {}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": [], "pending_verification": []}, "settings": {"bacs_debit_payments": {}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "Etc/UTC"}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "sepa_debit_payments": {}}, "type": "standard"}, "emitted_at": 1697627267884} +{"stream": "accounts", "data": {"id": "acct_1NGp6SD04fX0Aizk", "object": "account", "capabilities": {"acss_debit_payments": "active", "affirm_payments": "active", "afterpay_clearpay_payments": "active", "bancontact_payments": "active", "card_payments": "active", "cartes_bancaires_payments": "pending", "cashapp_payments": "active", "eps_payments": "active", "giropay_payments": "active", "ideal_payments": "active", "klarna_payments": "active", "link_payments": "active", "p24_payments": "active", "sepa_debit_payments": "active", "sofort_payments": "active", "transfers": "active", "us_bank_account_ach_payments": "active"}, "charges_enabled": true, "country": "US", "default_currency": "usd", "details_submitted": true, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "payouts_enabled": true, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "settings": {"bacs_debit_payments": {"display_name": null, "service_user_number": null}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"statement_descriptor_prefix": "AIRBYTE", "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": "Airbyte", "timezone": "Asia/Tbilisi"}, "payments": {"statement_descriptor": "WWW.AIRBYTE.COM", "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "sepa_debit_payments": {}}, "type": "standard"}, "emitted_at": 1697627267880} +{"stream": "accounts", "data": {"id": "acct_1MwD6tIyVv44cUB4", "object": "account", "business_profile": {"mcc": null, "name": null, "product_description": null, "support_address": null, "support_email": null, "support_phone": null, "support_url": null, "url": null}, "business_type": null, "capabilities": {"card_payments": "inactive", "transfers": "inactive"}, "charges_enabled": false, "country": "US", "created": 1681342196, "default_currency": "usd", "details_submitted": false, "email": "jenny.rosen@example.com", "external_accounts": {"object": "list", "data": [], "has_more": false, "total_count": 0, "url": "/v1/accounts/acct_1MwD6tIyVv44cUB4/external_accounts"}, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "metadata": {}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "pending_verification": []}, "settings": {"bacs_debit_payments": {"display_name": null, "service_user_number": null}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"decline_on": {"avs_failure": false, "cvc_failure": false}, "statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "Etc/UTC"}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "payouts": {"debit_negative_balances": false, "schedule": {"delay_days": 2, "interval": "daily"}, "statement_descriptor": null}, "sepa_debit_payments": {}}, "tos_acceptance": {"date": null, "ip": null, "user_agent": null}, "type": "custom"}, "emitted_at": 1697627267882} +{"stream": "accounts", "data": {"id": "acct_1Jx8unEYmRTj5on1", "object": "account", "business_profile": {"mcc": null, "name": "Airbyte", "support_address": null, "support_email": null, "support_phone": null, "support_url": null, "url": null}, "capabilities": {}, "charges_enabled": false, "controller": {"type": "account"}, "country": "US", "default_currency": "usd", "details_submitted": false, "email": null, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "metadata": {}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": [], "pending_verification": []}, "settings": {"bacs_debit_payments": {"display_name": null, "service_user_number": null}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "Etc/UTC"}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "sepa_debit_payments": {}}, "type": "standard"}, "emitted_at": 1697627267884} {"stream": "shipping_rates", "data": {"id": "shr_1NXgplEcXtiJtvvhA1ntV782", "object": "shipping_rate", "active": true, "created": 1690274589, "delivery_estimate": "{'maximum': {'unit': 'business_day', 'value': 14}, 'minimum': {'unit': 'business_day', 'value': 10}}", "display_name": "Test Ground Shipping", "fixed_amount": {"amount": 999, "currency": "usd"}, "livemode": false, "metadata": {}, "tax_behavior": "inclusive", "tax_code": "txcd_92010001", "type": "fixed_amount"}, "emitted_at": 1697627269309} {"stream": "balance_transactions", "data": {"id": "txn_1KVQhfEcXtiJtvvhF7ox3YEm", "object": "balance_transaction", "amount": -9164, "available_on": 1645488000, "created": 1645406919, "currency": "usd", "description": "STRIPE PAYOUT", "exchange_rate": null, "fee": 0, "fee_details": [], "net": -9164, "reporting_category": "payout", "source": "po_1KVQhfEcXtiJtvvhZlUkl08U", "status": "available", "type": "payout"}, "emitted_at": 1697627270253} {"stream": "balance_transactions", "data": {"id": "txn_3K9FSOEcXtiJtvvh0KoS5mx7", "object": "balance_transaction", "amount": 5300, "available_on": 1640649600, "created": 1640120473, "currency": "usd", "description": null, "exchange_rate": null, "fee": 184, "fee_details": [{"amount": 184, "application": null, "currency": "usd", "description": "Stripe processing fees", "type": "stripe_fee"}], "net": 5116, "reporting_category": "charge", "source": "ch_3K9FSOEcXtiJtvvh0zxb7clc", "status": "available", "type": "charge"}, "emitted_at": 1697627270254} @@ -17,7 +17,7 @@ {"stream": "file_links", "data": {"id": "link_1KnfIiEcXtiJtvvhCNceSyei", "object": "file_link", "created": 1649752936, "expired": false, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfY1FvanBFTmt0dUdrRWJXTHBpUlVYVUtu007305bsv3"}, "emitted_at": 1697627273833} {"stream": "file_links", "data": {"id": "link_1KnfIbEcXtiJtvvhyBLUqkSt", "object": "file_link", "created": 1649752929, "expired": false, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfaXh1blBqMmY0MzI3SHZWbUZIeFVGU3Nl0022JjupYq"}, "emitted_at": 1697627273834} {"stream": "file_links", "data": {"id": "link_1KnfIUEcXtiJtvvh0ktKHfWz", "object": "file_link", "created": 1649752922, "expired": false, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfNzhlbE9MUGNYbkJzMkRLSWdEcnhvY3FH00DK5jBVaH"}, "emitted_at": 1697627273835} -{"stream": "checkout_sessions", "data": {"id": "cs_test_a1uSLwxkrTLjGhRXgzJweMwh09uvSZcWIkGLcIqDXzYADowSPwkAmJUrAN", "object": "checkout.session", "after_expiration": null, "allow_promotion_codes": null, "amount_subtotal": 3400, "amount_total": 3400, "automatic_tax": {"enabled": false, "status": null}, "billing_address_collection": null, "cancel_url": null, "client_reference_id": null, "client_secret": null, "consent": null, "consent_collection": null, "created": 1697627124, "currency": "usd", "currency_conversion": null, "custom_fields": [], "custom_text": {"shipping_address": null, "submit": null, "terms_of_service_acceptance": null}, "customer": null, "customer_creation": "always", "customer_details": null, "customer_email": null, "expires_at": 1697713523, "invoice": null, "invoice_creation": {"enabled": false, "invoice_data": {"account_tax_ids": null, "custom_fields": null, "description": null, "footer": null, "metadata": {}, "rendering_options": null}}, "livemode": false, "locale": null, "metadata": {}, "mode": "payment", "payment_intent": "pi_3O2XZ1EcXtiJtvvh0zWGn33E", "payment_link": null, "payment_method_collection": "always", "payment_method_configuration_details": {"id": "pmc_1MC0oMEcXtiJtvvhmhbSUwTJ", "parent": null}, "payment_method_options": {"us_bank_account": {"financial_connections": {"permissions": ["payment_method"], "prefetch": []}, "verification_method": "automatic"}, "wechat_pay": {"app_id": null, "client": "web"}}, "payment_method_types": ["card", "alipay", "klarna", "link", "us_bank_account", "wechat_pay", "cashapp"], "payment_status": "unpaid", "phone_number_collection": {"enabled": false}, "recovered_from": null, "setup_intent": null, "shipping_address_collection": null, "shipping_cost": null, "shipping_details": null, "shipping_options": [], "status": "expired", "submit_type": null, "subscription": null, "success_url": "https://example.com/success", "total_details": {"amount_discount": 0, "amount_shipping": 0, "amount_tax": 0}, "ui_mode": "hosted", "url": null, "updated": 1697713523}, "emitted_at": 1697627275062} +{"stream": "checkout_sessions", "data": {"id": "cs_test_a1uSLwxkrTLjGhRXgzJweMwh09uvSZcWIkGLcIqDXzYADowSPwkAmJUrAN", "object": "checkout.session", "after_expiration": null, "allow_promotion_codes": null, "amount_subtotal": 3400, "amount_total": 3400, "automatic_tax": {"enabled": false, "status": null}, "billing_address_collection": null, "cancel_url": null, "client_reference_id": null, "client_secret": null, "consent": null, "consent_collection": null, "created": 1697627124, "currency": "usd", "currency_conversion": null, "custom_fields": [], "custom_text": {"shipping_address": null, "submit": null, "terms_of_service_acceptance": null}, "customer": null, "customer_creation": "always", "customer_details": null, "customer_email": null, "expires_at": 1697713523, "invoice": null, "invoice_creation": {"enabled": false, "invoice_data": {"account_tax_ids": null, "custom_fields": null, "description": null, "footer": null, "metadata": {}, "rendering_options": null}}, "livemode": false, "locale": null, "metadata": {}, "mode": "payment", "payment_intent": "pi_3O2XZ1EcXtiJtvvh0zWGn33E", "payment_link": null, "payment_method_collection": "always", "payment_method_configuration_details": {"id": "pmc_1MC0oMEcXtiJtvvhmhbSUwTJ", "parent": null}, "payment_method_options": {"us_bank_account": {"financial_connections": {"permissions": ["payment_method"], "prefetch": []}, "verification_method": "automatic"}, "wechat_pay": {"app_id": null, "client": "web"}}, "payment_method_types": ["card", "alipay", "klarna", "link", "us_bank_account", "wechat_pay", "cashapp"], "payment_status": "unpaid", "phone_number_collection": {"enabled": false}, "recovered_from": null, "setup_intent": null, "shipping_address_collection": null, "shipping_cost": null, "shipping_details": null, "shipping_options": [], "status": "expired", "submit_type": null, "subscription": null, "success_url": "https://example.com/success", "total_details": {"amount_discount": 0, "amount_shipping": 0, "amount_tax": 0}, "ui_mode": "hosted", "url": null, "updated": 1697627124}, "emitted_at": 1697627275062} {"stream": "credit_notes", "data": {"id": "cn_1NGPwmEcXtiJtvvhNXwHpgJF", "object": "credit_note", "amount": 8400, "amount_shipping": 0, "created": 1686158100, "currency": "usd", "customer": "cus_Kou8knsO3qQOwU", "customer_balance_transaction": null, "discount_amount": "0", "discount_amounts": [], "effective_at": 1686158100, "invoice": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "lines": {"object": "list", "data": [{"id": "cnli_1NGPwmEcXtiJtvvhcL7yEIBJ", "object": "credit_note_line_item", "amount": 8400, "amount_excluding_tax": 8400, "description": "a box of parsnips", "discount_amount": 0, "discount_amounts": [], "invoice_line_item": "il_1K9GKLEcXtiJtvvhhHaYMebN", "livemode": false, "quantity": 1, "tax_amounts": [], "tax_rates": [], "type": "invoice_line_item", "unit_amount": 8400, "unit_amount_decimal": 8400.0, "unit_amount_excluding_tax": 8400.0}], "has_more": false, "url": "/v1/credit_notes/cn_1NGPwmEcXtiJtvvhNXwHpgJF/lines"}, "livemode": false, "memo": null, "metadata": {}, "number": "CA35DF83-0001-CN-01", "out_of_band_amount": null, "pdf": "https://pay.stripe.com/credit_notes/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9PMlV3dFlJelh4NHM1R0VIWnhMR3RjWUtlejFlRWtILDg4MTY4MDc20200Sa50llWu/pdf?s=ap", "reason": null, "refund": null, "shipping_cost": null, "status": "issued", "subtotal": 8400, "subtotal_excluding_tax": 8400, "tax_amounts": [], "total": 8400, "total_excluding_tax": 8400, "type": "pre_payment", "voided_at": null, "updated": 1686158100}, "emitted_at": 1697627276386} {"stream": "customers", "data": {"id": "cus_LIiHR6omh14Xdg", "object": "customer", "address": {"city": "san francisco", "country": "US", "line1": "san francisco", "line2": "", "postal_code": "", "state": "CA"}, "balance": 0, "created": 1646998902, "currency": "usd", "default_source": "card_1MSHU1EcXtiJtvvhytSN6V54", "delinquent": false, "description": "test", "discount": null, "email": "test@airbyte_integration_test.com", "invoice_prefix": "09A6A98F", "invoice_settings": {"custom_fields": null, "default_payment_method": null, "footer": null, "rendering_options": null}, "livemode": false, "metadata": {}, "name": "Test", "next_invoice_sequence": 1, "phone": null, "preferred_locales": [], "shipping": {"address": {"city": "", "country": "US", "line1": "", "line2": "", "postal_code": "", "state": ""}, "name": "", "phone": ""}, "tax_exempt": "none", "test_clock": null, "updated": 1646998902}, "emitted_at": 1697627278433} {"stream": "customers", "data": {"id": "cus_Kou8knsO3qQOwU", "object": "customer", "address": null, "balance": 0, "created": 1640123795, "currency": "usd", "default_source": "src_1MSID8EcXtiJtvvhxIT9lXRy", "delinquent": false, "description": null, "discount": null, "email": "edward.gao+stripe-test-customer-1@airbyte.io", "invoice_prefix": "CA35DF83", "invoice_settings": {"custom_fields": null, "default_payment_method": null, "footer": null, "rendering_options": null}, "livemode": false, "metadata": {}, "name": "edgao-test-customer-1", "next_invoice_sequence": 2, "phone": null, "preferred_locales": [], "shipping": null, "tax_exempt": "none", "test_clock": null, "updated": 1640123795}, "emitted_at": 1697627278435} @@ -50,9 +50,9 @@ {"stream": "subscription_schedule", "data": {"id": "sub_sched_1O2Dg0EcXtiJtvvh7GtbtIhP", "object": "subscription_schedule", "application": null, "canceled_at": null, "completed_at": null, "created": 1697550676, "current_phase": {"end_date": 1705499476, "start_date": 1697550676}, "customer": "cus_NGoTFiJFVbSsvZ", "default_settings": {"application_fee_percent": null, "automatic_tax": {"enabled": false}, "billing_cycle_anchor": "automatic", "billing_thresholds": null, "collection_method": "charge_automatically", "default_payment_method": null, "default_source": null, "description": "Test Test", "invoice_settings": "{'days_until_due': None}", "on_behalf_of": null, "transfer_data": null}, "end_behavior": "cancel", "livemode": false, "metadata": {}, "phases": [{"add_invoice_items": [], "application_fee_percent": null, "automatic_tax": {"enabled": true}, "billing_cycle_anchor": null, "billing_thresholds": null, "collection_method": "charge_automatically", "coupon": null, "currency": "usd", "default_payment_method": null, "default_tax_rates": [], "description": "Test Test", "end_date": 1705499476, "invoice_settings": "{'days_until_due': None}", "items": [{"billing_thresholds": null, "metadata": {}, "plan": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "price": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "quantity": 1, "tax_rates": []}], "metadata": {}, "on_behalf_of": null, "proration_behavior": "create_prorations", "start_date": 1697550676, "transfer_data": null, "trial_end": null}], "released_at": null, "released_subscription": null, "renewal_interval": null, "status": "active", "subscription": "sub_1O2Dg0EcXtiJtvvhz7Q4zS0n", "test_clock": null, "updated": 1697550676}, "emitted_at": 1697627312079} {"stream": "transfers", "data": {"id": "tr_1NH18zEcXtiJtvvhnd827cNO", "object": "transfer", "amount": 10000, "amount_reversed": 0, "balance_transaction": "txn_1NH190EcXtiJtvvhBO3PeR7p", "created": 1686301085, "currency": "usd", "description": null, "destination": "acct_1Jx8unEYmRTj5on1", "destination_payment": "py_1NH18zEYmRTj5on1GkCCsqLK", "livemode": false, "metadata": {}, "reversals": {"object": "list", "data": [], "has_more": false, "total_count": 0.0, "url": "/v1/transfers/tr_1NH18zEcXtiJtvvhnd827cNO/reversals"}, "reversed": false, "source_transaction": null, "source_type": "card", "transfer_group": null, "updated": 1686301085}, "emitted_at": 1697627313262} {"stream": "transfers", "data": {"id": "tr_1NGoaCEcXtiJtvvhjmHtOGOm", "object": "transfer", "amount": 100, "amount_reversed": 100, "balance_transaction": "txn_1NGoaDEcXtiJtvvhsZrNMsdJ", "created": 1686252800, "currency": "usd", "description": null, "destination": "acct_1Jx8unEYmRTj5on1", "destination_payment": "py_1NGoaCEYmRTj5on1LAlAIG3a", "livemode": false, "metadata": {}, "reversals": {"object": "list", "data": [{"id": "trr_1NGolCEcXtiJtvvhOYPck3CP", "object": "transfer_reversal", "amount": 100, "balance_transaction": "txn_1NGolCEcXtiJtvvhZRy4Kd5S", "created": 1686253482, "currency": "usd", "destination_payment_refund": "pyr_1NGolBEYmRTj5on1STal3rmp", "metadata": {}, "source_refund": null, "transfer": "tr_1NGoaCEcXtiJtvvhjmHtOGOm"}], "has_more": false, "total_count": 1.0, "url": "/v1/transfers/tr_1NGoaCEcXtiJtvvhjmHtOGOm/reversals"}, "reversed": true, "source_transaction": null, "source_type": "card", "transfer_group": "ORDER10", "updated": 1686252800}, "emitted_at": 1697627313264} -{"stream": "refunds", "data": {"id": "re_3MVuZyEcXtiJtvvh0A6rSbeJ", "object": "refund", "amount": 200000, "balance_transaction": "txn_3MVuZyEcXtiJtvvh0v0QyAMx", "charge": "ch_3MVuZyEcXtiJtvvh0tiVC7DI", "created": 1675074488, "currency": "usd", "metadata": {}, "payment_intent": "pi_3MVuZyEcXtiJtvvh07Ehi4cx", "reason": "fraudulent", "receipt_number": "3278-5368", "source_transfer_reversal": null, "status": "succeeded", "transfer_reversal": null, "updated": 1675074488}, "emitted_at": 1697627314206} -{"stream": "refunds", "data": {"id": "re_3NcwAGEcXtiJtvvh1UT4PBe6", "object": "refund", "amount": 600, "balance_transaction": "txn_3NcwAGEcXtiJtvvh1AcNi3Ma", "charge": "ch_3NcwAGEcXtiJtvvh1m0SSmfQ", "created": 1692782173, "currency": "usd", "metadata": {}, "payment_intent": "pi_3NcwAGEcXtiJtvvh1olHTPmH", "reason": null, "receipt_number": null, "source_transfer_reversal": null, "status": "succeeded", "transfer_reversal": null, "updated": 1692782173}, "emitted_at": 1697627314485} -{"stream": "refunds", "data": {"id": "re_3MngeoEcXtiJtvvh0c4KeMOd", "object": "refund", "amount": 540, "balance_transaction": "txn_3MngeoEcXtiJtvvh0Cz3qwU2", "charge": "ch_3MngeoEcXtiJtvvh0SBFQWe2", "created": 1683889626, "currency": "usd", "metadata": {}, "payment_intent": "pi_3MngeoEcXtiJtvvh0B7Tcbr4", "reason": "requested_by_customer", "receipt_number": null, "source_transfer_reversal": null, "status": "succeeded", "transfer_reversal": null, "updated": 1683889626}, "emitted_at": 1697627314486} +{"stream": "refunds", "data": {"id": "re_3MVuZyEcXtiJtvvh0A6rSbeJ", "object": "refund", "amount": 200000, "balance_transaction": "txn_3MVuZyEcXtiJtvvh0v0QyAMx", "charge": "ch_3MVuZyEcXtiJtvvh0tiVC7DI", "created": 1675074488, "currency": "usd", "metadata": {}, "payment_intent": "pi_3MVuZyEcXtiJtvvh07Ehi4cx", "reason": "fraudulent", "receipt_number": "3278-5368", "source_transfer_reversal": null, "status": "succeeded", "transfer_reversal": null}, "emitted_at": 1697627314206} +{"stream": "refunds", "data": {"id": "re_3NcwAGEcXtiJtvvh1UT4PBe6", "object": "refund", "amount": 600, "balance_transaction": "txn_3NcwAGEcXtiJtvvh1AcNi3Ma", "charge": "ch_3NcwAGEcXtiJtvvh1m0SSmfQ", "created": 1692782173, "currency": "usd", "metadata": {}, "payment_intent": "pi_3NcwAGEcXtiJtvvh1olHTPmH", "reason": null, "receipt_number": null, "source_transfer_reversal": null, "status": "succeeded", "transfer_reversal": null}, "emitted_at": 1697627314485} +{"stream": "refunds", "data": {"id": "re_3MngeoEcXtiJtvvh0c4KeMOd", "object": "refund", "amount": 540, "balance_transaction": "txn_3MngeoEcXtiJtvvh0Cz3qwU2", "charge": "ch_3MngeoEcXtiJtvvh0SBFQWe2", "created": 1683889626, "currency": "usd", "metadata": {}, "payment_intent": "pi_3MngeoEcXtiJtvvh0B7Tcbr4", "reason": "requested_by_customer", "receipt_number": null, "source_transfer_reversal": null, "status": "succeeded", "transfer_reversal": null}, "emitted_at": 1697627314486} {"stream": "payment_intents", "data": {"id": "pi_3K9FSOEcXtiJtvvh0AEIFllC", "object": "payment_intent", "amount": 5300, "amount_capturable": 0, "amount_details": {"tip": {}}, "amount_received": 5300, "application": null, "application_fee_amount": null, "automatic_payment_methods": null, "canceled_at": null, "cancellation_reason": null, "capture_method": "automatic", "client_secret": "pi_3K9FSOEcXtiJtvvh0AEIFllC_secret_uPUtIaSltgtW0qK7mLD0uF2Mr", "confirmation_method": "automatic", "created": 1640120472, "currency": "usd", "customer": null, "description": null, "invoice": null, "last_payment_error": null, "latest_charge": "ch_3K9FSOEcXtiJtvvh0zxb7clc", "livemode": false, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": null, "payment_method_configuration_details": null, "payment_method_options": {"card": {"installments": null, "mandate_options": null, "network": null, "request_three_d_secure": "automatic"}}, "payment_method_types": ["card"], "processing": null, "receipt_email": null, "review": null, "setup_future_usage": null, "shipping": null, "source": "src_1K9FSOEcXtiJtvvhHGu1qtOx", "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null, "updated": 1640120472}, "emitted_at": 1697627315508} {"stream": "payment_intents", "data": {"id": "pi_3K9F5DEcXtiJtvvh16scJMp6", "object": "payment_intent", "amount": 4200, "amount_capturable": 0, "amount_details": {"tip": {}}, "amount_received": 4200, "application": null, "application_fee_amount": null, "automatic_payment_methods": null, "canceled_at": null, "cancellation_reason": null, "capture_method": "automatic", "client_secret": "pi_3K9F5DEcXtiJtvvh16scJMp6_secret_YwhzCTpXtfcKYeklXnPnysRRi", "confirmation_method": "automatic", "created": 1640119035, "currency": "usd", "customer": null, "description": "edgao test", "invoice": null, "last_payment_error": null, "latest_charge": "ch_3K9F5DEcXtiJtvvh1w2MaTpj", "livemode": false, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": null, "payment_method_configuration_details": null, "payment_method_options": {"card": {"installments": null, "mandate_options": null, "network": null, "request_three_d_secure": "automatic"}}, "payment_method_types": ["card"], "processing": null, "receipt_email": null, "review": null, "setup_future_usage": null, "shipping": null, "source": "src_1K9F5CEcXtiJtvvhrsZdur8Y", "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null, "updated": 1640119035}, "emitted_at": 1697627315511} {"stream": "payment_intents", "data": {"id": "pi_3K9F4mEcXtiJtvvh18NKhEuo", "object": "payment_intent", "amount": 4200, "amount_capturable": 0, "amount_details": {"tip": {}}, "amount_received": 0, "application": null, "application_fee_amount": null, "automatic_payment_methods": null, "canceled_at": null, "cancellation_reason": null, "capture_method": "automatic", "client_secret": "pi_3K9F4mEcXtiJtvvh18NKhEuo_secret_pfUt7CTkPjVdJacycm0bMpdLt", "confirmation_method": "automatic", "created": 1640119008, "currency": "usd", "customer": null, "description": "edgao test", "invoice": null, "last_payment_error": {"charge": "ch_3K9F4mEcXtiJtvvh1kUzxjwN", "code": "card_declined", "decline_code": "test_mode_live_card", "doc_url": "https://stripe.com/docs/error-codes/card-declined", "message": "Your card was declined. Your request was in test mode, but used a non test (live) card. For a list of valid test cards, visit: https://stripe.com/docs/testing.", "source": {"id": "src_1K9F4hEcXtiJtvvhrUEwvCyi", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": "unchecked", "dynamic_last4": null, "exp_month": 9, "exp_year": 2028, "fingerprint": "Re3p4j8issXA77iI", "funding": "credit", "last4": "8097", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_b3v8YqNMLGykB120fqv2Tjhq", "created": 1640119003, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "type": "card_error"}, "latest_charge": "ch_3K9F4mEcXtiJtvvh1kUzxjwN", "livemode": false, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": null, "payment_method_configuration_details": null, "payment_method_options": {"card": {"installments": null, "mandate_options": null, "network": null, "request_three_d_secure": "automatic"}}, "payment_method_types": ["card"], "processing": null, "receipt_email": null, "review": null, "setup_future_usage": null, "shipping": null, "source": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "requires_payment_method", "transfer_data": null, "transfer_group": null, "updated": 1640119008}, "emitted_at": 1697627315513} diff --git a/airbyte-integrations/connectors/source-stripe/metadata.yaml b/airbyte-integrations/connectors/source-stripe/metadata.yaml index 1aaaa9ee97c4..f521fb172278 100644 --- a/airbyte-integrations/connectors/source-stripe/metadata.yaml +++ b/airbyte-integrations/connectors/source-stripe/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: e094cb9a-26de-4645-8761-65c0c425d1de - dockerImageTag: 4.5.4 + dockerImageTag: 5.0.0 dockerRepository: airbyte/source-stripe documentationUrl: https://docs.airbyte.com/integrations/sources/stripe githubIssueLabel: source-stripe @@ -33,6 +33,11 @@ data: schema refresh of all effected streams is required to use the new cursor format. upgradeDeadline: "2023-09-14" + 5.0.0: + message: + Version 5.0.0 introduces fixes for the `CheckoutSessions`, `CheckoutSessionsLineItems` and `Refunds` streams. The cursor field is changed for the `CheckoutSessionsLineItems` and `Refunds` streams. This will prevent data loss during incremental syncs. + Also, the `Invoices`, `Subscriptions` and `SubscriptionSchedule` stream schemas have been updated. + upgradeDeadline: "2023-11-30" suggestedStreams: streams: - customers diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions_line_items.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions_line_items.json index 09c6e9e28f3e..b00f6569d12e 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions_line_items.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions_line_items.json @@ -5,6 +5,8 @@ "id": { "type": ["null", "string"] }, "checkout_session_id": { "type": ["null", "string"] }, "checkout_session_expires_at": { "type": ["null", "integer"] }, + "checkout_session_created": { "type": ["null", "integer"] }, + "checkout_session_updated": { "type": ["null", "integer"] }, "object": { "type": ["null", "string"] }, "amount_subtotal": { "type": ["null", "integer"] }, "amount_tax": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json index 33d407425ea0..6959909cc77f 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json @@ -499,10 +499,7 @@ "type": ["null", "integer"] }, "default_tax_rates": { - "type": ["null", "array"], - "items": { - "$ref": "tax_rates.json" - } + "$ref": "tax_rates.json" }, "total_excluding_tax": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_schedule.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_schedule.json index 9f187d82924c..cc14a57138fd 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_schedule.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_schedule.json @@ -134,11 +134,7 @@ "type": ["null", "string"] }, "default_tax_rates": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "additionalProperties": true - } + "$ref": "tax_rates.json" }, "description": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json index 5d06810fb4c0..1a720f6fd034 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json @@ -323,10 +323,7 @@ } }, "default_tax_rates": { - "type": ["null", "array"], - "items": { - "$ref": "tax_rates.json" - } + "$ref": "tax_rates.json" }, "pause_collection": { "type": ["null", "object"], diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py index 5683c875497e..449eb53f99db 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py @@ -22,11 +22,11 @@ from airbyte_cdk.utils.traced_exception import AirbyteTracedException from airbyte_protocol.models import SyncMode from source_stripe.streams import ( - CheckoutSessionsLineItems, CreatedCursorIncrementalStripeStream, CustomerBalanceTransactions, Events, IncrementalStripeStream, + ParentIncrementalStipeSubStream, Persons, SetupAttempts, StripeLazySubStream, @@ -197,10 +197,9 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: subscription_items = StripeLazySubStream( name="subscription_items", path="subscription_items", - extra_request_params=lambda self, stream_slice, *args, **kwargs: {"subscription": stream_slice[self.parent_id]}, + extra_request_params=lambda self, stream_slice, *args, **kwargs: {"subscription": stream_slice["parent"]["id"]}, parent=subscriptions, use_cache=USE_CACHE, - parent_id="subscription_id", sub_items_attr="items", **args, ) @@ -238,8 +237,22 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: ], **args, ) + checkout_sessions = UpdatedCursorIncrementalStripeStream( + name="checkout_sessions", + path="checkout/sessions", + use_cache=USE_CACHE, + legacy_cursor_field="created", + event_types=[ + "checkout.session.async_payment_failed", + "checkout.session.async_payment_succeeded", + "checkout.session.completed", + "checkout.session.expired", + ], + **args, + ) + streams = [ - CheckoutSessionsLineItems(**incremental_args), + checkout_sessions, CustomerBalanceTransactions(**args), Events(**incremental_args), UpdatedCursorIncrementalStripeStream( @@ -267,19 +280,10 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: CreatedCursorIncrementalStripeStream(name="balance_transactions", path="balance_transactions", **incremental_args), CreatedCursorIncrementalStripeStream(name="files", path="files", **incremental_args), CreatedCursorIncrementalStripeStream(name="file_links", path="file_links", **incremental_args), - UpdatedCursorIncrementalStripeStream( - name="checkout_sessions", - path="checkout/sessions", - use_cache=USE_CACHE, - legacy_cursor_field="expires_at", - event_types=[ - "checkout.session.async_payment_failed", - "checkout.session.async_payment_succeeded", - "checkout.session.completed", - "checkout.session.expired", - ], - **args, - ), + # The Refunds stream does not utilize the Events API as it created issues with data loss during the incremental syncs. + # Therefore, we're using the regular API with the `created` cursor field. A bug has been filed with Stripe. + # See more at https://github.com/airbytehq/oncall/issues/3090, https://github.com/airbytehq/oncall/issues/3428 + CreatedCursorIncrementalStripeStream(name="refunds", path="refunds", **incremental_args), UpdatedCursorIncrementalStripeStream( name="payment_methods", path="payment_methods", @@ -396,9 +400,6 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: **args, ), transfers, - IncrementalStripeStream( - name="refunds", path="refunds", use_cache=USE_CACHE, event_types=["refund.created", "refund.updated"], **args - ), IncrementalStripeStream( name="payment_intents", path="payment_intents", @@ -449,45 +450,56 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: ), UpdatedCursorIncrementalStripeLazySubStream( name="application_fees_refunds", - path=lambda self, stream_slice, *args, **kwargs: f"application_fees/{stream_slice[self.parent_id]}/refunds", + path=lambda self, stream_slice, *args, **kwargs: f"application_fees/{stream_slice['parent']['id']}/refunds", parent=application_fees, event_types=["application_fee.refund.updated"], - parent_id="refund_id", sub_items_attr="refunds", - add_parent_id=True, **args, ), UpdatedCursorIncrementalStripeLazySubStream( name="bank_accounts", - path=lambda self, stream_slice, *args, **kwargs: f"customers/{stream_slice[self.parent_id]}/sources", + path=lambda self, stream_slice, *args, **kwargs: f"customers/{stream_slice['parent']['id']}/sources", parent=self.customers(expand_items=["data.sources"], **args), event_types=["customer.source.created", "customer.source.expiring", "customer.source.updated", "customer.source.deleted"], legacy_cursor_field=None, - parent_id="customer_id", sub_items_attr="sources", extra_request_params={"object": "bank_account"}, response_filter=lambda record: record["object"] == "bank_account", **args, ), + ParentIncrementalStipeSubStream( + name="checkout_sessions_line_items", + path=lambda self, stream_slice, *args, **kwargs: f"checkout/sessions/{stream_slice['parent']['id']}/line_items", + parent=checkout_sessions, + expand_items=["data.discounts", "data.taxes"], + cursor_field="checkout_session_updated", + slice_data_retriever=lambda record, stream_slice: { + "checkout_session_id": stream_slice["parent"]["id"], + "checkout_session_expires_at": stream_slice["parent"]["expires_at"], + "checkout_session_created": stream_slice["parent"]["created"], + "checkout_session_updated": stream_slice["parent"]["updated"], + **record, + }, + **args, + ), StripeLazySubStream( name="invoice_line_items", - path=lambda self, stream_slice, *args, **kwargs: f"invoices/{stream_slice[self.parent_id]}/lines", + path=lambda self, stream_slice, *args, **kwargs: f"invoices/{stream_slice['parent']['id']}/lines", parent=invoices, - parent_id="invoice_id", sub_items_attr="lines", - add_parent_id=True, + slice_data_retriever=lambda record, stream_slice: {"invoice_id": stream_slice["parent"]["id"], **record}, **args, ), subscription_items, StripeSubStream( name="transfer_reversals", - path=lambda self, stream_slice, *args, **kwargs: f"transfers/{stream_slice.get('parent', {}).get('id')}/reversals", + path=lambda self, stream_slice, *args, **kwargs: f"transfers/{stream_slice['parent']['id']}/reversals", parent=transfers, **args, ), StripeSubStream( name="usage_records", - path=lambda self, stream_slice, *args, **kwargs: f"subscription_items/{stream_slice.get('parent', {}).get('id')}/usage_record_summaries", + path=lambda self, stream_slice, *args, **kwargs: f"subscription_items/{stream_slice['parent']['id']}/usage_record_summaries", parent=subscription_items, primary_key=None, **args, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml b/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml index 5a31b610cd27..719177412a96 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml @@ -42,8 +42,8 @@ connectionSpecification: description: >- When set, the connector will always re-export data from the past N days, where N is the value set here. This is useful if your data is frequently updated - after creation. Applies only to streams that do not support event-based incremental syncs: CheckoutSessionLineItems, - Events, SetupAttempts, ShippingRates, BalanceTransactions, Files, FileLinks. More info here order: 3 slice_range: diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py index 460059716a75..a8b7feaef07f 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py @@ -25,41 +25,54 @@ class IRecordExtractor(ABC): @abstractmethod - def extract_records(self, records: Iterable[MutableMapping]) -> Iterable[Mapping]: + def extract_records(self, records: Iterable[MutableMapping], stream_slice: Optional[Mapping[str, Any]] = None) -> Iterable[Mapping]: pass class DefaultRecordExtractor(IRecordExtractor): - def __init__(self, response_filter: Optional[Callable] = None): - self._response_filter = response_filter or (lambda x: x) + def __init__(self, response_filter: Optional[Callable] = None, slice_data_retriever: Optional[Callable] = None): + self._response_filter = response_filter or (lambda record: record) + self._slice_data_retriever = slice_data_retriever or (lambda record, *_: record) - def extract_records(self, records: Iterable[MutableMapping]) -> Iterable[MutableMapping]: - yield from filter(self._response_filter, records) + def extract_records( + self, records: Iterable[MutableMapping], stream_slice: Optional[Mapping[str, Any]] = None + ) -> Iterable[MutableMapping]: + yield from filter(self._response_filter, map(lambda x: self._slice_data_retriever(x, stream_slice), records)) class EventRecordExtractor(DefaultRecordExtractor): - def __init__(self, cursor_field: str, response_filter: Optional[Callable] = None): - super().__init__(response_filter) + def __init__(self, cursor_field: str, response_filter: Optional[Callable] = None, slice_data_retriever: Optional[Callable] = None): + super().__init__(response_filter, slice_data_retriever) self.cursor_field = cursor_field - def extract_records(self, records: Iterable[MutableMapping]) -> Iterable[MutableMapping]: + def extract_records( + self, records: Iterable[MutableMapping], stream_slice: Optional[Mapping[str, Any]] = None + ) -> Iterable[MutableMapping]: for record in records: item = record["data"]["object"] item[self.cursor_field] = record["created"] if record["type"].endswith(".deleted"): item["is_deleted"] = True if self._response_filter(item): - yield item + yield self._slice_data_retriever(item, stream_slice) class UpdatedCursorIncrementalRecordExtractor(DefaultRecordExtractor): - def __init__(self, cursor_field: str, legacy_cursor_field: Optional[str], response_filter: Optional[Callable] = None): - super().__init__(response_filter) + def __init__( + self, + cursor_field: str, + legacy_cursor_field: Optional[str], + response_filter: Optional[Callable] = None, + slice_data_retriever: Optional[Callable] = None, + ): + super().__init__(response_filter, slice_data_retriever) self.cursor_field = cursor_field self.legacy_cursor_field = legacy_cursor_field - def extract_records(self, records: Iterable[MutableMapping]) -> Iterable[MutableMapping]: - records = super().extract_records(records) + def extract_records( + self, records: Iterable[MutableMapping], stream_slice: Optional[Mapping[str, Any]] = None + ) -> Iterable[MutableMapping]: + records = super().extract_records(records, stream_slice) for record in records: if self.cursor_field in record: yield record @@ -126,13 +139,14 @@ def __init__( expand_items: Optional[List[str]] = None, extra_request_params: Optional[Union[Mapping[str, Any], Callable]] = None, response_filter: Optional[Callable] = None, + slice_data_retriever: Optional[Callable] = None, primary_key: Optional[str] = "id", **kwargs, ): self.account_id = account_id self.start_date = start_date self.slice_range = slice_range or self.DEFAULT_SLICE_RANGE - self._record_extractor = record_extractor or DefaultRecordExtractor(response_filter) + self._record_extractor = record_extractor or DefaultRecordExtractor(response_filter, slice_data_retriever) self._name = name self._path = path self._use_cache = use_cache @@ -174,7 +188,7 @@ def parse_response( stream_slice: Optional[Mapping[str, Any]] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Iterable[Mapping[str, Any]]: - yield from self.record_extractor.extract_records(response.json().get("data", [])) + yield from self.record_extractor.extract_records(response.json().get("data", []), stream_slice) def request_headers(self, **kwargs) -> Mapping[str, Any]: headers = {"Stripe-Version": STRIPE_API_VERSION} @@ -251,7 +265,8 @@ def stream_slices( def get_start_timestamp(self, stream_state) -> int: start_point = self.start_date - start_point = max(start_point, stream_state.get(self.cursor_field, 0)) + # we use +1 second because date range is inclusive + start_point = max(start_point, stream_state.get(self.cursor_field, 0) + 1) if start_point and self.lookback_window_days: self.logger.info(f"Applying lookback window of {self.lookback_window_days} days to stream {self.name}") @@ -471,100 +486,6 @@ def read_records( yield from self.parent_stream.read_records(sync_mode, cursor_field, stream_slice, stream_state) -class CheckoutSessionsLineItems(CreatedCursorIncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/checkout/sessions/line_items - """ - - cursor_field = "checkout_session_expires_at" - - @property - def expand_items(self) -> Optional[List[str]]: - return ["data.discounts", "data.taxes"] - - @property - def checkout_session(self): - return UpdatedCursorIncrementalStripeStream( - name="checkout_sessions", - path="checkout/sessions", - use_cache=USE_CACHE, - legacy_cursor_field="expires_at", - event_types=[ - "checkout.session.async_payment_failed", - "checkout.session.async_payment_succeeded", - "checkout.session.completed", - "checkout.session.expired", - ], - authenticator=self.authenticator, - account_id=self.account_id, - start_date=self.start_date, - slice_range=self.slice_range, - ) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # https://stripe.com/docs/api/checkout/sessions/create#create_checkout_session-expires_at - # 'expires_at' - can be anywhere from 1 to 24 hours after Checkout Session creation. - # thus we should always add 1 day to lookback window to avoid possible checkout_sessions losses - self.lookback_window_days = self.lookback_window_days + 1 - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs): - return f"checkout/sessions/{stream_slice['checkout_session_id']}/line_items" - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - # override to not refer to slice values - params = { - "limit": 100, - **self.extra_request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), - } - if self.expand_items: - params["expand[]"] = self.expand_items - if next_page_token: - params.update(next_page_token) - return params - - def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: - checkout_session_state = None - if stream_state: - checkout_session_state = {"expires_at": stream_state["checkout_session_expires_at"]} - for checkout_session in self.checkout_session.read_records( - sync_mode=SyncMode.full_refresh, stream_state=checkout_session_state, stream_slice={} - ): - yield { - "checkout_session_id": checkout_session["id"], - "expires_at": checkout_session["expires_at"], - } - - @property - def raise_on_http_errors(self): - return False - - def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping]: - if response.status_code == 404: - self.logger.warning(response.json()) - return - response.raise_for_status() - - response_json = response.json() - data = response_json.get("data", []) - if data and stream_slice: - self.logger.info(f"stream_slice: {stream_slice}") - cs_id = stream_slice.get("checkout_session_id", None) - cs_expires_at = stream_slice.get("expires_at", None) - for e in data: - e["checkout_session_id"] = cs_id - e["checkout_session_expires_at"] = cs_expires_at - yield from data - - class CustomerBalanceTransactions(StripeStream): """ API docs: https://stripe.com/docs/api/customer_balance_transactions/list @@ -721,17 +642,6 @@ class StripeLazySubStream(StripeStream, HttpSubStream): } """ - @property - def add_parent_id(self) -> bool: - return self._add_parent_id - - @property - def parent_id(self) -> str: - """ - :return: string with attribute name - """ - return self._parent_id - @property def sub_items_attr(self) -> str: """ @@ -743,14 +653,10 @@ def sub_items_attr(self) -> str: def __init__( self, *args, - add_parent_id: bool = False, - parent_id: Optional[str] = None, sub_items_attr: Optional[str] = None, **kwargs, ): super().__init__(*args, **kwargs) - self._add_parent_id = add_parent_id - self._parent_id = parent_id self._sub_items_attr = sub_items_attr @property @@ -767,24 +673,16 @@ def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs): return params def read_records(self, sync_mode: SyncMode, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: - parent_record = stream_slice["parent"] - items_obj = parent_record.get(self.sub_items_attr, {}) + items_obj = stream_slice["parent"].get(self.sub_items_attr, {}) if not items_obj: return - items = list(self.record_extractor.extract_records(items_obj.get("data", []))) - - # get next pages items_next_pages = [] + items = list(self.record_extractor.extract_records(items_obj.get("data", []), stream_slice)) if items_obj.get("has_more") and items: - stream_slice = {self.parent_id: parent_record["id"], "starting_after": items[-1]["id"]} + stream_slice = {"starting_after": items[-1]["id"], **stream_slice} items_next_pages = super().read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice, **kwargs) - - for item in chain(items, items_next_pages): - if self.add_parent_id: - # add reference to parent object when item doesn't have it already - item[self.parent_id] = parent_record["id"] - yield item + yield from chain(items, items_next_pages) class IncrementalStripeLazySubStreamSelector(IStreamSelector): @@ -797,6 +695,11 @@ def get_parent_stream(self, stream_state: Mapping[str, Any]) -> StripeStream: class UpdatedCursorIncrementalStripeLazySubStream(StripeStream, ABC): + """ + This stream uses StripeLazySubStream under the hood to run full refresh or initial incremental syncs. + In case of subsequent incremental syncs, it uses the UpdatedCursorIncrementalStripeStream class. + """ + def __init__( self, parent: StripeStream, @@ -804,8 +707,6 @@ def __init__( cursor_field: str = "updated", legacy_cursor_field: Optional[str] = "created", event_types: Optional[List[str]] = None, - parent_id: Optional[str] = None, - add_parent_id: bool = False, sub_items_attr: Optional[str] = None, response_filter: Optional[Callable] = None, **kwargs, @@ -823,8 +724,6 @@ def __init__( self.lazy_substream = StripeLazySubStream( *args, parent=parent, - parent_id=parent_id, - add_parent_id=add_parent_id, sub_items_attr=sub_items_attr, record_extractor=UpdatedCursorIncrementalRecordExtractor( cursor_field=cursor_field, legacy_cursor_field=legacy_cursor_field, response_filter=response_filter @@ -866,3 +765,60 @@ def read_records( yield from self.parent_stream.read_records( sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state ) + + +class ParentIncrementalStipeSubStream(StripeSubStream): + """ + This stream differs from others in that it runs parent stream in exactly same sync mode it is run itself to generate stream slices. + It also uses regular /v1 API endpoints to sync data no matter what the sync mode is. This means that the event-based API can only + be utilized by the parent stream. + """ + + @property + def cursor_field(self) -> str: + return self._cursor_field + + def __init__(self, cursor_field: str, *args, **kwargs): + self._cursor_field = cursor_field + super().__init__(*args, **kwargs) + + def stream_slices( + self, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + stream_state = stream_state or {} + if stream_state: + # state is shared between self and parent, but cursor fields are different + stream_state = {self.parent.cursor_field: stream_state.get(self.cursor_field, 0)} + parent_stream_slices = self.parent.stream_slices(sync_mode=sync_mode, cursor_field=cursor_field, stream_state=stream_state) + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + for record in parent_records: + yield {"parent": record} + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + return {self.cursor_field: max(current_stream_state.get(self.cursor_field, 0), latest_record[self.cursor_field])} + + @property + def availability_strategy(self) -> Optional[AvailabilityStrategy]: + return StripeSubStreamAvailabilityStrategy() + + @property + def raise_on_http_errors(self) -> bool: + return False + + def parse_response(self, response: requests.Response, *args, **kwargs) -> Iterable[Mapping[str, Any]]: + if response.status_code == 200: + return super().parse_response(response, *args, **kwargs) + if response.status_code == 404: + # When running incremental sync with state, the returned parent object very likely will not contain sub-items + # as the events API does not support expandable items. Parent class will try getting sub-items from this object, + # then from its own API. In case there are no sub-items at all for this entity, API will raise 404 error. + self.logger.warning( + "Data was not found for URL: {response.request.url}. " + "If this is a path for getting child attributes like /v1/checkout/sessions//line_items when running " + "the incremental sync, you may safely ignore this warning." + ) + return [] + response.raise_for_status() diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py index 143331e06992..5f942b152157 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py @@ -7,7 +7,7 @@ import freezegun import pendulum import pytest -from source_stripe.streams import CheckoutSessionsLineItems, CustomerBalanceTransactions, Persons, SetupAttempts +from source_stripe.streams import CustomerBalanceTransactions, Persons, SetupAttempts def read_from_stream(stream, sync_mode, state): @@ -170,40 +170,185 @@ def test_lazy_substream_data_is_filtered( assert record["object"] == expected_object -@freezegun.freeze_time("2023-08-23T15:00:15Z") -def test_created_cursor_incremental_stream(requests_mock, stream_by_name, config): - config["start_date"] = str(pendulum.now().subtract(months=23)) - stream = stream_by_name("balance_transactions", {"lookback_window_days": 14, **config}) - requests_mock.get( - "/v1/balance_transactions", - [ +balance_transactions_api_objects = [ + {"id": "txn_1KVQhfEcXtiJtvvhF7ox3YEm", "object": "balance_transaction", "amount": 435, "created": 1653299388, "status": "available"}, + {"id": "txn_tiJtvvhF7ox3YEmKvVQhfEcX", "object": "balance_transaction", "amount": -9164, "created": 1679568588, "status": "available"}, +] + + +refunds_api_objects = [ + { + "id": "re_3NYB8LAHLf1oYfwN3EZRDIfF", + "object": "refund", + "amount": 100, + "charge": "ch_3NYB8LAHLf1oYfwN3P6BxdKj", + "created": 1653299388, + "currency": "usd", + }, + { + "id": "re_Lf1oYfwN3EZRDIfF3NYB8LAH", + "object": "refund", + "amount": 15, + "charge": "ch_YfwN3P6BxdKj3NYB8LAHLf1o", + "created": 1679568588, + "currency": "eur", + }, +] + + +@pytest.mark.parametrize( + "requests_mock_map, expected_records, expected_slices, stream_name, sync_mode, state", + ( + ( { - "json": { - "data": [{"id": "txn_1KVQhfEcXtiJtvvhF7ox3YEm", "object": "balance_transaction", "amount": 435, "status": "available"}], - "has_more": False, - } + "/v1/balance_transactions": [ + { + "json": { + "data": [balance_transactions_api_objects[0]], + "has_more": False, + } + }, + { + "json": { + "data": [balance_transactions_api_objects[-1]], + "has_more": False, + } + }, + ], + }, + [ + { + "id": "txn_1KVQhfEcXtiJtvvhF7ox3YEm", + "object": "balance_transaction", + "amount": 435, + "created": 1653299388, + "status": "available", + }, + { + "id": "txn_tiJtvvhF7ox3YEmKvVQhfEcX", + "object": "balance_transaction", + "amount": -9164, + "created": 1679568588, + "status": "available", + }, + ], + [{"created[gte]": 1631199615, "created[lte]": 1662735615}, {"created[gte]": 1662735616, "created[lte]": 1692802815}], + "balance_transactions", + "full_refresh", + {}, + ), + ( + { + "/v1/balance_transactions": [ + { + "json": { + "data": [balance_transactions_api_objects[-1]], + "has_more": False, + } + }, + ], }, + [ + { + "id": "txn_tiJtvvhF7ox3YEmKvVQhfEcX", + "object": "balance_transaction", + "amount": -9164, + "created": 1679568588, + "status": "available", + }, + ], + [{"created[gte]": 1665308989, "created[lte]": 1692802815}], + "balance_transactions", + "incremental", + {"created": 1666518588}, + ), + ( { - "json": { - "data": [ - {"id": "txn_tiJtvvhF7ox3YEmKvVQhfEcX", "object": "balance_transaction", "amount": -9164, "status": "available"} - ], - "has_more": False, - } + "/v1/refunds": [ + { + "json": { + "data": [refunds_api_objects[0]], + "has_more": False, + } + }, + { + "json": { + "data": [refunds_api_objects[-1]], + "has_more": False, + } + }, + ], }, - ], - ) + [ + { + "id": "re_3NYB8LAHLf1oYfwN3EZRDIfF", + "object": "refund", + "amount": 100, + "charge": "ch_3NYB8LAHLf1oYfwN3P6BxdKj", + "created": 1653299388, + "currency": "usd", + }, + { + "id": "re_Lf1oYfwN3EZRDIfF3NYB8LAH", + "object": "refund", + "amount": 15, + "charge": "ch_YfwN3P6BxdKj3NYB8LAHLf1o", + "created": 1679568588, + "currency": "eur", + }, + ], + [{"created[gte]": 1631199615, "created[lte]": 1662735615}, {"created[gte]": 1662735616, "created[lte]": 1692802815}], + "refunds", + "full_refresh", + {}, + ), + ( + { + "/v1/refunds": [ + { + "json": { + "data": [refunds_api_objects[-1]], + "has_more": False, + } + }, + ], + }, + [ + { + "id": "re_Lf1oYfwN3EZRDIfF3NYB8LAH", + "object": "refund", + "amount": 15, + "charge": "ch_YfwN3P6BxdKj3NYB8LAHLf1o", + "created": 1679568588, + "currency": "eur", + } + ], + [{"created[gte]": 1665308989, "created[lte]": 1692802815}], + "refunds", + "incremental", + {"created": 1666518588}, + ), + ), +) +@freezegun.freeze_time("2023-08-23T15:00:15Z") +def test_created_cursor_incremental_stream( + requests_mock, requests_mock_map, stream_by_name, expected_records, expected_slices, stream_name, sync_mode, state, config +): + config["start_date"] = str(pendulum.now().subtract(months=23)) + stream = stream_by_name(stream_name, {"lookback_window_days": 14, **config}) + for url, response in requests_mock_map.items(): + requests_mock.get(url, response) - slices = list(stream.stream_slices("full_refresh")) - assert slices == [{"created[gte]": 1631199615, "created[lte]": 1662735615}, {"created[gte]": 1662735616, "created[lte]": 1692802815}] - records = [] + slices = list(stream.stream_slices(sync_mode, stream_state=state)) + assert slices == expected_slices + records = read_from_stream(stream, sync_mode, state) + assert records == expected_records + for record in records: + assert bool(record[stream.cursor_field]) + call_history = iter(requests_mock.request_history) for slice_ in slices: - for record in stream.read_records("full_refresh", stream_slice=slice_): - records.append(record) - assert records == [ - {"id": "txn_1KVQhfEcXtiJtvvhF7ox3YEm", "object": "balance_transaction", "amount": 435, "status": "available"}, - {"id": "txn_tiJtvvhF7ox3YEmKvVQhfEcX", "object": "balance_transaction", "amount": -9164, "status": "available"}, - ] + call = next(call_history) + assert urlencode(slice_) in call.url @pytest.mark.parametrize( @@ -213,8 +358,8 @@ def test_created_cursor_incremental_stream(requests_mock, stream_by_name, config ("2020-01-01T00:00:00Z", 14, 0, {}, "2019-12-18T00:00:00Z"), ("2020-01-01T00:00:00Z", 0, 30, {}, "2023-07-24T15:00:15Z"), ("2020-01-01T00:00:00Z", 14, 30, {}, "2023-07-24T15:00:15Z"), - ("2020-01-01T00:00:00Z", 0, 0, {"created": pendulum.parse("2022-07-17T00:00:00Z").int_timestamp}, "2022-07-17T00:00:00Z"), - ("2020-01-01T00:00:00Z", 14, 0, {"created": pendulum.parse("2022-07-17T00:00:00Z").int_timestamp}, "2022-07-03T00:00:00Z"), + ("2020-01-01T00:00:00Z", 0, 0, {"created": pendulum.parse("2022-07-17T00:00:00Z").int_timestamp}, "2022-07-17T00:00:01Z"), + ("2020-01-01T00:00:00Z", 14, 0, {"created": pendulum.parse("2022-07-17T00:00:00Z").int_timestamp}, "2022-07-03T00:00:01Z"), ("2020-01-01T00:00:00Z", 0, 30, {"created": pendulum.parse("2022-07-17T00:00:00Z").int_timestamp}, "2023-07-24T15:00:15Z"), ("2020-01-01T00:00:00Z", 14, 30, {"created": pendulum.parse("2022-07-17T00:00:00Z").int_timestamp}, "2023-07-24T15:00:15Z"), ), @@ -326,50 +471,6 @@ def test_updated_cursor_incremental_stream_read_w_state(requests_mock, stream_by assert records == [{"object": "credit_note", "invoice": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "created": 1653341716, "updated": 1691629292}] -def test_checkout_session_line_items(requests_mock): - - session_id_missed = "cs_test_a165K4wNihuJlp2u3tknuohrvjAxyXFUB7nxZH3lwXRKJsadNEvIEWMUJ9" - session_id_exists = "cs_test_a1RjRHNyGUQOFVF3OkL8V8J0lZUASyVoCtsnZYG74VrBv3qz4245BLA1BP" - - response_sessions = { - "data": [{"id": session_id_missed, "expires_at": 100_000}, {"id": session_id_exists, "expires_at": 100_000}], - "has_more": False, - "object": "list", - "url": "/v1/checkout/sessions", - } - - response_sessions_line_items = { - "data": [{"id": "li_1JpAUUIEn5WyEQxnfGJT5MbL"}], - "has_more": False, - "object": "list", - "url": "/v1/checkout/sessions/{}/line_items".format(session_id_exists), - } - - response_error = { - "error": { - "code": "resource_missing", - "doc_url": "https://stripe.com/docs/error-codes/resource-missing", - "message": "No such checkout session: '{}'".format(session_id_missed), - "param": "session", - "type": "invalid_request_error", - } - } - - requests_mock.get("https://api.stripe.com/v1/checkout/sessions", json=response_sessions) - requests_mock.get( - "https://api.stripe.com/v1/checkout/sessions/{}/line_items".format(session_id_exists), json=response_sessions_line_items - ) - requests_mock.get( - "https://api.stripe.com/v1/checkout/sessions/{}/line_items".format(session_id_missed), json=response_error, status_code=404 - ) - - stream = CheckoutSessionsLineItems(start_date=100_100, account_id=None) - records = [] - for slice_ in stream.stream_slices(sync_mode="full_refresh"): - records.extend(stream.read_records(sync_mode="full_refresh", stream_slice=slice_)) - assert len(records) == 1 - - def test_customer_balance_transactions_stream_slices(requests_mock, stream_args): stream_args["start_date"] = pendulum.now().subtract(days=1).int_timestamp requests_mock.get( @@ -627,7 +728,6 @@ def test_subscription_items_extra_request_params(requests_mock, stream_by_name, "livemode": False, } ], - "has_more": False, }, ) requests_mock.get( @@ -668,3 +768,333 @@ def test_subscription_items_extra_request_params(requests_mock, stream_by_name, ] assert len(requests_mock.request_history) == 2 assert "subscription=sub_1OApco2eZvKYlo2CEDCzwLrE" in requests_mock.request_history[-1].url + + +checkout_session_api_response = { + "/v1/checkout/sessions": { + "object": "list", + "url": "/v1/checkout/sessions", + "has_more": False, + "data": [ + { + "id": "cs_test_a1yxusdFIgDDkWTaKn6JTYniMDBzrmnBiXH8oRSExZt7tcbIzIEoZk1Lre", + "object": "checkout.session", + "created": 1699647441, + "expires_at": 1699647441, + "payment_intent": "pi_1Gt0KQ2eZvKYlo2CeWXUgmhy", + "status": "open", + "line_items": { + "object": "list", + "has_more": False, + "url": "/v1/checkout/sessions", + "data": [ + { + "id": "li_1OB18o2eZvKYlo2CObYam50U", + "object": "item", + "amount_discount": 0, + "amount_subtotal": 0, + "amount_tax": 0, + "amount_total": 0, + "currency": "usd", + } + ], + }, + }, + { + "id": "cs_test_XH8oRSExZt7tcbIzIEoZk1Lrea1yxusdFIgDDkWTaKn6JTYniMDBzrmnBi", + "object": "checkout.session", + "created": 1699744164, + "expires_at": 1699644174, + "payment_intent": "pi_lo2CeWXUgmhy1Gt0KQ2eZvKY", + "status": "open", + "line_items": { + "object": "list", + "has_more": False, + "url": "/v1/checkout/sessions", + "data": [ + { + "id": "li_KYlo2CObYam50U1OB18o2eZv", + "object": "item", + "amount_discount": 0, + "amount_subtotal": 0, + "amount_tax": 0, + "amount_total": 0, + "currency": "usd", + } + ], + }, + }, + ], + } +} + + +checkout_session_line_items_api_response = { + "/v1/checkout/sessions/cs_test_a1yxusdFIgDDkWTaKn6JTYniMDBzrmnBiXH8oRSExZt7tcbIzIEoZk1Lre/line_items": { + "object": "list", + "has_more": False, + "data": [ + { + "id": "li_1OB18o2eZvKYlo2CObYam50U", + "object": "item", + "amount_discount": 0, + "amount_subtotal": 0, + "amount_tax": 0, + "amount_total": 0, + "currency": "usd", + } + ], + "link": "/v1/checkout/sessions/cs_test_a1yxusdFIgDDkWTaKn6JTYniMDBzrmnBiXH8oRSExZt7tcbIzIEoZk1Lre/line_items", + }, + "/v1/checkout/sessions/cs_test_XH8oRSExZt7tcbIzIEoZk1Lrea1yxusdFIgDDkWTaKn6JTYniMDBzrmnBi/line_items": { + "object": "list", + "has_more": False, + "url": "/v1/checkout/sessions/cs_test_XH8oRSExZt7tcbIzIEoZk1Lrea1yxusdFIgDDkWTaKn6JTYniMDBzrmnBi/line_items", + "data": [ + { + "id": "li_KYlo2CObYam50U1OB18o2eZv", + "object": "item", + "amount_discount": 0, + "amount_subtotal": 0, + "amount_tax": 0, + "amount_total": 0, + "currency": "usd", + } + ], + }, +} + + +checkout_session_events_response = { + "/v1/events": { + "data": [ + { + "id": "evt_1NdNFoEcXtiJtvvhBP5mxQmL", + "object": "event", + "api_version": "2020-08-27", + "created": 1699902016, + "data": { + "object": { + "object": "checkout_session", + "checkout_session": "cs_test_a1yxusdFIgDDkWTaKn6JTYniMDBzrmnBiXH8oRSExZt7tcbIzIEoZk1Lre", + "created": 1653341716, + "id": "cs_test_a1yxusdFIgDDkWTaKn6JTYniMDBzrmnBiXH8oRSExZt7tcbIzIEoZk1Lre", + "expires_at": 1692896410, + } + }, + "type": "checkout.session.completed", + }, + { + "id": "evt_XtiJtvvhBP5mxQmL1NdNFoEc", + "object": "event", + "api_version": "2020-08-27", + "created": 1699901630, + "data": { + "object": { + "object": "checkout_session", + "checkout_session": "cs_test_XH8oRSExZt7tcbIzIEoZk1Lrea1yxusdFIgDDkWTaKn6JTYniMDBzrmnBi", + "created": 1653341716, + "id": "cs_test_XH8oRSExZt7tcbIzIEoZk1Lrea1yxusdFIgDDkWTaKn6JTYniMDBzrmnBi", + "expires_at": 1692896410, + } + }, + "type": "checkout.session.completed", + }, + ], + "has_more": False, + }, +} + + +@pytest.mark.parametrize( + "requests_mock_map, stream_name, sync_mode, state, expected_slices", + ( + ( + checkout_session_api_response, + "checkout_sessions_line_items", + "full_refresh", + {}, + [ + { + "parent": { + "id": "cs_test_a1yxusdFIgDDkWTaKn6JTYniMDBzrmnBiXH8oRSExZt7tcbIzIEoZk1Lre", + "object": "checkout.session", + "created": 1699647441, + "updated": 1699647441, + "expires_at": 1699647441, + "payment_intent": "pi_1Gt0KQ2eZvKYlo2CeWXUgmhy", + "status": "open", + "line_items": { + "object": "list", + "has_more": False, + "url": "/v1/checkout/sessions", + "data": [ + { + "id": "li_1OB18o2eZvKYlo2CObYam50U", + "object": "item", + "amount_discount": 0, + "amount_subtotal": 0, + "amount_tax": 0, + "amount_total": 0, + "currency": "usd", + } + ], + }, + } + }, + { + "parent": { + "id": "cs_test_XH8oRSExZt7tcbIzIEoZk1Lrea1yxusdFIgDDkWTaKn6JTYniMDBzrmnBi", + "object": "checkout.session", + "created": 1699744164, + "updated": 1699744164, + "expires_at": 1699644174, + "payment_intent": "pi_lo2CeWXUgmhy1Gt0KQ2eZvKY", + "status": "open", + "line_items": { + "object": "list", + "has_more": False, + "url": "/v1/checkout/sessions", + "data": [ + { + "id": "li_KYlo2CObYam50U1OB18o2eZv", + "object": "item", + "amount_discount": 0, + "amount_subtotal": 0, + "amount_tax": 0, + "amount_total": 0, + "currency": "usd", + } + ], + }, + } + }, + ], + ), + ( + checkout_session_events_response, + "checkout_sessions_line_items", + "incremental", + {"checkout_session_updated": 1685898010}, + [ + { + "parent": { + "object": "checkout_session", + "checkout_session": "cs_test_a1yxusdFIgDDkWTaKn6JTYniMDBzrmnBiXH8oRSExZt7tcbIzIEoZk1Lre", + "created": 1653341716, + "id": "cs_test_a1yxusdFIgDDkWTaKn6JTYniMDBzrmnBiXH8oRSExZt7tcbIzIEoZk1Lre", + "expires_at": 1692896410, + "updated": 1699902016, + } + }, + { + "parent": { + "object": "checkout_session", + "checkout_session": "cs_test_XH8oRSExZt7tcbIzIEoZk1Lrea1yxusdFIgDDkWTaKn6JTYniMDBzrmnBi", + "created": 1653341716, + "updated": 1699901630, + "id": "cs_test_XH8oRSExZt7tcbIzIEoZk1Lrea1yxusdFIgDDkWTaKn6JTYniMDBzrmnBi", + "expires_at": 1692896410, + } + }, + ], + ), + ), +) +@freezegun.freeze_time("2023-08-23T15:00:15") +def test_parent_incremental_substream_stream_slices( + requests_mock, requests_mock_map, stream_by_name, stream_name, sync_mode, state, expected_slices +): + for url, response in requests_mock_map.items(): + requests_mock.get(url, json=response) + + stream = stream_by_name(stream_name) + slices = stream.stream_slices(sync_mode, stream_state=state) + assert list(slices) == expected_slices + + +checkout_session_line_items_slice_to_record_data_map = { + "id": "checkout_session_id", + "expires_at": "checkout_session_expires_at", + "created": "checkout_session_created", + "updated": "checkout_session_updated", +} + + +@pytest.mark.parametrize( + "requests_mock_map, stream_name, sync_mode, state, mapped_fields", + ( + ( + {**checkout_session_api_response, **checkout_session_line_items_api_response}, + "checkout_sessions_line_items", + "full_refresh", + {}, + checkout_session_line_items_slice_to_record_data_map, + ), + ( + {**checkout_session_events_response, **checkout_session_line_items_api_response}, + "checkout_sessions_line_items", + "incremental", + {"checkout_session_updated": 1685898010}, + checkout_session_line_items_slice_to_record_data_map, + ), + ), +) +def test_parent_incremental_substream_records_contain_data_from_slice( + requests_mock, requests_mock_map, stream_by_name, stream_name, sync_mode, state, mapped_fields +): + for url, response in requests_mock_map.items(): + requests_mock.get(url, json=response) + + stream = stream_by_name(stream_name) + for slice_ in stream.stream_slices(sync_mode, stream_state=state): + for record in stream.read_records(sync_mode, stream_slice=slice_, stream_state=state): + for key, value in mapped_fields.items(): + assert slice_["parent"][key] == record[value] + + +@pytest.mark.parametrize( + "requests_mock_map, stream_name, state", + ( + ( + { + "/v1/events": ( + { + "data": [ + { + "id": "evt_1NdNFoEcXtiJtvvhBP5mxQmL", + "object": "event", + "api_version": "2020-08-27", + "created": 1699902016, + "data": { + "object": { + "object": "checkout_session", + "checkout_session": "cs_1K9GK0EcXtiJtvvhSo2LvGqT", + "created": 1653341716, + "id": "cs_1K9GK0EcXtiJtvvhSo2LvGqT", + "expires_at": 1692896410, + } + }, + "type": "checkout.session.completed", + } + ], + "has_more": False, + }, + 200, + ), + "/v1/checkout/sessions/cs_1K9GK0EcXtiJtvvhSo2LvGqT/line_items": ({}, 404), + }, + "checkout_sessions_line_items", + {"checkout_session_updated": 1686934810}, + ), + ), +) +@freezegun.freeze_time("2023-08-23T15:00:15") +def test_parent_incremental_substream_handles_404(requests_mock, requests_mock_map, stream_by_name, stream_name, state, caplog): + for url, (response, status) in requests_mock_map.items(): + requests_mock.get(url, json=response, status_code=status) + + stream = stream_by_name(stream_name) + records = read_from_stream(stream, "incremental", state) + assert records == [] + assert "Data was not found for URL" in caplog.text diff --git a/docs/integrations/sources/stripe-migrations.md b/docs/integrations/sources/stripe-migrations.md index 5dc7fa19f9b5..60f4be4d4ab4 100644 --- a/docs/integrations/sources/stripe-migrations.md +++ b/docs/integrations/sources/stripe-migrations.md @@ -1,5 +1,20 @@ # Stripe Migration Guide +## Upgrading to 5.0.0 + +This change fixes multiple incremental sync issues with the `Refunds`, `Checkout Sessions` and `Checkout Sessions Line Items` streams: + - `Refunds` stream was not syncing data in the incremental sync mode. Cursor field has been updated to "created" to allow for incremental syncs. Because of the changed cursor field of the `Refunds` stream, incremental syncs will not reflect every update of the records that have been previously replicated. Only newly created records will be synced. To always have the up-to-date data, users are encouraged to make use of the lookback window. + - `CheckoutSessions` stream had been missing data for one day when using the incremental sync mode after a reset; this has been resolved. + - `CheckoutSessionsLineItems` previously had potential data loss. It has been updated to use a new cursor field `checkout_session_updated`. + - Incremental streams with the `created` cursor had been duplicating some data; this has been fixed. + +Stream schema update is a breaking change as well as changing the cursor field for the `Refunds` and the `CheckoutSessionsLineItems` stream. A schema refresh and data reset of all effected streams is required after the update is applied. + +Also, this update affects three more streams: `Invoices`, `Subscriptions`, `SubscriptionSchedule`. Schemas are changed in this update so that the declared data types would match the actual data. + +Stream schema update is a breaking change as well as changing the cursor field for the `Refunds` and the `CheckoutSessionsLineItems` stream. A schema refresh and data reset of all effected streams is required after the update is applied. +Because of the changed cursor field of the `Refunds` stream, incremental syncs will not reflect every update of the records that have been previously replicated. Only newly created records will be synced. To always have the up-to-date data, users are encouraged to make use of the lookback window. + ## Upgrading to 4.0.0 A major update of most streams to support event-based incremental sync mode. This allows the connector to pull not only the newly created data since the last sync, but the modified data as well. diff --git a/docs/integrations/sources/stripe.md b/docs/integrations/sources/stripe.md index bde9f9d6d0e6..3f949c6847e6 100644 --- a/docs/integrations/sources/stripe.md +++ b/docs/integrations/sources/stripe.md @@ -36,7 +36,7 @@ For more information on Stripe API Keys, see the [Stripe documentation](https:// 5. For **Account ID**, enter your Stripe Account ID. This ID begins with `acct_`, and can be found in the top-right corner of your Stripe [account settings page](https://dashboard.stripe.com/settings/account). 6. For **Secret Key**, enter the restricted key you created for the connection. 7. For **Replication Start Date**, use the provided datepicker or enter a UTC date and time programmatically in the format `YYYY-MM-DDTHH:mm:ssZ`. The data added on and after this date will be replicated. -8. (Optional) For **Lookback Window**, you may specify a number of days from the present day to reread data. This allows the connector to retrieve data that might have been updated after its initial creation, and is useful for handling any post-transaction adjustments. This applies only to streams that do not support event-based incremental syncs, please see the list below. +8. (Optional) For **Lookback Window**, you may specify a number of days from the present day to reread data. This allows the connector to retrieve data that might have been updated after its initial creation, and is useful for handling any post-transaction adjustments. This applies only to streams that do not support event-based incremental syncs, please see [the list below](#troubleshooting). - Leaving the **Lookback Window** at its default value of 0 means Airbyte will not re-export data after it has been synced. - Setting the **Lookback Window** to 1 means Airbyte will re-export data from the past day, capturing any changes made in the last 24 hours. @@ -143,26 +143,27 @@ Please be aware: this also means that any change older than 30 days will not be ::: ### Troubleshooting -:::note + Since the Stripe API does not allow querying objects which were updated since the last sync, the Stripe connector uses the Events API under the hood to implement incremental syncs and export data based on its update date. However, not all the entities are supported by the Events API, so the Stripe connector uses the `created` field or its analogue to query for new data in your Stripe account. These are the entities synced based on the date of creation: -- `BalanceTransactions` -- `CheckoutSessionLineItems` (cursor field is `checkout_session_expires_at`) +- `Balance Transactions` - `Events` -- `FileLinks` +- `File Links` - `Files` -- `SetupAttempts` -- `ShippingRates` +- `Refunds` +- `Setup Attempts` +- `Shipping Rates` On the other hand, the following streams use the `updated` field value as a cursor: - `Application Fees` - `Application Fee Refunds` - `Authorizations` -- `Bank accounts` +- `Bank Accounts` - `Cardholders` - `Cards` - `Charges` - `Checkout Sessions` +- `Checkout Session Line Items` (cursor field is `checkout_session_updated`) - `Coupons` - `Credit Notes` - `Customer Balance Transactions` @@ -180,7 +181,6 @@ On the other hand, the following streams use the `updated` field value as a curs - `Plans` - `Prices` - `Products` -- `Refunds` - `Reviews` - `Setup Intents` - `Subscription Schedule` @@ -188,9 +188,9 @@ On the other hand, the following streams use the `updated` field value as a curs - `Top Ups` - `Transactions` - `Transfers` -::: -:::note +## Incremental deletes + The Stripe API also provides a way to implement incremental deletes for a limited number of streams: - `Bank Accounts` - `Coupons` @@ -206,98 +206,99 @@ The Stripe API also provides a way to implement incremental deletes for a limite - `Subscriptions` Each record is marked with `is_deleted` flag when the appropriate event happens upstream. -::: - * Check out common troubleshooting issues for the Stripe source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). +### Data type mapping + ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| -| 4.5.4 | 2023-11-16 | [32284](https://github.com/airbytehq/airbyte/pull/32284/) | Enable client-side rate limiting | -| 4.5.3 | 2023-11-14 | [32473](https://github.com/airbytehq/airbyte/pull/32473/) | Have all full_refresh stream syncs be concurrent | -| 4.5.2 | 2023-11-03 | [32146](https://github.com/airbytehq/airbyte/pull/32146/) | Fix multiple BankAccount issues | -| 4.5.1 | 2023-11-01 | [32056](https://github.com/airbytehq/airbyte/pull/32056/) | Use CDK version 0.52.8 | -| 4.5.0 | 2023-10-25 | [31327](https://github.com/airbytehq/airbyte/pull/31327/) | Use concurrent CDK when running in full-refresh | -| 4.4.2 | 2023-10-24 | [31764](https://github.com/airbytehq/airbyte/pull/31764) | Base image migration: remove Dockerfile and use the python-connector-base image | -| 4.4.1 | 2023-10-18 | [31553](https://github.com/airbytehq/airbyte/pull/31553) | Adjusted `Setup Attempts` and extended `Checkout Sessions` stream schemas | -| 4.4.0 | 2023-10-04 | [31046](https://github.com/airbytehq/airbyte/pull/31046) | Added margins field to invoice_line_items stream. | -| 4.3.1 | 2023-09-27 | [30800](https://github.com/airbytehq/airbyte/pull/30800) | Handle permission issues a non breaking | -| 4.3.0 | 2023-09-26 | [30752](https://github.com/airbytehq/airbyte/pull/30752) | Do not sync upcoming invoices, extend stream schemas | -| 4.2.0 | 2023-09-21 | [30660](https://github.com/airbytehq/airbyte/pull/30660) | Fix updated state for the incremental syncs | -| 4.1.1 | 2023-09-15 | [30494](https://github.com/airbytehq/airbyte/pull/30494) | Fix datatype of invoices.lines property | -| 4.1.0 | 2023-08-29 | [29950](https://github.com/airbytehq/airbyte/pull/29950) | Implement incremental deletes, add suggested streams | -| 4.0.1 | 2023-09-07 | [30254](https://github.com/airbytehq/airbyte/pull/30254) | Fix cursorless incremental streams | -| 4.0.0 | 2023-08-15 | [29330](https://github.com/airbytehq/airbyte/pull/29330) | Implement incremental syncs based on date of update | -| 3.17.4 | 2023-08-15 | [29425](https://github.com/airbytehq/airbyte/pull/29425) | Revert 3.17.3 | -| 3.17.3 | 2023-08-01 | [28911](https://github.com/airbytehq/airbyte/pull/28911) | Revert 3.17.2 and fix atm_fee property | -| 3.17.2 | 2023-08-01 | [28911](https://github.com/airbytehq/airbyte/pull/28911) | Fix stream schemas, remove custom 403 error handling | -| 3.17.1 | 2023-08-01 | [28887](https://github.com/airbytehq/airbyte/pull/28887) | Fix `Invoices` schema | -| 3.17.0 | 2023-07-28 | [26127](https://github.com/airbytehq/airbyte/pull/26127) | Add `Prices` stream | -| 3.16.0 | 2023-07-27 | [28776](https://github.com/airbytehq/airbyte/pull/28776) | Add new fields to stream schemas | -| 3.15.0 | 2023-07-09 | [28709](https://github.com/airbytehq/airbyte/pull/28709) | Remove duplicate streams | -| 3.14.0 | 2023-07-09 | [27217](https://github.com/airbytehq/airbyte/pull/27217) | Add `ShippingRates` stream | -| 3.13.0 | 2023-07-18 | [28466](https://github.com/airbytehq/airbyte/pull/28466) | Pin source API version | -| 3.12.0 | 2023-05-20 | [26208](https://github.com/airbytehq/airbyte/pull/26208) | Add new stream `Persons` | -| 3.11.0 | 2023-06-26 | [27734](https://github.com/airbytehq/airbyte/pull/27734) | License Update: Elv2 stream | -| 3.10.0 | 2023-06-22 | [27132](https://github.com/airbytehq/airbyte/pull/27132) | Add `CreditNotes` stream | -| 3.9.1 | 2023-06-20 | [27522](https://github.com/airbytehq/airbyte/pull/27522) | Fix formatting | -| 3.9.0 | 2023-06-19 | [27362](https://github.com/airbytehq/airbyte/pull/27362) | Add new Streams: Transfer Reversals, Setup Attempts, Usage Records, Transactions | -| 3.8.0 | 2023-06-12 | [27238](https://github.com/airbytehq/airbyte/pull/27238) | Add `Topups` stream; Add `Files` stream; Add `FileLinks` stream | -| 3.7.0 | 2023-06-06 | [27083](https://github.com/airbytehq/airbyte/pull/27083) | Add new Streams: Authorizations, Cardholders, Cards, Payment Methods, Reviews | -| 3.6.0 | 2023-05-24 | [25893](https://github.com/airbytehq/airbyte/pull/25893) | Add `ApplicationFeesRefunds` stream with parent `ApplicationFees` | -| 3.5.0 | 2023-05-20 | [22859](https://github.com/airbytehq/airbyte/pull/22859) | Add stream `Early Fraud Warnings` | -| 3.4.3 | 2023-05-10 | [25965](https://github.com/airbytehq/airbyte/pull/25965) | Fix Airbyte date-time data-types | -| 3.4.2 | 2023-05-04 | [25795](https://github.com/airbytehq/airbyte/pull/25795) | Added `CDK TypeTransformer` to guarantee declared JSON Schema data-types | -| 3.4.1 | 2023-04-24 | [23389](https://github.com/airbytehq/airbyte/pull/23389) | Add `customer_tax_ids` to `Invoices` | -| 3.4.0 | 2023-03-20 | [23963](https://github.com/airbytehq/airbyte/pull/23963) | Add `SetupIntents` stream | -| 3.3.0 | 2023-04-12 | [25136](https://github.com/airbytehq/airbyte/pull/25136) | Add stream `Accounts` | -| 3.2.0 | 2023-04-10 | [23624](https://github.com/airbytehq/airbyte/pull/23624) | Add new stream `Subscription Schedule` | -| 3.1.0 | 2023-03-10 | [19906](https://github.com/airbytehq/airbyte/pull/19906) | Expand `tiers` when syncing `Plans` streams | -| 3.0.5 | 2023-03-25 | [22866](https://github.com/airbytehq/airbyte/pull/22866) | Specified date formatting in specification | -| 3.0.4 | 2023-03-24 | [24471](https://github.com/airbytehq/airbyte/pull/24471) | Fix stream slices for single sliced streams | -| 3.0.3 | 2023-03-17 | [24179](https://github.com/airbytehq/airbyte/pull/24179) | Get customer's attributes safely | -| 3.0.2 | 2023-03-13 | [24051](https://github.com/airbytehq/airbyte/pull/24051) | Cache `customers` stream; Do not request transactions of customers with zero balance. | -| 3.0.1 | 2023-02-22 | [22898](https://github.com/airbytehq/airbyte/pull/22898) | Add missing column to Subscriptions stream | -| 3.0.0 | 2023-02-21 | [23295](https://github.com/airbytehq/airbyte/pull/23295) | Fix invoice schema | -| 2.0.0 | 2023-02-14 | [22312](https://github.com/airbytehq/airbyte/pull/22312) | Another fix of `Invoices` stream schema + Remove http urls from openapi_spec.json | -| 1.0.2 | 2023-02-09 | [22659](https://github.com/airbytehq/airbyte/pull/22659) | Set `AvailabilityStrategy` for all streams | -| 1.0.1 | 2023-01-27 | [22042](https://github.com/airbytehq/airbyte/pull/22042) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 1.0.0 | 2023-01-25 | [21858](https://github.com/airbytehq/airbyte/pull/21858) | Update the `Subscriptions` and `Invoices` stream schemas | -| 0.1.40 | 2022-10-20 | [18228](https://github.com/airbytehq/airbyte/pull/18228) | Update the `PaymentIntents` stream schema | -| 0.1.39 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream states. | -| 0.1.38 | 2022-09-09 | [16537](https://github.com/airbytehq/airbyte/pull/16537) | Fix `redeem_by` field type for `customers` stream | -| 0.1.37 | 2022-08-16 | [15686](https://github.com/airbytehq/airbyte/pull/15686) | Fix the bug when the stream couldn't be fetched due to limited permission set, if so - it should be skipped | -| 0.1.36 | 2022-08-04 | [15292](https://github.com/airbytehq/airbyte/pull/15292) | Implement slicing | -| 0.1.35 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from spec and schema | -| 0.1.34 | 2022-07-01 | [14357](https://github.com/airbytehq/airbyte/pull/14357) | Add external account streams - | -| 0.1.33 | 2022-06-06 | [13449](https://github.com/airbytehq/airbyte/pull/13449) | Add semi-incremental support for CheckoutSessions and CheckoutSessionsLineItems streams, fixed big in StripeSubStream, added unittests, updated docs | -| 0.1.32 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | -| 0.1.31 | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | Update connector to use a `spec.yaml` | -| 0.1.30 | 2022-03-21 | [11286](https://github.com/airbytehq/airbyte/pull/11286) | Minor corrections to documentation and connector specification | -| 0.1.29 | 2022-03-08 | [10359](https://github.com/airbytehq/airbyte/pull/10359) | Improved performance for streams with substreams: invoice_line_items, subscription_items, bank_accounts | -| 0.1.28 | 2022-02-08 | [10165](https://github.com/airbytehq/airbyte/pull/10165) | Improve 404 handling for `CheckoutSessionsLineItems` stream | -| 0.1.27 | 2021-12-28 | [9148](https://github.com/airbytehq/airbyte/pull/9148) | Fix `date`, `arrival\_date` fields | -| 0.1.26 | 2021-12-21 | [8992](https://github.com/airbytehq/airbyte/pull/8992) | Fix type `events.request` in schema | -| 0.1.25 | 2021-11-25 | [8250](https://github.com/airbytehq/airbyte/pull/8250) | Rearrange setup fields | -| 0.1.24 | 2021-11-08 | [7729](https://github.com/airbytehq/airbyte/pull/7729) | Include tax data in `checkout_sessions_line_items` stream | -| 0.1.23 | 2021-11-08 | [7729](https://github.com/airbytehq/airbyte/pull/7729) | Correct `payment_intents` schema | -| 0.1.22 | 2021-11-05 | [7345](https://github.com/airbytehq/airbyte/pull/7345) | Add 3 new streams | -| 0.1.21 | 2021-10-07 | [6841](https://github.com/airbytehq/airbyte/pull/6841) | Fix missing `start_date` argument + update json files for SAT | -| 0.1.20 | 2021-09-30 | [6017](https://github.com/airbytehq/airbyte/pull/6017) | Add lookback_window_days parameter | -| 0.1.19 | 2021-09-27 | [6466](https://github.com/airbytehq/airbyte/pull/6466) | Use `start_date` parameter in incremental streams | -| 0.1.18 | 2021-09-14 | [6004](https://github.com/airbytehq/airbyte/pull/6004) | Fix coupons and subscriptions stream schemas by removing incorrect timestamp formatting | -| 0.1.17 | 2021-09-14 | [6004](https://github.com/airbytehq/airbyte/pull/6004) | Add `PaymentIntents` stream | -| 0.1.16 | 2021-07-28 | [4980](https://github.com/airbytehq/airbyte/pull/4980) | Remove Updated field from schemas | -| 0.1.15 | 2021-07-21 | [4878](https://github.com/airbytehq/airbyte/pull/4878) | Fix incorrect percent_off and discounts data filed types | -| 0.1.14 | 2021-07-09 | [4669](https://github.com/airbytehq/airbyte/pull/4669) | Subscriptions Stream now returns all kinds of subscriptions \(including expired and canceled\) | -| 0.1.13 | 2021-07-03 | [4528](https://github.com/airbytehq/airbyte/pull/4528) | Remove regex for acc validation | -| 0.1.12 | 2021-06-08 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | -| 0.1.11 | 2021-05-30 | [3744](https://github.com/airbytehq/airbyte/pull/3744) | Fix types in schema | -| 0.1.10 | 2021-05-28 | [3728](https://github.com/airbytehq/airbyte/pull/3728) | Update data types to be number instead of int | -| 0.1.9 | 2021-05-13 | [3367](https://github.com/airbytehq/airbyte/pull/3367) | Add acceptance tests for connected accounts | -| 0.1.8 | 2021-05-11 | [3566](https://github.com/airbytehq/airbyte/pull/3368) | Bump CDK connectors | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 5.0.0 | 2023-11-14 | [32286](https://github.com/airbytehq/airbyte/pull/32286/) | Fix multiple issues regarding usage of the incremental sync mode for the `Refunds`, `CheckoutSessions`, `CheckoutSessionsLineItems` streams. Fix schemas for the streams: `Invoices`, `Subscriptions`, `SubscriptionSchedule` | +| 4.5.4 | 2023-11-16 | [32284](https://github.com/airbytehq/airbyte/pull/32284/) | Enable client-side rate limiting | +| 4.5.3 | 2023-11-14 | [32473](https://github.com/airbytehq/airbyte/pull/32473/) | Have all full_refresh stream syncs be concurrent | +| 4.5.2 | 2023-11-03 | [32146](https://github.com/airbytehq/airbyte/pull/32146/) | Fix multiple BankAccount issues | +| 4.5.1 | 2023-11-01 | [32056](https://github.com/airbytehq/airbyte/pull/32056/) | Use CDK version 0.52.8 | +| 4.5.0 | 2023-10-25 | [31327](https://github.com/airbytehq/airbyte/pull/31327/) | Use concurrent CDK when running in full-refresh | +| 4.4.2 | 2023-10-24 | [31764](https://github.com/airbytehq/airbyte/pull/31764) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 4.4.1 | 2023-10-18 | [31553](https://github.com/airbytehq/airbyte/pull/31553) | Adjusted `Setup Attempts` and extended `Checkout Sessions` stream schemas | +| 4.4.0 | 2023-10-04 | [31046](https://github.com/airbytehq/airbyte/pull/31046) | Added margins field to invoice_line_items stream. | +| 4.3.1 | 2023-09-27 | [30800](https://github.com/airbytehq/airbyte/pull/30800) | Handle permission issues a non breaking | +| 4.3.0 | 2023-09-26 | [30752](https://github.com/airbytehq/airbyte/pull/30752) | Do not sync upcoming invoices, extend stream schemas | +| 4.2.0 | 2023-09-21 | [30660](https://github.com/airbytehq/airbyte/pull/30660) | Fix updated state for the incremental syncs | +| 4.1.1 | 2023-09-15 | [30494](https://github.com/airbytehq/airbyte/pull/30494) | Fix datatype of invoices.lines property | +| 4.1.0 | 2023-08-29 | [29950](https://github.com/airbytehq/airbyte/pull/29950) | Implement incremental deletes, add suggested streams | +| 4.0.1 | 2023-09-07 | [30254](https://github.com/airbytehq/airbyte/pull/30254) | Fix cursorless incremental streams | +| 4.0.0 | 2023-08-15 | [29330](https://github.com/airbytehq/airbyte/pull/29330) | Implement incremental syncs based on date of update | +| 3.17.4 | 2023-08-15 | [29425](https://github.com/airbytehq/airbyte/pull/29425) | Revert 3.17.3 | +| 3.17.3 | 2023-08-01 | [28911](https://github.com/airbytehq/airbyte/pull/28911) | Revert 3.17.2 and fix atm_fee property | +| 3.17.2 | 2023-08-01 | [28911](https://github.com/airbytehq/airbyte/pull/28911) | Fix stream schemas, remove custom 403 error handling | +| 3.17.1 | 2023-08-01 | [28887](https://github.com/airbytehq/airbyte/pull/28887) | Fix `Invoices` schema | +| 3.17.0 | 2023-07-28 | [26127](https://github.com/airbytehq/airbyte/pull/26127) | Add `Prices` stream | +| 3.16.0 | 2023-07-27 | [28776](https://github.com/airbytehq/airbyte/pull/28776) | Add new fields to stream schemas | +| 3.15.0 | 2023-07-09 | [28709](https://github.com/airbytehq/airbyte/pull/28709) | Remove duplicate streams | +| 3.14.0 | 2023-07-09 | [27217](https://github.com/airbytehq/airbyte/pull/27217) | Add `ShippingRates` stream | +| 3.13.0 | 2023-07-18 | [28466](https://github.com/airbytehq/airbyte/pull/28466) | Pin source API version | +| 3.12.0 | 2023-05-20 | [26208](https://github.com/airbytehq/airbyte/pull/26208) | Add new stream `Persons` | +| 3.11.0 | 2023-06-26 | [27734](https://github.com/airbytehq/airbyte/pull/27734) | License Update: Elv2 stream | +| 3.10.0 | 2023-06-22 | [27132](https://github.com/airbytehq/airbyte/pull/27132) | Add `CreditNotes` stream | +| 3.9.1 | 2023-06-20 | [27522](https://github.com/airbytehq/airbyte/pull/27522) | Fix formatting | +| 3.9.0 | 2023-06-19 | [27362](https://github.com/airbytehq/airbyte/pull/27362) | Add new Streams: Transfer Reversals, Setup Attempts, Usage Records, Transactions | +| 3.8.0 | 2023-06-12 | [27238](https://github.com/airbytehq/airbyte/pull/27238) | Add `Topups` stream; Add `Files` stream; Add `FileLinks` stream | +| 3.7.0 | 2023-06-06 | [27083](https://github.com/airbytehq/airbyte/pull/27083) | Add new Streams: Authorizations, Cardholders, Cards, Payment Methods, Reviews | +| 3.6.0 | 2023-05-24 | [25893](https://github.com/airbytehq/airbyte/pull/25893) | Add `ApplicationFeesRefunds` stream with parent `ApplicationFees` | +| 3.5.0 | 2023-05-20 | [22859](https://github.com/airbytehq/airbyte/pull/22859) | Add stream `Early Fraud Warnings` | +| 3.4.3 | 2023-05-10 | [25965](https://github.com/airbytehq/airbyte/pull/25965) | Fix Airbyte date-time data-types | +| 3.4.2 | 2023-05-04 | [25795](https://github.com/airbytehq/airbyte/pull/25795) | Added `CDK TypeTransformer` to guarantee declared JSON Schema data-types | +| 3.4.1 | 2023-04-24 | [23389](https://github.com/airbytehq/airbyte/pull/23389) | Add `customer_tax_ids` to `Invoices` | +| 3.4.0 | 2023-03-20 | [23963](https://github.com/airbytehq/airbyte/pull/23963) | Add `SetupIntents` stream | +| 3.3.0 | 2023-04-12 | [25136](https://github.com/airbytehq/airbyte/pull/25136) | Add stream `Accounts` | +| 3.2.0 | 2023-04-10 | [23624](https://github.com/airbytehq/airbyte/pull/23624) | Add new stream `Subscription Schedule` | +| 3.1.0 | 2023-03-10 | [19906](https://github.com/airbytehq/airbyte/pull/19906) | Expand `tiers` when syncing `Plans` streams | +| 3.0.5 | 2023-03-25 | [22866](https://github.com/airbytehq/airbyte/pull/22866) | Specified date formatting in specification | +| 3.0.4 | 2023-03-24 | [24471](https://github.com/airbytehq/airbyte/pull/24471) | Fix stream slices for single sliced streams | +| 3.0.3 | 2023-03-17 | [24179](https://github.com/airbytehq/airbyte/pull/24179) | Get customer's attributes safely | +| 3.0.2 | 2023-03-13 | [24051](https://github.com/airbytehq/airbyte/pull/24051) | Cache `customers` stream; Do not request transactions of customers with zero balance. | +| 3.0.1 | 2023-02-22 | [22898](https://github.com/airbytehq/airbyte/pull/22898) | Add missing column to Subscriptions stream | +| 3.0.0 | 2023-02-21 | [23295](https://github.com/airbytehq/airbyte/pull/23295) | Fix invoice schema | +| 2.0.0 | 2023-02-14 | [22312](https://github.com/airbytehq/airbyte/pull/22312) | Another fix of `Invoices` stream schema + Remove http urls from openapi_spec.json | +| 1.0.2 | 2023-02-09 | [22659](https://github.com/airbytehq/airbyte/pull/22659) | Set `AvailabilityStrategy` for all streams | +| 1.0.1 | 2023-01-27 | [22042](https://github.com/airbytehq/airbyte/pull/22042) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 1.0.0 | 2023-01-25 | [21858](https://github.com/airbytehq/airbyte/pull/21858) | Update the `Subscriptions` and `Invoices` stream schemas | +| 0.1.40 | 2022-10-20 | [18228](https://github.com/airbytehq/airbyte/pull/18228) | Update the `PaymentIntents` stream schema | +| 0.1.39 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream states. | +| 0.1.38 | 2022-09-09 | [16537](https://github.com/airbytehq/airbyte/pull/16537) | Fix `redeem_by` field type for `customers` stream | +| 0.1.37 | 2022-08-16 | [15686](https://github.com/airbytehq/airbyte/pull/15686) | Fix the bug when the stream couldn't be fetched due to limited permission set, if so - it should be skipped | +| 0.1.36 | 2022-08-04 | [15292](https://github.com/airbytehq/airbyte/pull/15292) | Implement slicing | +| 0.1.35 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from spec and schema | +| 0.1.34 | 2022-07-01 | [14357](https://github.com/airbytehq/airbyte/pull/14357) | Add external account streams - | +| 0.1.33 | 2022-06-06 | [13449](https://github.com/airbytehq/airbyte/pull/13449) | Add semi-incremental support for CheckoutSessions and CheckoutSessionsLineItems streams, fixed big in StripeSubStream, added unittests, updated docs | +| 0.1.32 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | +| 0.1.31 | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | Update connector to use a `spec.yaml` | +| 0.1.30 | 2022-03-21 | [11286](https://github.com/airbytehq/airbyte/pull/11286) | Minor corrections to documentation and connector specification | +| 0.1.29 | 2022-03-08 | [10359](https://github.com/airbytehq/airbyte/pull/10359) | Improved performance for streams with substreams: invoice_line_items, subscription_items, bank_accounts | +| 0.1.28 | 2022-02-08 | [10165](https://github.com/airbytehq/airbyte/pull/10165) | Improve 404 handling for `CheckoutSessionsLineItems` stream | +| 0.1.27 | 2021-12-28 | [9148](https://github.com/airbytehq/airbyte/pull/9148) | Fix `date`, `arrival\_date` fields | +| 0.1.26 | 2021-12-21 | [8992](https://github.com/airbytehq/airbyte/pull/8992) | Fix type `events.request` in schema | +| 0.1.25 | 2021-11-25 | [8250](https://github.com/airbytehq/airbyte/pull/8250) | Rearrange setup fields | +| 0.1.24 | 2021-11-08 | [7729](https://github.com/airbytehq/airbyte/pull/7729) | Include tax data in `checkout_sessions_line_items` stream | +| 0.1.23 | 2021-11-08 | [7729](https://github.com/airbytehq/airbyte/pull/7729) | Correct `payment_intents` schema | +| 0.1.22 | 2021-11-05 | [7345](https://github.com/airbytehq/airbyte/pull/7345) | Add 3 new streams | +| 0.1.21 | 2021-10-07 | [6841](https://github.com/airbytehq/airbyte/pull/6841) | Fix missing `start_date` argument + update json files for SAT | +| 0.1.20 | 2021-09-30 | [6017](https://github.com/airbytehq/airbyte/pull/6017) | Add lookback_window_days parameter | +| 0.1.19 | 2021-09-27 | [6466](https://github.com/airbytehq/airbyte/pull/6466) | Use `start_date` parameter in incremental streams | +| 0.1.18 | 2021-09-14 | [6004](https://github.com/airbytehq/airbyte/pull/6004) | Fix coupons and subscriptions stream schemas by removing incorrect timestamp formatting | +| 0.1.17 | 2021-09-14 | [6004](https://github.com/airbytehq/airbyte/pull/6004) | Add `PaymentIntents` stream | +| 0.1.16 | 2021-07-28 | [4980](https://github.com/airbytehq/airbyte/pull/4980) | Remove Updated field from schemas | +| 0.1.15 | 2021-07-21 | [4878](https://github.com/airbytehq/airbyte/pull/4878) | Fix incorrect percent_off and discounts data filed types | +| 0.1.14 | 2021-07-09 | [4669](https://github.com/airbytehq/airbyte/pull/4669) | Subscriptions Stream now returns all kinds of subscriptions \(including expired and canceled\) | +| 0.1.13 | 2021-07-03 | [4528](https://github.com/airbytehq/airbyte/pull/4528) | Remove regex for acc validation | +| 0.1.12 | 2021-06-08 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | +| 0.1.11 | 2021-05-30 | [3744](https://github.com/airbytehq/airbyte/pull/3744) | Fix types in schema | +| 0.1.10 | 2021-05-28 | [3728](https://github.com/airbytehq/airbyte/pull/3728) | Update data types to be number instead of int | +| 0.1.9 | 2021-05-13 | [3367](https://github.com/airbytehq/airbyte/pull/3367) | Add acceptance tests for connected accounts | +| 0.1.8 | 2021-05-11 | [3566](https://github.com/airbytehq/airbyte/pull/3368) | Bump CDK connectors | From 3da9d84b03d5a650390283cc6e36305249d49dcd Mon Sep 17 00:00:00 2001 From: Stephane Geneix <147216312+stephane-airbyte@users.noreply.github.com> Date: Thu, 16 Nov 2023 10:14:27 -0800 Subject: [PATCH 089/124] bump source-mysql-strict-encrypt to 3.1.7 (#32611) --- .../connectors/source-mysql-strict-encrypt/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/source-mysql-strict-encrypt/metadata.yaml index 885c843f1f95..e7aa7fb15b86 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/metadata.yaml @@ -11,7 +11,7 @@ data: connectorSubtype: database connectorType: source definitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad - dockerImageTag: 3.1.5 + dockerImageTag: 3.1.7 dockerRepository: airbyte/source-mysql-strict-encrypt githubIssueLabel: source-mysql icon: mysql.svg From c477dc48463436edeb818c1db7edbc0fd2f3fc69 Mon Sep 17 00:00:00 2001 From: Patrick Nilan Date: Thu, 16 Nov 2023 10:29:56 -0800 Subject: [PATCH 090/124] Paypal Transaction Source: Updates QL to 200 (#32390) --- .../connectors/source-paypal-transaction/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml b/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml index b5d777e36eda..1821fdddaddc 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml +++ b/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml @@ -1,6 +1,6 @@ data: ab_internal: - ql: 400 + ql: 200 sl: 200 allowedHosts: hosts: From 8b07903f75585f5958c2f5acc926dbb5377496d2 Mon Sep 17 00:00:00 2001 From: Patrick Nilan Date: Thu, 16 Nov 2023 10:30:07 -0800 Subject: [PATCH 091/124] Chargebee Source: Updated QL in metadata (#32382) --- airbyte-integrations/connectors/source-chargebee/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-chargebee/metadata.yaml b/airbyte-integrations/connectors/source-chargebee/metadata.yaml index 52df4ecc5974..854035925e39 100644 --- a/airbyte-integrations/connectors/source-chargebee/metadata.yaml +++ b/airbyte-integrations/connectors/source-chargebee/metadata.yaml @@ -1,6 +1,6 @@ data: ab_internal: - ql: 400 + ql: 200 sl: 200 allowedHosts: hosts: From 1cc5e533ba486c340eac1caed0b03aa573290347 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Thu, 16 Nov 2023 15:56:39 -0300 Subject: [PATCH 092/124] Source Cart: fix schema and allow to use connector on Airbyte Cloud (#32517) --- .../connectors/source-cart/Dockerfile | 2 +- .../source-cart/acceptance-test-config.yml | 71 +++++++++++-------- .../connectors/source-cart/metadata.yaml | 4 +- .../source_cart/schemas/addresses.json | 5 ++ .../source_cart/schemas/customers_cart.json | 16 ++++- .../source_cart/schemas/order_items.json | 65 +++++++++++++++++ .../source_cart/schemas/order_payments.json | 2 + .../source_cart/schemas/order_statuses.json | 5 +- .../source_cart/schemas/orders.json | 19 +++++ .../source_cart/schemas/products.json | 2 + docs/integrations/sources/cart.md | 1 + 11 files changed, 158 insertions(+), 34 deletions(-) diff --git a/airbyte-integrations/connectors/source-cart/Dockerfile b/airbyte-integrations/connectors/source-cart/Dockerfile index 4323b70cdd86..d526813a96ab 100644 --- a/airbyte-integrations/connectors/source-cart/Dockerfile +++ b/airbyte-integrations/connectors/source-cart/Dockerfile @@ -21,5 +21,5 @@ COPY source_cart ./source_cart ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.1 +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/source-cart diff --git a/airbyte-integrations/connectors/source-cart/acceptance-test-config.yml b/airbyte-integrations/connectors/source-cart/acceptance-test-config.yml index 3ab3140291d3..52803b05481a 100644 --- a/airbyte-integrations/connectors/source-cart/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-cart/acceptance-test-config.yml @@ -1,38 +1,51 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-cart:dev -tests: +test_strictness_level: low +acceptance_tests: spec: - - spec_path: "source_cart/spec.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.6" + tests: + - spec_path: "source_cart/spec.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.6" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - timeout_seconds: 180 + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + timeout_seconds: 180 discovery: - - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.6" + tests: + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.6" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 1800 + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 1800 + empty_streams: + - name: "order_payments" + bypass_reason: "no data" + - name: "products" + bypass_reason: "no data" incremental: - - config_path: "secrets/config_central_api_router.json" - configured_catalog_path: "integration_tests/configured_catalog_wo_order_statuses.json" - future_state_path: "integration_tests/abnormal_state.json" - timeout_seconds: 1800 - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" - timeout_seconds: 1800 + tests: + # - config_path: "secrets/config_central_api_router.json" + # configured_catalog_path: "integration_tests/configured_catalog_wo_order_statuses.json" + # future_state_path: "integration_tests/abnormal_state.json" + # timeout_seconds: 1800 + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + timeout_seconds: 1800 full_refresh: - - config_path: "secrets/config_central_api_router.json" - configured_catalog_path: "integration_tests/configured_catalog_wo_order_statuses.json" - timeout_seconds: 1800 - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 1800 + tests: + - config_path: "secrets/config_central_api_router.json" + configured_catalog_path: "integration_tests/configured_catalog_wo_order_statuses.json" + timeout_seconds: 1800 + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 1800 diff --git a/airbyte-integrations/connectors/source-cart/metadata.yaml b/airbyte-integrations/connectors/source-cart/metadata.yaml index 8f75a5db4ceb..191e92810614 100644 --- a/airbyte-integrations/connectors/source-cart/metadata.yaml +++ b/airbyte-integrations/connectors/source-cart/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: bb1a6d31-6879-4819-a2bd-3eed299ea8e2 - dockerImageTag: 0.2.1 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-cart githubIssueLabel: source-cart icon: cart.svg @@ -10,7 +10,7 @@ data: name: Cart.com registries: cloud: - enabled: false + enabled: true oss: enabled: true releaseStage: alpha diff --git a/airbyte-integrations/connectors/source-cart/source_cart/schemas/addresses.json b/airbyte-integrations/connectors/source-cart/source_cart/schemas/addresses.json index 98fe097f1779..e7d377656a16 100644 --- a/airbyte-integrations/connectors/source-cart/source_cart/schemas/addresses.json +++ b/airbyte-integrations/connectors/source-cart/source_cart/schemas/addresses.json @@ -1,5 +1,7 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" @@ -10,6 +12,9 @@ "address_line_1": { "type": ["string", "null"] }, + "address_type": { + "type": ["string", "null"] + }, "address_line_2": { "type": ["string", "null"] }, diff --git a/airbyte-integrations/connectors/source-cart/source_cart/schemas/customers_cart.json b/airbyte-integrations/connectors/source-cart/source_cart/schemas/customers_cart.json index 8520252b4485..23c4e341dce3 100644 --- a/airbyte-integrations/connectors/source-cart/source_cart/schemas/customers_cart.json +++ b/airbyte-integrations/connectors/source-cart/source_cart/schemas/customers_cart.json @@ -1,5 +1,7 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" @@ -7,6 +9,12 @@ "customer_number": { "type": ["string", "null"] }, + "credit_limit": { + "type": ["string", "null"] + }, + "payment_net_term": { + "type": ["string", "null"] + }, "last_name": { "type": ["string", "null"] }, @@ -38,7 +46,13 @@ "type": ["integer", "null"] }, "is_no_tax_customer": { - "type": "boolean" + "type": ["boolean", "null"] + }, + "is_inactive": { + "type": ["boolean", "null"] + }, + "lock_default_address": { + "type": ["boolean", "null"] }, "comments": { "type": ["string", "null"] diff --git a/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_items.json b/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_items.json index e803c78c5ac1..b7223e79fd55 100644 --- a/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_items.json +++ b/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_items.json @@ -1,5 +1,7 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" @@ -81,6 +83,69 @@ }, "warehouse_id": { "type": ["integer", "null"] + }, + "configuration": { + "type": ["string", "null"] + }, + "description": { + "type": ["string", "null"] + }, + "discount_amount": { + "type": ["number", "null"] + }, + "discount_percentage": { + "type": ["number", "null"] + }, + "fitment": { + "type": ["string", "null"] + }, + "is_non_shipping_item": { + "type": ["boolean", "null"] + }, + "item_number_full": { + "type": ["string", "null"] + }, + "order_shipping_address_id": { + "type": ["string", "null"] + }, + "personalizations": { + "type": ["array", "null"] + }, + "selected_shipping_method": { + "type": ["string", "null"] + }, + "selected_shipping_method_id": { + "type": ["string", "null"] + }, + "selected_shipping_provider_service": { + "type": ["string", "null"] + }, + "shipping_total": { + "type": ["string", "null"] + }, + "status": { + "type": ["string", "null"] + }, + "tax": { + "type": ["number", "null"] + }, + "tax_code": { + "type": ["string", "null"] + }, + "variant_inventory_id": { + "type": ["string", "null"] + }, + "shipping_classification_code": { + "type": ["string", "null"] + }, + "variants": { + "type": ["array", "null"] + }, + "vendor_store_id": { + "type": ["integer", "null"] + }, + "weight_unit": { + "type": ["string", "null"] } } } diff --git a/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_payments.json b/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_payments.json index f4dee9743008..ab2f2c844d71 100644 --- a/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_payments.json +++ b/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_payments.json @@ -1,5 +1,7 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" diff --git a/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_statuses.json b/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_statuses.json index eb7182c2f368..b77422eb2f54 100644 --- a/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_statuses.json +++ b/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_statuses.json @@ -1,5 +1,7 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" }, "name": { "type": ["null", "string"] }, @@ -13,6 +15,7 @@ "created_at": { "type": ["null", "string"] }, "is_fully_refunded": { "type": ["null", "boolean"] }, "is_partially_refunded": { "type": ["null", "boolean"] }, - "is_quote_status": { "type": ["null", "boolean"] } + "is_quote_status": { "type": ["null", "boolean"] }, + "is_partially_shipped": { "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-cart/source_cart/schemas/orders.json b/airbyte-integrations/connectors/source-cart/source_cart/schemas/orders.json index e5e7091efda4..f1ebdb8b5b9d 100644 --- a/airbyte-integrations/connectors/source-cart/source_cart/schemas/orders.json +++ b/airbyte-integrations/connectors/source-cart/source_cart/schemas/orders.json @@ -1,5 +1,7 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" @@ -7,6 +9,23 @@ "customer_id": { "type": ["integer", "null"] }, + "delivery_tax": { + "type": ["string", "null"] + }, + "entered_by_type": { + "type": ["string", "null"] + }, + "shipping_selections": { + "type": ["array", "null"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} + } + }, + "sales_agent_user_id": { + "type": ["string", "null"] + }, "customer_type_id": { "type": ["integer", "null"] }, diff --git a/airbyte-integrations/connectors/source-cart/source_cart/schemas/products.json b/airbyte-integrations/connectors/source-cart/source_cart/schemas/products.json index ed1473eb08a6..5d0ac08fa31a 100644 --- a/airbyte-integrations/connectors/source-cart/source_cart/schemas/products.json +++ b/airbyte-integrations/connectors/source-cart/source_cart/schemas/products.json @@ -1,5 +1,7 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" diff --git a/docs/integrations/sources/cart.md b/docs/integrations/sources/cart.md index 0559e6c7f3a3..90a618b956c6 100644 --- a/docs/integrations/sources/cart.md +++ b/docs/integrations/sources/cart.md @@ -50,6 +50,7 @@ Please follow these [steps](https://developers.cart.com/docs/rest-api/docs/READM | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------- | +| 0.3.0 | 2023-11-14 | [23317](https://github.com/airbytehq/airbyte/pull/23317) | Update schemas | | 0.2.1 | 2023-02-22 | [23317](https://github.com/airbytehq/airbyte/pull/23317) | Remove support for incremental for `order_statuses` stream | | 0.2.0 | 2022-09-21 | [16612](https://github.com/airbytehq/airbyte/pull/16612) | Source Cart.com: implement Central API Router access method and improve backoff policy | | 0.1.6 | 2022-07-15 | [14752](https://github.com/airbytehq/airbyte/pull/14752) | Add `order_statuses` stream | From b3f474c3ecd288b35e5022caa2b278d49c2518b6 Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Thu, 16 Nov 2023 14:54:53 -0800 Subject: [PATCH 093/124] Update License documentation (#32575) --- LICENSE | 26 ++++++----- docs/project-overview/licenses/license-faq.md | 46 +++++++++++++------ 2 files changed, 47 insertions(+), 25 deletions(-) diff --git a/LICENSE b/LICENSE index 814fd88f57f3..0df58b4829be 100644 --- a/LICENSE +++ b/LICENSE @@ -1,14 +1,17 @@ Airbyte monorepo uses multiple licenses. The license for a particular work is defined with following prioritized rules: + 1. License directly present in the file 2. LICENSE file in the same directory as the work -3. First LICENSE found when exploring parent directories up to the project top level directory -4. Defaults to Elastic License 2.0 +3. A `license` property defined in the `metadata.yaml` configuration file found when exploring parent directories (most connectors) +4. First LICENSE found when exploring parent directories up to the project top level directory +5. Defaults to Elastic License 2.0 If you have any question regarding licenses, just visit our [FAQ](https://airbyte.io/license-faq) or [contact us](mailto:license@airbyte.io). ------------------------------------------------------------------------------------- +--- + MIT License Copyright (c) 2020 Airbyte, Inc. @@ -31,7 +34,8 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------------------------------------------------------------------------------------- +--- + Elastic License 2.0 (ELv2) **Acceptance** @@ -65,16 +69,16 @@ If you use the software in violation of these terms, such use is not licensed, a As far as the law allows, the software comes as is, without any warranty or condition, and the licensor will not be liable to you for any damages arising out of these terms or the use or nature of the software, under any kind of legal claim. **Definitions** -The *licensor* is the entity offering these terms, and the *software* is the software the licensor makes available under these terms, including any portion of it. +The _licensor_ is the entity offering these terms, and the _software_ is the software the licensor makes available under these terms, including any portion of it. -*you* refers to the individual or entity agreeing to these terms. +_you_ refers to the individual or entity agreeing to these terms. -*your company* is any legal entity, sole proprietorship, or other kind of organization that you work for, plus all organizations that have control over, are under the control of, or are under common control with that organization. *control* means ownership of substantially all the assets of an entity, or the power to direct its management and policies by vote, contract, or otherwise. Control can be direct or indirect. +_your company_ is any legal entity, sole proprietorship, or other kind of organization that you work for, plus all organizations that have control over, are under the control of, or are under common control with that organization. _control_ means ownership of substantially all the assets of an entity, or the power to direct its management and policies by vote, contract, or otherwise. Control can be direct or indirect. -*your licenses* are all the licenses granted to you for the software under these terms. +_your licenses_ are all the licenses granted to you for the software under these terms. -*use* means anything you do with the software requiring one of your licenses. +_use_ means anything you do with the software requiring one of your licenses. -*trademark* means trademarks, service marks, and similar rights. +_trademark_ means trademarks, service marks, and similar rights. ------------------------------------------------------------------------------------- +--- diff --git a/docs/project-overview/licenses/license-faq.md b/docs/project-overview/licenses/license-faq.md index 837ae5a5fd3d..6865094e4ba4 100644 --- a/docs/project-overview/licenses/license-faq.md +++ b/docs/project-overview/licenses/license-faq.md @@ -1,16 +1,19 @@ # License FAQ ## Airbyte Licensing Overview -* **Airbyte Connectors** are open sourced and available under the MIT License. -* **Airbyte Protocol** is open sourced and available under the MIT License. -* **Airbyte CDK** (Connector Development Kit) is open sourced and available under the MIT License. -* **Airbyte Core** is licensed under the Elastic License 2.0 (ELv2). -* **Airbyte Cloud & Airbyte Enterprise** are both closed source and require a commercial license from Airbyte. + +- **Airbyte Connectors** are open sourced and available under the [MIT](https://opensource.org/license/mit/) or [Elastic License 2.0 (ELv2)](https://www.elastic.co/licensing/elastic-license/faq) License. Each connector's `metadata.yaml` file contains more information. +- **Airbyte Protocol** is open sourced and available under the MIT License. +- **Airbyte CDK** (Connector Development Kit) is open sourced and available under the MIT License. +- **Airbyte Core** is licensed under the Elastic License 2.0 (ELv2). +- **Airbyte Cloud & Airbyte Enterprise** are both closed source and require a commercial license from Airbyte. ![Diagram of license structure](../../.gitbook/assets/license_faq_diagram.png) ## About Elastic License 2.0 (ELv2) + ELv2 is a simple, non-copyleft license, allowing for the right to “use, copy, distribute, make available, and prepare derivative works of the software”. Anyone can use Airbyte, free of charge. You can run the software at scale on your infrastructure. There are only three high-level limitations. You cannot: + 1. Provide the products to others as a managed service ([read more](#what-is-the-managed-service-use-case-that-is-not-allowed-under-elv2)); 2. Circumvent the license key functionality or remove/obscure features protected by license keys; or 3. Remove or obscure any licensing, copyright, or other notices. @@ -20,60 +23,75 @@ In case you want to work with Airbyte without these limitations, we offer altern [View License](elv2-license.md) ## FAQ + ### What limitations does ELv2 impose on my use of Airbyte? + If you are an Airbyte Cloud customer, nothing changes for you. For open-source users, everyone can continue to use Airbyte as they are doing today: no limitations on volume, number of users, number of connections… There are only a few high-level limitations. You cannot: + 1. Provide the products to others as a managed service. For example, you cannot sell a cloud service that provides users with direct access to Airbyte. You can sell access to applications built and run using Airbyte ([read more](#what-is-the-managed-service-use-case-that-is-not-allowed-under-elv2)). 2. Circumvent the license key functionality or remove/obscure features protected by license keys. For example, our code may contain watermarks or keys to unlock proprietary functionality. Those elements of our code will be marked in our source code. You can’t remove or change them. ### Why did Airbyte adopt ELv2? + We are releasing Airbyte Cloud, a managed version of Airbyte that will offer alternatives to how our users operate Airbyte, including additional features and new execution models. We want to find a great way to execute our mission to commoditize data integration with open source and our ambition to create a sustainable business. -ELv2 gives us the best of both worlds. +ELv2 gives us the best of both worlds. On one hand, our users can continue to use Airbyte freely, and on the other hand, we can safely create a sustainable business and continue to invest in our community, project and product. We don’t have to worry about other large companies taking the product to monetize it for themselves, thus hurting our community. ### Will Airbyte connectors continue to be open source? + Our own connectors remain open-source, and our contributors can also develop their own connectors and continue to choose whichever license they prefer. This is our way to accomplish Airbyte’s vision of commoditizing data integration: access to data shouldn’t be behind a paywall. Also, we want Airbyte’s licensing to work well with applications that are integrated using connectors. We are continuously investing in Airbyte's data protocol and all the tooling around it. The Connector Development Kit (CDK), which helps our community and our team build and maintain connectors at scale, is a cornerstone of our commoditization strategy and also remains open-source. ### How do I continue to contribute to Airbyte under ELv2? + Airbyte’s projects are available here. Anyone can contribute to any of these projects (including those licensed with ELv2). We are introducing a Contributor License Agreement that you will have to sign with your first contribution. ### When will ELv2 be effective? + ELv2 will apply from the following Airbyte core version as of September 27, 2021: version 0.30.0. ### What is the “managed service” use case that is not allowed under ELv2? -We chose ELv2 because it is very permissive with what you can do with the software. + +We chose ELv2 because it is very permissive with what you can do with the software. You can basically build ANY product on top of Airbyte as long as you don’t: -* Host Airbyte yourself and sell it as an ELT/ETL tool, or a replacement for the Airbyte solution. -* Sell a product that directly exposes Airbyte’s UI or API. + +- Host Airbyte yourself and sell it as an ELT/ETL tool, or a replacement for the Airbyte solution. +- Sell a product that directly exposes Airbyte’s UI or API. Here is a non-exhaustive list of what you can do (without providing your customers direct access to Airbyte functionality): -* I am creating an analytics platform and I want to use Airbyte to bring data in on behalf of my customers. -* I am building my internal data stack and I want my team to be able to interact with Airbyte to configure the pipelines through the UI or the API. -* ... + +- I am creating an analytics platform and I want to use Airbyte to bring data in on behalf of my customers. +- I am building my internal data stack and I want my team to be able to interact with Airbyte to configure the pipelines through the UI or the API. +- ... ### My company has a policy against using code that restricts commercial use – can I still use Airbyte under ELv2? -You can use software under ELv2 for your commercial business, you simply cannot offer it as a managed service. + +You can use software under ELv2 for your commercial business, you simply cannot offer it as a managed service. ### As a Data Agency, I currently use Airbyte to fulfill my customer needs. How does ELv2 affect me? + You can continue to use Airbyte, as long as you don’t offer it as a managed service. ### I started to use Airbyte to ingest my customer’s data. What should I do? + You can continue to use Airbyte, as long as you don’t offer it as a managed service. ### Can I customize ELv2 software? + Yes, you can customize ELv2 software. ELv2 is similar in this sense to permissive open-source licenses. You can modify the software, integrate the variant into your application, and operate the modified application, as long as you don’t go against any of the limitations. ### Why didn’t you use a closed-source license for Airbyte Core? + We want to provide developers with free access to our Airbyte Core source code — including rights to modify it. Since this wouldn’t be possible with a closed-source license, we decided to use the more permissive ELv2. ### Is there any revenue sharing for those who create Airbyte connectors? -We will be introducing a new participative model in the next few months. There are still a lot of details to figure out, but the general idea is that maintainers of connectors would have the option to obtain a share of revenue when the connectors are being used in the paid version of Airbyte. In exchange, maintainers would be responsible for SLAs, new features, and bug fixes for the said connector. +We will be introducing a new participative model in the next few months. There are still a lot of details to figure out, but the general idea is that maintainers of connectors would have the option to obtain a share of revenue when the connectors are being used in the paid version of Airbyte. In exchange, maintainers would be responsible for SLAs, new features, and bug fixes for the said connector. From ddd233bcb3b376c2f644c25534c3c4f8630ca68f Mon Sep 17 00:00:00 2001 From: Alexandre Cuoci Date: Thu, 16 Nov 2023 18:09:28 -0500 Subject: [PATCH 094/124] self-managed docs rewrite (#32605) --- .../on-kubernetes-via-helm.md | 35 ------ .../assets}/okta-app-integration-name.png | Bin .../okta-create-new-app-integration.png | Bin .../assets}/okta-login-redirect-uris.png | Bin docs/enterprise-setup/self-managed/README.md | 9 ++ .../self-managed/implementation-guide.md | 103 ++++++++++++++++++ .../self-managed/sso.md} | 25 ++--- docusaurus/redirects.yml | 4 +- docusaurus/sidebars.js | 19 +++- 9 files changed, 142 insertions(+), 53 deletions(-) rename docs/{assets/docs => enterprise-setup/assets}/okta-app-integration-name.png (100%) rename docs/{assets/docs => enterprise-setup/assets}/okta-create-new-app-integration.png (100%) rename docs/{assets/docs => enterprise-setup/assets}/okta-login-redirect-uris.png (100%) create mode 100644 docs/enterprise-setup/self-managed/README.md create mode 100644 docs/enterprise-setup/self-managed/implementation-guide.md rename docs/{airbyte-enterprise.md => enterprise-setup/self-managed/sso.md} (54%) diff --git a/docs/deploying-airbyte/on-kubernetes-via-helm.md b/docs/deploying-airbyte/on-kubernetes-via-helm.md index 12f192db3174..818dec3f78f5 100644 --- a/docs/deploying-airbyte/on-kubernetes-via-helm.md +++ b/docs/deploying-airbyte/on-kubernetes-via-helm.md @@ -122,41 +122,6 @@ After specifying your own configuration, run the following command: helm install --values path/to/values.yaml %release_name% airbyte/airbyte ``` -### (Early Access) Airbyte Enterprise deployment - -[Airbyte Enterprise](/airbyte-enterprise) is in an early access stage for select priority users. Once you [are qualified for an Airbyte Enterprise license key](https://airbyte.com/company/talk-to-sales), you can install Airbyte Enterprise via helm by following these steps: - -1. Checkout the latest revision of the [airbyte-platform repository](https://github.com/airbytehq/airbyte-platform) - -2. Add your Airbyte Enterprise license key and [auth configuration details](/airbyte-enterprise#single-sign-on-sso) to a file called `airbyte.yml` in the `configs` directory of `airbyte-platform`. You can copy `airbyte.sample.yml` to use as a template: - -```sh -cp configs/airbyte.sample.yml configs/airbyte.yml -``` - -Then, open up `airbyte.yml` in your text editor to fill in the indicated fields. - -:::caution - -For now, auth configurations aren't easy to modify once initially installed, so please double check them to make sure they're accurate before proceeding! This will be improved in the near future. - -::: - -3. Make sure your helm repository is up to date: - -```text -helm repo update -``` - -4. Install Airbyte Enterprise on helm using the following command: - -```text -./tools/bin/install_airbyte_pro_on_helm.sh -``` - -The default release name is `airbyte-pro`. You can change this via the `RELEASE_NAME` environment -variable. - ## Migrate from old charts to new ones Starting from `0.39.37-alpha` we've revisited helm charts structure and separated all components of airbyte into their own independent charts, thus by allowing our developers to test single component without deploying airbyte as a whole and by upgrading single component at a time. diff --git a/docs/assets/docs/okta-app-integration-name.png b/docs/enterprise-setup/assets/okta-app-integration-name.png similarity index 100% rename from docs/assets/docs/okta-app-integration-name.png rename to docs/enterprise-setup/assets/okta-app-integration-name.png diff --git a/docs/assets/docs/okta-create-new-app-integration.png b/docs/enterprise-setup/assets/okta-create-new-app-integration.png similarity index 100% rename from docs/assets/docs/okta-create-new-app-integration.png rename to docs/enterprise-setup/assets/okta-create-new-app-integration.png diff --git a/docs/assets/docs/okta-login-redirect-uris.png b/docs/enterprise-setup/assets/okta-login-redirect-uris.png similarity index 100% rename from docs/assets/docs/okta-login-redirect-uris.png rename to docs/enterprise-setup/assets/okta-login-redirect-uris.png diff --git a/docs/enterprise-setup/self-managed/README.md b/docs/enterprise-setup/self-managed/README.md new file mode 100644 index 000000000000..30d1e7e05598 --- /dev/null +++ b/docs/enterprise-setup/self-managed/README.md @@ -0,0 +1,9 @@ +# Airbyte Self-Managed + +[Airbyte Self-Managed](https://airbyte.com/solutions/airbyte-enterprise) is a self-hosted version of Airbyte with additional features for enterprise customers. Airbyte Enterprise is in an early access stage for select priority users. + +A valid license key is required to get started with Airbyte Enterprise. [Talk to sales](https://airbyte.com/company/talk-to-sales) to receive your license key. + +The following pages outline how to: +1. [Deploy Airbyte Enterprise using Kubernetes](./implementation-guide.md) +2. [Configure Okta for Single Sign-On (SSO) with Airbyte Enterprise](./sso.md) diff --git a/docs/enterprise-setup/self-managed/implementation-guide.md b/docs/enterprise-setup/self-managed/implementation-guide.md new file mode 100644 index 000000000000..3b5c2da9c5d2 --- /dev/null +++ b/docs/enterprise-setup/self-managed/implementation-guide.md @@ -0,0 +1,103 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Implementation Guide + +[Airbyte Self-Managed](./README.md) is in an early access stage for select priority users. Once you [are qualified for an Airbyte Self Managed license key](https://airbyte.com/company/talk-to-sales), you can deploy Airbyte with the following instructions. + +Airbyte Self Managed must be deployed using Kubernetes. This is to enable Airbyte's best performance and scale. The core components \(api server, scheduler, etc\) run as deployments while the scheduler launches connector-related pods on different nodes. + +## Prerequisites + +There are three prerequisites to deploying Self-Managed: installing [helm](https://helm.sh/docs/intro/install/), a Kubernetes cluster, and having configured `kubectl` to connect to the cluster. + +For production, we recommend deploying to EKS, GKE or AKS. If you are doing some local testing, follow the cluster setup instructions outlined [here](../../deploying-airbyte/on-kubernetes-via-helm.md#cluster-setup). + +To install `kubectl`, please follow [these instructions](https://kubernetes.io/docs/tasks/tools/). To configure `kubectl` to connect to your cluster by using `kubectl use-context my-cluster-name`, see the following: + +
+ Configure kubectl to connect to your cluster + + +
    +
  1. Configure gcloud with gcloud auth login.
    2. +
  2. On the Google Cloud Console, the cluster page will have a "Connect" button, with a command to run locally: gcloud container clusters get-credentials $CLUSTER_NAME --zone $ZONE_NAME --project $PROJECT_NAME
    3. +
  3. Use kubectl config get-contexts to show the contexts available.
    4. +
  4. Run kubectl config use-context $GKE_CONTEXT to access the cluster from kubectl.
    5. +
+ + +
    +
  1. Configure your AWS CLI to connect to your project.
    2. +
  2. Install eksctl.
    3. +
  3. Run eksctl utils write-kubeconfig --cluster=$CLUSTER_NAME to make the context available to kubectl.
    4. +
  4. Use kubectl config get-contexts to show the contexts available.
    5. +
  5. Run kubectl config use-context $EKS_CONTEXT to access the cluster with kubectl.
    6. +
+ + +
+ +## Deploy Airbyte Self-Managed + +### Add Airbyte Helm Repository + +Follow these instructions to add the Airbyte helm repository: +1. Run `helm repo add airbyte https://airbytehq.github.io/helm-charts`, where `airbyte` is the name of the repository that will be indexed locally. +2. Perform the repo indexing process, and ensure your helm repository is up-to-date by running `helm repo update`. +3. You can then browse all charts uploaded to your repository by running `helm search repo airbyte`. + +### Clone & Configure Airbyte + + +1. `git clone` the latest revision of the [airbyte-platform repository](https://github.com/airbytehq/airbyte-platform) + +2. Create a new `airbyte.yml` file in the `configs` directory of the `airbyte-platform` folder. You may also copy `airbyte.sample.yml` to use as a template: + +```sh +cp configs/airbyte.sample.yml configs/airbyte.yml +``` + +3. Add your Airbyte Enterprise license key to your `airbyte.yml`. + +4. Add your [auth details](/airbyte-enterprise#single-sign-on-sso) to your `airbyte.yml`. Auth configurations aren't easy to modify after Airbyte is installed, so please double check them to make sure they're accurate before proceeding. + +
+ Configuring auth in your airbyte.yml file + +To configure SSO with Okta, add the following at the end of your `airbyte.yml` file: + +``` +auth: + identity-providers: + - type: okta + domain: $OKTA_DOMAIN + app-name: $OKTA_APP_INTEGRATION_NAME + client-id: $OKTA_CLIENT_ID + client-secret: $OKTA_CLIENT_SECRET +``` + +To configure basic auth (deploy without SSO), remove the entire `auth:` section from your airbyte.yml config file. You will authenticate with the instance admin user and password included in the your `airbyte.yml`. + +
+ +### Install Airbyte Self Managed + +Install Airbyte Enterprise on helm using the following command: + +```text +./tools/bin/install_airbyte_pro_on_helm.sh +``` + +The default release name is `airbyte-pro`. You can change this via the `RELEASE_NAME` environment +variable. + +### Customizing your Airbyte Self Managed Deployment + +In order to customize your deployment, you need to create `values.yaml` file in a local folder and populate it with default configuration override values. A `values.yaml` example can be located in [charts/airbyte](https://github.com/airbytehq/airbyte-platform/blob/main/charts/airbyte/values.yaml) folder of the Airbyte repository. + +After specifying your own configuration, run the following command: + +```text +./tools/bin/install_airbyte_pro_on_helm.sh --values path/to/values.yaml $RELEASE_NAME airbyte/airbyte +``` \ No newline at end of file diff --git a/docs/airbyte-enterprise.md b/docs/enterprise-setup/self-managed/sso.md similarity index 54% rename from docs/airbyte-enterprise.md rename to docs/enterprise-setup/self-managed/sso.md index d717cd41d3ef..a7295e60ecf5 100644 --- a/docs/airbyte-enterprise.md +++ b/docs/enterprise-setup/self-managed/sso.md @@ -1,14 +1,12 @@ -# Airbyte Enterprise +# Using Single Sign-On (SSO) -[Airbyte Enterprise](https://airbyte.com/solutions/airbyte-enterprise) is a self-managed version of Airbyte with additional features for enterprise customers. Airbyte Enterprise is in an early access stage for select priority users. A valid license key is required to get started with Airbyte Enterprise. [Talk to sales](https://airbyte.com/company/talk-to-sales) to receive your license key. +Leverage your existing identity provider to enable employees to access your Airbyte instance using their corporate credentials, simplifying user provisioning. Enabling Single Sign-On extends Airbyte Self Managed to support multiple users, and multiple teams all on one instance. -The following instructions outline how to: -1. Configure Okta for Single Sign-On (SSO) with Airbyte Enterprise -2. Deploy Airbyte Enterprise using Kubernetes (License Key Required) +Airbyte Self Managed currently supports SSO via OIDC with [Okta](https://www.okta.com/) as an IdP. Support for Azure Active Directory and connecting via SAML are both coming soon. Please talk to us to learn more about upcoming [enterprise features](https://airbyte.com/company/talk-to-sales). -## Single Sign-On (SSO) - -Airbyte Enterprise supports Single Sign-On, allowing an organization to manage user access to their Airbyte Enterprise instance through the configuration of an Identity Provider (IdP). Airbyte Enterprise currently supports SSO via OIDC with [Okta](https://www.okta.com/) as an IdP. +The following instructions walk you through: +1. [Setting up the Okta OIDC App Integration to be used by your Airbyte instance](#setting-up-okta-for-sso) +2. [Configuring Airbyte Self-Managed to use SSO](#deploying-airbyte-enterprise-with-okta) ### Setting up Okta for SSO @@ -16,13 +14,13 @@ You will need to create a new Okta OIDC App Integration for your Airbyte instanc You should create an app integration with **OIDC - OpenID Connect** as the sign-in method and **Web Application** as the application type: -![Screenshot of Okta app integration creation modal](./assets/docs/okta-create-new-app-integration.png) +![Screenshot of Okta app integration creation modal](../assets/okta-create-new-app-integration.png) #### App integration name Please choose a URL-friendly app integraiton name without spaces or special characters, such as `my-airbyte-app`: -![Screenshot of Okta app integration name](./assets/docs/okta-app-integration-name.png) +![Screenshot of Okta app integration name](../assets/okta-app-integration-name.png) Spaces or special characters in this field could result in invalid redirect URIs. @@ -42,13 +40,14 @@ Sign-out redirect URIs /auth/realms/airbyte/broker//endpoint/logout_response ``` -![Okta app integration name screenshot](./assets/docs/okta-login-redirect-uris.png) +![Okta app integration name screenshot](../assets/okta-login-redirect-uris.png) _Example values_ `` should point to where your Airbyte instance will be available, including the http/https protocol. -## Deploying Airbyte Enterprise with Okta + +## Deploying Airbyte Self-Managed with Okta Once your Okta app is set up, you're ready to deploy Airbyte with SSO. Take note of the following configuration values, as you will need them to configure Airbyte to use your new Okta SSO app integration: @@ -57,4 +56,4 @@ Once your Okta app is set up, you're ready to deploy Airbyte with SSO. Take note - Client ID - Client Secret -Visit [Airbyte Enterprise deployment](/deploying-airbyte/on-kubernetes-via-helm#early-access-airbyte-enterprise-deployment) for instructions on how to deploy Airbyte Enterprise using `kubernetes`, `kubectl` and `helm`. +Visit the [implementation guide](./implementation-guide.md) for instructions on how to deploy Airbyte Enterprise using `kubernetes`, `kubectl` and `helm`. diff --git a/docusaurus/redirects.yml b/docusaurus/redirects.yml index b69386db8c1d..28a7f499bc15 100644 --- a/docusaurus/redirects.yml +++ b/docusaurus/redirects.yml @@ -1,6 +1,8 @@ # A list of URLs that should be redirected to new pathes - from: /airbyte-pro - to: /airbyte-enterprise + to: /enterprise-setup/self-managed/ +- from: /airbyte-enterprise + to: /enterprise-setup/self-managed/ - from: /upgrading-airbyte to: /operator-guides/upgrading-airbyte - from: /catalog diff --git a/docusaurus/sidebars.js b/docusaurus/sidebars.js index 2b915c7b4bb4..61e5dde2146a 100644 --- a/docusaurus/sidebars.js +++ b/docusaurus/sidebars.js @@ -418,6 +418,19 @@ const deployAirbyte = { ], }; +const airbyteSelfManaged = { + type: "category", + label: "Airbyte Self Managed", + link: { + type: "doc", + id: "enterprise-setup/self-managed/README", + }, + items: [ + "enterprise-setup/self-managed/implementation-guide", + "enterprise-setup/self-managed/sso", + ] +} + const operatorGuide = { type: "category", label: "Manage Airbyte", @@ -518,10 +531,8 @@ module.exports = { type: "doc", id: "troubleshooting", }, - { - type: "doc", - id: "airbyte-enterprise", - }, + sectionHeader("Enterprise Setup"), + airbyteSelfManaged, sectionHeader("Developer Guides"), { type: "doc", From 159dfe69a4bd44979a7bbbf2398e26ff46636419 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 16 Nov 2023 15:53:40 -0800 Subject: [PATCH 095/124] Hotfix: Fix formatting issue (#32619) --- airbyte-integrations/connectors/source-bing-ads/metadata.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml index 599a7e5c7206..1c29083b42ff 100644 --- a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml @@ -25,11 +25,11 @@ data: name: Bing Ads registries: cloud: - dockerImageTag: 1.13.0 #https://airbytehq-team.slack.com/archives/C0662JB7XPU + dockerImageTag: 1.13.0 #https://airbytehq-team.slack.com/archives/C0662JB7XPU enabled: true oss: enabled: true - dockerImageTag: 1.13.0 #https://airbytehq-team.slack.com/archives/C0662JB7XPU + dockerImageTag: 1.13.0 #https://airbytehq-team.slack.com/archives/C0662JB7XPU releaseStage: generally_available releases: breakingChanges: From b6e5aff10cc938443ad018b65abf8fd577012997 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 16 Nov 2023 16:08:38 -0800 Subject: [PATCH 096/124] CI: Add a test version of the approve and merge command that requires required checks to be passing (#32612) Co-authored-by: bnchrch --- .github/workflows/airbyte-ci-tests.yml | 1 + .../approve-and-merge-demo-dispatch.yml | 87 +++++++++++++++++++ .github/workflows/format_fix.yml | 1 + .github/workflows/gradle.yml | 1 + .github/workflows/slash-commands.yml | 1 + 5 files changed, 91 insertions(+) create mode 100644 .github/workflows/approve-and-merge-demo-dispatch.yml diff --git a/.github/workflows/airbyte-ci-tests.yml b/.github/workflows/airbyte-ci-tests.yml index f11f6947b469..7089532b095d 100644 --- a/.github/workflows/airbyte-ci-tests.yml +++ b/.github/workflows/airbyte-ci-tests.yml @@ -13,6 +13,7 @@ on: - synchronize jobs: run-airbyte-ci-tests: + # Note if you are changing this name you must also change it in the approve-and-merge-dispatch.yml workflow name: Run Airbyte CI tests runs-on: "ci-runner-connector-test-large-dagger-0-6-4" steps: diff --git a/.github/workflows/approve-and-merge-demo-dispatch.yml b/.github/workflows/approve-and-merge-demo-dispatch.yml new file mode 100644 index 000000000000..5bed14cee2c2 --- /dev/null +++ b/.github/workflows/approve-and-merge-demo-dispatch.yml @@ -0,0 +1,87 @@ +name: Approve and Merge Command Dispatch + +# Note: We have a two stage dispatch so that we can wait for the formatters to run before approving and merging. +on: + repository_dispatch: + types: [approve-and-merge-demo-command] +jobs: + checkFormat: + runs-on: ubuntu-latest + steps: + - name: Wait for formatters to succeed + id: wait-for-formatters + uses: lewagon/wait-on-check-action@v1.3.1 + with: + ref: ${{ github.ref }} + check-name: "Apply All Formatting Rules" + repo-token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + wait-interval: 30 + - name: Comment if formatters failed + if: failure() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.client_payload.github.payload.comment.id }} + body: | + > Error: Formatters failed. Please fix the errors and try again. + + checkGradle: + runs-on: ubuntu-latest + steps: + - name: Wait for Gradle Check to succeed + id: wait-for-gradle + uses: lewagon/wait-on-check-action@v1.3.1 + with: + ref: ${{ github.ref }} + check-name: "Gradle Check" + repo-token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + wait-interval: 60 + - name: Comment if Gradle Check failed + if: failure() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.client_payload.github.payload.comment.id }} + body: | + > Error: Gradle Check failed. Please fix the errors and try again. + + checkAirbyteCI: + runs-on: ubuntu-latest + steps: + - name: Wait for Airbyte CI tests to succeed + id: wait-for-gradle + uses: lewagon/wait-on-check-action@v1.3.1 + with: + ref: ${{ github.ref }} + check-name: "Run Airbyte CI tests" + repo-token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + wait-interval: 30 + - name: Comment if Airbyte CI tests failed + if: failure() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.client_payload.github.payload.comment.id }} + body: | + > Error: Airbyte CI tests failed. Please fix the errors and try again. + + approveAndMergeDispatch: + runs-on: ubuntu-latest + needs: [checkFormat, checkGradle, checkAirbyteCI] + steps: + - name: Auto Approve Slash Command Dispatch + uses: peter-evans/slash-command-dispatch@v3 + id: scd + with: + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + permission: write + issue-type: pull-request + repository: airbytehq/airbyte-cloud + dispatch-type: repository + commands: | + approve-and-merge + + - name: Edit comment with error message + if: steps.scd.outputs.error-message + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.client_payload.github.payload.comment.id }} + body: | + > Error: ${{ steps.scd.outputs.error-message }} diff --git a/.github/workflows/format_fix.yml b/.github/workflows/format_fix.yml index d689a8891063..58761fef432e 100644 --- a/.github/workflows/format_fix.yml +++ b/.github/workflows/format_fix.yml @@ -11,6 +11,7 @@ on: jobs: format-fix: runs-on: "ci-runner-connector-format-medium-dagger-0-6-4" + # Note if you are changing this name you must also change it in the approve-and-merge-dispatch.yml workflow name: "Apply All Formatting Rules" timeout-minutes: 40 steps: diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 13fa98a6acaa..695ccde96fa1 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -54,6 +54,7 @@ jobs: # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. needs: start-check-runner # required to start the main job when the runner is ready runs-on: ${{ needs.start-check-runner.outputs.label }} # run the job on the newly created runner + # Note if you are changing this name you must also change it in the approve-and-merge-dispatch.yml workflow name: Gradle Check timeout-minutes: 30 steps: diff --git a/.github/workflows/slash-commands.yml b/.github/workflows/slash-commands.yml index 22029f06baba..91dc9d0fb8d5 100644 --- a/.github/workflows/slash-commands.yml +++ b/.github/workflows/slash-commands.yml @@ -29,6 +29,7 @@ jobs: publish-java-cdk legacy-publish connector-performance + approve-and-merge-demo static-args: | repo=${{ steps.getref.outputs.repo }} gitref=${{ steps.getref.outputs.ref }} From 5c87e1c54a4a595ec86ad726632d9dec967e04bf Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 16 Nov 2023 16:19:30 -0800 Subject: [PATCH 097/124] CI TEST: Benign docstring change (#32620) --- .github/workflows/approve-and-merge-demo-dispatch.yml | 1 + .github/workflows/slash-commands.yml | 2 +- .../connectors/pipelines/pipelines/airbyte_ci/test/commands.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/approve-and-merge-demo-dispatch.yml b/.github/workflows/approve-and-merge-demo-dispatch.yml index 5bed14cee2c2..5ab290d6230c 100644 --- a/.github/workflows/approve-and-merge-demo-dispatch.yml +++ b/.github/workflows/approve-and-merge-demo-dispatch.yml @@ -4,6 +4,7 @@ name: Approve and Merge Command Dispatch on: repository_dispatch: types: [approve-and-merge-demo-command] + jobs: checkFormat: runs-on: ubuntu-latest diff --git a/.github/workflows/slash-commands.yml b/.github/workflows/slash-commands.yml index 91dc9d0fb8d5..d589eee4b49a 100644 --- a/.github/workflows/slash-commands.yml +++ b/.github/workflows/slash-commands.yml @@ -17,7 +17,7 @@ jobs: - name: Slash Command Dispatch id: scd - uses: peter-evans/slash-command-dispatch@v2 + uses: peter-evans/slash-command-dispatch@v3 with: token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} permission: write diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index 205498c331da..f6073542ca2f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -18,7 +18,7 @@ @pass_pipeline_context @click_ignore_unused_kwargs async def test(pipeline_context: ClickPipelineContext): - """Runs the tests for the given airbyte-ci package. + """Runs the tests for the given airbyte-ci package Args: pipeline_context (ClickPipelineContext): The context object. From 796f4fa1ba70d79a54b2f2e8c13bb1c59751d7cd Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 16 Nov 2023 16:27:30 -0800 Subject: [PATCH 098/124] CI TEST: Benign docstring change (#32621) --- .github/workflows/slash-commands.yml | 13 +++++++++++-- .../pipelines/pipelines/airbyte_ci/test/commands.py | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/slash-commands.yml b/.github/workflows/slash-commands.yml index d589eee4b49a..8df9fb7d3342 100644 --- a/.github/workflows/slash-commands.yml +++ b/.github/workflows/slash-commands.yml @@ -15,7 +15,7 @@ jobs: echo ref="$(echo $pr_info | jq -r '.head.ref')" >> $GITHUB_OUTPUT echo repo="$(echo $pr_info | jq -r '.head.repo.full_name')" >> $GITHUB_OUTPUT - - name: Slash Command Dispatch + - name: Slash Command Dispatch (Workflow) id: scd uses: peter-evans/slash-command-dispatch@v3 with: @@ -29,13 +29,22 @@ jobs: publish-java-cdk legacy-publish connector-performance - approve-and-merge-demo static-args: | repo=${{ steps.getref.outputs.repo }} gitref=${{ steps.getref.outputs.ref }} comment-id=${{ github.event.comment.id }} dispatch-type: workflow + - name: Slash Command Dispatch (Repository) + id: scdr + uses: peter-evans/slash-command-dispatch@v3 + with: + token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + permission: write + commands: | + approve-and-merge-demo + dispatch-type: repository + - name: Edit comment with error message if: steps.scd.outputs.error-message uses: peter-evans/create-or-update-comment@v1 diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index f6073542ca2f..205498c331da 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -18,7 +18,7 @@ @pass_pipeline_context @click_ignore_unused_kwargs async def test(pipeline_context: ClickPipelineContext): - """Runs the tests for the given airbyte-ci package + """Runs the tests for the given airbyte-ci package. Args: pipeline_context (ClickPipelineContext): The context object. From 88cc7c8dfeb0995ecc85f256ab9930b71ef2b4c0 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 16 Nov 2023 16:49:19 -0800 Subject: [PATCH 099/124] CI TEST: Benign docstring change (#32622) --- .../approve-and-merge-demo-dispatch.yml | 44 ++----------------- .../pipelines/airbyte_ci/test/commands.py | 2 +- 2 files changed, 4 insertions(+), 42 deletions(-) diff --git a/.github/workflows/approve-and-merge-demo-dispatch.yml b/.github/workflows/approve-and-merge-demo-dispatch.yml index 5ab290d6230c..6e2532a647c9 100644 --- a/.github/workflows/approve-and-merge-demo-dispatch.yml +++ b/.github/workflows/approve-and-merge-demo-dispatch.yml @@ -14,7 +14,7 @@ jobs: uses: lewagon/wait-on-check-action@v1.3.1 with: ref: ${{ github.ref }} - check-name: "Apply All Formatting Rules" + check-name: "Automatic Formatting on PRs" repo-token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} wait-interval: 30 - name: Comment if formatters failed @@ -23,49 +23,11 @@ jobs: with: comment-id: ${{ github.event.client_payload.github.payload.comment.id }} body: | - > Error: Formatters failed. Please fix the errors and try again. - - checkGradle: - runs-on: ubuntu-latest - steps: - - name: Wait for Gradle Check to succeed - id: wait-for-gradle - uses: lewagon/wait-on-check-action@v1.3.1 - with: - ref: ${{ github.ref }} - check-name: "Gradle Check" - repo-token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} - wait-interval: 60 - - name: Comment if Gradle Check failed - if: failure() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.client_payload.github.payload.comment.id }} - body: | - > Error: Gradle Check failed. Please fix the errors and try again. - - checkAirbyteCI: - runs-on: ubuntu-latest - steps: - - name: Wait for Airbyte CI tests to succeed - id: wait-for-gradle - uses: lewagon/wait-on-check-action@v1.3.1 - with: - ref: ${{ github.ref }} - check-name: "Run Airbyte CI tests" - repo-token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} - wait-interval: 30 - - name: Comment if Airbyte CI tests failed - if: failure() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.client_payload.github.payload.comment.id }} - body: | - > Error: Airbyte CI tests failed. Please fix the errors and try again. + > Error: Formatters failed. Ensure formatting is passing before using approve-and-merge. approveAndMergeDispatch: runs-on: ubuntu-latest - needs: [checkFormat, checkGradle, checkAirbyteCI] + needs: [checkFormat] steps: - name: Auto Approve Slash Command Dispatch uses: peter-evans/slash-command-dispatch@v3 diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index 205498c331da..f6073542ca2f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -18,7 +18,7 @@ @pass_pipeline_context @click_ignore_unused_kwargs async def test(pipeline_context: ClickPipelineContext): - """Runs the tests for the given airbyte-ci package. + """Runs the tests for the given airbyte-ci package Args: pipeline_context (ClickPipelineContext): The context object. From 95c8667859a533b9bfe1561ca61b71e0984a01ae Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 16 Nov 2023 17:01:11 -0800 Subject: [PATCH 100/124] CI TEST: Benign docstring change (#32623) --- .github/workflows/approve-and-merge-demo-dispatch.yml | 2 +- .../connectors/pipelines/pipelines/airbyte_ci/test/commands.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/approve-and-merge-demo-dispatch.yml b/.github/workflows/approve-and-merge-demo-dispatch.yml index 6e2532a647c9..d4c158d80248 100644 --- a/.github/workflows/approve-and-merge-demo-dispatch.yml +++ b/.github/workflows/approve-and-merge-demo-dispatch.yml @@ -14,7 +14,7 @@ jobs: uses: lewagon/wait-on-check-action@v1.3.1 with: ref: ${{ github.ref }} - check-name: "Automatic Formatting on PRs" + check-name: "Apply All Formatting Rules" repo-token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} wait-interval: 30 - name: Comment if formatters failed diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index f6073542ca2f..205498c331da 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -18,7 +18,7 @@ @pass_pipeline_context @click_ignore_unused_kwargs async def test(pipeline_context: ClickPipelineContext): - """Runs the tests for the given airbyte-ci package + """Runs the tests for the given airbyte-ci package. Args: pipeline_context (ClickPipelineContext): The context object. From 70cb2306477ac0779b90d96771dd8ff1d0bb89c6 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 16 Nov 2023 17:39:15 -0800 Subject: [PATCH 101/124] CI TEST: Benign docstring change (#32624) --- .github/workflows/approve-and-merge-demo-dispatch.yml | 10 +++------- .../pipelines/pipelines/airbyte_ci/test/commands.py | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/approve-and-merge-demo-dispatch.yml b/.github/workflows/approve-and-merge-demo-dispatch.yml index d4c158d80248..1e542e38bd0a 100644 --- a/.github/workflows/approve-and-merge-demo-dispatch.yml +++ b/.github/workflows/approve-and-merge-demo-dispatch.yml @@ -9,14 +9,10 @@ jobs: checkFormat: runs-on: ubuntu-latest steps: - - name: Wait for formatters to succeed - id: wait-for-formatters - uses: lewagon/wait-on-check-action@v1.3.1 + - uses: maael/confirm-checks-action with: - ref: ${{ github.ref }} - check-name: "Apply All Formatting Rules" - repo-token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} - wait-interval: 30 + github_token: ${{secrets.GH_PAT_APPROVINGTON_OCTAVIA}} + checks: 'Apply All Formatting Rules' - name: Comment if formatters failed if: failure() uses: peter-evans/create-or-update-comment@v1 diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index 205498c331da..f6073542ca2f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -18,7 +18,7 @@ @pass_pipeline_context @click_ignore_unused_kwargs async def test(pipeline_context: ClickPipelineContext): - """Runs the tests for the given airbyte-ci package. + """Runs the tests for the given airbyte-ci package Args: pipeline_context (ClickPipelineContext): The context object. From 251bba1ff09203cdb057b7a3d6164f98649b3773 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 16 Nov 2023 17:44:23 -0800 Subject: [PATCH 102/124] CI TEST: Benign docstring change (#32628) --- .github/workflows/approve-and-merge-demo-dispatch.yml | 4 ++-- .../pipelines/pipelines/airbyte_ci/test/commands.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/approve-and-merge-demo-dispatch.yml b/.github/workflows/approve-and-merge-demo-dispatch.yml index 1e542e38bd0a..a129bad50ec7 100644 --- a/.github/workflows/approve-and-merge-demo-dispatch.yml +++ b/.github/workflows/approve-and-merge-demo-dispatch.yml @@ -1,4 +1,4 @@ -name: Approve and Merge Command Dispatch +name: Approve and Merge Demo Command Dispatch # Note: We have a two stage dispatch so that we can wait for the formatters to run before approving and merging. on: @@ -12,7 +12,7 @@ jobs: - uses: maael/confirm-checks-action with: github_token: ${{secrets.GH_PAT_APPROVINGTON_OCTAVIA}} - checks: 'Apply All Formatting Rules' + checks: "Apply All Formatting Rules" - name: Comment if formatters failed if: failure() uses: peter-evans/create-or-update-comment@v1 diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index f6073542ca2f..205498c331da 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -18,7 +18,7 @@ @pass_pipeline_context @click_ignore_unused_kwargs async def test(pipeline_context: ClickPipelineContext): - """Runs the tests for the given airbyte-ci package + """Runs the tests for the given airbyte-ci package. Args: pipeline_context (ClickPipelineContext): The context object. From 44450c3840ace7efcec7372b01a1a08e30cdbeb1 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 16 Nov 2023 18:13:42 -0800 Subject: [PATCH 103/124] Test CI: Benign docstring change (#32629) --- .github/workflows/approve-and-merge-demo-dispatch.yml | 10 +++++++--- .../pipelines/pipelines/airbyte_ci/test/commands.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/approve-and-merge-demo-dispatch.yml b/.github/workflows/approve-and-merge-demo-dispatch.yml index a129bad50ec7..5458e3cee8c4 100644 --- a/.github/workflows/approve-and-merge-demo-dispatch.yml +++ b/.github/workflows/approve-and-merge-demo-dispatch.yml @@ -9,10 +9,14 @@ jobs: checkFormat: runs-on: ubuntu-latest steps: - - uses: maael/confirm-checks-action + - name: Wait for formatters to succeed + id: wait-for-formatters + uses: lewagon/wait-on-check-action@v1.3.1 with: - github_token: ${{secrets.GH_PAT_APPROVINGTON_OCTAVIA}} - checks: "Apply All Formatting Rules" + ref: ${{ github.ref }} + check-name: "Apply All Formatting Rules" + repo-token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + wait-interval: 30 - name: Comment if formatters failed if: failure() uses: peter-evans/create-or-update-comment@v1 diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index 205498c331da..f6073542ca2f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -18,7 +18,7 @@ @pass_pipeline_context @click_ignore_unused_kwargs async def test(pipeline_context: ClickPipelineContext): - """Runs the tests for the given airbyte-ci package. + """Runs the tests for the given airbyte-ci package Args: pipeline_context (ClickPipelineContext): The context object. From 0e7f30b76e24c825b6eb6fb838541f8b262edd59 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 16 Nov 2023 18:34:11 -0800 Subject: [PATCH 104/124] TEST CI: Benign docstring change (#32630) --- .github/workflows/approve-and-merge-demo-dispatch.yml | 8 ++++++-- .github/workflows/slash-commands.yml | 4 ++++ .../pipelines/pipelines/airbyte_ci/test/commands.py | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/approve-and-merge-demo-dispatch.yml b/.github/workflows/approve-and-merge-demo-dispatch.yml index 5458e3cee8c4..e19d16e46698 100644 --- a/.github/workflows/approve-and-merge-demo-dispatch.yml +++ b/.github/workflows/approve-and-merge-demo-dispatch.yml @@ -9,12 +9,16 @@ jobs: checkFormat: runs-on: ubuntu-latest steps: + - name: Dump the client payload context + env: + PAYLOAD_CONTEXT: ${{ toJson(github.event.client_payload) }} + run: echo "$PAYLOAD_CONTEXT" - name: Wait for formatters to succeed id: wait-for-formatters uses: lewagon/wait-on-check-action@v1.3.1 with: - ref: ${{ github.ref }} - check-name: "Apply All Formatting Rules" + ref: ${{ github.event.client_payload.slash_command.args.named.gitref }} + check-name: "Check for formatting errors on" repo-token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} wait-interval: 30 - name: Comment if formatters failed diff --git a/.github/workflows/slash-commands.yml b/.github/workflows/slash-commands.yml index 8df9fb7d3342..90106603c9af 100644 --- a/.github/workflows/slash-commands.yml +++ b/.github/workflows/slash-commands.yml @@ -43,6 +43,10 @@ jobs: permission: write commands: | approve-and-merge-demo + static-args: | + repo=${{ steps.getref.outputs.repo }} + gitref=${{ steps.getref.outputs.ref }} + comment-id=${{ github.event.comment.id }} dispatch-type: repository - name: Edit comment with error message diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index f6073542ca2f..205498c331da 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -18,7 +18,7 @@ @pass_pipeline_context @click_ignore_unused_kwargs async def test(pipeline_context: ClickPipelineContext): - """Runs the tests for the given airbyte-ci package + """Runs the tests for the given airbyte-ci package. Args: pipeline_context (ClickPipelineContext): The context object. From 4c6e6a0f936ca0d6843f2cd7288ae559f9d38575 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Thu, 16 Nov 2023 18:40:37 -0800 Subject: [PATCH 105/124] TEST CI: Benign docstring change (#32631) --- .github/workflows/approve-and-merge-demo-dispatch.yml | 8 ++------ .github/workflows/slash-commands.yml | 4 ---- .../pipelines/pipelines/airbyte_ci/test/commands.py | 2 +- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/approve-and-merge-demo-dispatch.yml b/.github/workflows/approve-and-merge-demo-dispatch.yml index e19d16e46698..234a038558e0 100644 --- a/.github/workflows/approve-and-merge-demo-dispatch.yml +++ b/.github/workflows/approve-and-merge-demo-dispatch.yml @@ -9,16 +9,12 @@ jobs: checkFormat: runs-on: ubuntu-latest steps: - - name: Dump the client payload context - env: - PAYLOAD_CONTEXT: ${{ toJson(github.event.client_payload) }} - run: echo "$PAYLOAD_CONTEXT" - name: Wait for formatters to succeed id: wait-for-formatters uses: lewagon/wait-on-check-action@v1.3.1 with: - ref: ${{ github.event.client_payload.slash_command.args.named.gitref }} - check-name: "Check for formatting errors on" + ref: ${{ github.event.client_payload.pull_request.head.ref }} + check-name: "Apply All Formatting Rules" repo-token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} wait-interval: 30 - name: Comment if formatters failed diff --git a/.github/workflows/slash-commands.yml b/.github/workflows/slash-commands.yml index 90106603c9af..8df9fb7d3342 100644 --- a/.github/workflows/slash-commands.yml +++ b/.github/workflows/slash-commands.yml @@ -43,10 +43,6 @@ jobs: permission: write commands: | approve-and-merge-demo - static-args: | - repo=${{ steps.getref.outputs.repo }} - gitref=${{ steps.getref.outputs.ref }} - comment-id=${{ github.event.comment.id }} dispatch-type: repository - name: Edit comment with error message diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py index 205498c331da..f6073542ca2f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -18,7 +18,7 @@ @pass_pipeline_context @click_ignore_unused_kwargs async def test(pipeline_context: ClickPipelineContext): - """Runs the tests for the given airbyte-ci package. + """Runs the tests for the given airbyte-ci package Args: pipeline_context (ClickPipelineContext): The context object. From f8ae6f639152010ad4c60a6aaf1abeef294e6cf9 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Fri, 17 Nov 2023 08:13:01 +0200 Subject: [PATCH 106/124] :bug: Source Amazon Seller Partner: fix OAuth (#32550) Co-authored-by: davydov-d --- .../source-amazon-seller-partner/main.py | 2 + .../metadata.yaml | 2 +- .../config_migrations.py | 79 +++++++++++++++++++ .../source_amazon_seller_partner/spec.json | 40 +++++++--- .../unit_tests/test_migrations.py | 41 ++++++++++ .../test_migrations/migrated_config.json | 9 +++ .../test_migrations/not_migrated_config.json | 8 ++ .../sources/amazon-seller-partner.md | 36 +++++++-- 8 files changed, 199 insertions(+), 18 deletions(-) create mode 100644 airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/config_migrations.py create mode 100644 airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py create mode 100644 airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations/migrated_config.json create mode 100644 airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations/not_migrated_config.json diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/main.py b/airbyte-integrations/connectors/source-amazon-seller-partner/main.py index a09a9063026c..f5089129f6a6 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/main.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/main.py @@ -7,7 +7,9 @@ from airbyte_cdk.entrypoint import launch from source_amazon_seller_partner import SourceAmazonSellerPartner +from source_amazon_seller_partner.config_migrations import MigrateAccountType if __name__ == "__main__": source = SourceAmazonSellerPartner() + MigrateAccountType.migrate(sys.argv[1:], source) launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml index 2bad20c6b106..3f39e4aae628 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: api connectorType: source definitionId: e55879a8-0ef8-4557-abcf-ab34c53ec460 - dockerImageTag: 2.0.0 + dockerImageTag: 2.0.1 dockerRepository: airbyte/source-amazon-seller-partner documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-seller-partner githubIssueLabel: source-amazon-seller-partner diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/config_migrations.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/config_migrations.py new file mode 100644 index 000000000000..5d2daf748f6c --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/config_migrations.py @@ -0,0 +1,79 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import logging +from typing import Any, List, Mapping + +from airbyte_cdk.config_observation import create_connector_config_control_message +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + +from .source import SourceAmazonSellerPartner + +logger = logging.getLogger("airbyte_logger") + + +class MigrateAccountType: + """ + This class stands for migrating the config at runtime, + while providing the backward compatibility when falling back to the previous source version. + + Specifically, starting from `2.0.1`, the `account_type` property becomes required. + For those connector configs that do not contain this key, the default value of `Seller` will be used. + Reverse operation is not needed as this field is ignored in previous versions of the connector. + """ + + message_repository: MessageRepository = InMemoryMessageRepository() + migration_key: str = "account_type" + + @classmethod + def _should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + This method determines whether config requires migration. + Returns: + > True, if the transformation is neccessary + > False, otherwise. + """ + return cls.migration_key not in config + + @classmethod + def _populate_with_default_value(cls, config: Mapping[str, Any], source: SourceAmazonSellerPartner = None) -> Mapping[str, Any]: + config[cls.migration_key] = "Seller" + return config + + @classmethod + def _modify_and_save(cls, config_path: str, source: SourceAmazonSellerPartner, config: Mapping[str, Any]) -> Mapping[str, Any]: + # modify the config + migrated_config = cls._populate_with_default_value(config, source) + # save the config + source.write_config(migrated_config, config_path) + # return modified config + return migrated_config + + @classmethod + def _emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + # add the Airbyte Control Message to message repo + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + # emit the Airbyte Control Message from message queue to stdout + for message in cls.message_repository.consume_queue(): + print(message.json(exclude_unset=True)) + + @classmethod + def migrate(cls, args: List[str], source: SourceAmazonSellerPartner) -> None: + """ + This method checks the input args, should the config be migrated, + transform if neccessary and emit the CONTROL message. + """ + # get config path + config_path = AirbyteEntrypoint(source).extract_config(args) + # proceed only if `--config` arg is provided + if config_path: + # read the existing config + config = source.read_config(config_path) + # migration check + if cls._should_migrate(config): + cls._emit_control_message( + cls._modify_and_save(config_path, source, config), + ) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json index afcd6279342f..d64b1ee1d86f 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json @@ -7,6 +7,7 @@ "required": [ "aws_environment", "region", + "account_type", "lwa_app_id", "lwa_client_secret", "refresh_token", @@ -59,10 +60,18 @@ "type": "string", "order": 2 }, + "account_type": { + "title": "AWS Seller Partner Account Type", + "description": "Type of the Account you're going to authorize the Airbyte application by", + "enum": ["Seller", "Vendor"], + "default": "Seller", + "type": "string", + "order": 3 + }, "lwa_app_id": { "title": "LWA Client Id", "description": "Your Login with Amazon Client ID.", - "order": 3, + "order": 4, "airbyte_secret": true, "type": "string" }, @@ -70,14 +79,14 @@ "title": "LWA Client Secret", "description": "Your Login with Amazon Client Secret.", "airbyte_secret": true, - "order": 4, + "order": 5, "type": "string" }, "refresh_token": { "title": "Refresh Token", "description": "The Refresh Token obtained via OAuth flow authorization.", "airbyte_secret": true, - "order": 5, + "order": 6, "type": "string" }, "replication_start_date": { @@ -85,7 +94,7 @@ "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "examples": ["2017-01-25T00:00:00Z"], - "order": 6, + "order": 7, "type": "string", "format": "date-time" }, @@ -94,7 +103,7 @@ "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data after this date will not be replicated.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$|^$", "examples": ["2017-01-25T00:00:00Z"], - "order": 7, + "order": 8, "type": "string", "format": "date-time" }, @@ -104,7 +113,7 @@ "description": "Will be used for stream slicing for initial full_refresh sync when no updated state is present for reports that support sliced incremental sync.", "default": 90, "minimum": 1, - "order": 8 + "order": 9 }, "report_options": { "title": "Report Options", @@ -113,7 +122,7 @@ "{\"GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT\": {\"reportPeriod\": \"WEEK\"}}", "{\"GET_SOME_REPORT\": {\"custom\": \"true\"}}" ], - "order": 9, + "order": 10, "type": "string" }, "max_wait_seconds": { @@ -121,7 +130,7 @@ "description": "Sometimes report can take up to 30 minutes to generate. This will set the limit for how long to wait for a successful report.", "default": 500, "examples": ["500", "1980"], - "order": 10, + "order": 11, "minimum": 1, "type": "integer" }, @@ -132,7 +141,7 @@ "{\"GET_SALES_AND_TRAFFIC_REPORT\": {\"availability_sla_days\": 3}}", "{\"GET_SOME_REPORT\": {\"custom\": \"true\"}}" ], - "order": 11, + "order": 12, "type": "string" } } @@ -142,6 +151,19 @@ "predicate_key": ["auth_type"], "predicate_value": "oauth2.0", "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "properties": { + "region": { + "type": "string", + "path_in_connector_config": ["region"] + }, + "account_type": { + "type": "string", + "path_in_connector_config": ["account_type"] + } + } + }, "complete_oauth_output_specification": { "type": "object", "additionalProperties": false, diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py new file mode 100644 index 000000000000..52af77133e47 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py @@ -0,0 +1,41 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import json +from typing import Any, Mapping + +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source +from source_amazon_seller_partner.config_migrations import MigrateAccountType +from source_amazon_seller_partner.source import SourceAmazonSellerPartner + +CMD = "check" +TEST_NOT_MIGRATED_CONFIG_PATH = "unit_tests/test_migrations/not_migrated_config.json" +TEST_MIGRATED_CONFIG_PATH = "unit_tests/test_migrations/migrated_config.json" +SOURCE: Source = SourceAmazonSellerPartner() + + +def load_config(config_path: str = TEST_NOT_MIGRATED_CONFIG_PATH) -> Mapping[str, Any]: + with open(config_path, "r") as config: + return json.load(config) + + +def test_migrate_config(capsys): + config = load_config(TEST_NOT_MIGRATED_CONFIG_PATH) + assert "acount_type" not in config + migration_instance = MigrateAccountType() + migration_instance.migrate([CMD, "--config", TEST_NOT_MIGRATED_CONFIG_PATH], SOURCE) + control_msg = json.loads(capsys.readouterr().out) + assert control_msg["type"] == Type.CONTROL.value + assert control_msg["control"]["type"] == OrchestratorType.CONNECTOR_CONFIG.value + migrated_config = control_msg["control"]["connectorConfig"]["config"] + assert migrated_config["account_type"] == "Seller" + + +def test_should_not_migrate(): + config = load_config(TEST_MIGRATED_CONFIG_PATH) + assert config["account_type"] + migration_instance = MigrateAccountType() + assert not migration_instance._should_migrate(config) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations/migrated_config.json b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations/migrated_config.json new file mode 100644 index 000000000000..3b65000693d3 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations/migrated_config.json @@ -0,0 +1,9 @@ +{ + "refresh_token": "refresh_token", + "lwa_app_id": "amzn1.application-oa2-client.lwa_app_id", + "lwa_client_secret": "amzn1.oa2-cs.v1.lwa_client_secret", + "replication_start_date": "2022-09-01T00:00:00Z", + "aws_environment": "PRODUCTION", + "account_type": "Vendor", + "region": "US" +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations/not_migrated_config.json b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations/not_migrated_config.json new file mode 100644 index 000000000000..e7f89850ba5b --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations/not_migrated_config.json @@ -0,0 +1,8 @@ +{ + "refresh_token": "refresh_token", + "lwa_app_id": "amzn1.application-oa2-client.lwa_app_id", + "lwa_client_secret": "amzn1.oa2-cs.v1.lwa_client_secret", + "replication_start_date": "2022-09-01T00:00:00Z", + "aws_environment": "PRODUCTION", + "region": "US" +} diff --git a/docs/integrations/sources/amazon-seller-partner.md b/docs/integrations/sources/amazon-seller-partner.md index c5da26fc4d3d..1023cee1379b 100644 --- a/docs/integrations/sources/amazon-seller-partner.md +++ b/docs/integrations/sources/amazon-seller-partner.md @@ -4,20 +4,39 @@ This page guides you through the process of setting up the Amazon Seller Partner ## Prerequisites +- Amazon Selling Partner account + + + +**For Airbyte Cloud:** + +- AWS Environment +- AWS Region +- Granted OAuth access +- Replication Start Date + + + + +**For Airbyte Open Source:** + - AWS Environment - AWS Region -- LWA Client ID (LWA App ID)** -- LWA Client Secret** -- Refresh token** - Replication Start Date + -**not required for Airbyte Cloud +## Setup Guide ## Step 1: Set up Amazon Seller Partner -1. [Register](https://developer-docs.amazon.com/sp-api/docs/registering-your-application) Amazon Seller Partner application. + + +**Airbyte Open Source setup steps** + +- [Register](https://developer-docs.amazon.com/sp-api/docs/registering-your-application) Amazon Seller Partner application. - The application must be published as Amazon does not allow external parties such as Airbyte to access draft applications. -2. [Create](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html) IAM user. + + ## Step 2: Set up the source connector in Airbyte @@ -28,7 +47,7 @@ This page guides you through the process of setting up the Amazon Seller Partner 3. On the source setup page, select **Amazon Seller Partner** from the Source type dropdown and enter a name for this connector. 4. Click `Authenticate your account`. 5. Log in and Authorize to your Amazon Seller Partner account. -6. Paste all other data to required fields using your IAM user. +6. Paste all other data to required fields. 7. Click `Set up source`. **For Airbyte Open Source:** @@ -37,7 +56,7 @@ This page guides you through the process of setting up the Amazon Seller Partner 2. Go to local Airbyte page. 3. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. 4. On the Set up the source page, enter the name for the Amazon Seller Partner connector and select **Amazon Seller Partner** from the Source type dropdown. -5. Paste all data to required fields using your IAM user and developer account. +5. Paste all data to required fields. 6. Click `Set up source`. ## Supported sync modes @@ -124,6 +143,7 @@ So, for any value that exceeds the limit, the `period_in_days` will be automatic | Version | Date | Pull Request | Subject | |:---------|:-----------|:--------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `2.0.1` | 2023-11-16 | [\#32550](https://github.com/airbytehq/airbyte/pull/32550) | Fix the OAuth flow | | `2.0.0` | 2023-11-23 | [\#32355](https://github.com/airbytehq/airbyte/pull/32355) | Remove Brand Analytics from Airbyte Cloud, permanently remove deprecated FBA reports | | `1.6.2` | 2023-11-14 | [\#32508](https://github.com/airbytehq/airbyte/pull/32508) | Do not use AWS signature as it is no longer required by the Amazon API | | `1.6.1` | 2023-11-13 | [\#32457](https://github.com/airbytehq/airbyte/pull/32457) | Fix report decompression | From d475beba195253099e55d09389472fa06d13be08 Mon Sep 17 00:00:00 2001 From: Yevhenii Date: Fri, 17 Nov 2023 12:22:20 +0000 Subject: [PATCH 107/124] =?UTF-8?q?=F0=9F=90=9BCDK:=20Fix=20of=20generate?= =?UTF-8?q?=20the=20error=20message=20using=20=5Ftry=5Fget=5Ferror=20based?= =?UTF-8?q?=20on=20list=20of=20errors=20(#32545)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sources/declarative/requesters/http_requester.py | 5 ++++- .../sources/declarative/requesters/test_http_requester.py | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py index 445325f77b2d..a3a8ec657090 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py @@ -538,7 +538,8 @@ def _try_get_error(value: Any) -> Any: if isinstance(value, str): return value elif isinstance(value, list): - return ", ".join(_try_get_error(v) for v in value) + error_list = [_try_get_error(v) for v in value] + return ", ".join(v for v in error_list if v is not None) elif isinstance(value, dict): new_value = ( value.get("message") @@ -547,6 +548,8 @@ def _try_get_error(value: Any) -> Any: or value.get("errors") or value.get("failures") or value.get("failure") + or value.get("details") + or value.get("detail") ) return _try_get_error(new_value) return None diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py index ecec2379693f..0800ff62d8f6 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py @@ -686,14 +686,19 @@ def test_raise_on_http_errors(mocker, error): ({"error": {"message": "something broke"}}, "something broke"), ({"error": "err-001", "message": "something broke"}, "something broke"), ({"failure": {"message": "something broke"}}, "something broke"), + ({"detail": {"message": "something broke"}}, "something broke"), ({"error": {"errors": [{"message": "one"}, {"message": "two"}, {"message": "three"}]}}, "one, two, three"), ({"errors": ["one", "two", "three"]}, "one, two, three"), + ({"errors": [None, {}, "third error", 9002.09]}, "third error"), ({"messages": ["one", "two", "three"]}, "one, two, three"), ({"errors": [{"message": "one"}, {"message": "two"}, {"message": "three"}]}, "one, two, three"), ({"error": [{"message": "one"}, {"message": "two"}, {"message": "three"}]}, "one, two, three"), ({"errors": [{"error": "one"}, {"error": "two"}, {"error": "three"}]}, "one, two, three"), ({"failures": [{"message": "one"}, {"message": "two"}, {"message": "three"}]}, "one, two, three"), + ({"details": [{"message": "one"}, {"message": "two"}, {"message": "three"}]}, "one, two, three"), + ({"details": ["one", 10087, True]}, "one"), (["one", "two", "three"], "one, two, three"), + ({"detail": False}, None), ([{"error": "one"}, {"error": "two"}, {"error": "three"}], "one, two, three"), ({"error": True}, None), ({"something_else": "hi"}, None), From 03cfe9b9104986e7824a6017572c8997b6617f78 Mon Sep 17 00:00:00 2001 From: yevhenii-ldv Date: Fri, 17 Nov 2023 12:30:09 +0000 Subject: [PATCH 108/124] =?UTF-8?q?=F0=9F=A4=96=20Bump=20patch=20version?= =?UTF-8?q?=20of=20Python=20CDK?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-cdk/python/.bumpversion.cfg | 2 +- airbyte-cdk/python/CHANGELOG.md | 3 +++ airbyte-cdk/python/Dockerfile | 4 ++-- airbyte-cdk/python/setup.py | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/airbyte-cdk/python/.bumpversion.cfg b/airbyte-cdk/python/.bumpversion.cfg index d93cc7ac0f15..c1c6f6b90cc0 100644 --- a/airbyte-cdk/python/.bumpversion.cfg +++ b/airbyte-cdk/python/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.53.8 +current_version = 0.53.9 commit = False [bumpversion:file:setup.py] diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index fc944faa64c8..591ed0c349e0 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.53.9 +Fix of generate the error message using _try_get_error based on list of errors + ## 0.53.8 Vector DB CDK: Remove CDC records, File CDK: Update unstructured parser diff --git a/airbyte-cdk/python/Dockerfile b/airbyte-cdk/python/Dockerfile index 21618bcb99ff..6942df8c0eea 100644 --- a/airbyte-cdk/python/Dockerfile +++ b/airbyte-cdk/python/Dockerfile @@ -10,7 +10,7 @@ RUN apk --no-cache upgrade \ && apk --no-cache add tzdata build-base # install airbyte-cdk -RUN pip install --prefix=/install airbyte-cdk==0.53.8 +RUN pip install --prefix=/install airbyte-cdk==0.53.9 # build a clean environment FROM base @@ -32,5 +32,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] # needs to be the same as CDK -LABEL io.airbyte.version=0.53.8 +LABEL io.airbyte.version=0.53.9 LABEL io.airbyte.name=airbyte/source-declarative-manifest diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 8c2f8921ae71..bfb4ec5d274c 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -36,7 +36,7 @@ name="airbyte-cdk", # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be # updated if our semver format changes such as using release candidate versions. - version="0.53.8", + version="0.53.9", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", From 831a5b01cee662011fc1aecaef3234afde69f4e0 Mon Sep 17 00:00:00 2001 From: Richard Gourley Date: Fri, 17 Nov 2023 13:39:59 +0100 Subject: [PATCH 109/124] Small typo fixed - An JSON -> A JSON (#32637) --- docs/understanding-airbyte/airbyte-protocol.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/understanding-airbyte/airbyte-protocol.md b/docs/understanding-airbyte/airbyte-protocol.md index 17c742722882..66c0bc4f10ed 100644 --- a/docs/understanding-airbyte/airbyte-protocol.md +++ b/docs/understanding-airbyte/airbyte-protocol.md @@ -143,7 +143,7 @@ The `discover` method detects and describes the _structure_ of the data in the d 1. `config` - A configuration JSON object that has been validated using `ConnectorSpecification#connectionSpecification` (see [ActorSpecification](#actor-specification) for information on `connectionSpecification`). 2. `configured catalog` - A `ConfiguredAirbyteCatalog` is built on top of the `catalog` returned by `discover`. The `ConfiguredAirbyteCatalog` specifies HOW the data in the catalog should be replicated. The catalog is documented in the [Catalog Section](#catalog). -3. `state` - An JSON object that represents a checkpoint in the replication. This object is only ever written or read by the source, so it is a JSON blob with whatever information is necessary to keep track of how much of the data source has already been read (learn more in the [State & Checkpointing](#state--checkpointing) Section). +3. `state` - A JSON object that represents a checkpoint in the replication. This object is only ever written or read by the source, so it is a JSON blob with whatever information is necessary to keep track of how much of the data source has already been read (learn more in the [State & Checkpointing](#state--checkpointing) Section). #### Output: From 4481be264653708b841baee7bab04e844386078f Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk <35109939+tolik0@users.noreply.github.com> Date: Fri, 17 Nov 2023 14:45:35 +0200 Subject: [PATCH 110/124] =?UTF-8?q?=E2=9C=A8=20Source=20Pinterest:=20Marke?= =?UTF-8?q?d=20start=20date=20as=20optional;=20add=20missing=20fields;=20a?= =?UTF-8?q?dd=20suggested=20streams=20(#32592)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../connectors/source-pinterest/metadata.yaml | 13 +++- .../connectors/source-pinterest/setup.py | 2 +- .../source_pinterest/schemas/ad_groups.json | 61 +++++++++++++++++++ .../source_pinterest/schemas/ads.json | 3 + .../source_pinterest/schemas/board_pins.json | 16 +++-- .../schemas/board_section_pins.json | 3 + .../source_pinterest/schemas/campaigns.json | 15 +++++ .../source_pinterest/source.py | 2 +- .../source_pinterest/spec.json | 1 - docs/integrations/sources/pinterest.md | 57 ++++++++--------- 10 files changed, 132 insertions(+), 41 deletions(-) diff --git a/airbyte-integrations/connectors/source-pinterest/metadata.yaml b/airbyte-integrations/connectors/source-pinterest/metadata.yaml index 4763dd34d13a..82a76edc0a4b 100644 --- a/airbyte-integrations/connectors/source-pinterest/metadata.yaml +++ b/airbyte-integrations/connectors/source-pinterest/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 5cb7e5fe-38c2-11ec-8d3d-0242ac130003 - dockerImageTag: 0.7.2 + dockerImageTag: 0.8.0 dockerRepository: airbyte/source-pinterest connectorBuildOptions: baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c @@ -19,6 +19,17 @@ data: oss: enabled: true releaseStage: generally_available + suggestedStreams: + streams: + - campaign_analytics + - ad_account_analytics + - ad_analytics + - campaigns + - ad_accounts + - ads + - user_account_analytics + - ad_group_analytics + - ad_groups documentationUrl: https://docs.airbyte.com/integrations/sources/pinterest tags: - language:python diff --git a/airbyte-integrations/connectors/source-pinterest/setup.py b/airbyte-integrations/connectors/source-pinterest/setup.py index eac9cebacb4b..5da646d8e719 100644 --- a/airbyte-integrations/connectors/source-pinterest/setup.py +++ b/airbyte-integrations/connectors/source-pinterest/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.2", "pendulum~=2.1.2"] +MAIN_REQUIREMENTS = ["airbyte-cdk", "pendulum~=2.1.2"] TEST_REQUIREMENTS = [ "pytest~=6.1", diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_groups.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_groups.json index 3f24d99ee067..4ab16b8d9676 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_groups.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_groups.json @@ -116,6 +116,67 @@ }, "updated_time": { "type": ["null", "number"] + }, + "optimization_goal_metadata": { + "type": ["null", "object"], + "properties": { + "conversion_tag_v3_goal_metadata": { + "type": ["null", "object"], + "properties": { + "attribution_windows": { + "type": ["null", "object"], + "properties": { + "click_window_days": { + "type": ["null", "integer"] + }, + "engagement_window_days": { + "type": ["null", "integer"] + }, + "view_window_days": { + "type": ["null", "integer"] + } + } + }, + "conversion_event": { + "type": ["null", "string"] + }, + "conversion_tag_id": { + "type": ["null", "string"] + }, + "cpa_goal_value_in_micro_currency": { + "type": ["null", "string"] + }, + "is_roas_optimized": { + "type": ["null", "boolean"] + }, + "learning_mode_type": { + "type": ["null", "string"] + } + } + }, + "frequency_goal_metadata": { + "type": ["null", "object"], + "properties": { + "frequency": { + "type": ["null", "integer"] + }, + "timerange": { + "type": ["null", "string"] + } + } + }, + "scrollup_goal_metadata": { + "type": ["null", "object"], + "properties": { + "scrollup_goal_value_in_micro_currency": { + "type": ["null", "string"] + } + } + } + } + }, + "bid_strategy_type": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ads.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ads.json index 2b385cce892d..d5f238bd9b10 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ads.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ads.json @@ -91,6 +91,9 @@ "view_tracking_url": { "type": ["null", "string"] }, + "lead_form_id": { + "type": ["null", "string"] + }, "ad_account_id": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_pins.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_pins.json index 2989e890ced2..55a5c52fbd48 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_pins.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_pins.json @@ -11,15 +11,7 @@ "format": "date-time" }, "creative_type": { - "type": ["null", "string"], - "enum": [ - "REGULAR", - "VIDEO", - "CAROUSEL", - "MAX_VIDEO", - "SHOP_THE_PIN", - "IDEA" - ] + "type": ["null", "string"] }, "is_standard": { "type": ["null", "boolean"] @@ -77,6 +69,12 @@ "type": ["null", "string"] } } + }, + "pin_metrics": { + "type": ["null", "object"] + }, + "has_been_promoted": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_section_pins.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_section_pins.json index 74bdf144cb8a..603145526fa6 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_section_pins.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_section_pins.json @@ -35,6 +35,9 @@ } } }, + "pin_metrics": { + "type": ["null", "object"] + }, "media": { "type": ["null", "object"], "properties": { diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaigns.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaigns.json index 561bebf0d971..cb91bc3af2d7 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaigns.json @@ -69,6 +69,21 @@ }, "type": { "type": ["null", "string"] + }, + "start_time": { + "type": ["null", "integer"] + }, + "end_time": { + "type": ["null", "integer"] + }, + "summary_status": { + "type": ["null", "string"] + }, + "is_campaign_budget_optimization": { + "type": ["null", "boolean"] + }, + "is_flexible_daily_budgets": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py index 828e07f22b53..b8f9b693a9ab 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py @@ -59,7 +59,7 @@ def _validate_and_transform(self, config: Mapping[str, Any], amount_of_days_allo today = pendulum.today() latest_date_allowed_by_api = today.subtract(days=amount_of_days_allowed_for_lookup) - start_date = config["start_date"] + start_date = config.get("start_date") if not start_date: config["start_date"] = latest_date_allowed_by_api else: diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/spec.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/spec.json index 835d983074c2..c19b80a65901 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/spec.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/spec.json @@ -4,7 +4,6 @@ "$schema": "https://json-schema.org/draft-07/schema#", "title": "Pinterest Spec", "type": "object", - "required": ["start_date"], "additionalProperties": true, "properties": { "start_date": { diff --git a/docs/integrations/sources/pinterest.md b/docs/integrations/sources/pinterest.md index aa599ef5389e..5b8f94e40de1 100644 --- a/docs/integrations/sources/pinterest.md +++ b/docs/integrations/sources/pinterest.md @@ -84,32 +84,33 @@ The connector is restricted by the Pinterest [requests limitation](https://devel ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------| :------------------------------------------------------- |:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :------------------------------------------------------- |:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.8.0 | 2023-11-16 | [32592](https://github.com/airbytehq/airbyte/pull/32592) | Make start_date optional; add suggested streams; add missing fields | | 0.7.2 | 2023-11-08 | [32299](https://github.com/airbytehq/airbyte/pull/32299) | added default `AvailabilityStrategy`, fixed bug which cases duplicated requests, added new streams: Catalogs, CatalogsFeeds, CatalogsProductGroups, Audiences, Keywords, ConversionTags, CustomerLists, CampaignTargetingReport, AdvertizerReport, AdvertizerTargetingReport, AdGroupReport, AdGroupTargetingReport, PinPromotionReport, PinPromotionTargetingReport, ProductGroupReport, ProductGroupTargetingReport, ProductItemReport, KeywordReport | -| 0.7.1 | 2023-11-01 | [32078](https://github.com/airbytehq/airbyte/pull/32078) | handle non json response | -| 0.7.0 | 2023-10-25 | [31876](https://github.com/airbytehq/airbyte/pull/31876) | Migrated to base image, removed token based authentication mthod becuase access_token is valid for 1 day only | -| 0.6.0 | 2023-07-25 | [28672](https://github.com/airbytehq/airbyte/pull/28672) | Add report stream for `CAMPAIGN` level | -| 0.5.3 | 2023-07-05 | [27964](https://github.com/airbytehq/airbyte/pull/27964) | Add `id` field to `owner` field in `ad_accounts` stream | -| 0.5.2 | 2023-06-02 | [26949](https://github.com/airbytehq/airbyte/pull/26949) | Update `BoardPins` stream with `note` property | -| 0.5.1 | 2023-05-11 | [25984](https://github.com/airbytehq/airbyte/pull/25984) | Add pattern for start_date | -| 0.5.0 | 2023-05-17 | [26188](https://github.com/airbytehq/airbyte/pull/26188) | Add `product_tags` field to the `BoardPins` stream | -| 0.4.0 | 2023-05-16 | [26112](https://github.com/airbytehq/airbyte/pull/26112) | Add `is_standard` field to the `BoardPins` stream | -| 0.3.0 | 2023-05-09 | [25915](https://github.com/airbytehq/airbyte/pull/25915) | Add `creative_type` field to the `BoardPins` stream | -| 0.2.6 | 2023-04-26 | [25548](https://github.com/airbytehq/airbyte/pull/25548) | Fix `format` issue for `boards` stream schema for fields with `date-time` | -| 0.2.5 | 2023-04-19 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Update `AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP` to 89 days | -| 0.2.4 | 2023-02-25 | [23457](https://github.com/airbytehq/airbyte/pull/23457) | Add missing columns for analytics streams for pinterest source | -| 0.2.3 | 2023-03-01 | [23649](https://github.com/airbytehq/airbyte/pull/23649) | Fix for `HTTP - 400 Bad Request` when requesting data >= 90 days | -| 0.2.2 | 2023-01-27 | [22020](https://github.com/airbytehq/airbyte/pull/22020) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 0.2.1 | 2022-12-15 | [20532](https://github.com/airbytehq/airbyte/pull/20532) | Bump CDK version | -| 0.2.0 | 2022-12-13 | [20242](https://github.com/airbytehq/airbyte/pull/20242) | Add data-type normalization up to the schemas declared | -| 0.1.9 | 2022-09-06 | [15074](https://github.com/airbytehq/airbyte/pull/15074) | Add filter based on statuses | -| 0.1.8 | 2022-10-21 | [18285](https://github.com/airbytehq/airbyte/pull/18285) | Fix type of `start_date` | -| 0.1.7 | 2022-09-29 | [17387](https://github.com/airbytehq/airbyte/pull/17387) | Set `start_date` dynamically based on API restrictions. | -| 0.1.6 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Use CDK 0.1.89 | -| 0.1.5 | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state | -| 0.1.4 | 2022-09-06 | [16161](https://github.com/airbytehq/airbyte/pull/16161) | Add ability to handle `429 - Too Many Requests` error with respect to `Max Rate Limit Exceeded Error` | -| 0.1.3 | 2022-09-02 | [16271](https://github.com/airbytehq/airbyte/pull/16271) | Add support of `OAuth2.0` authentication method | -| 0.1.2 | 2021-12-22 | [10223](https://github.com/airbytehq/airbyte/pull/10223) | Fix naming of `AD_ID` and `AD_ACCOUNT_ID` fields | -| 0.1.1 | 2021-12-22 | [9043](https://github.com/airbytehq/airbyte/pull/9043) | Update connector fields title/description | -| 0.1.0 | 2021-10-29 | [7493](https://github.com/airbytehq/airbyte/pull/7493) | Release Pinterest CDK Connector | +| 0.7.1 | 2023-11-01 | [32078](https://github.com/airbytehq/airbyte/pull/32078) | handle non json response | +| 0.7.0 | 2023-10-25 | [31876](https://github.com/airbytehq/airbyte/pull/31876) | Migrated to base image, removed token based authentication mthod becuase access_token is valid for 1 day only | +| 0.6.0 | 2023-07-25 | [28672](https://github.com/airbytehq/airbyte/pull/28672) | Add report stream for `CAMPAIGN` level | +| 0.5.3 | 2023-07-05 | [27964](https://github.com/airbytehq/airbyte/pull/27964) | Add `id` field to `owner` field in `ad_accounts` stream | +| 0.5.2 | 2023-06-02 | [26949](https://github.com/airbytehq/airbyte/pull/26949) | Update `BoardPins` stream with `note` property | +| 0.5.1 | 2023-05-11 | [25984](https://github.com/airbytehq/airbyte/pull/25984) | Add pattern for start_date | +| 0.5.0 | 2023-05-17 | [26188](https://github.com/airbytehq/airbyte/pull/26188) | Add `product_tags` field to the `BoardPins` stream | +| 0.4.0 | 2023-05-16 | [26112](https://github.com/airbytehq/airbyte/pull/26112) | Add `is_standard` field to the `BoardPins` stream | +| 0.3.0 | 2023-05-09 | [25915](https://github.com/airbytehq/airbyte/pull/25915) | Add `creative_type` field to the `BoardPins` stream | +| 0.2.6 | 2023-04-26 | [25548](https://github.com/airbytehq/airbyte/pull/25548) | Fix `format` issue for `boards` stream schema for fields with `date-time` | +| 0.2.5 | 2023-04-19 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Update `AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP` to 89 days | +| 0.2.4 | 2023-02-25 | [23457](https://github.com/airbytehq/airbyte/pull/23457) | Add missing columns for analytics streams for pinterest source | +| 0.2.3 | 2023-03-01 | [23649](https://github.com/airbytehq/airbyte/pull/23649) | Fix for `HTTP - 400 Bad Request` when requesting data >= 90 days | +| 0.2.2 | 2023-01-27 | [22020](https://github.com/airbytehq/airbyte/pull/22020) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.2.1 | 2022-12-15 | [20532](https://github.com/airbytehq/airbyte/pull/20532) | Bump CDK version | +| 0.2.0 | 2022-12-13 | [20242](https://github.com/airbytehq/airbyte/pull/20242) | Add data-type normalization up to the schemas declared | +| 0.1.9 | 2022-09-06 | [15074](https://github.com/airbytehq/airbyte/pull/15074) | Add filter based on statuses | +| 0.1.8 | 2022-10-21 | [18285](https://github.com/airbytehq/airbyte/pull/18285) | Fix type of `start_date` | +| 0.1.7 | 2022-09-29 | [17387](https://github.com/airbytehq/airbyte/pull/17387) | Set `start_date` dynamically based on API restrictions. | +| 0.1.6 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Use CDK 0.1.89 | +| 0.1.5 | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state | +| 0.1.4 | 2022-09-06 | [16161](https://github.com/airbytehq/airbyte/pull/16161) | Add ability to handle `429 - Too Many Requests` error with respect to `Max Rate Limit Exceeded Error` | +| 0.1.3 | 2022-09-02 | [16271](https://github.com/airbytehq/airbyte/pull/16271) | Add support of `OAuth2.0` authentication method | +| 0.1.2 | 2021-12-22 | [10223](https://github.com/airbytehq/airbyte/pull/10223) | Fix naming of `AD_ID` and `AD_ACCOUNT_ID` fields | +| 0.1.1 | 2021-12-22 | [9043](https://github.com/airbytehq/airbyte/pull/9043) | Update connector fields title/description | +| 0.1.0 | 2021-10-29 | [7493](https://github.com/airbytehq/airbyte/pull/7493) | Release Pinterest CDK Connector | From 9d1a4ba04e8d508c196d45dd2e5e13eb890fa85a Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 17 Nov 2023 14:51:49 +0100 Subject: [PATCH 111/124] Deprecate langchain destination (#32455) --- .../connectors/destination-langchain/Dockerfile | 2 +- .../connectors/destination-langchain/metadata.yaml | 2 +- .../connectors/destination-langchain/setup.py | 2 +- docs/integrations/destinations/langchain.md | 13 +++++++++++++ 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/destination-langchain/Dockerfile b/airbyte-integrations/connectors/destination-langchain/Dockerfile index b4b48cdd04c2..30452c2628ac 100644 --- a/airbyte-integrations/connectors/destination-langchain/Dockerfile +++ b/airbyte-integrations/connectors/destination-langchain/Dockerfile @@ -42,5 +42,5 @@ COPY destination_langchain ./destination_langchain ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/destination-langchain diff --git a/airbyte-integrations/connectors/destination-langchain/metadata.yaml b/airbyte-integrations/connectors/destination-langchain/metadata.yaml index a76d4126868f..f8db27c1afe0 100644 --- a/airbyte-integrations/connectors/destination-langchain/metadata.yaml +++ b/airbyte-integrations/connectors/destination-langchain/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: database connectorType: destination definitionId: cf98d52c-ba5a-4dfd-8ada-c1baebfa6e73 - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 dockerRepository: airbyte/destination-langchain githubIssueLabel: destination-langchain icon: langchain.svg diff --git a/airbyte-integrations/connectors/destination-langchain/setup.py b/airbyte-integrations/connectors/destination-langchain/setup.py index 80f25bd65f1e..5446952fc464 100644 --- a/airbyte-integrations/connectors/destination-langchain/setup.py +++ b/airbyte-integrations/connectors/destination-langchain/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk==0.51.10", "langchain", "openai", "requests", diff --git a/docs/integrations/destinations/langchain.md b/docs/integrations/destinations/langchain.md index 31a9ddcae93d..4ac1fe151906 100644 --- a/docs/integrations/destinations/langchain.md +++ b/docs/integrations/destinations/langchain.md @@ -1,5 +1,17 @@ # Vector Database (powered by LangChain) +:::warning +The vector db destination destination has been split into separate destinations per vector database. This destination will not receive any further updates and is not subject to SLAs. The separate destinations support all features of this destination and are actively maintained. Please migrate to the respective destination as soon as possible. + +Please use the respective destination for the vector database you want to use to ensure you receive updates and support. + +To following databases are supported: +* [Pinecone](https://docs.airbyte.com/integrations/destinations/pinecone) +* [Weaviate](https://docs.airbyte.com/integrations/destinations/weaviate) +* [Milvus](https://docs.airbyte.com/integrations/destinations/milvus) +* [Chroma](https://docs.airbyte.com/integrations/destinations/chroma) +* [Qdrant](https://docs.airbyte.com/integrations/destinations/qdrant) +::: ## Overview @@ -140,6 +152,7 @@ Please make sure that Docker Desktop has access to `/tmp` (and `/private` on a M | Version | Date | Pull Request | Subject | |:--------| :--------- |:--------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.1.2 | 2023-11-13 | [#32455](https://github.com/airbytehq/airbyte/pull/32455) | Fix build | | 0.1.1 | 2023-09-01 | [#30282](https://github.com/airbytehq/airbyte/pull/30282) | Use embedders from CDK | | 0.1.0 | 2023-09-01 | [#30080](https://github.com/airbytehq/airbyte/pull/30080) | Fix bug with potential data loss on append+dedup syncing. 🚨 Streams using append+dedup mode need to be reset after upgrade. | | 0.0.8 | 2023-08-21 | [#29515](https://github.com/airbytehq/airbyte/pull/29515) | Clean up generated schema spec | From 0391d374a0d6f9b49e82f8f3dac11bd67ac98efb Mon Sep 17 00:00:00 2001 From: Artem Inzhyyants <36314070+artem1205@users.noreply.github.com> Date: Fri, 17 Nov 2023 14:58:31 +0100 Subject: [PATCH 112/124] =?UTF-8?q?=F0=9F=90=9B=20Source=20Bing=20Ads:=20F?= =?UTF-8?q?ix=20start=20date=20parsing=20from=20stream=20state=20(#32597)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../integration_tests/expected_records.jsonl | 4 ++-- .../connectors/source-bing-ads/metadata.yaml | 2 +- .../source_bing_ads/report_streams.py | 2 +- .../source-bing-ads/unit_tests/test_reports.py | 15 ++++++--------- docs/integrations/sources/bing-ads.md | 1 + 5 files changed, 11 insertions(+), 13 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl index 62627cf0aeae..7a61e57866e3 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl @@ -15,8 +15,8 @@ {"stream":"campaign_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"TimePeriod":"2023-11-05","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Syndicated search partners","DeliveredMatchType":"Exact","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","CampaignStatus":"Active","CampaignLabels":null,"Impressions":9,"Clicks":1,"Ctr":11.11,"Spend":0.03,"CostPerConversion":null,"QualityScore":5.0,"AdRelevance":3.0,"LandingPageExperience":1.0,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"CustomParameters":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.03,"AveragePosition":0.0,"AverageCpm":3.33,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"LowQualityClicks":0,"LowQualityClicksPercent":0.0,"LowQualityImpressions":0,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"BudgetName":null,"BudgetStatus":null,"BudgetAssociationStatus":"Current","HistoricalQualityScore":5.0,"HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":1.0},"emitted_at":1699954081143} {"stream":"campaign_impression_performance_report_daily","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-11-07","CampaignStatus":"Active","CampaignName":"Airbyte test","CampaignId":531016227,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":10,"Clicks":1,"Ctr":10.0,"AverageCpc":0.33,"Spend":0.33,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"LowQualityClicks":0,"LowQualityClicksPercent":0.0,"LowQualityImpressions":9,"LowQualityImpressionsPercent":47.37,"LowQualityConversions":0,"LowQualityConversionRate":null,"DeviceType":"Computer","ImpressionSharePercent":3.37,"ImpressionLostToBudgetPercent":85.19,"ImpressionLostToRankAggPercent":11.45,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"HistoricalQualityScore":null,"HistoricalExpectedCtr":null,"HistoricalAdRelevance":null,"HistoricalLandingPageExperience":null,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":0,"CampaignLabels":null,"ExactMatchImpressionSharePercent":null,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":6.02,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":14.63,"TopImpressionShareLostToBudgetPercent":77.24,"AbsoluteTopImpressionShareLostToRankPercent":15.66,"AbsoluteTopImpressionShareLostToBudgetPercent":78.31,"TopImpressionSharePercent":8.13,"AbsoluteTopImpressionRatePercent":50.0,"TopImpressionRatePercent":100.0,"BaseCampaignId":531016227,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AverageCpm":33.0,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1699954182626} {"stream":"campaign_impression_performance_report_weekly","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-11-05","CampaignStatus":"Active","CampaignName":"Airbyte test","CampaignId":531016227,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":10,"Clicks":1,"Ctr":10.0,"AverageCpc":0.33,"Spend":0.33,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"LowQualityClicks":0,"LowQualityClicksPercent":0.0,"LowQualityImpressions":9,"LowQualityImpressionsPercent":47.37,"LowQualityConversions":0,"LowQualityConversionRate":null,"DeviceType":"Computer","ImpressionSharePercent":10.87,"ImpressionLostToBudgetPercent":17.05,"ImpressionLostToRankAggPercent":72.08,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"HistoricalQualityScore":null,"HistoricalExpectedCtr":null,"HistoricalAdRelevance":null,"HistoricalLandingPageExperience":null,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":0,"CampaignLabels":null,"ExactMatchImpressionSharePercent":29.07,"ClickSharePercent":2.89,"AbsoluteTopImpressionSharePercent":8.88,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":76.51,"TopImpressionShareLostToBudgetPercent":9.99,"AbsoluteTopImpressionShareLostToRankPercent":81.99,"AbsoluteTopImpressionShareLostToBudgetPercent":9.13,"TopImpressionSharePercent":13.5,"AbsoluteTopImpressionRatePercent":50.0,"TopImpressionRatePercent":100.0,"BaseCampaignId":531016227,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AverageCpm":33.0,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1699954211223} -{"stream":"keyword_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"KeywordId":84801135055365,"Keyword":"connector","AdId":84800390693061,"TimePeriod":"2023-11-07","CurrencyCode":"USD","DeliveredMatchType":"Phrase","AdDistribution":"Search","DeviceType":"Computer","Language":"German","Network":"Syndicated search partners","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","KeywordStatus":"Active","HistoricalExpectedCtr":null,"HistoricalAdRelevance":null,"HistoricalLandingPageExperience":null,"HistoricalQualityScore":null,"Impressions":1,"Clicks":0,"Ctr":0.0,"CurrentMaxCpc":2.27,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"QualityImpact":0.0,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"Mainline1Bid":null,"MainlineBid":1.18,"FirstPageBid":0.51,"FinalUrlSuffix":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1700078149400} -{"stream":"keyword_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"KeywordId":84801135055365,"Keyword":"connector","AdId":84800390693061,"TimePeriod":"2023-11-05","CurrencyCode":"USD","DeliveredMatchType":"Exact","AdDistribution":"Search","DeviceType":"Computer","Language":"English","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","KeywordStatus":"Active","Impressions":2,"Clicks":0,"Ctr":0.0,"CurrentMaxCpc":2.27,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"QualityImpact":0.0,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"Mainline1Bid":null,"MainlineBid":1.18,"FirstPageBid":0.51,"FinalUrlSuffix":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1700078299436} +{"stream":"keyword_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"KeywordId":84801135055365,"Keyword":"connector","AdId":84800390693061,"TimePeriod":"2023-11-07","CurrencyCode":"USD","DeliveredMatchType":"Phrase","AdDistribution":"Search","DeviceType":"Computer","Language":"German","Network":"Syndicated search partners","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","KeywordStatus":"Active","HistoricalExpectedCtr":null,"HistoricalAdRelevance":null,"HistoricalLandingPageExperience":null,"HistoricalQualityScore":null,"Impressions":1,"Clicks":0,"Ctr":0.0,"CurrentMaxCpc":2.27,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"QualityImpact":0.0,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"Mainline1Bid":null,"MainlineBid":1.11,"FirstPageBid":0.48,"FinalUrlSuffix":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1700167471396} +{"stream":"keyword_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"KeywordId":84801135055365,"Keyword":"connector","AdId":84800390693061,"TimePeriod":"2023-11-05","CurrencyCode":"USD","DeliveredMatchType":"Exact","AdDistribution":"Search","DeviceType":"Computer","Language":"English","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","KeywordStatus":"Active","Impressions":2,"Clicks":0,"Ctr":0.0,"CurrentMaxCpc":2.27,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"QualityImpact":0.0,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"Mainline1Bid":null,"MainlineBid":1.11,"FirstPageBid":0.48,"FinalUrlSuffix":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1700167573859} {"stream":"geographic_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"TimePeriod":"2023-11-07","Country":"Australia","CurrencyCode":"USD","DeliveredMatchType":"Broad","AdDistribution":"Search","DeviceType":"Computer","Language":"English","Network":"Syndicated search partners","DeviceOS":"Windows","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","MetroArea":null,"State":"New South Wales","City":null,"AdGroupName":"keywords","Ctr":0.0,"ProximityTargetLocation":null,"Radius":"0","Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"LocationType":"Physical location","MostSpecificLocation":"2000","AccountStatus":"Active","CampaignStatus":"Active","AdGroupStatus":"Active","County":null,"PostalCode":"2000","LocationId":"122395","BaseCampaignId":"531016227","AllCostPerConversion":null,"AllReturnOnAdSpend":null,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":"100.00","AllConversionsQualified":"0.00","ViewThroughConversionsQualified":null,"Neighborhood":null,"ViewThroughRevenue":"0.00","CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null,"AssetGroupStatus":null,"Clicks":0,"Spend":0.0,"Impressions":1,"CostPerConversion":null,"AccountName":"Airbyte","AccountNumber":"F149MJ18","CampaignName":"Airbyte test","Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1699956863587} {"stream":"geographic_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"TimePeriod":"2023-11-05","Country":"Argentina","CurrencyCode":"USD","DeliveredMatchType":"Exact","AdDistribution":"Search","DeviceType":"Computer","Language":"Spanish","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","MetroArea":null,"State":"Buenos Aires Province","City":null,"AdGroupName":"keywords","Ctr":0.0,"ProximityTargetLocation":null,"Radius":"0","Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"LocationType":"Physical location","MostSpecificLocation":"Buenos Aires Province","AccountStatus":"Active","CampaignStatus":"Active","AdGroupStatus":"Active","County":null,"PostalCode":null,"LocationId":"141965","BaseCampaignId":"531016227","AllCostPerConversion":null,"AllReturnOnAdSpend":null,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":"100.00","AllConversionsQualified":"0.00","ViewThroughConversionsQualified":null,"Neighborhood":null,"ViewThroughRevenue":"0.00","CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null,"AssetGroupStatus":null,"Clicks":0,"Spend":0.0,"Impressions":1,"CostPerConversion":null,"AccountName":"Airbyte","AccountNumber":"F149MJ18","CampaignName":"Airbyte test","Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1699953673210} {"stream":"age_gender_audience_report_daily","data":{"AccountId":180519267,"AgeGroup":"Unknown","Gender":"Unknown","TimePeriod":"2023-11-07","AllConversions":0,"AccountName":"Airbyte","AccountNumber":"F149MJ18","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"AdDistribution":"Search","Impressions":3,"Clicks":1,"Conversions":0.0,"Spend":0.79,"Revenue":0.0,"ExtendedCost":0.0,"Assists":0,"Language":"German","AccountStatus":"Active","CampaignStatus":"Active","AdGroupStatus":"Active","BaseCampaignId":"531016227","AllRevenue":0.0,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":33.33,"TopImpressionRatePercent":100.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0},"emitted_at":1699954406862} diff --git a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml index 1c29083b42ff..8d64cd443fa7 100644 --- a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml @@ -16,7 +16,7 @@ data: connectorSubtype: api connectorType: source definitionId: 47f25999-dd5e-4636-8c39-e7cea2453331 - dockerImageTag: 2.0.0 + dockerImageTag: 2.0.1 dockerRepository: airbyte/source-bing-ads documentationUrl: https://docs.airbyte.com/integrations/sources/bing-ads githubIssueLabel: source-bing-ads diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py index 2be5dd890834..5a0d1c5818a3 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py @@ -154,7 +154,7 @@ def request_params( def get_start_date(self, stream_state: Mapping[str, Any] = None, account_id: str = None): if stream_state and account_id: if stream_state.get(account_id, {}).get(self.cursor_field): - return pendulum.parse(self.get_report_record_timestamp(stream_state[account_id][self.cursor_field])) + return pendulum.parse(stream_state[account_id][self.cursor_field]) return self.client.reports_start_date diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py index 44c5329bd83a..a10ba7f94022 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py @@ -200,7 +200,7 @@ def test_get_report_record_timestamp_hourly(stream_report_hourly_cls): def test_report_get_start_date_wo_stream_state(): expected_start_date = "2020-01-01" - test_report = TestReport() + test_report = GeographicPerformanceReportDaily(client=Mock(), config=TEST_CONFIG) test_report.client.reports_start_date = "2020-01-01" stream_state = {} account_id = "123" @@ -209,20 +209,18 @@ def test_report_get_start_date_wo_stream_state(): def test_report_get_start_date_with_stream_state(): expected_start_date = pendulum.parse("2023-04-17T21:29:57") - test_report = TestReport() - test_report.cursor_field = "cursor_field" + test_report = GeographicPerformanceReportDaily(client=Mock(), config=TEST_CONFIG) test_report.client.reports_start_date = "2020-01-01" - stream_state = {"123": {"cursor_field": "2023-04-17T21:29:57+00:00"}} + stream_state = {"123": {"TimePeriod": "2023-04-17T21:29:57+00:00"}} account_id = "123" assert expected_start_date == test_report.get_start_date(stream_state, account_id) def test_report_get_start_date_performance_report_with_stream_state(): expected_start_date = pendulum.parse("2023-04-07T21:29:57") - test_report = TestPerformanceReport() - test_report.cursor_field = "cursor_field" + test_report = GeographicPerformanceReportDaily(client=Mock(), config=TEST_CONFIG) test_report.config = {"lookback_window": 10} - stream_state = {"123": {"cursor_field": "2023-04-17T21:29:57+00:00"}} + stream_state = {"123": {"TimePeriod": "2023-04-17T21:29:57+00:00"}} account_id = "123" assert expected_start_date == test_report.get_start_date(stream_state, account_id) @@ -230,8 +228,7 @@ def test_report_get_start_date_performance_report_with_stream_state(): def test_report_get_start_date_performance_report_wo_stream_state(): days_to_subtract = 10 reports_start_date = pendulum.parse("2021-04-07T00:00:00") - test_report = TestPerformanceReport() - test_report.cursor_field = "cursor_field" + test_report = GeographicPerformanceReportDaily(client=Mock(), config=TEST_CONFIG) test_report.client.reports_start_date = reports_start_date test_report.config = {"lookback_window": days_to_subtract} stream_state = {} diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md index cab52dc73a5b..a7a972931394 100644 --- a/docs/integrations/sources/bing-ads.md +++ b/docs/integrations/sources/bing-ads.md @@ -208,6 +208,7 @@ The Bing Ads API limits the number of requests for all Microsoft Advertising cli | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.0.1 | 2023-11-16 | [32597](https://github.com/airbytehq/airbyte/pull/32597) | Fix start date parsing from stream state | | 2.0.0 | 2023-11-07 | [31995](https://github.com/airbytehq/airbyte/pull/31995) | Schema update for Accounts, Campaigns and Search Query Performance Report streams. Convert `date` and `date-time` fields to standard `RFC3339` | | 1.13.0 | 2023-11-13 | [32306](https://github.com/airbytehq/airbyte/pull/32306) | Add Custom reports and decrease backoff max tries number | | 1.12.1 | 2023-11-10 | [32422](https://github.com/airbytehq/airbyte/pull/32422) | Normalize numeric values in reports | From a745a374023afbab84a37f6fd22603d11a7fe8ac Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk <35109939+tolik0@users.noreply.github.com> Date: Fri, 17 Nov 2023 16:23:20 +0200 Subject: [PATCH 113/124] =?UTF-8?q?=F0=9F=90=9B=20Source=20Instagram:=20Ch?= =?UTF-8?q?ange=20start=5Fdate=20type=20to=20date;=20fix=20docs=20(#32627)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../source-instagram/integration_tests/spec.json | 3 ++- .../connectors/source-instagram/metadata.yaml | 2 +- .../source-instagram/source_instagram/source.py | 2 +- docs/integrations/sources/instagram.md | 13 +++++++------ 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/airbyte-integrations/connectors/source-instagram/integration_tests/spec.json b/airbyte-integrations/connectors/source-instagram/integration_tests/spec.json index cfed60215dc4..f3fbd6e9dc22 100644 --- a/airbyte-integrations/connectors/source-instagram/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-instagram/integration_tests/spec.json @@ -10,7 +10,8 @@ "description": "The date from which you'd like to replicate data for User Insights, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated. If left blank, the start date will be set to 2 years before the present date.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "examples": ["2017-01-25T00:00:00Z"], - "type": "string" + "type": "string", + "format": "date-time" }, "access_token": { "title": "Access Token", diff --git a/airbyte-integrations/connectors/source-instagram/metadata.yaml b/airbyte-integrations/connectors/source-instagram/metadata.yaml index 2e1403daea66..48c76e1a70d2 100644 --- a/airbyte-integrations/connectors/source-instagram/metadata.yaml +++ b/airbyte-integrations/connectors/source-instagram/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6acf6b55-4f1e-4fca-944e-1a3caef8aba8 - dockerImageTag: 1.0.15 + dockerImageTag: 1.0.16 dockerRepository: airbyte/source-instagram githubIssueLabel: source-instagram icon: instagram.svg diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/source.py b/airbyte-integrations/connectors/source-instagram/source_instagram/source.py index 73ce1fb5b5f4..4a41d013c1a9 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/source.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/source.py @@ -18,7 +18,7 @@ class ConnectorConfig(BaseModel): class Config: title = "Source Instagram" - start_date: Optional[str] = Field( + start_date: Optional[datetime] = Field( description="The date from which you'd like to replicate data for User Insights, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated. If left blank, the start date will be set to 2 years before the present date.", pattern="^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", examples=["2017-01-25T00:00:00Z"], diff --git a/docs/integrations/sources/instagram.md b/docs/integrations/sources/instagram.md index b392c4737016..307acbd686ac 100644 --- a/docs/integrations/sources/instagram.md +++ b/docs/integrations/sources/instagram.md @@ -10,9 +10,10 @@ This page contains the setup guide and reference information for the Instagram s - [Meta for Developers account](https://developers.facebook.com) - [Instagram business account](https://www.facebook.com/business/help/898752960195806) to your Facebook page +- [Facebook ad account ID number](https://www.facebook.com/business/help/1492627900875762) (you'll use this to configure Instagram as a source in Airbyte - [Instagram Graph API](https://developers.facebook.com/docs/instagram-api/) to your Facebook app -- [Facebook OAuth Reference](https://developers.facebook.com/docs/instagram-basic-display-api/reference) -- [Facebook ad account ID number](https://www.facebook.com/business/help/1492627900875762) (you'll use this to configure Instagram as a source in Airbyte) +- [Facebook Instagram OAuth Reference](https://developers.facebook.com/docs/instagram-basic-display-api/reference) + ## Setup Guide @@ -28,7 +29,7 @@ This page contains the setup guide and reference information for the Instagram s 4. Enter a name for your source. 5. Click **Authenticate your Instagram account**. 6. Log in and authorize the Instagram account. -7. Enter the **Start Date** in YYYY-MM-DDTHH:mm:ssZ format. All data generated after this date will be replicated. If this field is blank, Airbyte will replicate all data. +7. (Optional) Enter the **Start Date** in YYYY-MM-DDTHH:mm:ssZ format. All data generated after this date will be replicated. If left blank, the start date will be set to 2 years before the present date. 8. Click **Set up source**. @@ -40,9 +41,8 @@ This page contains the setup guide and reference information for the Instagram s 2. Click **Sources** and then click **+ New source**. 3. On the Set up the source page, select **Instagram** from the **Source type** dropdown. 4. Enter a name for your source. -5. Click **Authenticate your Instagram account**. -6. Log in and authorize the Instagram account. -7. Enter the **Start Date** in YYYY-MM-DDTHH:mm:ssZ format. All data generated after this date will be replicated. If this field is blank, Airbyte will replicate all data. +5. Enter **Access Token** generated using [Graph API Explorer](https://developers.facebook.com/tools/explorer/) or [by using an app you can create on Facebook](https://developers.facebook.com/docs/instagram-api/getting-started) with the required permissions: instagram_basic, instagram_manage_insights, pages_show_list, pages_read_engagement. +7. (Optional) Enter the **Start Date** in YYYY-MM-DDTHH:mm:ssZ format. All data generated after this date will be replicated. If left blank, the start date will be set to 2 years before the present date. 8. Click **Set up source**. @@ -93,6 +93,7 @@ AirbyteRecords are required to conform to the [Airbyte type](https://docs.airbyt | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------| +| 1.0.16 | 2023-11-17 | [32627](https://github.com/airbytehq/airbyte/pull/32627) | Fix start_date type; fix docs | | 1.0.15 | 2023-11-14 | [32494](https://github.com/airbytehq/airbyte/pull/32494) | Marked start_date as optional; set max retry time to 10 minutes; add suggested streams | | 1.0.14 | 2023-11-13 | [32423](https://github.com/airbytehq/airbyte/pull/32423) | Capture media_product_type column in media and stories stream | | 1.0.13 | 2023-11-10 | [32245](https://github.com/airbytehq/airbyte/pull/32245) | Add skipping reading MediaInsights stream if an error code 10 is received | From cc7f01925397c7d773f8cae33eb19651bf697c85 Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk <35109939+tolik0@users.noreply.github.com> Date: Fri, 17 Nov 2023 17:46:37 +0200 Subject: [PATCH 114/124] =?UTF-8?q?=F0=9F=9A=A8=F0=9F=9A=A8=E2=9C=A8=20Sou?= =?UTF-8?q?rce=20Instagram:=20Add=20primary=20keys=20for=20UserLifetimeIns?= =?UTF-8?q?ights=20and=20UserInsights;=20add=20airbyte=5Ftype=20to=20times?= =?UTF-8?q?tamp=20fields=20=20(#32500)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../integration_tests/expected_records.jsonl | 14 ++++---- .../connectors/source-instagram/metadata.yaml | 9 +++++- .../source_instagram/schemas/media.json | 6 ++-- .../source_instagram/schemas/stories.json | 3 +- .../schemas/user_insights.json | 3 +- .../schemas/user_lifetime_insights.json | 3 +- .../source_instagram/streams.py | 31 ++++++++++++++---- .../source-instagram/unit_tests/conftest.py | 1 + .../unit_tests/test_source.py | 3 ++ .../unit_tests/test_streams.py | 32 ++++++++++++++----- .../sources/instagram-migrations.md | 9 ++++++ docs/integrations/sources/instagram.md | 1 + 12 files changed, 88 insertions(+), 27 deletions(-) create mode 100644 docs/integrations/sources/instagram-migrations.md diff --git a/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl index 0a60185b5dbf..d90034355c7a 100644 --- a/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl @@ -1,7 +1,7 @@ -{"stream": "users", "data": {"id": "17841408147298757", "biography": "Airbyte is the new open-source data integration platform that consolidates your data into your warehouses.", "profile_picture_url": "https://scontent-iev1-1.xx.fbcdn.net/v/t51.2885-15/153169696_890787328349641_8382928081987798464_n.jpg?_nc_cat=111&_nc_sid=7d201b&_nc_ohc=uQq3P1OLNOYAX_JjmNN&_nc_ht=scontent-iev1-1.xx&edm=AL-3X8kEAAAA&oh=00_AfBI4aspXBrxU-bYTD-qnPWh7ex05YFFAOl_24u7JxLYrw&oe=6558D73E", "username": "airbytehq", "followers_count": 1253, "name": "Jean Lafleur", "ig_id": 8070063576, "media_count": 258, "follows_count": 14, "website": "https://www.airbyte.io/", "page_id": "144706962067225"}, "emitted_at": 1700004246764} -{"stream": "media", "data": {"id": "17884386203808767", "media_product_type": "REELS", "shortcode": "CtZs0Y3v2lx", "permalink": "https://www.instagram.com/reel/CtZs0Y3v2lx/", "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/B34BFFBB0614049AD69F066D153FDD8C_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=107&vs=986202625710684_1200838240&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC9CMzRCRkZCQjA2MTQwNDlBRDY5RjA2NkQxNTNGREQ4Q192aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dDQm9HQlV3a2JxUWwtY0JBRnZGTnFBUkdQeHpicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJtDf4euHnbtAFQIoAkMzLBdAUBtDlYEGJRgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfAJ_4aiqzpxj20QL_aetXfmjmA8nRmz27vnAzpiARGK5w&oe=6555EADA&_nc_sid=1d576d", "timestamp": "2023-06-12T19:20:02+0000", "media_type": "VIDEO", "caption": "Terraform Explained Part 1\n.\n.\n.\n#airbyte #dataengineering #tech #terraform #cloud #cloudengineer #coding #reels", "comments_count": 2, "username": "airbytehq", "is_comment_enabled": true, "owner": {"id": "17841408147298757"}, "like_count": 9, "ig_id": "3123724930722523505", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/353022694_609901831117241_2447211336606431614_n.jpg?_nc_cat=100&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=1ZTHPkRhzl8AX-hZcw_&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfAzQkg0OB_775OS9F7QSmHxKMrjBSNFi8Rx24OISWSTTQ&oe=655888CE", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1700004200656} -{"stream": "media", "data": {"id": "17864256500936159", "media_product_type": "REELS", "shortcode": "CscAR5EsRgA", "permalink": "https://www.instagram.com/reel/CscAR5EsRgA/", "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/BE4F848CC97FBA35A1AE1B1150B989A7_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=110&vs=6290041361087047_1877877688&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC9CRTRGODQ4Q0M5N0ZCQTM1QTFBRTFCMTE1MEI5ODlBN192aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dEaE94aFJJdk1BWGZaWURBQXQyS0FLWWxOSlhicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJrD%2B6LaRwf1AFQIoAkMzLBdARDmZmZmZmhgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfB3i72i3aoV6KoK_SkI7W93z4rQLdbYHatg-KzPo0ADCg&oe=655556A6&_nc_sid=1d576d", "timestamp": "2023-05-19T20:08:33+0000", "media_type": "VIDEO", "caption": "When and why you should be using Rust for Data Engineering! \n\n#rust #airbyte #coding #programming #tech #dataengineering #data", "comments_count": 0, "username": "airbytehq", "is_comment_enabled": true, "owner": {"id": "17841408147298757"}, "like_count": 7, "ig_id": "3106359072491902976", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/347441626_604256678433845_716271787932876577_n.jpg?_nc_cat=108&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=2ACJfSHiIRkAX8S0ZFU&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfDJSGXmZXkQnQZmkrVUi4nadhEddZxH5LUNtELipGu4Dw&oe=655947D9", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1700004200657} -{"stream": "media", "data": {"id": "17964324206288599", "media_product_type": "REELS", "shortcode": "CsUe2iqpQif", "permalink": "https://www.instagram.com/reel/CsUe2iqpQif/", "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/274503D36EA0F6E79A7CF3797A8D5985_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNTc2LmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=106&vs=1336282350269744_3931649106&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC8yNzQ1MDNEMzZFQTBGNkU3OUE3Q0YzNzk3QThENTk4NV92aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dQdzNzaFRId3VlSlBFWURBSDFmTjUzcUNhd0JicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJrDwmtqO44lAFQIoAkMzLBdAIewIMSbpeRgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfB1HP_4v5ndbtfq_6eWT0cxo0vqzO9F6mu5ZS-q4IRDzg&oe=65558FF2&_nc_sid=1d576d", "timestamp": "2023-05-16T22:01:45+0000", "media_type": "VIDEO", "caption": "We've all been there right? \ud83e\udd23\n\n#airbyte #data #dataengineering #datascience #dataanalytics #tech #softwareengineer", "comments_count": 0, "username": "airbytehq", "is_comment_enabled": true, "owner": {"id": "17841408147298757"}, "like_count": 13, "ig_id": "3104241732634871967", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/347429218_1848940842145573_5975413208994727174_n.jpg?_nc_cat=101&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=3He_36rMQuYAX9Pz0NM&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfCAgX5HrHW8grC2x_VzJyCf2lUTViJCmwNy0uStHB-YFg&oe=6559347C", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1700004200657} -{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_city", "date": "2023-11-14T08:00:00+0000", "value": {"London, England": 8, "Sydney, New South Wales": 19, "Algiers, Algiers Province": 4, "Casablanca, Grand Casablanca": 4, "S\u00e3o Paulo, S\u00e3o Paulo (state)": 14, "Rio de Janeiro, Rio de Janeiro (state)": 5, "Hong Kong, Hong Kong": 4, "Berlin, Berlin": 7, "Kolkata, West Bengal": 4, "Phoenix, Arizona": 3, "Tulsa, Oklahoma": 4, "Lagos, Lagos State": 18, "Skopje, Municipality of Centar (Skopje)": 4, "Ahmedabad, Gujarat": 4, "Moscow, Moscow": 5, "Karachi, Sindh": 4, "Bogot\u00e1, Distrito Especial": 5, "Dar es Salaam, Dar es Salaam": 7, "Jakarta, Jakarta": 10, "Accra, Greater Accra Region": 4, "Buenos Aires, Ciudad Aut\u00f3noma de Buenos Aires": 9, "Melbourne, Victoria": 7, "Delhi, Delhi": 6, "Gurugram, Haryana": 6, "Kuala Lumpur, Kuala Lumpur": 4, "Los Angeles, California": 5, "Lima, Lima Region": 5, "Istanbul, Istanbul Province": 9, "Abuja, Federal Capital Territory": 5, "Chennai, Tamil Nadu": 6, "Bangkok, Bangkok": 5, "Mexico City, Distrito Federal": 7, "Cape Town, Western Cape": 5, "San Francisco, California": 6, "Greater Noida, Uttar Pradesh": 3, "Tehran, Tehran Province": 4, "New York, New York": 13, "Cairo, Cairo Governorate": 4, "Santiago, Santiago Metropolitan Region": 6, "Dubai, Dubai": 8, "Mumbai, Maharashtra": 8, "Bangalore, Karnataka": 18, "Singapore, Singapore": 6, "Hyderabad, Telangana": 7, "San Diego, California": 6}}, "emitted_at": 1700004246978} -{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_country", "date": "2023-11-14T08:00:00+0000", "value": {"DE": 31, "HK": 4, "FI": 5, "TW": 5, "RU": 9, "TZ": 8, "FR": 11, "SA": 8, "BR": 64, "SE": 6, "MA": 6, "SG": 6, "DZ": 6, "ID": 29, "GB": 45, "CA": 24, "US": 264, "GH": 4, "EG": 10, "AE": 9, "CH": 7, "IN": 125, "ZA": 16, "IQ": 6, "CL": 9, "IR": 12, "GR": 6, "IT": 19, "MX": 24, "MY": 9, "CO": 11, "ES": 12, "VE": 9, "AR": 23, "AT": 4, "TH": 7, "AU": 35, "PE": 5, "PH": 7, "NG": 30, "TN": 6, "PK": 10, "PL": 5, "TR": 10, "NL": 13}}, "emitted_at": 1700004246980} -{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_gender_age", "date": "2023-11-14T08:00:00+0000", "value": {"F.18-24": 11, "F.25-34": 75, "F.35-44": 72, "F.45-54": 17, "F.55-64": 1, "F.65+": 2, "M.13-17": 2, "M.18-24": 50, "M.25-34": 367, "M.35-44": 226, "M.45-54": 83, "M.55-64": 20, "M.65+": 12, "U.18-24": 18, "U.25-34": 67, "U.35-44": 43, "U.45-54": 19, "U.55-64": 5}}, "emitted_at": 1700004246980} +{"stream": "users", "data": {"id": "17841408147298757", "website": "https://www.airbyte.io/", "ig_id": 8070063576, "followers_count": 1252, "name": "Jean Lafleur", "media_count": 258, "username": "airbytehq", "follows_count": 14, "biography": "Airbyte is the new open-source data integration platform that consolidates your data into your warehouses.", "profile_picture_url": "https://scontent-iev1-1.xx.fbcdn.net/v/t51.2885-15/153169696_890787328349641_8382928081987798464_n.jpg?_nc_cat=111&_nc_sid=7d201b&_nc_ohc=DFFn_25gYVMAX8nPfUd&_nc_ht=scontent-iev1-1.xx&edm=AL-3X8kEAAAA&oh=00_AfBHQPJ5aiFU1qw88d3gTF5jmg-Rpd5TX_gxAQt3jrSA4g&oe=655CCBBE", "page_id": "144706962067225"}, "emitted_at": 1700230802579} +{"stream": "media", "data": {"id": "17884386203808767", "caption": "Terraform Explained Part 1\n.\n.\n.\n#airbyte #dataengineering #tech #terraform #cloud #cloudengineer #coding #reels", "ig_id": "3123724930722523505", "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/B34BFFBB0614049AD69F066D153FDD8C_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=107&vs=986202625710684_1200838240&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC9CMzRCRkZCQjA2MTQwNDlBRDY5RjA2NkQxNTNGREQ4Q192aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dDQm9HQlV3a2JxUWwtY0JBRnZGTnFBUkdQeHpicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJtDf4euHnbtAFQIoAkMzLBdAUBtDlYEGJRgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfBPpWnNa8TFbux-TpRO48bJGSkaIKPFOnmXhcv39jLd_A&oe=6559369A&_nc_sid=1d576d", "owner": {"id": "17841408147298757"}, "shortcode": "CtZs0Y3v2lx", "username": "airbytehq", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/353022694_609901831117241_2447211336606431614_n.jpg?_nc_cat=100&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=1ZTHPkRhzl8AX-hZcw_&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfBdTKQTru0U2JNSqNnuPN0cWYv1u6o6t6u3EHIFteUV7w&oe=655C7D4E", "is_comment_enabled": true, "permalink": "https://www.instagram.com/reel/CtZs0Y3v2lx/", "timestamp": "2023-06-12T19:20:02+00:00", "like_count": 9, "comments_count": 2, "media_product_type": "REELS", "media_type": "VIDEO", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1700230757119} +{"stream": "media", "data": {"id": "17864256500936159", "caption": "When and why you should be using Rust for Data Engineering! \n\n#rust #airbyte #coding #programming #tech #dataengineering #data", "ig_id": "3106359072491902976", "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/BE4F848CC97FBA35A1AE1B1150B989A7_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=110&vs=6290041361087047_1877877688&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC9CRTRGODQ4Q0M5N0ZCQTM1QTFBRTFCMTE1MEI5ODlBN192aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dEaE94aFJJdk1BWGZaWURBQXQyS0FLWWxOSlhicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJrD%2B6LaRwf1AFQIoAkMzLBdARDmZmZmZmhgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfC6GeTJWR8KJZ3-eb1-faBZ8P8G8AFyswEDdD4gFzmPMg&oe=65594B26&_nc_sid=1d576d", "owner": {"id": "17841408147298757"}, "shortcode": "CscAR5EsRgA", "username": "airbytehq", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/347441626_604256678433845_716271787932876577_n.jpg?_nc_cat=108&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=jLyY4sWj0v0AX-iadbF&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfA-x6QyIXxT7o_lEwDH0k7tDb_bgCGeP61AseCpluCtPA&oe=655D3C59", "is_comment_enabled": true, "permalink": "https://www.instagram.com/reel/CscAR5EsRgA/", "timestamp": "2023-05-19T20:08:33+00:00", "like_count": 7, "comments_count": 0, "media_product_type": "REELS", "media_type": "VIDEO", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1700230757120} +{"stream": "media", "data": {"id": "17964324206288599", "caption": "We've all been there right? \ud83e\udd23\n\n#airbyte #data #dataengineering #datascience #dataanalytics #tech #softwareengineer", "ig_id": "3104241732634871967", "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/274503D36EA0F6E79A7CF3797A8D5985_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNTc2LmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=106&vs=1336282350269744_3931649106&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC8yNzQ1MDNEMzZFQTBGNkU3OUE3Q0YzNzk3QThENTk4NV92aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dQdzNzaFRId3VlSlBFWURBSDFmTjUzcUNhd0JicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJrDwmtqO44lAFQIoAkMzLBdAIewIMSbpeRgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfACHaQfoSJ_vMXbm4Xw3gmWnG_vnJgUsIYUePDdtIUS-w&oe=6558DBB2&_nc_sid=1d576d", "owner": {"id": "17841408147298757"}, "shortcode": "CsUe2iqpQif", "username": "airbytehq", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/347429218_1848940842145573_5975413208994727174_n.jpg?_nc_cat=101&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=Y6VzeGH_9lkAX_wkzpd&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfDil0e2W7Iqq0-d7rf9JkdOluS7U2C3nhK17EfQ3c07fw&oe=655D28FC", "is_comment_enabled": true, "permalink": "https://www.instagram.com/reel/CsUe2iqpQif/", "timestamp": "2023-05-16T22:01:45+00:00", "like_count": 13, "comments_count": 0, "media_product_type": "REELS", "media_type": "VIDEO", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1700230757120} +{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_city", "date": "2023-11-17T08:00:00+00:00", "value": {"London, England": 7, "Sydney, New South Wales": 19, "Atlanta, Georgia": 4, "Algiers, Algiers Province": 4, "Caracas, Capital District": 4, "S\u00e3o Paulo, S\u00e3o Paulo (state)": 14, "Rio de Janeiro, Rio de Janeiro (state)": 5, "Hong Kong, Hong Kong": 4, "Berlin, Berlin": 8, "Kolkata, West Bengal": 5, "Tulsa, Oklahoma": 4, "Lagos, Lagos State": 16, "Dili, Timor-Leste": 3, "Ahmedabad, Gujarat": 4, "Skopje, Municipality of Centar (Skopje)": 4, "Moscow, Moscow": 5, "Karachi, Sindh": 4, "Bogot\u00e1, Distrito Especial": 5, "Dar es Salaam, Dar es Salaam": 7, "Jakarta, Jakarta": 10, "Accra, Greater Accra Region": 4, "Buenos Aires, Ciudad Aut\u00f3noma de Buenos Aires": 9, "Melbourne, Victoria": 7, "Gurugram, Haryana": 6, "Delhi, Delhi": 6, "Kuala Lumpur, Kuala Lumpur": 4, "Los Angeles, California": 5, "Lima, Lima Region": 4, "Istanbul, Istanbul Province": 9, "Chennai, Tamil Nadu": 6, "Abuja, Federal Capital Territory": 7, "Bangkok, Bangkok": 5, "Mexico City, Distrito Federal": 7, "Cape Town, Western Cape": 5, "San Francisco, California": 6, "Tehran, Tehran Province": 4, "New York, New York": 14, "Cairo, Cairo Governorate": 4, "Santiago, Santiago Metropolitan Region": 6, "Dubai, Dubai": 8, "Mumbai, Maharashtra": 8, "Bangalore, Karnataka": 18, "Singapore, Singapore": 6, "Hyderabad, Telangana": 7, "San Diego, California": 6}}, "emitted_at": 1700230802791} +{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_country", "date": "2023-11-17T08:00:00+00:00", "value": {"DE": 31, "HK": 4, "TW": 5, "FI": 5, "RU": 9, "TZ": 8, "FR": 10, "SA": 8, "BR": 64, "SE": 6, "SG": 6, "MA": 6, "DZ": 6, "ID": 29, "GB": 45, "CA": 24, "US": 264, "GH": 4, "EG": 10, "AE": 9, "CH": 7, "IN": 125, "ZA": 16, "IQ": 6, "CL": 9, "IR": 12, "GR": 6, "IT": 19, "MX": 24, "MY": 9, "CO": 11, "ES": 13, "VE": 9, "AR": 23, "AT": 4, "TH": 7, "AU": 35, "PE": 4, "PH": 7, "NG": 30, "TN": 6, "PK": 10, "PL": 5, "TR": 10, "NL": 13}}, "emitted_at": 1700230802792} +{"stream": "user_lifetime_insights", "data": {"page_id": "144706962067225", "business_account_id": "17841408147298757", "metric": "audience_gender_age", "date": "2023-11-17T08:00:00+00:00", "value": {"F.18-24": 11, "F.25-34": 75, "F.35-44": 72, "F.45-54": 17, "F.55-64": 1, "F.65+": 2, "M.13-17": 2, "M.18-24": 50, "M.25-34": 365, "M.35-44": 228, "M.45-54": 83, "M.55-64": 20, "M.65+": 12, "U.18-24": 18, "U.25-34": 67, "U.35-44": 42, "U.45-54": 19, "U.55-64": 5}}, "emitted_at": 1700230802792} diff --git a/airbyte-integrations/connectors/source-instagram/metadata.yaml b/airbyte-integrations/connectors/source-instagram/metadata.yaml index 48c76e1a70d2..13fb9d4bc74a 100644 --- a/airbyte-integrations/connectors/source-instagram/metadata.yaml +++ b/airbyte-integrations/connectors/source-instagram/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6acf6b55-4f1e-4fca-944e-1a3caef8aba8 - dockerImageTag: 1.0.16 + dockerImageTag: 2.0.0 dockerRepository: airbyte/source-instagram githubIssueLabel: source-instagram icon: instagram.svg @@ -19,6 +19,13 @@ data: oss: enabled: true releaseStage: generally_available + releases: + breakingChanges: + 2.0.0: + message: + This release introduces a default primary key for the streams UserLifetimeInsights and UserInsights. + Additionally, the format of timestamp fields has been updated in the UserLifetimeInsights, UserInsights, Media and Stories streams to include timezone information. + upgradeDeadline: "2023-12-03" suggestedStreams: streams: - media diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media.json b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media.json index 4185de5f66cb..03c77796f5a0 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media.json +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media.json @@ -53,7 +53,8 @@ }, "timestamp": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "username": { "type": ["null", "string"] @@ -94,7 +95,8 @@ }, "timestamp": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "username": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/stories.json b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/stories.json index 7fc7fa7b40a3..876edf95ea41 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/stories.json +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/stories.json @@ -47,7 +47,8 @@ }, "timestamp": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "username": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_insights.json b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_insights.json index fe98eafcccbf..91bc309d8eb6 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_insights.json +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_insights.json @@ -9,7 +9,8 @@ }, "date": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "follower_count": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_lifetime_insights.json b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_lifetime_insights.json index eb9bb57fc720..4cb5092f5ace 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_lifetime_insights.json +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_lifetime_insights.json @@ -9,7 +9,8 @@ }, "date": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "metric": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py index 9c07d98bb083..bf5d39de1e1c 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py @@ -11,6 +11,7 @@ import pendulum from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import IncrementalMixin, Stream +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from cached_property import cached_property from facebook_business.adobjects.igmedia import IGMedia from facebook_business.exceptions import FacebookRequestError @@ -19,6 +20,24 @@ from .common import remove_params_from_url +class DatetimeTransformerMixin: + transformer: TypeTransformer = TypeTransformer(TransformConfig.CustomSchemaNormalization) + + @staticmethod + @transformer.registerCustomTransform + def custom_transform_datetime_rfc3339(original_value, field_schema): + """ + Transform datetime string to RFC 3339 format + """ + if original_value and field_schema.get("format") == "date-time" and field_schema.get("airbyte_type") == "timestamp_with_timezone": + # Parse the ISO format timestamp + dt = pendulum.parse(original_value) + + # Convert to RFC 3339 format + return dt.to_rfc3339_string() + return original_value + + class InstagramStream(Stream, ABC): """Base stream class""" @@ -121,10 +140,10 @@ def read_records( yield self.transform(record) -class UserLifetimeInsights(InstagramStream): +class UserLifetimeInsights(DatetimeTransformerMixin, InstagramStream): """Docs: https://developers.facebook.com/docs/instagram-api/reference/ig-user/insights""" - primary_key = None + primary_key = ["business_account_id", "metric", "date"] LIFETIME_METRICS = ["audience_city", "audience_country", "audience_gender_age", "audience_locale"] period = "lifetime" @@ -156,7 +175,7 @@ def request_params( return params -class UserInsights(InstagramIncrementalStream): +class UserInsights(DatetimeTransformerMixin, InstagramIncrementalStream): """Docs: https://developers.facebook.com/docs/instagram-api/reference/ig-user/insights""" METRICS_BY_PERIOD = { @@ -176,7 +195,7 @@ class UserInsights(InstagramIncrementalStream): "lifetime": ["online_followers"], } - primary_key = None + primary_key = ["business_account_id", "date"] cursor_field = "date" # For some metrics we can only get insights not older than 30 days, it is Facebook policy @@ -295,7 +314,7 @@ def _state_has_legacy_format(self, state: Mapping[str, Any]) -> bool: return False -class Media(InstagramStream): +class Media(DatetimeTransformerMixin, InstagramStream): """Children objects can only be of the media_type == "CAROUSEL_ALBUM". And children object does not support INVALID_CHILDREN_FIELDS fields, so they are excluded when trying to get child objects to avoid the error @@ -403,7 +422,7 @@ def _get_insights(self, item, account_id) -> Optional[MutableMapping[str, Any]]: raise error -class Stories(InstagramStream): +class Stories(DatetimeTransformerMixin, InstagramStream): """Docs: https://developers.facebook.com/docs/instagram-api/reference/ig-user/stories""" def read_records( diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py b/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py index 7b9fd1db150e..a065d01b77cf 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py @@ -35,6 +35,7 @@ def config_fixture(): def some_config_fixture(account_id): return {"start_date": "2021-01-23T00:00:00Z", "access_token": "unknown_token"} + @fixture(scope="session", name="some_config_future_date") def some_config_future_date_fixture(account_id): return {"start_date": "2030-01-23T00:00:00Z", "access_token": "unknown_token"} diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py b/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py index 2cdca11b4f8f..add26ad1a33f 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py @@ -31,11 +31,14 @@ def test_check_connection_empty_config(api): assert not ok assert error_msg + def test_check_connection_invalid_config_future_date(api, some_config_future_date): ok, error_msg = SourceInstagram().check_connection(logger, config=some_config_future_date) assert not ok assert error_msg + + def test_check_connection_no_date_config(api, some_config): some_config.pop("start_date") ok, error_msg = SourceInstagram().check_connection(logger, config=some_config) diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py index 39fc889e7e48..19470cb9c22b 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py @@ -9,6 +9,7 @@ from airbyte_cdk.models import SyncMode from facebook_business import FacebookAdsApi, FacebookSession from source_instagram.streams import ( + DatetimeTransformerMixin, InstagramStream, Media, MediaInsights, @@ -32,15 +33,11 @@ def test_clear_url(config): def test_state_outdated(api, config): - assert UserInsights(api=api, start_date=config["start_date"])._state_has_legacy_format( - {"state": MagicMock()} - ) + assert UserInsights(api=api, start_date=config["start_date"])._state_has_legacy_format({"state": MagicMock()}) def test_state_is_not_outdated(api, config): - assert not UserInsights(api=api, start_date=config["start_date"])._state_has_legacy_format( - {"state": {}} - ) + assert not UserInsights(api=api, start_date=config["start_date"])._state_has_legacy_format({"state": {}}) def test_media_get_children(api, requests_mock, some_config): @@ -208,9 +205,9 @@ def test_user_lifetime_insights_read(api, config, user_insight_data, requests_mo @pytest.mark.parametrize( "values,expected", [ - ({"end_time": "test_end_time", "value": "test_value"}, {"date": "test_end_time", "value": "test_value"}), + ({"end_time": "2020-05-04T07:00:00+0000", "value": "test_value"}, {"date": "2020-05-04T07:00:00+0000", "value": "test_value"}), ({"value": "test_value"}, {"date": None, "value": "test_value"}), - ({"end_time": "test_end_time"}, {"date": "test_end_time", "value": None}), + ({"end_time": "2020-05-04T07:00:00+0000"}, {"date": "2020-05-04T07:00:00+0000", "value": None}), ({}, {"date": None, "value": None}), ], ids=[ @@ -363,3 +360,22 @@ def test_exit_gracefully(api, config, requests_mock, caplog): assert not records assert requests_mock.call_count == 6 # 4 * 1 per `metric_to_period` map + 1 `summary` request + 1 `business_account_id` request assert "Stopping syncing stream 'user_insights'" in caplog.text + + +@pytest.mark.parametrize( + "original_value, field_schema, expected", + [ + ("2020-01-01T12:00:00Z", {"format": "date-time", "airbyte_type": "timestamp_with_timezone"}, "2020-01-01T12:00:00+00:00"), + ("2020-05-04T07:00:00+0000", {"format": "date-time", "airbyte_type": "timestamp_with_timezone"}, "2020-05-04T07:00:00+00:00"), + (None, {"format": "date-time", "airbyte_type": "timestamp_with_timezone"}, None), + ("2020-01-01T12:00:00", {"format": "date-time", "airbyte_type": "timestamp_without_timezone"}, "2020-01-01T12:00:00"), + ("2020-01-01T14:00:00", {"format": "date-time"}, "2020-01-01T14:00:00"), + ("2020-02-03T12:00:00", {"type": "string"}, "2020-02-03T12:00:00"), + ], +) +def test_custom_transform_datetime_rfc3339(original_value, field_schema, expected): + # Call the static method + result = DatetimeTransformerMixin.custom_transform_datetime_rfc3339(original_value, field_schema) + + # Assert the result matches the expected output + assert result == expected diff --git a/docs/integrations/sources/instagram-migrations.md b/docs/integrations/sources/instagram-migrations.md new file mode 100644 index 000000000000..f9009b09e3b5 --- /dev/null +++ b/docs/integrations/sources/instagram-migrations.md @@ -0,0 +1,9 @@ +# Instagram Migration Guide + +## Upgrading to 2.0.0 + +This release adds a default primary key for the streams UserLifetimeInsights and UserInsights, and updates the format of timestamp fields in the UserLifetimeInsights, UserInsights, Media and Stories streams to include timezone information. + +To ensure uninterrupted syncs, users should: +- Refresh the source schema +- Reset affected streams \ No newline at end of file diff --git a/docs/integrations/sources/instagram.md b/docs/integrations/sources/instagram.md index 307acbd686ac..7b4999945fd4 100644 --- a/docs/integrations/sources/instagram.md +++ b/docs/integrations/sources/instagram.md @@ -93,6 +93,7 @@ AirbyteRecords are required to conform to the [Airbyte type](https://docs.airbyt | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------| +| 2.0.0 | 2023-11-17 | [32500](https://github.com/airbytehq/airbyte/pull/32500) | Add primary keys for UserLifetimeInsights and UserInsights; add airbyte_type to timestamp fields | | 1.0.16 | 2023-11-17 | [32627](https://github.com/airbytehq/airbyte/pull/32627) | Fix start_date type; fix docs | | 1.0.15 | 2023-11-14 | [32494](https://github.com/airbytehq/airbyte/pull/32494) | Marked start_date as optional; set max retry time to 10 minutes; add suggested streams | | 1.0.14 | 2023-11-13 | [32423](https://github.com/airbytehq/airbyte/pull/32423) | Capture media_product_type column in media and stories stream | From e69e3decc1f00f8e6ce2673f11dfd5691de2ed89 Mon Sep 17 00:00:00 2001 From: Artem Inzhyyants <36314070+artem1205@users.noreply.github.com> Date: Fri, 17 Nov 2023 17:47:12 +0100 Subject: [PATCH 115/124] Source Bing Ads: unpin version (#32643) --- .../source-bing-ads/integration_tests/expected_records.jsonl | 4 ++-- airbyte-integrations/connectors/source-bing-ads/metadata.yaml | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl index 7a61e57866e3..73359a964320 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl @@ -15,8 +15,8 @@ {"stream":"campaign_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"TimePeriod":"2023-11-05","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Syndicated search partners","DeliveredMatchType":"Exact","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","CampaignStatus":"Active","CampaignLabels":null,"Impressions":9,"Clicks":1,"Ctr":11.11,"Spend":0.03,"CostPerConversion":null,"QualityScore":5.0,"AdRelevance":3.0,"LandingPageExperience":1.0,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"CustomParameters":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.03,"AveragePosition":0.0,"AverageCpm":3.33,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"LowQualityClicks":0,"LowQualityClicksPercent":0.0,"LowQualityImpressions":0,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"BudgetName":null,"BudgetStatus":null,"BudgetAssociationStatus":"Current","HistoricalQualityScore":5.0,"HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":1.0},"emitted_at":1699954081143} {"stream":"campaign_impression_performance_report_daily","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-11-07","CampaignStatus":"Active","CampaignName":"Airbyte test","CampaignId":531016227,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":10,"Clicks":1,"Ctr":10.0,"AverageCpc":0.33,"Spend":0.33,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"LowQualityClicks":0,"LowQualityClicksPercent":0.0,"LowQualityImpressions":9,"LowQualityImpressionsPercent":47.37,"LowQualityConversions":0,"LowQualityConversionRate":null,"DeviceType":"Computer","ImpressionSharePercent":3.37,"ImpressionLostToBudgetPercent":85.19,"ImpressionLostToRankAggPercent":11.45,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"HistoricalQualityScore":null,"HistoricalExpectedCtr":null,"HistoricalAdRelevance":null,"HistoricalLandingPageExperience":null,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":0,"CampaignLabels":null,"ExactMatchImpressionSharePercent":null,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":6.02,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":14.63,"TopImpressionShareLostToBudgetPercent":77.24,"AbsoluteTopImpressionShareLostToRankPercent":15.66,"AbsoluteTopImpressionShareLostToBudgetPercent":78.31,"TopImpressionSharePercent":8.13,"AbsoluteTopImpressionRatePercent":50.0,"TopImpressionRatePercent":100.0,"BaseCampaignId":531016227,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AverageCpm":33.0,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1699954182626} {"stream":"campaign_impression_performance_report_weekly","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-11-05","CampaignStatus":"Active","CampaignName":"Airbyte test","CampaignId":531016227,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":10,"Clicks":1,"Ctr":10.0,"AverageCpc":0.33,"Spend":0.33,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"LowQualityClicks":0,"LowQualityClicksPercent":0.0,"LowQualityImpressions":9,"LowQualityImpressionsPercent":47.37,"LowQualityConversions":0,"LowQualityConversionRate":null,"DeviceType":"Computer","ImpressionSharePercent":10.87,"ImpressionLostToBudgetPercent":17.05,"ImpressionLostToRankAggPercent":72.08,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"HistoricalQualityScore":null,"HistoricalExpectedCtr":null,"HistoricalAdRelevance":null,"HistoricalLandingPageExperience":null,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":0,"CampaignLabels":null,"ExactMatchImpressionSharePercent":29.07,"ClickSharePercent":2.89,"AbsoluteTopImpressionSharePercent":8.88,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":76.51,"TopImpressionShareLostToBudgetPercent":9.99,"AbsoluteTopImpressionShareLostToRankPercent":81.99,"AbsoluteTopImpressionShareLostToBudgetPercent":9.13,"TopImpressionSharePercent":13.5,"AbsoluteTopImpressionRatePercent":50.0,"TopImpressionRatePercent":100.0,"BaseCampaignId":531016227,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AverageCpm":33.0,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":null,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1699954211223} -{"stream":"keyword_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"KeywordId":84801135055365,"Keyword":"connector","AdId":84800390693061,"TimePeriod":"2023-11-07","CurrencyCode":"USD","DeliveredMatchType":"Phrase","AdDistribution":"Search","DeviceType":"Computer","Language":"German","Network":"Syndicated search partners","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","KeywordStatus":"Active","HistoricalExpectedCtr":null,"HistoricalAdRelevance":null,"HistoricalLandingPageExperience":null,"HistoricalQualityScore":null,"Impressions":1,"Clicks":0,"Ctr":0.0,"CurrentMaxCpc":2.27,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"QualityImpact":0.0,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"Mainline1Bid":null,"MainlineBid":1.11,"FirstPageBid":0.48,"FinalUrlSuffix":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1700167471396} -{"stream":"keyword_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"KeywordId":84801135055365,"Keyword":"connector","AdId":84800390693061,"TimePeriod":"2023-11-05","CurrencyCode":"USD","DeliveredMatchType":"Exact","AdDistribution":"Search","DeviceType":"Computer","Language":"English","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","KeywordStatus":"Active","Impressions":2,"Clicks":0,"Ctr":0.0,"CurrentMaxCpc":2.27,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"QualityImpact":0.0,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"Mainline1Bid":null,"MainlineBid":1.11,"FirstPageBid":0.48,"FinalUrlSuffix":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1700167573859} +{"stream":"keyword_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"KeywordId":84801135055365,"Keyword":"connector","AdId":84800390693061,"TimePeriod":"2023-11-07","CurrencyCode":"USD","DeliveredMatchType":"Phrase","AdDistribution":"Search","DeviceType":"Computer","Language":"German","Network":"Syndicated search partners","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","KeywordStatus":"Active","HistoricalExpectedCtr":null,"HistoricalAdRelevance":null,"HistoricalLandingPageExperience":null,"HistoricalQualityScore":null,"Impressions":1,"Clicks":0,"Ctr":0.0,"CurrentMaxCpc":2.27,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"QualityImpact":0.0,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"Mainline1Bid":null,"MainlineBid":1.0,"FirstPageBid":0.43,"FinalUrlSuffix":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1700237754157} +{"stream":"keyword_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"KeywordId":84801135055365,"Keyword":"connector","AdId":84800390693061,"TimePeriod":"2023-11-05","CurrencyCode":"USD","DeliveredMatchType":"Exact","AdDistribution":"Search","DeviceType":"Computer","Language":"English","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","KeywordStatus":"Active","Impressions":2,"Clicks":0,"Ctr":0.0,"CurrentMaxCpc":2.27,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"QualityImpact":0.0,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"Mainline1Bid":null,"MainlineBid":1.0,"FirstPageBid":0.43,"FinalUrlSuffix":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1700237801690} {"stream":"geographic_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"TimePeriod":"2023-11-07","Country":"Australia","CurrencyCode":"USD","DeliveredMatchType":"Broad","AdDistribution":"Search","DeviceType":"Computer","Language":"English","Network":"Syndicated search partners","DeviceOS":"Windows","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","MetroArea":null,"State":"New South Wales","City":null,"AdGroupName":"keywords","Ctr":0.0,"ProximityTargetLocation":null,"Radius":"0","Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"LocationType":"Physical location","MostSpecificLocation":"2000","AccountStatus":"Active","CampaignStatus":"Active","AdGroupStatus":"Active","County":null,"PostalCode":"2000","LocationId":"122395","BaseCampaignId":"531016227","AllCostPerConversion":null,"AllReturnOnAdSpend":null,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":"100.00","AllConversionsQualified":"0.00","ViewThroughConversionsQualified":null,"Neighborhood":null,"ViewThroughRevenue":"0.00","CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null,"AssetGroupStatus":null,"Clicks":0,"Spend":0.0,"Impressions":1,"CostPerConversion":null,"AccountName":"Airbyte","AccountNumber":"F149MJ18","CampaignName":"Airbyte test","Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1699956863587} {"stream":"geographic_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"TimePeriod":"2023-11-05","Country":"Argentina","CurrencyCode":"USD","DeliveredMatchType":"Exact","AdDistribution":"Search","DeviceType":"Computer","Language":"Spanish","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","MetroArea":null,"State":"Buenos Aires Province","City":null,"AdGroupName":"keywords","Ctr":0.0,"ProximityTargetLocation":null,"Radius":"0","Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"LocationType":"Physical location","MostSpecificLocation":"Buenos Aires Province","AccountStatus":"Active","CampaignStatus":"Active","AdGroupStatus":"Active","County":null,"PostalCode":null,"LocationId":"141965","BaseCampaignId":"531016227","AllCostPerConversion":null,"AllReturnOnAdSpend":null,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":"100.00","AllConversionsQualified":"0.00","ViewThroughConversionsQualified":null,"Neighborhood":null,"ViewThroughRevenue":"0.00","CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null,"AssetGroupStatus":null,"Clicks":0,"Spend":0.0,"Impressions":1,"CostPerConversion":null,"AccountName":"Airbyte","AccountNumber":"F149MJ18","CampaignName":"Airbyte test","Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1699953673210} {"stream":"age_gender_audience_report_daily","data":{"AccountId":180519267,"AgeGroup":"Unknown","Gender":"Unknown","TimePeriod":"2023-11-07","AllConversions":0,"AccountName":"Airbyte","AccountNumber":"F149MJ18","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"AdDistribution":"Search","Impressions":3,"Clicks":1,"Conversions":0.0,"Spend":0.79,"Revenue":0.0,"ExtendedCost":0.0,"Assists":0,"Language":"German","AccountStatus":"Active","CampaignStatus":"Active","AdGroupStatus":"Active","BaseCampaignId":"531016227","AllRevenue":0.0,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":33.33,"TopImpressionRatePercent":100.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0},"emitted_at":1699954406862} diff --git a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml index 8d64cd443fa7..102607de6664 100644 --- a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml @@ -25,11 +25,9 @@ data: name: Bing Ads registries: cloud: - dockerImageTag: 1.13.0 #https://airbytehq-team.slack.com/archives/C0662JB7XPU enabled: true oss: enabled: true - dockerImageTag: 1.13.0 #https://airbytehq-team.slack.com/archives/C0662JB7XPU releaseStage: generally_available releases: breakingChanges: From 4e51b7d645fbe256bacae189612b22c1aef5cba1 Mon Sep 17 00:00:00 2001 From: Christo Grabowski <108154848+ChristoGrab@users.noreply.github.com> Date: Fri, 17 Nov 2023 12:04:54 -0500 Subject: [PATCH 116/124] Source Mailchimp: Handle empty fields in Reports stream (#32543) --- .../connectors/source-mailchimp/metadata.yaml | 2 +- .../source_mailchimp/schemas/reports.json | 2 +- .../source_mailchimp/streams.py | 25 +++++++----- .../unit_tests/test_streams.py | 38 ++++++++++++++++++- docs/integrations/sources/mailchimp.md | 1 + 5 files changed, 56 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/source-mailchimp/metadata.yaml b/airbyte-integrations/connectors/source-mailchimp/metadata.yaml index 11ff4b1bbbcc..ef06fb27bda0 100644 --- a/airbyte-integrations/connectors/source-mailchimp/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailchimp/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: b03a9f3e-22a5-11eb-adc1-0242ac120002 - dockerImageTag: 0.8.2 + dockerImageTag: 0.8.3 dockerRepository: airbyte/source-mailchimp documentationUrl: https://docs.airbyte.com/integrations/sources/mailchimp githubIssueLabel: source-mailchimp diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/reports.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/reports.json index fe6dcf6a599b..34e513022879 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/reports.json +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/reports.json @@ -138,7 +138,7 @@ "description": "The number of unique opens divided by the total number of successful deliveries." }, "last_open": { - "type": "string", + "type": ["null", "string"], "format": "date-time", "title": "Last Open", "description": "The date and time of the last recorded open in ISO 8601 format." diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py index d5239e7f3ecf..27df31f5b05c 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py @@ -274,21 +274,28 @@ class Reports(IncrementalMailChimpStream): cursor_field = "send_time" data_field = "reports" + @staticmethod + def remove_empty_datetime_fields(record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + In some cases, the 'clicks.last_click' and 'opens.last_open' fields are returned as an empty string, + which causes validation errors on the `date-time` format. + To avoid this, we remove the fields if they are empty. + """ + clicks = record.get("clicks", {}) + opens = record.get("opens", {}) + if not clicks.get("last_click"): + clicks.pop("last_click", None) + if not opens.get("last_open"): + opens.pop("last_open", None) + return record + def path(self, **kwargs) -> str: return "reports" def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - response = super().parse_response(response, **kwargs) - - # In some cases, the 'last_click' field is returned as an empty string, - # which causes validation errors on the `date-time` format. - # To avoid this, we remove the field if it is empty. for record in response: - clicks = record.get("clicks", {}) - if not clicks.get("last_click"): - clicks.pop("last_click", None) - yield record + yield self.remove_empty_datetime_fields(record) class Segments(MailChimpListSubStream): diff --git a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_streams.py index 021f50470920..094eb4fe0bf5 100644 --- a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_streams.py @@ -10,7 +10,7 @@ import responses from airbyte_cdk.models import SyncMode from requests.exceptions import HTTPError -from source_mailchimp.streams import Campaigns, EmailActivity, ListMembers, Lists, Segments +from source_mailchimp.streams import Campaigns, EmailActivity, ListMembers, Lists, Reports, Segments from utils import read_full_refresh, read_incremental @@ -413,3 +413,39 @@ def test_403_error_handling( # Handle non-403 error except HTTPError as e: assert e.response.status_code == status_code + +@pytest.mark.parametrize( + "record, expected_return", + [ + ( + {"clicks": {"last_click": ""}, "opens": {"last_open": ""}}, + {"clicks": {}, "opens": {}}, + ), + ( + {"clicks": {"last_click": "2023-01-01T00:00:00.000Z"}, "opens": {"last_open": ""}}, + {"clicks": {"last_click": "2023-01-01T00:00:00.000Z"}, "opens": {}}, + ), + ( + {"clicks": {"last_click": ""}, "opens": {"last_open": "2023-01-01T00:00:00.000Z"}}, + {"clicks": {}, "opens": {"last_open": "2023-01-01T00:00:00.000Z"}}, + + ), + ( + {"clicks": {"last_click": "2023-01-01T00:00:00.000Z"}, "opens": {"last_open": "2023-01-01T00:00:00.000Z"}}, + {"clicks": {"last_click": "2023-01-01T00:00:00.000Z"}, "opens": {"last_open": "2023-01-01T00:00:00.000Z"}}, + ), + ], + ids=[ + "last_click and last_open empty", + "last_click empty", + "last_open empty", + "last_click and last_open not empty" + ] +) +def test_reports_remove_empty_datetime_fields(auth, record, expected_return): + """ + Tests that the Reports stream removes the 'clicks' and 'opens' fields from the response + when they are empty strings + """ + stream = Reports(authenticator=auth) + assert stream.remove_empty_datetime_fields(record) == expected_return, f"Expected: {expected_return}, Actual: {stream.remove_empty_datetime_fields(record)}" diff --git a/docs/integrations/sources/mailchimp.md b/docs/integrations/sources/mailchimp.md index d3d2551a8a9c..20523890da5e 100644 --- a/docs/integrations/sources/mailchimp.md +++ b/docs/integrations/sources/mailchimp.md @@ -76,6 +76,7 @@ Now that you have set up the Mailchimp source connector, check out the following | Version | Date | Pull Request | Subject | |---------|------------|----------------------------------------------------------|----------------------------------------------------------------------------| +| 0.8.3 | 2023-11-15 | [32543](https://github.com/airbytehq/airbyte/pull/32543) | Handle empty datetime fields in Reports stream | | 0.8.2 | 2023-11-13 | [32466](https://github.com/airbytehq/airbyte/pull/32466) | Improve error handling during connection check | | 0.8.1 | 2023-11-06 | [32226](https://github.com/airbytehq/airbyte/pull/32226) | Unmute expected records test after data anonymisation | | 0.8.0 | 2023-11-01 | [32032](https://github.com/airbytehq/airbyte/pull/32032) | Add ListMembers stream | From 6d54a5dc725ed98666e31c2cf46fed82b1eab42f Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 17 Nov 2023 19:18:24 +0100 Subject: [PATCH 117/124] Vector DBs: Mention langchain in docs (#32501) --- docs/integrations/destinations/chroma.md | 1 + docs/integrations/destinations/milvus.md | 2 +- docs/integrations/destinations/pinecone.md | 2 +- docs/integrations/destinations/qdrant.md | 1 + docs/integrations/destinations/weaviate.md | 2 +- 5 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/integrations/destinations/chroma.md b/docs/integrations/destinations/chroma.md index bd7357ffc70f..3e37bebba225 100644 --- a/docs/integrations/destinations/chroma.md +++ b/docs/integrations/destinations/chroma.md @@ -56,6 +56,7 @@ Make sure your Chroma database can be accessed by Airbyte. If your database is w You should now have all the requirements needed to configure Chroma as a destination in the UI. You'll need the following information to configure the Chroma destination: - (Required) **Text fields to embed** +- (Optional) **Text splitter** Options around configuring the chunking process provided by the [Langchain Python library](https://python.langchain.com/docs/get_started/introduction). - (Required) **Fields to store as metadata** - (Required) **Collection** The name of the collection in Chroma db to store your data - (Required) Authentication method diff --git a/docs/integrations/destinations/milvus.md b/docs/integrations/destinations/milvus.md index f5de8b2e04ad..2e7c225a0b8c 100644 --- a/docs/integrations/destinations/milvus.md +++ b/docs/integrations/destinations/milvus.md @@ -37,7 +37,7 @@ You'll need the following information to configure the destination: ### Processing -Each record will be split into text fields and meta fields as configured in the "Processing" section. All text fields are concatenated into a single string and then split into chunks of configured length. If specified, the metadata fields are stored as-is along with the embedded text chunks. +Each record will be split into text fields and meta fields as configured in the "Processing" section. All text fields are concatenated into a single string and then split into chunks of configured length. If specified, the metadata fields are stored as-is along with the embedded text chunks. Options around configuring the chunking process use the [Langchain Python library](https://python.langchain.com/docs/get_started/introduction). When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. diff --git a/docs/integrations/destinations/pinecone.md b/docs/integrations/destinations/pinecone.md index 6142e7935fcc..e060cf243bf0 100644 --- a/docs/integrations/destinations/pinecone.md +++ b/docs/integrations/destinations/pinecone.md @@ -46,7 +46,7 @@ All other fields are ignored. ### Processing -Each record will be split into text fields and meta fields as configured in the "Processing" section. All text fields are concatenated into a single string and then split into chunks of configured length. If specified, the metadata fields are stored as-is along with the embedded text chunks. Please note that meta data fields can only be used for filtering and not for retrieval and have to be of type string, number, boolean (all other values are ignored). Please note that there's a 40kb limit on the _total_ size of the metadata saved for each entry. +Each record will be split into text fields and meta fields as configured in the "Processing" section. All text fields are concatenated into a single string and then split into chunks of configured length. If specified, the metadata fields are stored as-is along with the embedded text chunks. Please note that meta data fields can only be used for filtering and not for retrieval and have to be of type string, number, boolean (all other values are ignored). Please note that there's a 40kb limit on the _total_ size of the metadata saved for each entry. Options around configuring the chunking process use the [Langchain Python library](https://python.langchain.com/docs/get_started/introduction). When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. diff --git a/docs/integrations/destinations/qdrant.md b/docs/integrations/destinations/qdrant.md index 549f26ada6d6..648d0b019283 100644 --- a/docs/integrations/destinations/qdrant.md +++ b/docs/integrations/destinations/qdrant.md @@ -45,6 +45,7 @@ Make sure your Qdrant database can be accessed by Airbyte. If your database is w You should now have all the requirements needed to configure Qdrant as a destination in the UI. You'll need the following information to configure the Qdrant destination: - (Required) **Text fields to embed** +- (Optional) **Text splitter** Options around configuring the chunking process provided by the [Langchain Python library](https://python.langchain.com/docs/get_started/introduction). - (Required) **Fields to store as metadata** - (Required) **Collection** The name of the collection in Qdrant db to store your data - (Required) **The field in the payload that contains the embedded text** diff --git a/docs/integrations/destinations/weaviate.md b/docs/integrations/destinations/weaviate.md index 0d914d136619..9e36ca07fe57 100644 --- a/docs/integrations/destinations/weaviate.md +++ b/docs/integrations/destinations/weaviate.md @@ -48,7 +48,7 @@ All other fields are serialized into their JSON representation. ### Processing -Each record will be split into text fields and metadata fields as configured in the "Processing" section. All text fields are concatenated into a single string and then split into chunks of configured length. If specified, the metadata fields are stored as-is along with the embedded text chunks. Please note that metadata fields can only be used for filtering and not for retrieval and have to be of type string, number, boolean (all other values are ignored). Please note that there's a 40kb limit on the _total_ size of the metadata saved for each entry. +Each record will be split into text fields and metadata fields as configured in the "Processing" section. All text fields are concatenated into a single string and then split into chunks of configured length. If specified, the metadata fields are stored as-is along with the embedded text chunks. Options around configuring the chunking process use the [Langchain Python library](https://python.langchain.com/docs/get_started/introduction). When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. From ab1490a38cd2823e0f7233db4abf586d19547c42 Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Fri, 17 Nov 2023 20:27:32 +0200 Subject: [PATCH 118/124] :bug: Source Stripe: update availability strategy (#32638) --- .../integration_tests/expected_records.jsonl | 4 +- .../connectors/source-stripe/metadata.yaml | 2 +- .../source_stripe/availability_strategy.py | 59 ++++++- .../source_stripe/stream_helpers.py | 41 +++++ .../source-stripe/source_stripe/streams.py | 46 ++++-- .../unit_tests/test_availability_strategy.py | 152 ++++++++++++++---- docs/integrations/sources/stripe.md | 3 +- 7 files changed, 261 insertions(+), 46 deletions(-) create mode 100644 airbyte-integrations/connectors/source-stripe/source_stripe/stream_helpers.py diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl index a84e541f0ecf..da9e7ed4ea18 100644 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl @@ -46,7 +46,7 @@ {"stream": "products", "data": {"id": "prod_KouQ5ez86yREmB", "object": "product", "active": true, "attributes": [], "created": 1640124902, "default_price": "price_1K9GbqEcXtiJtvvhJ3lZe4i5", "description": null, "features": [], "images": [], "livemode": false, "metadata": {}, "name": "edgao-test-product", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10000000", "type": "service", "unit_label": null, "updated": 1696839715, "url": null}, "emitted_at": 1697627307635} {"stream": "products", "data": {"id": "prod_NHcKselSHfKdfc", "object": "product", "active": true, "attributes": [], "created": 1675345504, "default_price": "price_1MX364EcXtiJtvvhE3WgTl4O", "description": "Test Product 1 description", "features": [], "images": ["https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfdjBOT09UaHRiNVl2WmJ6clNYRUlmcFFD00cCBRNHnV"], "livemode": false, "metadata": {}, "name": "Test Product 1", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10301000", "type": "service", "unit_label": null, "updated": 1696839789, "url": null}, "emitted_at": 1697627307877} {"stream": "products", "data": {"id": "prod_NCgx1XP2IFQyKF", "object": "product", "active": true, "attributes": [], "created": 1674209524, "default_price": null, "description": null, "features": [], "images": [], "livemode": false, "metadata": {}, "name": "tu", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10000000", "type": "service", "unit_label": null, "updated": 1696839225, "url": null}, "emitted_at": 1697627307879} -{"stream": "subscriptions", "data": {"id": "sub_1O2Dg0EcXtiJtvvhz7Q4zS0n", "object": "subscription", "application": null, "application_fee_percent": null, "automatic_tax": {"enabled": true}, "billing_cycle_anchor": 1697550676.0, "billing_thresholds": null, "cancel_at": 1705499476.0, "cancel_at_period_end": false, "canceled_at": 1697550676.0, "cancellation_details": {"comment": null, "feedback": null, "reason": "cancellation_requested"}, "collection_method": "charge_automatically", "created": 1697550676, "currency": "usd", "current_period_end": 1700229076.0, "current_period_start": 1697550676, "customer": "cus_NGoTFiJFVbSsvZ", "days_until_due": null, "default_payment_method": null, "default_source": null, "default_tax_rates": [], "description": null, "discount": null, "ended_at": null, "items": {"object": "list", "data": [{"id": "si_OptSP2o3XZUBpx", "object": "subscription_item", "billing_thresholds": null, "created": 1697550677, "metadata": {}, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "price": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "recurring": {"aggregate_usage": null, "interval": "month", "interval_count": 1, "trial_period_days": null, "usage_type": "licensed"}, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "recurring", "unit_amount": 600, "unit_amount_decimal": "600"}, "quantity": 1, "subscription": "sub_1O2Dg0EcXtiJtvvhz7Q4zS0n", "tax_rates": []}], "has_more": false, "total_count": 1.0, "url": "/v1/subscription_items?subscription=sub_1O2Dg0EcXtiJtvvhz7Q4zS0n"}, "latest_invoice": "in_1O2Dg0EcXtiJtvvhLe87VaYL", "livemode": false, "metadata": {}, "next_pending_invoice_item_invoice": null, "on_behalf_of": null, "pause_collection": null, "payment_settings": {"payment_method_options": null, "payment_method_types": null, "save_default_payment_method": null}, "pending_invoice_item_interval": null, "pending_setup_intent": null, "pending_update": null, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "quantity": 1, "schedule": "sub_sched_1O2Dg0EcXtiJtvvh7GtbtIhP", "start_date": 1697550676, "status": "active", "test_clock": null, "transfer_data": null, "trial_end": null, "trial_settings": {"end_behavior": {"missing_payment_method": "create_invoice"}}, "trial_start": null, "updated": 1697550676}, "emitted_at": 1697627310741} +{"stream": "subscriptions", "data": {"id": "sub_1O2Dg0EcXtiJtvvhz7Q4zS0n", "object": "subscription", "application": null, "application_fee_percent": null, "automatic_tax": {"enabled": true}, "billing_cycle_anchor": 1697550676.0, "billing_thresholds": null, "cancel_at": 1705499476.0, "cancel_at_period_end": false, "canceled_at": 1697550676.0, "cancellation_details": {"comment": null, "feedback": null, "reason": "cancellation_requested"}, "collection_method": "charge_automatically", "created": 1697550676, "currency": "usd", "current_period_end": 1702821076.0, "current_period_start": 1700229076, "customer": "cus_NGoTFiJFVbSsvZ", "days_until_due": null, "default_payment_method": null, "default_source": null, "default_tax_rates": [], "description": null, "discount": null, "ended_at": null, "items": {"object": "list", "data": [{"id": "si_OptSP2o3XZUBpx", "object": "subscription_item", "billing_thresholds": null, "created": 1697550677, "metadata": {}, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "price": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "recurring": {"aggregate_usage": null, "interval": "month", "interval_count": 1, "trial_period_days": null, "usage_type": "licensed"}, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "recurring", "unit_amount": 600, "unit_amount_decimal": "600"}, "quantity": 1, "subscription": "sub_1O2Dg0EcXtiJtvvhz7Q4zS0n", "tax_rates": []}], "has_more": false, "total_count": 1.0, "url": "/v1/subscription_items?subscription=sub_1O2Dg0EcXtiJtvvhz7Q4zS0n"}, "latest_invoice": "in_1ODSSHEcXtiJtvvhW5LllxDH", "livemode": false, "metadata": {}, "next_pending_invoice_item_invoice": null, "on_behalf_of": null, "pause_collection": null, "payment_settings": {"payment_method_options": null, "payment_method_types": null, "save_default_payment_method": null}, "pending_invoice_item_interval": null, "pending_setup_intent": null, "pending_update": null, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "quantity": 1, "schedule": "sub_sched_1O2Dg0EcXtiJtvvh7GtbtIhP", "start_date": 1697550676, "status": "active", "test_clock": null, "transfer_data": null, "trial_end": null, "trial_settings": {"end_behavior": {"missing_payment_method": "create_invoice"}}, "trial_start": null, "updated": 1697550676}, "emitted_at": 1700232971060} {"stream": "subscription_schedule", "data": {"id": "sub_sched_1O2Dg0EcXtiJtvvh7GtbtIhP", "object": "subscription_schedule", "application": null, "canceled_at": null, "completed_at": null, "created": 1697550676, "current_phase": {"end_date": 1705499476, "start_date": 1697550676}, "customer": "cus_NGoTFiJFVbSsvZ", "default_settings": {"application_fee_percent": null, "automatic_tax": {"enabled": false}, "billing_cycle_anchor": "automatic", "billing_thresholds": null, "collection_method": "charge_automatically", "default_payment_method": null, "default_source": null, "description": "Test Test", "invoice_settings": "{'days_until_due': None}", "on_behalf_of": null, "transfer_data": null}, "end_behavior": "cancel", "livemode": false, "metadata": {}, "phases": [{"add_invoice_items": [], "application_fee_percent": null, "automatic_tax": {"enabled": true}, "billing_cycle_anchor": null, "billing_thresholds": null, "collection_method": "charge_automatically", "coupon": null, "currency": "usd", "default_payment_method": null, "default_tax_rates": [], "description": "Test Test", "end_date": 1705499476, "invoice_settings": "{'days_until_due': None}", "items": [{"billing_thresholds": null, "metadata": {}, "plan": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "price": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "quantity": 1, "tax_rates": []}], "metadata": {}, "on_behalf_of": null, "proration_behavior": "create_prorations", "start_date": 1697550676, "transfer_data": null, "trial_end": null}], "released_at": null, "released_subscription": null, "renewal_interval": null, "status": "active", "subscription": "sub_1O2Dg0EcXtiJtvvhz7Q4zS0n", "test_clock": null, "updated": 1697550676}, "emitted_at": 1697627312079} {"stream": "transfers", "data": {"id": "tr_1NH18zEcXtiJtvvhnd827cNO", "object": "transfer", "amount": 10000, "amount_reversed": 0, "balance_transaction": "txn_1NH190EcXtiJtvvhBO3PeR7p", "created": 1686301085, "currency": "usd", "description": null, "destination": "acct_1Jx8unEYmRTj5on1", "destination_payment": "py_1NH18zEYmRTj5on1GkCCsqLK", "livemode": false, "metadata": {}, "reversals": {"object": "list", "data": [], "has_more": false, "total_count": 0.0, "url": "/v1/transfers/tr_1NH18zEcXtiJtvvhnd827cNO/reversals"}, "reversed": false, "source_transaction": null, "source_type": "card", "transfer_group": null, "updated": 1686301085}, "emitted_at": 1697627313262} {"stream": "transfers", "data": {"id": "tr_1NGoaCEcXtiJtvvhjmHtOGOm", "object": "transfer", "amount": 100, "amount_reversed": 100, "balance_transaction": "txn_1NGoaDEcXtiJtvvhsZrNMsdJ", "created": 1686252800, "currency": "usd", "description": null, "destination": "acct_1Jx8unEYmRTj5on1", "destination_payment": "py_1NGoaCEYmRTj5on1LAlAIG3a", "livemode": false, "metadata": {}, "reversals": {"object": "list", "data": [{"id": "trr_1NGolCEcXtiJtvvhOYPck3CP", "object": "transfer_reversal", "amount": 100, "balance_transaction": "txn_1NGolCEcXtiJtvvhZRy4Kd5S", "created": 1686253482, "currency": "usd", "destination_payment_refund": "pyr_1NGolBEYmRTj5on1STal3rmp", "metadata": {}, "source_refund": null, "transfer": "tr_1NGoaCEcXtiJtvvhjmHtOGOm"}], "has_more": false, "total_count": 1.0, "url": "/v1/transfers/tr_1NGoaCEcXtiJtvvhjmHtOGOm/reversals"}, "reversed": true, "source_transaction": null, "source_type": "card", "transfer_group": "ORDER10", "updated": 1686252800}, "emitted_at": 1697627313264} @@ -69,4 +69,4 @@ {"stream": "invoice_line_items", "data": {"id": "il_1MX2yfEcXtiJtvvhiunY2j1x", "object": "line_item", "amount": 25200, "amount_excluding_tax": 25200, "currency": "usd", "description": "edgao-test-product", "discount_amounts": [{"amount": 2520, "discount": "di_1MX2ysEcXtiJtvvh8ORqRVKm"}], "discountable": true, "discounts": ["di_1MX2ysEcXtiJtvvh8ORqRVKm"], "invoice_item": "ii_1MX2yfEcXtiJtvvhfhyOG7SP", "livemode": false, "metadata": {}, "period": {"end": 1675345045, "start": 1675345045}, "plan": null, "price": {"id": "price_1K9GbqEcXtiJtvvhJ3lZe4i5", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1640124902, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_KouQ5ez86yREmB", "recurring": null, "tax_behavior": "inclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 12600, "unit_amount_decimal": "12600"}, "proration": false, "proration_details": {"credited_items": null}, "quantity": 2, "subscription": null, "tax_amounts": [{"amount": 0, "inclusive": true, "tax_rate": "txr_1MX2yfEcXtiJtvvhVcMEMTRj", "taxability_reason": "not_collecting", "taxable_amount": 0}], "tax_rates": [], "type": "invoiceitem", "unit_amount_excluding_tax": "12600", "invoice_id": "in_1MX2yFEcXtiJtvvhMXhUCgKx"}, "emitted_at": 1697627336449} {"stream": "subscription_items", "data": {"id": "si_OptSP2o3XZUBpx", "object": "subscription_item", "billing_thresholds": null, "created": 1697550677, "metadata": {}, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "price": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "recurring": {"aggregate_usage": null, "interval": "month", "interval_count": 1, "trial_period_days": null, "usage_type": "licensed"}, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "recurring", "unit_amount": 600, "unit_amount_decimal": "600"}, "quantity": 1, "subscription": "sub_1O2Dg0EcXtiJtvvhz7Q4zS0n", "tax_rates": []}, "emitted_at": 1697627337431} {"stream": "transfer_reversals", "data": {"id": "trr_1NGolCEcXtiJtvvhOYPck3CP", "object": "transfer_reversal", "amount": 100, "balance_transaction": "txn_1NGolCEcXtiJtvvhZRy4Kd5S", "created": 1686253482, "currency": "usd", "destination_payment_refund": "pyr_1NGolBEYmRTj5on1STal3rmp", "metadata": {}, "source_refund": null, "transfer": "tr_1NGoaCEcXtiJtvvhjmHtOGOm"}, "emitted_at": 1697627338960} -{"stream": "usage_records", "data": {"id": "sis_1O4gIOEcXtiJtvvhmsoeBHkP", "object": "usage_record_summary", "invoice": null, "livemode": false, "period": {"end": null, "start": 1697550676}, "subscription_item": "si_OptSP2o3XZUBpx", "total_usage": 1}, "emitted_at": 1697627340175} +{"stream": "usage_records", "data": {"id": "sis_1ODTdwEcXtiJtvvhZChEVsbN", "object": "usage_record_summary", "invoice": null, "livemode": false, "period": {"end": null, "start": 1700229076}, "subscription_item": "si_OptSP2o3XZUBpx", "total_usage": 1}, "emitted_at": 1700233660884} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-stripe/metadata.yaml b/airbyte-integrations/connectors/source-stripe/metadata.yaml index f521fb172278..11fc73a40128 100644 --- a/airbyte-integrations/connectors/source-stripe/metadata.yaml +++ b/airbyte-integrations/connectors/source-stripe/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: e094cb9a-26de-4645-8761-65c0c425d1de - dockerImageTag: 5.0.0 + dockerImageTag: 5.0.1 dockerRepository: airbyte/source-stripe documentationUrl: https://docs.airbyte.com/integrations/sources/stripe githubIssueLabel: source-stripe diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/availability_strategy.py b/airbyte-integrations/connectors/source-stripe/source_stripe/availability_strategy.py index 9906c21a525a..6226ffc12ea9 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/availability_strategy.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/availability_strategy.py @@ -3,13 +3,16 @@ # import logging -from typing import Optional, Tuple +from typing import Any, Mapping, Optional, Tuple +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from requests import HTTPError +from .stream_helpers import get_first_record_for_slice, get_first_stream_slice + STRIPE_ERROR_CODES = { "more_permissions_required": "This is most likely due to insufficient permissions on the credentials in use. " "Try to grant required permissions/scopes or re-authenticate", @@ -20,6 +23,60 @@ class StripeAvailabilityStrategy(HttpAvailabilityStrategy): + def _check_availability_for_sync_mode( + self, + stream: Stream, + sync_mode: SyncMode, + logger: logging.Logger, + source: Optional["Source"], + stream_state: Optional[Mapping[str, Any]], + ) -> Tuple[bool, Optional[str]]: + try: + # Some streams need a stream slice to read records (e.g. if they have a SubstreamPartitionRouter) + # Streams that don't need a stream slice will return `None` as their first stream slice. + stream_slice = get_first_stream_slice(stream, sync_mode, stream_state) + except StopIteration: + # If stream_slices has no `next()` item (Note - this is different from stream_slices returning [None]!) + # This can happen when a substream's `stream_slices` method does a `for record in parent_records: yield ` + # without accounting for the case in which the parent stream is empty. + reason = f"Cannot attempt to connect to stream {stream.name} - no stream slices were found, likely because the parent stream is empty." + return False, reason + except HTTPError as error: + is_available, reason = self.handle_http_error(stream, logger, source, error) + if not is_available: + reason = f"Unable to get slices for {stream.name} stream, because of error in parent stream. {reason}" + return is_available, reason + + try: + get_first_record_for_slice(stream, sync_mode, stream_slice, stream_state) + return True, None + except StopIteration: + logger.info(f"Successfully connected to stream {stream.name}, but got 0 records.") + return True, None + except HTTPError as error: + is_available, reason = self.handle_http_error(stream, logger, source, error) + if not is_available: + reason = f"Unable to read {stream.name} stream. {reason}" + return is_available, reason + + def check_availability(self, stream: Stream, logger: logging.Logger, source: Optional["Source"]) -> Tuple[bool, Optional[str]]: + """ + Check stream availability by attempting to read the first record of the + stream. + + :param stream: stream + :param logger: source logger + :param source: (optional) source + :return: A tuple of (boolean, str). If boolean is true, then the stream + is available, and no str is required. Otherwise, the stream is unavailable + for some reason and the str should describe what went wrong and how to + resolve the unavailability, if possible. + """ + is_available, reason = self._check_availability_for_sync_mode(stream, SyncMode.full_refresh, logger, source, None) + if not is_available or not stream.supports_incremental: + return is_available, reason + return self._check_availability_for_sync_mode(stream, SyncMode.incremental, logger, source, {stream.cursor_field: 0}) + def handle_http_error( self, stream: Stream, logger: logging.Logger, source: Optional["Source"], error: HTTPError ) -> Tuple[bool, Optional[str]]: diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/stream_helpers.py b/airbyte-integrations/connectors/source-stripe/source_stripe/stream_helpers.py new file mode 100644 index 000000000000..dad073ae485b --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/stream_helpers.py @@ -0,0 +1,41 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Mapping, Optional + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.core import Stream, StreamData + + +def get_first_stream_slice(stream, sync_mode, stream_state) -> Optional[Mapping[str, Any]]: + """ + Gets the first stream_slice from a given stream's stream_slices. + :param stream: stream + :param sync_mode: sync_mode + :param stream_state: stream_state + :raises StopIteration: if there is no first slice to return (the stream_slices generator is empty) + :return: first stream slice from 'stream_slices' generator (`None` is a valid stream slice) + """ + # We wrap the return output of stream_slices() because some implementations return types that are iterable, + # but not iterators such as lists or tuples + slices = iter(stream.stream_slices(sync_mode=sync_mode, cursor_field=stream.cursor_field, stream_state=stream_state)) + return next(slices) + + +def get_first_record_for_slice( + stream: Stream, sync_mode: SyncMode, stream_slice: Optional[Mapping[str, Any]], stream_state: Optional[Mapping[str, Any]] +) -> StreamData: + """ + Gets the first record for a stream_slice of a stream. + :param stream: stream + :param sync_mode: sync_mode + :param stream_slice: stream_slice + :param stream_state: stream_state + :raises StopIteration: if there is no first record to return (the read_records generator is empty) + :return: StreamData containing the first record in the slice + """ + # We wrap the return output of read_records() because some implementations return types that are iterable, + # but not iterators such as lists or tuples + records_for_slice = iter(stream.read_records(sync_mode=sync_mode, stream_slice=stream_slice, stream_state=stream_state)) + return next(records_for_slice) diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py index a8b7feaef07f..f47f34d26bc6 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py @@ -15,6 +15,7 @@ from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import StreamData from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream +from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from source_stripe.availability_strategy import StripeAvailabilityStrategy, StripeSubStreamAvailabilityStrategy @@ -491,12 +492,9 @@ class CustomerBalanceTransactions(StripeStream): API docs: https://stripe.com/docs/api/customer_balance_transactions/list """ - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs): - return f"customers/{stream_slice['id']}/balance_transactions" - - @property - def customers(self) -> IncrementalStripeStream: - return IncrementalStripeStream( + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.parent = IncrementalStripeStream( name="customers", path="customers", use_cache=USE_CACHE, @@ -506,13 +504,19 @@ def customers(self) -> IncrementalStripeStream: start_date=self.start_date, ) + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs): + return f"customers/{stream_slice['id']}/balance_transactions" + + @property + def availability_strategy(self) -> Optional[AvailabilityStrategy]: + return StripeSubStreamAvailabilityStrategy() + def stream_slices( self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream = self.customers - slices = parent_stream.stream_slices(sync_mode=SyncMode.full_refresh) + slices = self.parent.stream_slices(sync_mode=SyncMode.full_refresh) for _slice in slices: - for customer in parent_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=_slice): + for customer in self.parent.read_records(sync_mode=SyncMode.full_refresh, stream_slice=_slice): # we use `get` here because some attributes may not be returned by some API versions if customer.get("next_invoice_sequence") == 1 and customer.get("balance") == 0: # We're making this check in order to speed up a sync. if a customer's balance is 0 and there are no @@ -547,6 +551,12 @@ def __init__(self, **kwargs): def path(self, **kwargs) -> str: return "setup_attempts" + @property + def availability_strategy(self) -> Optional[AvailabilityStrategy]: + # we use the default http availability strategy here because parent stream may lack data in the incremental stream mode + # and this stream would be marked inaccessible which is not actually true + return HttpAvailabilityStrategy() + def stream_slices( self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: @@ -586,6 +596,10 @@ def __init__(self, *args, **kwargs): parent = StripeStream(*args, name="accounts", path="accounts", use_cache=USE_CACHE, **kwargs) super().__init__(*args, parent=parent, **kwargs) + @property + def availability_strategy(self) -> Optional[AvailabilityStrategy]: + return StripeSubStreamAvailabilityStrategy() + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs): return f"accounts/{stream_slice['parent']['id']}/persons" @@ -597,7 +611,9 @@ def stream_slices( class StripeSubStream(StripeStream, HttpSubStream): - pass + @property + def availability_strategy(self) -> Optional[AvailabilityStrategy]: + return StripeSubStreamAvailabilityStrategy() class StripeLazySubStream(StripeStream, HttpSubStream): @@ -800,10 +816,6 @@ def stream_slices( def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: return {self.cursor_field: max(current_stream_state.get(self.cursor_field, 0), latest_record[self.cursor_field])} - @property - def availability_strategy(self) -> Optional[AvailabilityStrategy]: - return StripeSubStreamAvailabilityStrategy() - @property def raise_on_http_errors(self) -> bool: return False @@ -822,3 +834,9 @@ def parse_response(self, response: requests.Response, *args, **kwargs) -> Iterab ) return [] response.raise_for_status() + + @property + def availability_strategy(self) -> Optional[AvailabilityStrategy]: + # we use the default http availability strategy here because parent stream may lack data in the incremental stream mode + # and this stream would be marked inaccessible which is not actually true + return HttpAvailabilityStrategy() diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_availability_strategy.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_availability_strategy.py index 0f747acac434..ee41b71dd049 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_availability_strategy.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_availability_strategy.py @@ -3,34 +3,45 @@ # import logging +import urllib.parse +import pytest from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from source_stripe.availability_strategy import STRIPE_ERROR_CODES, StripeSubStreamAvailabilityStrategy from source_stripe.streams import IncrementalStripeStream, StripeLazySubStream -def test_traverse_over_substreams(mocker): +@pytest.fixture() +def stream_mock(mocker): + def _mocker(): + return mocker.Mock(stream_slices=mocker.Mock(return_value=[{}]), read_records=mocker.Mock(return_value=[{}])) + return _mocker + + +def test_traverse_over_substreams(stream_mock, mocker): # Mock base HttpAvailabilityStrategy to capture all the check_availability method calls - check_availability_mock = mocker.MagicMock() - check_availability_mock.return_value = (True, None) + check_availability_mock = mocker.MagicMock(return_value=(True, None)) + cdk_check_availability_mock = mocker.MagicMock(return_value=(True, None)) mocker.patch( - "airbyte_cdk.sources.streams.http.availability_strategy.HttpAvailabilityStrategy.check_availability", check_availability_mock + "source_stripe.availability_strategy.StripeAvailabilityStrategy.check_availability", check_availability_mock + ) + mocker.patch( + "airbyte_cdk.sources.streams.http.availability_strategy.HttpAvailabilityStrategy.check_availability", cdk_check_availability_mock ) - # Prepare tree of nested objects - root = mocker.Mock() + root = stream_mock() root.availability_strategy = HttpAvailabilityStrategy() root.parent = None - child_1 = mocker.Mock() + child_1 = stream_mock() child_1.availability_strategy = StripeSubStreamAvailabilityStrategy() child_1.parent = root - child_1_1 = mocker.Mock() + child_1_1 = stream_mock() child_1_1.availability_strategy = StripeSubStreamAvailabilityStrategy() child_1_1.parent = child_1 - child_1_1_1 = mocker.Mock() + child_1_1_1 = stream_mock() child_1_1_1.availability_strategy = StripeSubStreamAvailabilityStrategy() child_1_1_1.parent = child_1_1 @@ -38,39 +49,38 @@ def test_traverse_over_substreams(mocker): is_available, reason = child_1_1_1.availability_strategy.check_availability(child_1_1_1, mocker.Mock(), mocker.Mock()) assert is_available and reason is None - # Check availability strategy was called once for every nested object - assert check_availability_mock.call_count == 4 + assert check_availability_mock.call_count == 3 + assert cdk_check_availability_mock.call_count == 1 # Check each availability strategy was called with proper instance argument - assert id(check_availability_mock.call_args_list[0].args[0]) == id(root) - assert id(check_availability_mock.call_args_list[1].args[0]) == id(child_1) - assert id(check_availability_mock.call_args_list[2].args[0]) == id(child_1_1) - assert id(check_availability_mock.call_args_list[3].args[0]) == id(child_1_1_1) + assert id(cdk_check_availability_mock.call_args_list[0].args[0]) == id(root) + assert id(check_availability_mock.call_args_list[0].args[0]) == id(child_1) + assert id(check_availability_mock.call_args_list[1].args[0]) == id(child_1_1) + assert id(check_availability_mock.call_args_list[2].args[0]) == id(child_1_1_1) -def test_traverse_over_substreams_failure(mocker): +def test_traverse_over_substreams_failure(stream_mock, mocker): # Mock base HttpAvailabilityStrategy to capture all the check_availability method calls - check_availability_mock = mocker.MagicMock() - check_availability_mock.side_effect = [(True, None), (False, "child_1")] + check_availability_mock = mocker.MagicMock(side_effect=[(True, None), (False, "child_1")]) mocker.patch( - "airbyte_cdk.sources.streams.http.availability_strategy.HttpAvailabilityStrategy.check_availability", check_availability_mock + "source_stripe.availability_strategy.StripeAvailabilityStrategy.check_availability", check_availability_mock ) # Prepare tree of nested objects - root = mocker.Mock() + root = stream_mock() root.availability_strategy = HttpAvailabilityStrategy() root.parent = None - child_1 = mocker.Mock() + child_1 = stream_mock() child_1.availability_strategy = StripeSubStreamAvailabilityStrategy() child_1.parent = root - child_1_1 = mocker.Mock() + child_1_1 = stream_mock() child_1_1.availability_strategy = StripeSubStreamAvailabilityStrategy() child_1_1.parent = child_1 - child_1_1_1 = mocker.Mock() + child_1_1_1 = stream_mock() child_1_1_1.availability_strategy = StripeSubStreamAvailabilityStrategy() child_1_1_1.parent = child_1_1 @@ -83,15 +93,15 @@ def test_traverse_over_substreams_failure(mocker): assert check_availability_mock.call_count == 2 # Check each availability strategy was called with proper instance argument - assert id(check_availability_mock.call_args_list[0].args[0]) == id(root) - assert id(check_availability_mock.call_args_list[1].args[0]) == id(child_1) + assert id(check_availability_mock.call_args_list[0].args[0]) == id(child_1) + assert id(check_availability_mock.call_args_list[1].args[0]) == id(child_1_1) def test_substream_availability(mocker, stream_by_name): check_availability_mock = mocker.MagicMock() check_availability_mock.return_value = (True, None) mocker.patch( - "airbyte_cdk.sources.streams.http.availability_strategy.HttpAvailabilityStrategy.check_availability", check_availability_mock + "source_stripe.availability_strategy.StripeAvailabilityStrategy.check_availability", check_availability_mock ) stream = stream_by_name("invoice_line_items") is_available, reason = stream.availability_strategy.check_availability(stream, mocker.Mock(), mocker.Mock()) @@ -106,7 +116,7 @@ def test_substream_availability_no_parent(mocker, stream_by_name): check_availability_mock = mocker.MagicMock() check_availability_mock.return_value = (True, None) mocker.patch( - "airbyte_cdk.sources.streams.http.availability_strategy.HttpAvailabilityStrategy.check_availability", check_availability_mock + "source_stripe.availability_strategy.StripeAvailabilityStrategy.check_availability", check_availability_mock ) stream = stream_by_name("invoice_line_items") stream.parent = None @@ -125,3 +135,91 @@ def test_403_error_handling(stream_by_name, requests_mock): available, message = stream.check_availability(logger) assert not available assert STRIPE_ERROR_CODES[error_code] in message + + +@pytest.mark.parametrize( + "stream_name, endpoints, expected_calls", + ( + ( + "accounts", + { + "/v1/accounts": {"data": []} + }, + 1 + ), + ( + "refunds", + { + "/v1/refunds": {"data": []} + }, + 2 + ), + ( + "credit_notes", + { + "/v1/credit_notes": {"data": []}, "/v1/events": {"data": []} + }, + 2 + ), + ( + "charges", + { + "/v1/charges": {"data": []}, "/v1/events": {"data": []} + }, + 2 + ), + ( + "subscription_items", + { + "/v1/subscriptions": {"data": [{"id": 1}]}, + "/v1/events": {"data": []} + }, + 3 + ), + ( + "bank_accounts", + { + "/v1/customers": {"data": [{"id": 1}]}, + "/v1/events": {"data": []} + }, + 2 + ), + ( + "customer_balance_transactions", + { + "/v1/events": {"data": [{"data":{"object": {"id": 1}}, "created": 1, "type": "customer.updated"}]}, + "/v1/customers": {"data": [{"id": 1}]}, + "/v1/customers/1/balance_transactions": {"data": []} + }, + 4 + ), + ( + "transfer_reversals", + { + "/v1/transfers": {"data": [{"id": 1}]}, + "/v1/events": {"data": [{"data":{"object": {"id": 1}}, "created": 1, "type": "transfer.updated"}]}, + "/v1/transfers/1/reversals": {"data": []} + }, + 4 + ), + ( + "persons", + { + "/v1/accounts": {"data": [{"id": 1}]}, + "/v1/events": {"data": []}, + "/v1/accounts/1/persons": {"data": []} + }, + 4 + ) + ) +) +def test_availability_strategy_visits_endpoints(stream_by_name, stream_name, endpoints, expected_calls, requests_mock, mocker, config): + for endpoint, data in endpoints.items(): + requests_mock.get(endpoint, json=data) + stream = stream_by_name(stream_name, config) + is_available, reason = stream.check_availability(mocker.Mock(), mocker.Mock()) + assert (is_available, reason) == (True, None) + assert len(requests_mock.request_history) == expected_calls + + for call in requests_mock.request_history: + assert urllib.parse.urlparse(call.url).path in endpoints.keys() diff --git a/docs/integrations/sources/stripe.md b/docs/integrations/sources/stripe.md index 3f949c6847e6..c873eae04833 100644 --- a/docs/integrations/sources/stripe.md +++ b/docs/integrations/sources/stripe.md @@ -216,7 +216,8 @@ Each record is marked with `is_deleted` flag when the appropriate event happens | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 5.0.0 | 2023-11-14 | [32286](https://github.com/airbytehq/airbyte/pull/32286/) | Fix multiple issues regarding usage of the incremental sync mode for the `Refunds`, `CheckoutSessions`, `CheckoutSessionsLineItems` streams. Fix schemas for the streams: `Invoices`, `Subscriptions`, `SubscriptionSchedule` | +| 5.0.1 | 2023-11-17 | [32638](https://github.com/airbytehq/airbyte/pull/32638/) | Availability stretegy: check availability of both endpoints (if applicable) - common API + events API | +| 5.0.0 | 2023-11-16 | [32286](https://github.com/airbytehq/airbyte/pull/32286/) | Fix multiple issues regarding usage of the incremental sync mode for the `Refunds`, `CheckoutSessions`, `CheckoutSessionsLineItems` streams. Fix schemas for the streams: `Invoices`, `Subscriptions`, `SubscriptionSchedule` | | 4.5.4 | 2023-11-16 | [32284](https://github.com/airbytehq/airbyte/pull/32284/) | Enable client-side rate limiting | | 4.5.3 | 2023-11-14 | [32473](https://github.com/airbytehq/airbyte/pull/32473/) | Have all full_refresh stream syncs be concurrent | | 4.5.2 | 2023-11-03 | [32146](https://github.com/airbytehq/airbyte/pull/32146/) | Fix multiple BankAccount issues | From 428a68167fffa1b94c9e5e04346a854071e77401 Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Fri, 17 Nov 2023 10:48:03 -0800 Subject: [PATCH 119/124] JDBC sources: improve timestamptz handling (#32616) Co-authored-by: edgao --- .../source/redshift/RedshiftSource.java | 12 ++-- .../redshift/RedshiftSourceOperations.java | 25 +++++++ .../sources/RedshiftSourceOperationsTest.java | 69 +++++++++++++++++++ docs/integrations/sources/redshift.md | 1 + 4 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceOperationsTest.java diff --git a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java index e7258bc07e8b..d80a2558ef1b 100644 --- a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java +++ b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java @@ -47,10 +47,7 @@ public JsonNode toDatabaseConfig(final JsonNode redshiftConfig) { final ImmutableMap.Builder builder = ImmutableMap.builder() .put(JdbcUtils.USERNAME_KEY, redshiftConfig.get(JdbcUtils.USERNAME_KEY).asText()) .put(JdbcUtils.PASSWORD_KEY, redshiftConfig.get(JdbcUtils.PASSWORD_KEY).asText()) - .put(JdbcUtils.JDBC_URL_KEY, String.format(DatabaseDriver.REDSHIFT.getUrlFormatString(), - redshiftConfig.get(JdbcUtils.HOST_KEY).asText(), - redshiftConfig.get(JdbcUtils.PORT_KEY).asInt(), - redshiftConfig.get(JdbcUtils.DATABASE_KEY).asText())); + .put(JdbcUtils.JDBC_URL_KEY, getJdbcUrl(redshiftConfig)); if (redshiftConfig.has(JdbcUtils.SCHEMAS_KEY) && redshiftConfig.get(JdbcUtils.SCHEMAS_KEY).isArray()) { schemas = new ArrayList<>(); @@ -75,6 +72,13 @@ public JsonNode toDatabaseConfig(final JsonNode redshiftConfig) { .build()); } + public static String getJdbcUrl(final JsonNode redshiftConfig) { + return String.format(DatabaseDriver.REDSHIFT.getUrlFormatString(), + redshiftConfig.get(JdbcUtils.HOST_KEY).asText(), + redshiftConfig.get(JdbcUtils.PORT_KEY).asInt(), + redshiftConfig.get(JdbcUtils.DATABASE_KEY).asText()); + } + private void addSsl(final List additionalProperties) { additionalProperties.add("ssl=true"); additionalProperties.add("sslfactory=com.amazon.redshift.ssl.NonValidatingFactory"); diff --git a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSourceOperations.java b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSourceOperations.java index 487fe6da1c29..2f3b9f169ee3 100644 --- a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSourceOperations.java +++ b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSourceOperations.java @@ -14,6 +14,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; +import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import org.slf4j.Logger; @@ -23,6 +24,19 @@ public class RedshiftSourceOperations extends JdbcSourceOperations { private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftSourceOperations.class); + @Override + public void copyToJsonField(final ResultSet resultSet, final int colIndex, final ObjectNode json) throws SQLException { + if ("timestamptz".equalsIgnoreCase(resultSet.getMetaData().getColumnTypeName(colIndex))) { + // Massive hack. Sometimes the JDBCType is TIMESTAMP (i.e. without timezone) + // even though it _should_ be TIMESTAMP_WITH_TIMEZONE. + // Check for this case explicitly. + final String columnName = resultSet.getMetaData().getColumnName(colIndex); + putTimestampWithTimezone(json, columnName, resultSet, colIndex); + } else { + super.copyToJsonField(resultSet, colIndex, json); + } + } + @Override protected void putTime(final ObjectNode node, final String columnName, @@ -44,6 +58,17 @@ protected void setTimestamp(final PreparedStatement preparedStatement, final int preparedStatement.setTimestamp(parameterIndex, Timestamp.valueOf(date)); } + @Override + protected void putTimestampWithTimezone(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) + throws SQLException { + try { + super.putTimestampWithTimezone(node, columnName, resultSet, index); + } catch (final Exception e) { + final Instant instant = resultSet.getTimestamp(index).toInstant(); + node.put(columnName, instant.toString()); + } + } + @Override protected void setDate(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException { final LocalDate date = LocalDate.parse(value); diff --git a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceOperationsTest.java b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceOperationsTest.java new file mode 100644 index 000000000000..856f6e9d1e6e --- /dev/null +++ b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceOperationsTest.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.source.jdbc.JdbcDataSourceUtils; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.redshift.RedshiftSource; +import io.airbyte.integrations.source.redshift.RedshiftSourceOperations; +import java.nio.file.Path; +import java.sql.SQLException; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.util.List; +import javax.sql.DataSource; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class RedshiftSourceOperationsTest { + + private JdbcDatabase database; + + @BeforeEach + void setup() { + final JsonNode config = Jsons.deserialize(IOs.readFile(Path.of("secrets/config.json"))); + + final DataSource dataSource = DataSourceFactory.create( + config.get("username").asText(), + config.get("password").asText(), + DatabaseDriver.REDSHIFT.getDriverClassName(), + RedshiftSource.getJdbcUrl(config), + JdbcDataSourceUtils.getConnectionProperties(config)); + database = new DefaultJdbcDatabase(dataSource, new RedshiftSourceOperations()); + } + + @Test + void testTimestampWithTimezone() throws SQLException { + // CURRENT_TIMESTAMP is converted to a string by queryJsons. + // CAST(CURRENT_TIMESTAMP AS VARCHAR) does the timestamp -> string conversion on the server side. + // If queryJsons is implemented correctly, both timestamps should be the same. + final List result = database.queryJsons("SELECT CURRENT_TIMESTAMP, CAST(CURRENT_TIMESTAMP AS VARCHAR)"); + + final Instant clientSideParse = Instant.parse(result.get(0).get("timestamptz").asText()); + // Redshift's default timestamp format is "2023-11-17 17:50:36.746606+00", which Instant.parse() + // can't handle. Build a custom datetime formatter. + // (Redshift supports server-side timestamp formatting, but it doesn't provide a way to force + // HH:MM offsets, which are required by Instant.parse) + final Instant serverSideParse = new DateTimeFormatterBuilder() + .append(DateTimeFormatter.ISO_DATE) + .appendLiteral(' ') + .append(DateTimeFormatter.ISO_LOCAL_TIME) + // "X" represents a +/-HH offset + .appendPattern("X") + .toFormatter() + .parse(result.get(0).get("varchar").asText(), Instant::from); + assertEquals(serverSideParse, clientSideParse); + } + +} diff --git a/docs/integrations/sources/redshift.md b/docs/integrations/sources/redshift.md index dafe396d2684..f7d84b6e06d2 100644 --- a/docs/integrations/sources/redshift.md +++ b/docs/integrations/sources/redshift.md @@ -56,6 +56,7 @@ All Redshift connections are encrypted using SSL | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------- | +| (none) | 2023-11-17 | [32616](https://github.com/airbytehq/airbyte/pull/32616) | Improve timestamptz handling | | 0.4.0 | 2023-06-26 | [27737](https://github.com/airbytehq/airbyte/pull/27737) | License Update: Elv2 | | 0.3.17 | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase | | 0.3.16 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | From b7afdbe65089f680b902ff6f75a78715dde78a35 Mon Sep 17 00:00:00 2001 From: Alexandre Cuoci Date: Fri, 17 Nov 2023 15:06:51 -0500 Subject: [PATCH 120/124] Update implementation-guide.md (#32625) --- docs/enterprise-setup/self-managed/README.md | 16 ++++++++++++---- .../self-managed/implementation-guide.md | 4 ++-- docs/enterprise-setup/self-managed/sso.md | 1 - docusaurus/sidebars.js | 2 +- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/docs/enterprise-setup/self-managed/README.md b/docs/enterprise-setup/self-managed/README.md index 30d1e7e05598..21d5fedf047d 100644 --- a/docs/enterprise-setup/self-managed/README.md +++ b/docs/enterprise-setup/self-managed/README.md @@ -1,9 +1,17 @@ # Airbyte Self-Managed -[Airbyte Self-Managed](https://airbyte.com/solutions/airbyte-enterprise) is a self-hosted version of Airbyte with additional features for enterprise customers. Airbyte Enterprise is in an early access stage for select priority users. +[Airbyte Self-Managed](https://airbyte.com/product/airbyte-enterprise) is the best way to run Airbyte yourself. You get all 300+ pre-built connectors, data never leaves your environment, and Airbyte becomes self-serve in your organization with new tools to manage multiple users, and multiple teams using Airbyte all in one place. -A valid license key is required to get started with Airbyte Enterprise. [Talk to sales](https://airbyte.com/company/talk-to-sales) to receive your license key. +A valid license key is required to get started with Airbyte Self-Managed. [Talk to sales](https://airbyte.com/company/talk-to-sales) to receive your license key. The following pages outline how to: -1. [Deploy Airbyte Enterprise using Kubernetes](./implementation-guide.md) -2. [Configure Okta for Single Sign-On (SSO) with Airbyte Enterprise](./sso.md) +1. [Deploy Airbyte Self-Managed using Kubernetes](./implementation-guide.md) +2. [Configure Okta for Single Sign-On (SSO) with Airbyte Self-Managed](./sso.md) + +| Feature | Description | +|---------------------------|--------------------------------------------------------------------------------------------------------------| +| Premium Support | [Priority assistance](https://docs.airbyte.com/operator-guides/contact-support/#airbyte-enterprise-self-hosted-support) with deploying, managing and upgrading Airbyte or troubleshooting any connection issues. | +| User Management | [Okta SSO](./sso.md) to extend each Airbyte workspace to multiple users | +| Multiple Workspaces | Ability to create + manage multiple workspaces on one Airbyte instance | +| Role-Based Access Control | Isolate workspaces from one another with users roles scoped to individual workspaces | + diff --git a/docs/enterprise-setup/self-managed/implementation-guide.md b/docs/enterprise-setup/self-managed/implementation-guide.md index 3b5c2da9c5d2..882a024436bb 100644 --- a/docs/enterprise-setup/self-managed/implementation-guide.md +++ b/docs/enterprise-setup/self-managed/implementation-guide.md @@ -60,7 +60,7 @@ cp configs/airbyte.sample.yml configs/airbyte.yml 3. Add your Airbyte Enterprise license key to your `airbyte.yml`. -4. Add your [auth details](/airbyte-enterprise#single-sign-on-sso) to your `airbyte.yml`. Auth configurations aren't easy to modify after Airbyte is installed, so please double check them to make sure they're accurate before proceeding. +4. Add your [auth details](/enterprise-setup/self-managed/sso) to your `airbyte.yml`. Auth configurations aren't easy to modify after Airbyte is installed, so please double check them to make sure they're accurate before proceeding.
Configuring auth in your airbyte.yml file @@ -100,4 +100,4 @@ After specifying your own configuration, run the following command: ```text ./tools/bin/install_airbyte_pro_on_helm.sh --values path/to/values.yaml $RELEASE_NAME airbyte/airbyte -``` \ No newline at end of file +``` diff --git a/docs/enterprise-setup/self-managed/sso.md b/docs/enterprise-setup/self-managed/sso.md index a7295e60ecf5..55d7053736f7 100644 --- a/docs/enterprise-setup/self-managed/sso.md +++ b/docs/enterprise-setup/self-managed/sso.md @@ -46,7 +46,6 @@ _Example values_ `` should point to where your Airbyte instance will be available, including the http/https protocol. - ## Deploying Airbyte Self-Managed with Okta Once your Okta app is set up, you're ready to deploy Airbyte with SSO. Take note of the following configuration values, as you will need them to configure Airbyte to use your new Okta SSO app integration: diff --git a/docusaurus/sidebars.js b/docusaurus/sidebars.js index 61e5dde2146a..55f4497d1e22 100644 --- a/docusaurus/sidebars.js +++ b/docusaurus/sidebars.js @@ -420,7 +420,7 @@ const deployAirbyte = { const airbyteSelfManaged = { type: "category", - label: "Airbyte Self Managed", + label: "Airbyte Self-Managed", link: { type: "doc", id: "enterprise-setup/self-managed/README", From 8a678fb931fb524fa4150a0babd72a12e1e31039 Mon Sep 17 00:00:00 2001 From: midavadim Date: Fri, 17 Nov 2023 22:44:01 +0200 Subject: [PATCH 121/124] :tada: Source Pinterest add custom resports (#32601) Co-authored-by: midavadim --- .../acceptance-test-config.yml | 3 +- .../config_custom_report.json | 18 ++ .../configured_catalog_custom_report.json | 15 ++ .../integration_tests/expected_records.jsonl | 3 +- .../connectors/source-pinterest/metadata.yaml | 2 +- .../source_pinterest/reports/reports.py | 51 +++++ .../source_pinterest/source.py | 55 ++++- .../source_pinterest/spec.json | 214 ++++++++++++++++++ .../unit_tests/test_reports.py | 61 +++++ .../unit_tests/test_source.py | 2 +- docs/integrations/sources/pinterest.md | 53 ++--- 11 files changed, 438 insertions(+), 39 deletions(-) create mode 100644 airbyte-integrations/connectors/source-pinterest/integration_tests/config_custom_report.json create mode 100644 airbyte-integrations/connectors/source-pinterest/integration_tests/configured_catalog_custom_report.json diff --git a/airbyte-integrations/connectors/source-pinterest/acceptance-test-config.yml b/airbyte-integrations/connectors/source-pinterest/acceptance-test-config.yml index 768d1d550502..4eab013a5fad 100644 --- a/airbyte-integrations/connectors/source-pinterest/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-pinterest/acceptance-test-config.yml @@ -5,7 +5,8 @@ acceptance_tests: tests: - spec_path: source_pinterest/spec.json backward_compatibility_tests_config: - disable_for_version: "0.7.0" # removed non-working token based auth method + disable_for_version: "0.7.3" # added custom report + # disable_for_version: "0.7.0" # removed non-working token based auth method # disable_for_version: "0.5.0" # Add Pattern for "start_date" connection: tests: diff --git a/airbyte-integrations/connectors/source-pinterest/integration_tests/config_custom_report.json b/airbyte-integrations/connectors/source-pinterest/integration_tests/config_custom_report.json new file mode 100644 index 000000000000..c8991f049ddf --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/integration_tests/config_custom_report.json @@ -0,0 +1,18 @@ +{ + "client_id": "1111111", + "client_secret": "XXXX", + "refresh_token": "XXXXX" + "start_date": "2023-01-08", + "custom_reports": [{ + "name": "vadim_report", + "level": "AD_GROUP", + "granularity": "MONTH", + "click_window_days": 30, + "engagement_window_days": 30, + "view_window_days": 30, + "conversion_report_time": "TIME_OF_CONVERSION", + "attribution_types": ["INDIVIDUAL", "HOUSEHOLD"], + "columns": ["ADVERTISER_ID", "AD_ACCOUNT_ID", "AD_GROUP_ID", "CTR", "IMPRESSION_2"], + "start_date": "2023-01-08" + }] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-pinterest/integration_tests/configured_catalog_custom_report.json b/airbyte-integrations/connectors/source-pinterest/integration_tests/configured_catalog_custom_report.json new file mode 100644 index 000000000000..645099b98d0e --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/integration_tests/configured_catalog_custom_report.json @@ -0,0 +1,15 @@ +{ + "streams": [ + { + "stream": { + "name": "custom_vadim_report", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": [] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-pinterest/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-pinterest/integration_tests/expected_records.jsonl index dbba2254d507..fb45fd3c024b 100644 --- a/airbyte-integrations/connectors/source-pinterest/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-pinterest/integration_tests/expected_records.jsonl @@ -20,4 +20,5 @@ {"stream": "ad_group_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ENTITY_STATUS": "ACTIVE", "AD_GROUP_ID": "2680068678993", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "TOTAL_IMPRESSION_USER": 1.0, "DATE": "2023-10-29"}, "emitted_at": 1699895043538} {"stream": "ad_group_targeting_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ENTITY_STATUS": "ACTIVE", "AD_GROUP_ID": "2680068678993", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "TARGETING_VALUE": "TWOCOLUMN_FEED", "TARGETING_TYPE": "FEED_TYPE", "DATE": "2023-10-29"}, "emitted_at": 1699895106949} {"stream": "pin_promotion_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ENTITY_STATUS": "ACTIVE", "AD_GROUP_ID": "2680068678993", "AD_ID": "687218400210", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "PIN_ID": 6.66743919837295e+17, "PIN_PROMOTION_ID": 687218400210.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "TOTAL_IMPRESSION_USER": 1.0, "DATE": "2023-10-29"}, "emitted_at": 1699895200157} -{"stream": "pin_promotion_targeting_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ENTITY_STATUS": "ACTIVE", "AD_GROUP_ID": "2680068678993", "AD_ID": "687218400210", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "PIN_ID": 6.66743919837295e+17, "PIN_PROMOTION_ID": 687218400210.0, "TARGETING_VALUE": "Education > Subjects > Science > Applied Science > Technology", "TARGETING_TYPE": "TARGETED_INTEREST", "DATE": "2023-10-29"}, "emitted_at": 1699895289749} \ No newline at end of file +{"stream": "pin_promotion_targeting_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ENTITY_STATUS": "ACTIVE", "AD_GROUP_ID": "2680068678993", "AD_ID": "687218400210", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "PIN_ID": 6.66743919837295e+17, "PIN_PROMOTION_ID": 687218400210.0, "TARGETING_VALUE": "Education > Subjects > Science > Applied Science > Technology", "TARGETING_TYPE": "TARGETED_INTEREST", "DATE": "2023-10-29"}, "emitted_at": 1699895289749} +{"stream": "custom_vadim_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ID": "2680068678993", "IMPRESSION_2": 11.0, "DATE_RANGE": "2023-10-01 - 2023-10-31"}, "emitted_at": 1700158289892} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-pinterest/metadata.yaml b/airbyte-integrations/connectors/source-pinterest/metadata.yaml index 82a76edc0a4b..a0da1eb5a704 100644 --- a/airbyte-integrations/connectors/source-pinterest/metadata.yaml +++ b/airbyte-integrations/connectors/source-pinterest/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 5cb7e5fe-38c2-11ec-8d3d-0242ac130003 - dockerImageTag: 0.8.0 + dockerImageTag: 0.8.1 dockerRepository: airbyte/source-pinterest connectorBuildOptions: baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py index 98e4809bb296..04e85473ff21 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py @@ -8,6 +8,7 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional from urllib.parse import urljoin +import airbyte_cdk.sources.utils.casing as casing import backoff import requests from airbyte_cdk.models import SyncMode @@ -260,3 +261,53 @@ class KeywordReport(PinterestAnalyticsTargetingReportStream): @property def level(self): return "KEYWORD" + + +class CustomReport(PinterestAnalyticsTargetingReportStream): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + self._custom_class_name = f"Custom_{self.config['name']}" + self._level = self.config["level"] + self.granularity = self.config["granularity"] + self.click_window_days = self.config["click_window_days"] + self.engagement_window_days = self.config["engagement_window_days"] + self.view_window_days = self.config["view_window_days"] + self.conversion_report_time = self.config["conversion_report_time"] + self.attribution_types = self.config["attribution_types"] + self.columns = self.config["columns"] + + @property + def level(self): + return self._level + + @property + def name(self) -> str: + """We override stream name to let the user change it via configuration.""" + name = self._custom_class_name or self.__class__.__name__ + return casing.camel_to_snake(name) + + def request_body_json(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> Optional[Mapping]: + """Return the body of the API request in JSON format.""" + return { + "start_date": stream_slice["start_date"], + "end_date": stream_slice["end_date"], + "level": self.level, + "granularity": self.granularity, + "click_window_days": self.click_window_days, + "engagement_window_days": self.engagement_window_days, + "view_window_days": self.view_window_days, + "conversion_report_time": self.conversion_report_time, + "attribution_types": self.attribution_types, + "columns": self.columns, + } + + @property + def window_in_days(self): + """Docs: https://developers.pinterest.com/docs/api/v5/#operation/analytics/get_report""" + if self.granularity == "HOUR": + return 2 + elif self.level == "PRODUCT_ITEM": + return 31 + else: + return 185 diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py index b8f9b693a9ab..ea5af593ebf8 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py @@ -3,8 +3,9 @@ # import copy +import logging from base64 import standard_b64encode -from typing import Any, List, Mapping, Tuple +from typing import Any, List, Mapping, Tuple, Type import pendulum import requests @@ -21,6 +22,7 @@ AdvertizerReport, AdvertizerTargetingReport, CampaignTargetingReport, + CustomReport, KeywordReport, PinPromotionReport, PinPromotionTargetingReport, @@ -52,6 +54,8 @@ UserAccountAnalytics, ) +logger = logging.getLogger("airbyte") + class SourcePinterest(AbstractSource): def _validate_and_transform(self, config: Mapping[str, Any], amount_of_days_allowed_for_lookup: int = 89): @@ -60,20 +64,25 @@ def _validate_and_transform(self, config: Mapping[str, Any], amount_of_days_allo latest_date_allowed_by_api = today.subtract(days=amount_of_days_allowed_for_lookup) start_date = config.get("start_date") - if not start_date: - config["start_date"] = latest_date_allowed_by_api - else: + + # transform to datetime + if start_date and isinstance(start_date, str): try: - config["start_date"] = pendulum.from_format(config["start_date"], "YYYY-MM-DD") + config["start_date"] = pendulum.from_format(start_date, "YYYY-MM-DD") except ValueError: - message = "Entered `Start Date` does not match format YYYY-MM-DD" + message = f"Entered `Start Date` {start_date} does not match format YYYY-MM-DD" raise AirbyteTracedException( message=message, internal_message=message, failure_type=FailureType.config_error, ) - if (today - config["start_date"]).days > amount_of_days_allowed_for_lookup: - config["start_date"] = latest_date_allowed_by_api + + if not start_date or config["start_date"] < latest_date_allowed_by_api: + logger.info( + f"Current start_date: {start_date} does not meet API report requirements. Resetting start_date to: {latest_date_allowed_by_api}" + ) + config["start_date"] = latest_date_allowed_by_api + return config @staticmethod @@ -154,4 +163,32 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: ProductGroupTargetingReport(ad_accounts, config=report_config), KeywordReport(ad_accounts, config=report_config), ProductItemReport(ad_accounts, config=report_config), - ] + ] + self.get_custom_report_streams(ad_accounts, config=report_config) + + def get_custom_report_streams(self, parent, config: dict) -> List[Type[Stream]]: + """return custom report streams""" + custom_streams = [] + for report_config in config.get("custom_reports", []): + report_config["authenticator"] = config["authenticator"] + + # https://developers.pinterest.com/docs/api/v5/#operation/analytics/get_report + if report_config.get("granularity") == "HOUR": + # Otherwise: Response Code: 400 {"code":1,"message":"HOURLY request must be less than 3 days"} + amount_of_days_allowed_for_lookup = 2 + elif report_config.get("level") == "PRODUCT_ITEM": + amount_of_days_allowed_for_lookup = 91 + else: + amount_of_days_allowed_for_lookup = 913 + + start_date = report_config.get("start_date") + if not start_date: + report_config["start_date"] = config.get("start_date") + + report_config = self._validate_and_transform(report_config, amount_of_days_allowed_for_lookup) + + stream = CustomReport( + parent=parent, + config=report_config, + ) + custom_streams.append(stream) + return custom_streams diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/spec.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/spec.json index c19b80a65901..ad385a664b48 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/spec.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/spec.json @@ -60,6 +60,220 @@ } } ] + }, + "custom_reports": { + "title": "Custom Reports", + "description": "A list which contains ad statistics entries, each entry must have a name and can contains fields, breakdowns or action_breakdowns. Click on \"add\" to fill this field.", + "type": "array", + "items": { + "title": "ReportConfig", + "description": "Config for custom report", + "type": "object", + "required": ["name", "level", "granularity", "columns"], + "properties": { + "name": { + "title": "Name", + "description": "The name value of report", + "type": "string", + "order": 0 + }, + "level": { + "title": "Level", + "description": "Chosen level for API", + "default": "ADVERTISER", + "enum": ["ADVERTISER", "ADVERTISER_TARGETING", "CAMPAIGN", "CAMPAIGN_TARGETING", "AD_GROUP", "AD_GROUP_TARGETING", "PIN_PROMOTION", "PIN_PROMOTION_TARGETING", "KEYWORD", "PRODUCT_GROUP", "PRODUCT_GROUP_TARGETING", "PRODUCT_ITEM"], + "type": "string", + "order": 1 + }, + "granularity": { + "title": "Granularity", + "description": "Chosen granularity for API", + "default": "TOTAL", + "enum": ["TOTAL", "DAY", "HOUR", "WEEK", "MONTH"], + "type": "string", + "order": 2 + }, + "columns": { + "title": "Columns", + "description": "A list of chosen columns", + "default": [], + "type": "array", + "order": 3, + "items": { + "title": "ValidEnums", + "description": "An enumeration.", + "enum": [ + "ADVERTISER_ID", + "AD_ACCOUNT_ID", + "AD_GROUP_ENTITY_STATUS", + "AD_GROUP_ID", + "AD_ID", + "CAMPAIGN_DAILY_SPEND_CAP", + "CAMPAIGN_ENTITY_STATUS", + "CAMPAIGN_ID", + "CAMPAIGN_LIFETIME_SPEND_CAP", + "CAMPAIGN_NAME", + "CHECKOUT_ROAS", + "CLICKTHROUGH_1", + "CLICKTHROUGH_1_GROSS", + "CLICKTHROUGH_2", + "CPC_IN_MICRO_DOLLAR", + "CPM_IN_DOLLAR", + "CPM_IN_MICRO_DOLLAR", + "CTR", + "CTR_2", + "ECPCV_IN_DOLLAR", + "ECPCV_P95_IN_DOLLAR", + "ECPC_IN_DOLLAR", + "ECPC_IN_MICRO_DOLLAR", + "ECPE_IN_DOLLAR", + "ECPM_IN_MICRO_DOLLAR", + "ECPV_IN_DOLLAR", + "ECTR", + "EENGAGEMENT_RATE", + "ENGAGEMENT_1", + "ENGAGEMENT_2", + "ENGAGEMENT_RATE", + "IDEA_PIN_PRODUCT_TAG_VISIT_1", + "IDEA_PIN_PRODUCT_TAG_VISIT_2", + "IMPRESSION_1", + "IMPRESSION_1_GROSS", + "IMPRESSION_2", + "INAPP_CHECKOUT_COST_PER_ACTION", + "OUTBOUND_CLICK_1", + "OUTBOUND_CLICK_2", + "PAGE_VISIT_COST_PER_ACTION", + "PAGE_VISIT_ROAS", + "PAID_IMPRESSION", + "PIN_ID", + "PIN_PROMOTION_ID", + "REPIN_1", + "REPIN_2", + "REPIN_RATE", + "SPEND_IN_DOLLAR", + "SPEND_IN_MICRO_DOLLAR", + "TOTAL_CHECKOUT", + "TOTAL_CHECKOUT_VALUE_IN_MICRO_DOLLAR", + "TOTAL_CLICKTHROUGH", + "TOTAL_CLICK_ADD_TO_CART", + "TOTAL_CLICK_CHECKOUT", + "TOTAL_CLICK_CHECKOUT_VALUE_IN_MICRO_DOLLAR", + "TOTAL_CLICK_LEAD", + "TOTAL_CLICK_SIGNUP", + "TOTAL_CLICK_SIGNUP_VALUE_IN_MICRO_DOLLAR", + "TOTAL_CONVERSIONS", + "TOTAL_CUSTOM", + "TOTAL_ENGAGEMENT", + "TOTAL_ENGAGEMENT_CHECKOUT", + "TOTAL_ENGAGEMENT_CHECKOUT_VALUE_IN_MICRO_DOLLAR", + "TOTAL_ENGAGEMENT_LEAD", + "TOTAL_ENGAGEMENT_SIGNUP", + "TOTAL_ENGAGEMENT_SIGNUP_VALUE_IN_MICRO_DOLLAR", + "TOTAL_IDEA_PIN_PRODUCT_TAG_VISIT", + "TOTAL_IMPRESSION_FREQUENCY", + "TOTAL_IMPRESSION_USER", + "TOTAL_LEAD", + "TOTAL_OFFLINE_CHECKOUT", + "TOTAL_PAGE_VISIT", + "TOTAL_REPIN_RATE", + "TOTAL_SIGNUP", + "TOTAL_SIGNUP_VALUE_IN_MICRO_DOLLAR", + "TOTAL_VIDEO_3SEC_VIEWS", + "TOTAL_VIDEO_AVG_WATCHTIME_IN_SECOND", + "TOTAL_VIDEO_MRC_VIEWS", + "TOTAL_VIDEO_P0_COMBINED", + "TOTAL_VIDEO_P100_COMPLETE", + "TOTAL_VIDEO_P25_COMBINED", + "TOTAL_VIDEO_P50_COMBINED", + "TOTAL_VIDEO_P75_COMBINED", + "TOTAL_VIDEO_P95_COMBINED", + "TOTAL_VIEW_ADD_TO_CART", + "TOTAL_VIEW_CHECKOUT", + "TOTAL_VIEW_CHECKOUT_VALUE_IN_MICRO_DOLLAR", + "TOTAL_VIEW_LEAD", + "TOTAL_VIEW_SIGNUP", + "TOTAL_VIEW_SIGNUP_VALUE_IN_MICRO_DOLLAR", + "TOTAL_WEB_CHECKOUT", + "TOTAL_WEB_CHECKOUT_VALUE_IN_MICRO_DOLLAR", + "TOTAL_WEB_CLICK_CHECKOUT", + "TOTAL_WEB_CLICK_CHECKOUT_VALUE_IN_MICRO_DOLLAR", + "TOTAL_WEB_ENGAGEMENT_CHECKOUT", + "TOTAL_WEB_ENGAGEMENT_CHECKOUT_VALUE_IN_MICRO_DOLLAR", + "TOTAL_WEB_SESSIONS", + "TOTAL_WEB_VIEW_CHECKOUT", + "TOTAL_WEB_VIEW_CHECKOUT_VALUE_IN_MICRO_DOLLAR", + "VIDEO_3SEC_VIEWS_2", + "VIDEO_LENGTH", + "VIDEO_MRC_VIEWS_2", + "VIDEO_P0_COMBINED_2", + "VIDEO_P100_COMPLETE_2", + "VIDEO_P25_COMBINED_2", + "VIDEO_P50_COMBINED_2", + "VIDEO_P75_COMBINED_2", + "VIDEO_P95_COMBINED_2", + "WEB_CHECKOUT_COST_PER_ACTION", + "WEB_CHECKOUT_ROAS", + "WEB_SESSIONS_1", + "WEB_SESSIONS_2" + ] + } + }, + "click_window_days": { + "title": "Click window days", + "description": "Number of days to use as the conversion attribution window for a pin click action.", + "default": 30, + "enum": [0, 1, 7, 14, 30, 60], + "type": "integer", + "order": 4 + }, + "engagement_window_days": { + "title": "Engagement window days", + "description": "Number of days to use as the conversion attribution window for an engagement action.", + "default": [30], + "enum": [0, 1, 7, 14, 30, 60], + "type": "integer", + "order": 5 + }, + "view_window_days": { + "title": "View window days", + "description": "Number of days to use as the conversion attribution window for a view action.", + "default": [30], + "enum": [0, 1, 7, 14, 30, 60], + "type": "integer", + "order": 6 + }, + "conversion_report_time": { + "title": "Conversion report time", + "description": "The date by which the conversion metrics returned from this endpoint will be reported. There are two dates associated with a conversion event: the date that the user interacted with the ad, and the date that the user completed a conversion event..", + "default": "TIME_OF_AD_ACTION", + "enum": ["TIME_OF_AD_ACTION", "TIME_OF_CONVERSION"], + "type": "string", + "order": 7 + }, + "attribution_types": { + "title": "Attribution types", + "description": "List of types of attribution for the conversion report", + "default": ["INDIVIDUAL", "HOUSEHOLD"], + "type": "array", + "items": { + "title": "ValidEnums", + "description": "An enumeration.", + "enum": ["INDIVIDUAL", "HOUSEHOLD"] + }, + "order": 8 + }, + "start_date": { + "type": "string", + "title": "Start Date", + "description": "A date in the format YYYY-MM-DD. If you have not set a date, it would be defaulted to latest allowed date by report api (913 days from today).", + "format": "date", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "pattern_descriptor": "YYYY-MM-DD", + "examples": ["2022-07-28"], + "order": 9 + } + } + } } } }, diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py index 8cc1a4f96057..61ba1f1c61f0 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py @@ -1,12 +1,31 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import copy +import os +from unittest.mock import MagicMock +import pytest import responses from source_pinterest import SourcePinterest +from source_pinterest.reports import CampaignAnalyticsReport +from source_pinterest.reports.reports import ( + AdGroupReport, + AdGroupTargetingReport, + AdvertizerReport, + AdvertizerTargetingReport, + CampaignTargetingReport, + KeywordReport, + PinPromotionReport, + PinPromotionTargetingReport, + ProductGroupReport, + ProductGroupTargetingReport, + ProductItemReport, +) from source_pinterest.utils import get_analytics_columns from unit_tests.test_source import setup_responses +os.environ["REQUEST_CACHE_PATH"] = '/tmp' @responses.activate def test_request_body_json(analytics_report_stream, date_range): @@ -62,3 +81,45 @@ def test_streams(test_config): streams = source.streams(test_config) expected_streams_number = 32 assert len(streams) == expected_streams_number + +@responses.activate +def test_custom_streams(test_config): + config = copy.deepcopy(test_config) + config['custom_reports'] = [{ + "name": "vadim_report", + "level": "AD_GROUP", + "granularity": "MONTH", + "click_window_days": 30, + "engagement_window_days": 30, + "view_window_days": 30, + "conversion_report_time": "TIME_OF_CONVERSION", + "attribution_types": ["INDIVIDUAL", "HOUSEHOLD"], + "columns": ["ADVERTISER_ID", "AD_ACCOUNT_ID", "AD_GROUP_ID", "CTR", "IMPRESSION_2"], + "start_date": "2023-01-08" + }] + setup_responses() + source = SourcePinterest() + streams = source.streams(config) + expected_streams_number = 33 + assert len(streams) == expected_streams_number + +@pytest.mark.parametrize( + "report_name, expected_level", + [ + [CampaignAnalyticsReport, 'CAMPAIGN'], + [CampaignTargetingReport, 'CAMPAIGN_TARGETING'], + [AdvertizerReport, 'ADVERTISER'], + [AdvertizerTargetingReport, 'ADVERTISER_TARGETING'], + [AdGroupReport, 'AD_GROUP'], + [AdGroupTargetingReport, 'AD_GROUP_TARGETING'], + [PinPromotionReport, 'PIN_PROMOTION'], + [PinPromotionTargetingReport, 'PIN_PROMOTION_TARGETING'], + [ProductGroupReport, 'PRODUCT_GROUP'], + [ProductGroupTargetingReport, 'PRODUCT_GROUP_TARGETING'], + [ProductItemReport, 'PRODUCT_ITEM'], + [KeywordReport, 'KEYWORD'] + ], +) +def test_level(test_config, report_name, expected_level): + assert report_name(parent=None, config=MagicMock()).level == expected_level + diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py index d86620fad40c..2fd50933d8e7 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py @@ -36,7 +36,7 @@ def test_check_wrong_date_connection(wrong_date_config): logger_mock = MagicMock() with pytest.raises(AirbyteTracedException) as e: source.check_connection(logger_mock, wrong_date_config) - assert e.value.message == "Entered `Start Date` does not match format YYYY-MM-DD" + assert e.value.message == "Entered `Start Date` wrong_date_format does not match format YYYY-MM-DD" @responses.activate diff --git a/docs/integrations/sources/pinterest.md b/docs/integrations/sources/pinterest.md index 5b8f94e40de1..1aae30167248 100644 --- a/docs/integrations/sources/pinterest.md +++ b/docs/integrations/sources/pinterest.md @@ -86,31 +86,32 @@ The connector is restricted by the Pinterest [requests limitation](https://devel | Version | Date | Pull Request | Subject | |:--------|:-----------| :------------------------------------------------------- |:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.8.1 | 2023-11-16 | [32601](https://github.com/airbytehq/airbyte/pull/32601) | added ability to create custom reports | | 0.8.0 | 2023-11-16 | [32592](https://github.com/airbytehq/airbyte/pull/32592) | Make start_date optional; add suggested streams; add missing fields | | 0.7.2 | 2023-11-08 | [32299](https://github.com/airbytehq/airbyte/pull/32299) | added default `AvailabilityStrategy`, fixed bug which cases duplicated requests, added new streams: Catalogs, CatalogsFeeds, CatalogsProductGroups, Audiences, Keywords, ConversionTags, CustomerLists, CampaignTargetingReport, AdvertizerReport, AdvertizerTargetingReport, AdGroupReport, AdGroupTargetingReport, PinPromotionReport, PinPromotionTargetingReport, ProductGroupReport, ProductGroupTargetingReport, ProductItemReport, KeywordReport | -| 0.7.1 | 2023-11-01 | [32078](https://github.com/airbytehq/airbyte/pull/32078) | handle non json response | -| 0.7.0 | 2023-10-25 | [31876](https://github.com/airbytehq/airbyte/pull/31876) | Migrated to base image, removed token based authentication mthod becuase access_token is valid for 1 day only | -| 0.6.0 | 2023-07-25 | [28672](https://github.com/airbytehq/airbyte/pull/28672) | Add report stream for `CAMPAIGN` level | -| 0.5.3 | 2023-07-05 | [27964](https://github.com/airbytehq/airbyte/pull/27964) | Add `id` field to `owner` field in `ad_accounts` stream | -| 0.5.2 | 2023-06-02 | [26949](https://github.com/airbytehq/airbyte/pull/26949) | Update `BoardPins` stream with `note` property | -| 0.5.1 | 2023-05-11 | [25984](https://github.com/airbytehq/airbyte/pull/25984) | Add pattern for start_date | -| 0.5.0 | 2023-05-17 | [26188](https://github.com/airbytehq/airbyte/pull/26188) | Add `product_tags` field to the `BoardPins` stream | -| 0.4.0 | 2023-05-16 | [26112](https://github.com/airbytehq/airbyte/pull/26112) | Add `is_standard` field to the `BoardPins` stream | -| 0.3.0 | 2023-05-09 | [25915](https://github.com/airbytehq/airbyte/pull/25915) | Add `creative_type` field to the `BoardPins` stream | -| 0.2.6 | 2023-04-26 | [25548](https://github.com/airbytehq/airbyte/pull/25548) | Fix `format` issue for `boards` stream schema for fields with `date-time` | -| 0.2.5 | 2023-04-19 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Update `AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP` to 89 days | -| 0.2.4 | 2023-02-25 | [23457](https://github.com/airbytehq/airbyte/pull/23457) | Add missing columns for analytics streams for pinterest source | -| 0.2.3 | 2023-03-01 | [23649](https://github.com/airbytehq/airbyte/pull/23649) | Fix for `HTTP - 400 Bad Request` when requesting data >= 90 days | -| 0.2.2 | 2023-01-27 | [22020](https://github.com/airbytehq/airbyte/pull/22020) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 0.2.1 | 2022-12-15 | [20532](https://github.com/airbytehq/airbyte/pull/20532) | Bump CDK version | -| 0.2.0 | 2022-12-13 | [20242](https://github.com/airbytehq/airbyte/pull/20242) | Add data-type normalization up to the schemas declared | -| 0.1.9 | 2022-09-06 | [15074](https://github.com/airbytehq/airbyte/pull/15074) | Add filter based on statuses | -| 0.1.8 | 2022-10-21 | [18285](https://github.com/airbytehq/airbyte/pull/18285) | Fix type of `start_date` | -| 0.1.7 | 2022-09-29 | [17387](https://github.com/airbytehq/airbyte/pull/17387) | Set `start_date` dynamically based on API restrictions. | -| 0.1.6 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Use CDK 0.1.89 | -| 0.1.5 | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state | -| 0.1.4 | 2022-09-06 | [16161](https://github.com/airbytehq/airbyte/pull/16161) | Add ability to handle `429 - Too Many Requests` error with respect to `Max Rate Limit Exceeded Error` | -| 0.1.3 | 2022-09-02 | [16271](https://github.com/airbytehq/airbyte/pull/16271) | Add support of `OAuth2.0` authentication method | -| 0.1.2 | 2021-12-22 | [10223](https://github.com/airbytehq/airbyte/pull/10223) | Fix naming of `AD_ID` and `AD_ACCOUNT_ID` fields | -| 0.1.1 | 2021-12-22 | [9043](https://github.com/airbytehq/airbyte/pull/9043) | Update connector fields title/description | -| 0.1.0 | 2021-10-29 | [7493](https://github.com/airbytehq/airbyte/pull/7493) | Release Pinterest CDK Connector | +| 0.7.1 | 2023-11-01 | [32078](https://github.com/airbytehq/airbyte/pull/32078) | handle non json response | +| 0.7.0 | 2023-10-25 | [31876](https://github.com/airbytehq/airbyte/pull/31876) | Migrated to base image, removed token based authentication mthod becuase access_token is valid for 1 day only | +| 0.6.0 | 2023-07-25 | [28672](https://github.com/airbytehq/airbyte/pull/28672) | Add report stream for `CAMPAIGN` level | +| 0.5.3 | 2023-07-05 | [27964](https://github.com/airbytehq/airbyte/pull/27964) | Add `id` field to `owner` field in `ad_accounts` stream | +| 0.5.2 | 2023-06-02 | [26949](https://github.com/airbytehq/airbyte/pull/26949) | Update `BoardPins` stream with `note` property | +| 0.5.1 | 2023-05-11 | [25984](https://github.com/airbytehq/airbyte/pull/25984) | Add pattern for start_date | +| 0.5.0 | 2023-05-17 | [26188](https://github.com/airbytehq/airbyte/pull/26188) | Add `product_tags` field to the `BoardPins` stream | +| 0.4.0 | 2023-05-16 | [26112](https://github.com/airbytehq/airbyte/pull/26112) | Add `is_standard` field to the `BoardPins` stream | +| 0.3.0 | 2023-05-09 | [25915](https://github.com/airbytehq/airbyte/pull/25915) | Add `creative_type` field to the `BoardPins` stream | +| 0.2.6 | 2023-04-26 | [25548](https://github.com/airbytehq/airbyte/pull/25548) | Fix `format` issue for `boards` stream schema for fields with `date-time` | +| 0.2.5 | 2023-04-19 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Update `AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP` to 89 days | +| 0.2.4 | 2023-02-25 | [23457](https://github.com/airbytehq/airbyte/pull/23457) | Add missing columns for analytics streams for pinterest source | +| 0.2.3 | 2023-03-01 | [23649](https://github.com/airbytehq/airbyte/pull/23649) | Fix for `HTTP - 400 Bad Request` when requesting data >= 90 days | +| 0.2.2 | 2023-01-27 | [22020](https://github.com/airbytehq/airbyte/pull/22020) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.2.1 | 2022-12-15 | [20532](https://github.com/airbytehq/airbyte/pull/20532) | Bump CDK version | +| 0.2.0 | 2022-12-13 | [20242](https://github.com/airbytehq/airbyte/pull/20242) | Add data-type normalization up to the schemas declared | +| 0.1.9 | 2022-09-06 | [15074](https://github.com/airbytehq/airbyte/pull/15074) | Add filter based on statuses | +| 0.1.8 | 2022-10-21 | [18285](https://github.com/airbytehq/airbyte/pull/18285) | Fix type of `start_date` | +| 0.1.7 | 2022-09-29 | [17387](https://github.com/airbytehq/airbyte/pull/17387) | Set `start_date` dynamically based on API restrictions. | +| 0.1.6 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Use CDK 0.1.89 | +| 0.1.5 | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state | +| 0.1.4 | 2022-09-06 | [16161](https://github.com/airbytehq/airbyte/pull/16161) | Add ability to handle `429 - Too Many Requests` error with respect to `Max Rate Limit Exceeded Error` | +| 0.1.3 | 2022-09-02 | [16271](https://github.com/airbytehq/airbyte/pull/16271) | Add support of `OAuth2.0` authentication method | +| 0.1.2 | 2021-12-22 | [10223](https://github.com/airbytehq/airbyte/pull/10223) | Fix naming of `AD_ID` and `AD_ACCOUNT_ID` fields | +| 0.1.1 | 2021-12-22 | [9043](https://github.com/airbytehq/airbyte/pull/9043) | Update connector fields title/description | +| 0.1.0 | 2021-10-29 | [7493](https://github.com/airbytehq/airbyte/pull/7493) | Release Pinterest CDK Connector | From f0d68ba895932427b1cc96e23870d38cbc85881d Mon Sep 17 00:00:00 2001 From: Artem Inzhyyants <36314070+artem1205@users.noreply.github.com> Date: Fri, 17 Nov 2023 22:30:22 +0100 Subject: [PATCH 122/124] =?UTF-8?q?=F0=9F=90=9B=20Source=20Amazon=20Seller?= =?UTF-8?q?=20Partner:=20remove=20`max=5Fwait=5Fseconds`=20from=20spec=20(?= =?UTF-8?q?#32462)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../integration_tests/spec.json | 200 ------------------ .../metadata.yaml | 2 +- .../source-amazon-seller-partner/setup.py | 2 +- .../source_amazon_seller_partner/source.py | 1 - .../source_amazon_seller_partner/spec.json | 9 - .../source_amazon_seller_partner/streams.py | 15 +- .../unit_tests/test_finance_streams.py | 2 - .../unit_tests/test_order_items_stream.py | 1 - .../test_reports_stream_sales_and_traffic.py | 1 - .../test_reports_streams_settlement_report.py | 1 - .../unit_tests/test_transform_function.py | 1 - .../sources/amazon-seller-partner.md | 1 + 12 files changed, 8 insertions(+), 228 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/spec.json diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/spec.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/spec.json deleted file mode 100644 index 9c8e32370a3e..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/spec.json +++ /dev/null @@ -1,200 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/amazon-seller-partner", - "changelogUrl": "https://docs.airbyte.com/integrations/sources/amazon-seller-partner", - "connectionSpecification": { - "title": "Amazon Seller Partner Spec", - "type": "object", - "required": [ - "aws_environment", - "region", - "lwa_app_id", - "lwa_client_secret", - "refresh_token", - "replication_start_date" - ], - "additionalProperties": true, - "properties": { - "auth_type": { - "title": "Auth Type", - "const": "oauth2.0", - "order": 0, - "type": "string" - }, - "aws_environment": { - "title": "AWS Environment", - "description": "Select the AWS Environment.", - "enum": ["PRODUCTION", "SANDBOX"], - "default": "PRODUCTION", - "type": "string", - "order": 1 - }, - "region": { - "title": "AWS Region", - "description": "Select the AWS Region.", - "enum": [ - "AE", - "AU", - "BE", - "BR", - "CA", - "DE", - "EG", - "ES", - "FR", - "GB", - "IN", - "IT", - "JP", - "MX", - "NL", - "PL", - "SA", - "SE", - "SG", - "TR", - "UK", - "US" - ], - "default": "US", - "type": "string", - "order": 2 - }, - "aws_access_key": { - "title": "AWS Access Key", - "description": "Specifies the AWS access key used as part of the credentials to authenticate the user.", - "airbyte_secret": true, - "order": 3, - "type": "string" - }, - "aws_secret_key": { - "title": "AWS Secret Access Key", - "description": "Specifies the AWS secret key used as part of the credentials to authenticate the user.", - "airbyte_secret": true, - "order": 4, - "type": "string" - }, - "role_arn": { - "title": "Role ARN", - "description": "Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. (Needs permission to 'Assume Role' STS).", - "airbyte_secret": true, - "order": 5, - "type": "string" - }, - "lwa_app_id": { - "title": "LWA Client Id", - "description": "Your Login with Amazon Client ID.", - "order": 6, - "airbyte_secret": true, - "type": "string" - }, - "lwa_client_secret": { - "title": "LWA Client Secret", - "description": "Your Login with Amazon Client Secret.", - "airbyte_secret": true, - "order": 7, - "type": "string" - }, - "refresh_token": { - "title": "Refresh Token", - "description": "The Refresh Token obtained via OAuth flow authorization.", - "airbyte_secret": true, - "order": 8, - "type": "string" - }, - "replication_start_date": { - "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2017-01-25T00:00:00Z"], - "order": 9, - "type": "string" - }, - "replication_end_date": { - "title": "End Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data after this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$|^$", - "examples": ["2017-01-25T00:00:00Z"], - "order": 10, - "type": "string" - }, - "period_in_days": { - "title": "Period In Days", - "type": "integer", - "description": "Will be used for stream slicing for initial full_refresh sync when no updated state is present for reports that support sliced incremental sync.", - "default": 90, - "order": 11 - }, - "report_options": { - "title": "Report Options", - "description": "Additional information passed to reports. This varies by report type. Must be a valid json string.", - "examples": [ - "{\"GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT\": {\"reportPeriod\": \"WEEK\"}}", - "{\"GET_SOME_REPORT\": {\"custom\": \"true\"}}" - ], - "order": 12, - "type": "string" - }, - "max_wait_seconds": { - "title": "Max wait time for reports (in seconds)", - "description": "Sometimes report can take up to 30 minutes to generate. This will set the limit for how long to wait for a successful report.", - "default": 500, - "examples": ["500", "1980"], - "order": 13, - "type": "integer" - }, - "advanced_stream_options": { - "title": "Advanced Stream Options", - "description": "Additional information to configure report options. This varies by report type, not every report implement this kind of feature. Must be a valid json string.", - "examples": [ - "{\"GET_SALES_AND_TRAFFIC_REPORT\": {\"availability_sla_days\": 3}}", - "{\"GET_SOME_REPORT\": {\"custom\": \"true\"}}" - ], - "order": 14, - "type": "string" - } - } - }, - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "lwa_app_id": { - "type": "string" - }, - "lwa_client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "lwa_app_id": { - "type": "string", - "path_in_connector_config": ["lwa_app_id"] - }, - "lwa_client_secret": { - "type": "string", - "path_in_connector_config": ["lwa_client_secret"] - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml index 3f39e4aae628..80b56d5d014b 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: api connectorType: source definitionId: e55879a8-0ef8-4557-abcf-ab34c53ec460 - dockerImageTag: 2.0.1 + dockerImageTag: 2.0.2 dockerRepository: airbyte/source-amazon-seller-partner documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-seller-partner githubIssueLabel: source-amazon-seller-partner diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py b/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py index af80eec8c453..9b4396a6c472 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk", "pendulum~=2.1", "pycryptodome~=3.10", "xmltodict~=0.12"] +MAIN_REQUIREMENTS = ["airbyte-cdk", "xmltodict~=0.12"] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py index 5a565c21ad71..dfe04d11d35f 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py @@ -81,7 +81,6 @@ def _get_stream_kwargs(self, config: Mapping[str, Any]) -> Mapping[str, Any]: "marketplace_id": marketplace_id, "period_in_days": config.get("period_in_days", 90), "report_options": config.get("report_options"), - "max_wait_seconds": config.get("max_wait_seconds", 500), "replication_end_date": config.get("replication_end_date"), "advanced_stream_options": config.get("advanced_stream_options"), } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json index d64b1ee1d86f..f0f37d084ff1 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json @@ -125,15 +125,6 @@ "order": 10, "type": "string" }, - "max_wait_seconds": { - "title": "Max wait time for reports (in seconds)", - "description": "Sometimes report can take up to 30 minutes to generate. This will set the limit for how long to wait for a successful report.", - "default": 500, - "examples": ["500", "1980"], - "order": 11, - "minimum": 1, - "type": "integer" - }, "advanced_stream_options": { "title": "Advanced Stream Options", "description": "Additional information to configure report options. This varies by report type, not every report implement this kind of feature. Must be a valid json string.", diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py index 401332714252..ac7ff0485a74 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py @@ -38,7 +38,6 @@ def __init__( period_in_days: Optional[int], report_options: Optional[str], advanced_stream_options: Optional[str], - max_wait_seconds: Optional[int], replication_end_date: Optional[str], *args, **kwargs, @@ -132,6 +131,7 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late class ReportsAmazonSPStream(HttpStream, ABC): + max_wait_seconds = 3600 """ API docs: https://github.com/amzn/selling-partner-api-docs/blob/main/references/reports-api/reports_2020-09-04.md API model: https://github.com/amzn/selling-partner-api-models/blob/main/models/reports-api-model/reports_2020-09-04.json @@ -163,7 +163,6 @@ def __init__( marketplace_id: str, period_in_days: Optional[int], report_options: Optional[str], - max_wait_seconds: Optional[int], replication_end_date: Optional[str], advanced_stream_options: Optional[str], *args, @@ -176,7 +175,6 @@ def __init__( self.marketplace_id = marketplace_id self.period_in_days = max(period_in_days, self.replication_start_date_limit_in_days) # ensure old configs work as well self._report_options = report_options or "{}" - self.max_wait_seconds = max_wait_seconds self._advanced_stream_options = dict() self._http_method = "GET" if advanced_stream_options is not None: @@ -250,11 +248,11 @@ def _retrieve_report(self, report_id: str) -> Mapping[str, Any]: return report_payload @default_backoff_handler(factor=5, max_tries=5) - def download_and_decompress_report_document(self, url, payload): + def download_and_decompress_report_document(self, payload: dict) -> str: """ Unpacks a report document """ - report = requests.get(url) + report = requests.get(payload.get("url")) report.raise_for_status() if "compressionAlgorithm" in payload: return gzip.decompress(report.content).decode("iso-8859-1") @@ -265,7 +263,7 @@ def parse_response( ) -> Iterable[Mapping]: payload = response.json() - document = self.download_and_decompress_report_document(payload.get("url"), payload) + document = self.download_and_decompress_report_document(payload) document_records = self.parse_document(document) yield from document_records @@ -902,10 +900,7 @@ def parse_response( payload = response.json() - document = self.decompress_report_document( - payload.get("url"), - payload, - ) + document = self.download_and_decompress_report_document(payload) document_records = self.parse_document(document) # Not all (partial) responses include the request date, so adding it manually here diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py index 7621df97a0f2..5e84a2cf47f3 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py @@ -104,7 +104,6 @@ def _internal(start_date: str = START_DATE_1, end_date: str = END_DATE_1): period_in_days=0, report_options=None, advanced_stream_options=None, - max_wait_seconds=500, ) return stream @@ -123,7 +122,6 @@ def _internal(start_date: str = START_DATE_1, end_date: str = END_DATE_1): period_in_days=0, report_options=None, advanced_stream_options=None, - max_wait_seconds=500, ) return stream diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_order_items_stream.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_order_items_stream.py index 0c4a80a1bcf4..7b7d5c016a9a 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_order_items_stream.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_order_items_stream.py @@ -39,7 +39,6 @@ def _internal(): period_in_days=0, report_options=None, advanced_stream_options=None, - max_wait_seconds=500, ) return stream diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_stream_sales_and_traffic.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_stream_sales_and_traffic.py index c255aa92eee6..106b5b543785 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_stream_sales_and_traffic.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_stream_sales_and_traffic.py @@ -18,7 +18,6 @@ def test_stream_uses_advanced_options(): period_in_days=0, report_options=None, advanced_stream_options='{"GET_SALES_AND_TRAFFIC_REPORT":{"availability_sla_days": 3}}', - max_wait_seconds=500, ) assert stream.availability_sla_days == 3 diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py index a1f64cf10b64..77a21ef4bca8 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py @@ -86,7 +86,6 @@ def _internal(start_date: str = START_DATE_1, end_date: str = END_DATE_1): period_in_days=0, report_options=None, advanced_stream_options=None, - max_wait_seconds=500, ) return stream diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_transform_function.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_transform_function.py index 27d7ca2d3b36..9b2aab17a298 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_transform_function.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_transform_function.py @@ -16,7 +16,6 @@ def reports_stream(marketplace_id): period_in_days=0, report_options=None, advanced_stream_options=None, - max_wait_seconds=0, ) return stream diff --git a/docs/integrations/sources/amazon-seller-partner.md b/docs/integrations/sources/amazon-seller-partner.md index 1023cee1379b..cfc3348ab1ba 100644 --- a/docs/integrations/sources/amazon-seller-partner.md +++ b/docs/integrations/sources/amazon-seller-partner.md @@ -143,6 +143,7 @@ So, for any value that exceeds the limit, the `period_in_days` will be automatic | Version | Date | Pull Request | Subject | |:---------|:-----------|:--------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `2.0.2` | 2023-11-17 | [\#32462](https://github.com/airbytehq/airbyte/pull/32462) | Remove Max time option from specification; set default waiting time for reports to 1 hour | | `2.0.1` | 2023-11-16 | [\#32550](https://github.com/airbytehq/airbyte/pull/32550) | Fix the OAuth flow | | `2.0.0` | 2023-11-23 | [\#32355](https://github.com/airbytehq/airbyte/pull/32355) | Remove Brand Analytics from Airbyte Cloud, permanently remove deprecated FBA reports | | `1.6.2` | 2023-11-14 | [\#32508](https://github.com/airbytehq/airbyte/pull/32508) | Do not use AWS signature as it is no longer required by the Amazon API | From e94501b87be88127f000c6206ae309afaba2e963 Mon Sep 17 00:00:00 2001 From: Natalie Kwong <38087517+nataliekwong@users.noreply.github.com> Date: Fri, 17 Nov 2023 14:29:32 -0800 Subject: [PATCH 123/124] Update Cloud Docs (#32539) --- docs/cloud/core-concepts.md | 107 +++---------- .../getting-started-with-airbyte-cloud.md | 146 +++++++----------- .../configuring-connections.md | 19 +-- .../manage-airbyte-cloud-notifications.md | 7 +- .../manage-airbyte-cloud-workspace.md | 4 +- .../manage-connection-state.md | 2 +- .../managing-airbyte-cloud/manage-credits.md | 4 +- .../manage-schema-changes.md | 64 ++++---- .../review-sync-history.md | 35 ++--- .../understand-airbyte-cloud-limits.md | 2 +- docs/integrations/destinations/firestore.md | 31 +++- 11 files changed, 175 insertions(+), 246 deletions(-) diff --git a/docs/cloud/core-concepts.md b/docs/cloud/core-concepts.md index 9383c6ffd036..c3c949599ee8 100644 --- a/docs/cloud/core-concepts.md +++ b/docs/cloud/core-concepts.md @@ -1,6 +1,6 @@ # Core Concepts -Airbyte enables you to build data pipelines and replicate data from a source to a destination. You can configure how frequently the data is synced, what data is replicated, what format the data is written to in the destination, and if the data is stored in raw tables format or basic normalized (or JSON) format. +Airbyte enables you to build data pipelines and replicate data from a source to a destination. You can configure how frequently the data is synced, what data is replicated, and how the data is written to in the destination. This page describes the concepts you need to know to use Airbyte. @@ -18,49 +18,15 @@ An Airbyte component which pulls data from a source or pushes data to a destinat ## Connection -A connection is an automated data pipeline that replicates data from a source to a destination. - -Setting up a connection involves configuring the following parameters: - - - - - - - - - - - - - - - - - - - - - - - - - - -
Parameter - Description -
Sync schedule - When should a data sync be triggered? -
Destination Namespace and stream names - Where should the replicated data be written? -
Catalog selection - What data should be replicated from the source to the destination? -
Sync mode - How should the streams be replicated (read and written)? -
Optional transformations - How should Airbyte protocol messages (raw JSON blob) data be converted into other data representations? -
+A connection is an automated data pipeline that replicates data from a source to a destination. Setting up a connection enables configuration of the following parameters: +| Concept | Description | +|---------------------|---------------------------------------------------------------------------------------------------------------------| +| Replication Frequency | When should a data sync be triggered? | +| Destination Namespace and Stream Prefix | Where should the replicated data be written? | +| Catalog Selection | What data (streams and columns) should be replicated from the source to the destination? | +| Sync Mode | How should the streams be replicated (read and written)? | +| Schema Propagation | How should Airbyte handle schema drift in sources? | ## Stream A stream is a group of related records. @@ -82,49 +48,26 @@ Examples of fields: ## Namespace -Namespace is a group of streams in a source or destination. Common use cases for namespaces are enforcing permissions, segregating test and production data, and general data organization. - -A schema in a relational database system is an example of a namespace. - -In a source, the namespace is the location from where the data is replicated to the destination. - -In a destination, the namespace is the location where the replicated data is stored in the destination. Airbyte supports the following configuration options for destination namespaces: - - - - - - - - - - - - - - - - - - -
Configuration - Description -
Mirror source structure - Some sources (for example, databases) provide namespace information for a stream. If a source provides the namespace information, the destination will reproduce the same namespace when this configuration is set. For sources or streams where the source namespace is not known, the behavior will default to the "Destination default" option. -
Destination default - All streams will be replicated and stored in the default namespace defined on the destination settings page. For settings for popular destinations, see ​​Destination Connector Settings -
Custom format - All streams will be replicated and stored in a user-defined custom format. See Custom format for more details. -
+Namespace is a method of grouping streams in a source or destination. Namespaces are used to generally organize data, segregate tests and production data, and enforce permissions. In a relational database system, this is known as a schema. + +In a source, the namespace is the location from where the data is replicated to the destination. In a destination, the namespace is the location where the replicated data is stored in the destination. + +Airbyte supports the following configuration options for a connection: + + | Destination Namepsace | Description | +| ---------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| Destination default | All streams will be replicated to the single default namespace defined by the Destination. For more details, see ​​Destination Connector Settings | +| Mirror source structure | Some sources (for example, databases) provide namespace information for a stream. If a source provides namespace information, the destination will mirror the same namespace when this configuration is set. For sources or streams where the source namespace is not known, the behavior will default to the "Destination default" option. | +| Custom format | All streams will be replicated to a single user-defined namespace. See Custom format for more details | ## Connection sync modes A sync mode governs how Airbyte reads from a source and writes to a destination. Airbyte provides different sync modes to account for various use cases. -- **Full Refresh | Overwrite:** Sync all records from the source and replace data in destination by overwriting it. -- **Full Refresh | Append:** Sync all records from the source and add them to the destination without deleting any data. -- **Incremental Sync | Append:** Sync new records from the source and add them to the destination without deleting any data. -- **Incremental Sync | Append + Deduped:** Sync new records from the source and add them to the destination. Also provides a de-duplicated view mirroring the state of the stream in the source. +- **Full Refresh | Overwrite:** Sync all records from the source and replace data in destination by overwriting it each time. +- **Full Refresh | Append:** Sync all records from the source and add them to the destination without deleting any data. This creates a historical copy of all records each sync. +- **Incremental Sync | Append:** Sync new records from the source and add them to the destination without deleting any data. This enables efficient historical tracking over time of data. +- **Incremental Sync | Append + Deduped:** Sync new records from the source and add them to the destination. Also provides a de-duplicated view mirroring the state of the stream in the source. This is the most common replication use case. ## Normalization @@ -132,8 +75,6 @@ Normalization is the process of structuring data from the source into a format a Note that normalization is only relevant for the following relational database & warehouse destinations: -- BigQuery -- Snowflake - Redshift - Postgres - Oracle diff --git a/docs/cloud/getting-started-with-airbyte-cloud.md b/docs/cloud/getting-started-with-airbyte-cloud.md index 0c4cc4d284cc..2fecf212572f 100644 --- a/docs/cloud/getting-started-with-airbyte-cloud.md +++ b/docs/cloud/getting-started-with-airbyte-cloud.md @@ -11,7 +11,7 @@ To use Airbyte Cloud: Airbyte Cloud offers a 14-day free trial that begins after your first successful sync. For more information, see [Pricing](https://airbyte.com/pricing). :::note - If you are invited to a workspace, you cannot use your Google login to create a new Airbyte account. + If you are invited to a workspace, you currently cannot use your Google login to create a new Airbyte account. ::: 2. If you signed up using your email address, Airbyte will send you an email with a verification link. On clicking the link, you'll be taken to your new workspace. @@ -28,16 +28,8 @@ A source is an API, file, database, or data warehouse that you want to ingest da To set up a source: -:::note - -Set your [default data residency](https://docs.airbyte.com/cloud/managing-airbyte-cloud/manage-data-residency#choose-your-default-data-residency) before creating a new source to ensure your data is processed in the correct region. - -::: - -1. On the Airbyte Cloud dashboard, click **Sources** and then click **+ New source**. -2. On the Set up the source page, select the source you want to set up from the **Source** catalog. - - The fields relevant to your source are displayed. The Setup Guide provides information to help you fill out the fields for your selected source. +1. On the Airbyte Cloud dashboard, click **Sources**. +2. On the Set up the source page, select the source you want to set up from the **Source catalog**. Airbyte currently offers more than 200 source connectors in Cloud to choose from. Once you've selected the source, a Setup Guide will lead you through the authentication and setup of the source. 3. Click **Set up source**. @@ -49,11 +41,8 @@ A destination is a data warehouse, data lake, database, or an analytics tool whe To set up a destination: -1. On the Airbyte Cloud dashboard, click **Destinations** and then click **+ New destination**. -2. On the Set up the destination page, select the destination you want to set up from the **Destination** catalog. - - The fields relevant to your destination are displayed. The Setup Guide provides information to help you fill out the fields for your selected destination. - +1. On the Airbyte Cloud dashboard, click **Destinations**. +2. On the Set up the Destination page, select the destination you want to set up from the **Destination catalog**. Airbyte currently offers more than 38 destination connectors in Cloud to choose from. Once you've selected the destination, a Setup Guide will lead you through the authentication and setup of the source. 3. Click **Set up destination**. ## Set up a connection @@ -64,96 +53,73 @@ A connection is an automated data pipeline that replicates data from a source to Setting up a connection involves configuring the following parameters: -| Parameter | Description | +| Replication Setting | Description | | ---------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | -| Replication frequency | How often should the data sync? | -| [Data residency](https://docs.airbyte.com/cloud/managing-airbyte-cloud/manage-data-residency#choose-the-data-residency-for-a-connection) | Where should the data be processed? | -| Destination Namespace and stream prefix | Where should the replicated data be written? | -| Catalog selection | Which streams and fields should be replicated from the source to the destination? | -| Sync mode | How should the streams be replicated (read and written)? | - -For more information, see [Connections and Sync Modes](../understanding-airbyte/connections/README.md) and [Namespaces](../understanding-airbyte/namespaces.md) - -If you need to use [cron scheduling](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html): +| [Destination Namespace](../understanding-airbyte/namespaces.md) and stream prefix | Where should the replicated data be written to? | +| Replication Frequency | How often should the data sync? | +| [Data Residency](https://docs.airbyte.com/cloud/managing-airbyte-cloud/manage-data-residency#choose-the-data-residency-for-a-connection) | Where should the data be processed? | +| [Schema Propagation](https://docs.airbyte.com/cloud/managing-airbyte-cloud/manage-schema-changes) | Should schema drift be automated? | -1. In the **Replication Frequency** dropdown, click **Cron**. -2. Enter a cron expression and choose a time zone to create a sync schedule. - -:::note - -- Only one sync per connection can run at a time. -- If cron schedules a sync to run before the last one finishes, the scheduled sync will start after the last sync completes. +After configuring the connection settings, you will then define specifically what data will be synced. +:::info +A connection's schema consists of one or many streams. Each stream is most commonly associated with a database table or an API endpoint. Within a stream, there can be one or many fields or columns. ::: +| Catalog Selection | Description | +| ---------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| Stream Selection | Which streams should be replicated from the source to the destination? | +| Column Selection | Which fields should be included in the sync? | +| [Sync Mode](../understanding-airbyte/connections/README.md) | How should the streams be replicated (read and written)? | + To set up a connection: -:::note +:::tip Set your [default data residency](https://docs.airbyte.com/cloud/managing-airbyte-cloud/manage-data-residency#choose-your-default-data-residency) before creating a new connection to ensure your data is processed in the correct region. ::: 1. On the Airbyte Cloud dashboard, click **Connections** and then click **+ New connection**. -2. On the New connection page, select a source: +2. Select a source: - - To use an existing source, select your desired source from the **Sources**. Click the source to use it. - - To set up a new source, select "+ New source". Select a destination from the catalog. The fields relevant to your source are displayed. The Setup Guide provides information to help you fill out the fields for your selected source. Click **Set up source**. + - To use a data source you've already set up with Airbyte, select from the list of existing sources. Click the source to use it. + - To set up a new source, select **Set up a new source** and fill out the fields relevant to your source using the Setup Guide. 3. Select a destination: - - To use an existing destination, select your desired destination from the existing destinations. Click the destination to use it. - - To set up a new destination, select "+ New destination". Select a destination from the catalog. The fields relevant to your destination are displayed. The Setup Guide provides information to help you fill out the fields for your selected destination. Click **Set up destination**. - - The Set up the connection page is displayed. - -4. From the **Replication frequency** dropdown, select how often you want the data to sync from the source to the destination. - - **Note:** The default replication frequency is **Every 24 hours**. - -5. From the **Destination Namespace** dropdown, select the format in which you want to store the data in the destination: - - **Note:** The default configuration is **Mirror source structure**. - - - - - - - - - - - - - - - - - - -
Configuration - Description -
Mirror source structure - Some sources (for example, databases) provide namespace information for a stream. If a source provides the namespace information, the destination will reproduce the same namespace when this configuration is set. For sources or streams where the source namespace is not known, the behavior will default to the "Destination default" option -
Destination default - All streams will be replicated and stored in the default namespace defined on the Destination Settings page. For more information, see ​​Destination Connector Settings -
Custom format - All streams will be replicated and stored in a custom format. See Custom format for more details -
+ - To use a data source you've already set up with Airbyte, select from the list of existing destinations. Click the destination to use it. + - To set up a new destination, select **Set up a new destination** and fill out the fields relevant to your destination using the Setup Guide. + + Airbyte will scan the schema of the source, and then display the **Connection Configuration** page. + +4. From the **Replication frequency** dropdown, select how often you want the data to sync from the source to the destination. The default replication frequency is **Every 24 hours**. You can also set up [cron scheduling](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html). + + Reach out to [Sales](https://airbyte.com/company/talk-to-sales) if you require replication more frequently than once per hour. + +5. From the **Destination Namespace** dropdown, select the format in which you want to store the data in the destination. Note: The default configuration is **Destination default**. + +| Destination Namepsace | Description | +| ---------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| Destination default | All streams will be replicated to the single default namespace defined by the Destination. For more details, see ​​Destination Connector Settings | +| Mirror source structure | Some sources (for example, databases) provide namespace information for a stream. If a source provides namespace information, the destination will mirror the same namespace when this configuration is set. For sources or streams where the source namespace is not known, the behavior will default to the "Destination default" option. | +| Custom format | All streams will be replicated to a single user-defined namespace. See Custom format for more details | :::tip -To better understand the destination namespace configurations, see [Destination Namespace example](../understanding-airbyte/namespaces.md#examples) +To ensure your data is synced correctly, see our examples of how to use the [Destination Namespace](../understanding-airbyte/namespaces.md#examples) ::: -6. (Optional) In the **Destination Stream Prefix (Optional)** field, add a prefix to stream names (for example, adding a prefix `airbyte_` renames `projects` to `airbyte_projects`). -7. Activate the streams you want to sync: - - (Optional) If your source has many tables, type the name of the stream you want to enable in the **Search stream name** search box. -8. Configure the sync settings: +6. (Optional) In the **Destination Stream Prefix (Optional)** field, add a prefix to stream names. For example, adding a prefix `airbyte_` renames the stream `projects` to `airbyte_projects`. This is helpful if you are sending multiple connections to the same Destination Namespace to ensure connections do not conflict when writing to the destination. + +7. Select in the **Detect and propagate schema changes** dropdown whether Airbyte should propagate schema changes. See more details about how we handle [schema changes](https://docs.airbyte.com/cloud/managing-airbyte-cloud/manage-schema-changes). + + +8. Activate the streams you want to sync by toggling the **Sync** button on. Use the **Search stream name** search box to find streams quickly. If you want to sync all streams, bulk toggle to enable all streams. - 1. Toggle the **Sync** button to enable sync for the stream. - 2. **Source stream name**: The table name in the source - 3. **Sync mode**: Select how you want the data to be replicated from the source to the destination: +9. Configure the stream settings: + 1. **Data Destination**: Where the data will land in the destination + 2. **Stream**: The table name in the source + 3. **Sync mode**: How the data will be replicated from the source to the destination. For the source: @@ -165,23 +131,23 @@ To better understand the destination namespace configurations, see [Destination - Select **Overwrite** to erase the old data and replace it completely - Select **Append** to capture changes to your table **Note:** This creates duplicate records - - Select **Append + Deduped** to mirror your source while keeping records unique + - Select **Append + Deduped** to mirror your source while keeping records unique (most common) **Note:** Some sync modes may not yet be available for the source or destination. 4. **Cursor field**: Used in **Incremental** sync mode to determine which records to sync. Airbyte pre-selects the cursor field for you (example: updated date). If you have multiple cursor fields, select the one you want. 5. **Primary key**: Used in **Append + Deduped** sync mode to determine the unique identifier. - 6. Choose which fields to sync. By default, all fields are synced. + 6. Choose which fields or columns to sync. By default, all fields are synced. 10. Click **Set up connection**. -11. Airbyte tests the connection. If the sync is successful, the Connection page is displayed. +11. Airbyte tests the connectio setup. If the test is successful, Airbyte will save the configuration. If the Replication Frequency uses a preset schedule or CRON, your first sync will immediately begin! -## Verify the connection +## Verify the sync -Verify the sync by checking the logs: +Once the first sync has completed, you can verify the sync has completed by checking in Airbyte Cloud and in your destination. 1. On the Airbyte Cloud dashboard, click **Connections**. The list of connections is displayed. Click on the connection you just set up. -2. The Sync History is displayed. Click on the first log in the sync history to view the log details. +2. The **Job History** tab shows each sync run, along with the sync summary of data and rows moved. You can also manually trigger syncs or view detailed logs for each sync here. 3. Check the data at your destination. If you added a Destination Stream Prefix while setting up the connection, make sure to search for the stream name with the prefix. ## Allowlist IP addresses diff --git a/docs/cloud/managing-airbyte-cloud/configuring-connections.md b/docs/cloud/managing-airbyte-cloud/configuring-connections.md index 49e6fd43bbaf..4e95bac58714 100644 --- a/docs/cloud/managing-airbyte-cloud/configuring-connections.md +++ b/docs/cloud/managing-airbyte-cloud/configuring-connections.md @@ -1,8 +1,8 @@ # Configuring connections -After you have created a connection, you can change how your data syncs to the destination by modifying the [configuration settings](#configure-connection-settings) and the [stream settings](#modify-streams-in-your-connection). +A connection links a source to a destination and defines how your data will sync. After you have created a connection, you can modify any of the [configuration settings](#configure-connection-settings) or [stream settings](#modify-streams-in-your-connection). -## Configure connection settings +## Configure Connection Settings Configuring the connection settings allows you to manage various aspects of the sync, such as how often data syncs and where data is written. @@ -12,9 +12,7 @@ To configure these settings: 2. Click the **Replication** tab. -3. Click the **Configuration** dropdown. - -You can configure the following settings: +3. Click the **Configuration** dropdown to expand the options. :::note @@ -22,12 +20,15 @@ These settings apply to all streams in the connection. ::: +You can configure the following settings: + | Setting | Description | |--------------------------------------|-------------------------------------------------------------------------------------| | Replication frequency | How often the data syncs | | Destination namespace | Where the replicated data is written | | Destination stream prefix | How you identify streams from different connectors | | [Detect and propagate schema changes](https://docs.airbyte.com/cloud/managing-airbyte-cloud/manage-schema-changes/#review-non-breaking-schema-changes) | How Airbyte handles syncs when it detects schema changes in the source | +| Connection Data Residency | Where data will be processed | To use [cron scheduling](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html): @@ -39,13 +40,13 @@ To use [cron scheduling](http://www.quartz-scheduler.org/documentation/quartz-2. * Only one sync per connection can run at a time. * If a sync is scheduled to run before the previous sync finishes, the scheduled sync will start after the completion of the previous sync. -* Reach out to [Sales](https://airbyte.com/company/talk-to-sales) to enable syncs more frequently than once per hour. +* Reach out to [Sales](https://airbyte.com/company/talk-to-sales) if you require replication more frequently than once per hour. ::: ## Modify streams in your connection -In the **Activate the streams you want to sync** table, you can choose which streams to sync and how they are loaded to the destination. +In the **Activate the streams you want to sync** table, you choose which streams to sync and how they are loaded to the destination. :::info A connection's schema consists of one or many streams. Each stream is most commonly associated with a database table or an API endpoint. Within a stream, there can be one or many fields or columns. @@ -71,9 +72,9 @@ Source-defined cursors and primary keys are selected automatically and cannot be ::: -3. Click on a stream to display the stream details panel. +3. Click on a stream to display the stream details panel. You'll see each column we detect from the source. -4. Toggle individual fields to include or exclude them in the sync, or use the toggle in the table header to select all fields at once. +4. Toggle individual fields or columns to include or exclude them in the sync, or use the toggle in the table header to select all fields at once. :::info diff --git a/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications.md b/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications.md index 49a663b451c9..160b28d5f47e 100644 --- a/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications.md +++ b/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications.md @@ -10,8 +10,11 @@ This page provides guidance on how to manage notifications for Airbyte Cloud, al | Successful Syncs | A sync from any of your connections succeeds. Note that if sync runs frequently or if there are many syncs in the workspace these types of events can be noisy | Automated Connection Updates | A connection is updated automatically (ex. a source schema is automatically updated) | | Connection Updates Requiring Action | A connection update requires you to take action (ex. a breaking schema change is detected) | -| Sync Disabled Warning | A connection will be disabled soon due to repeated failures. It has failed 50 times consecutively or there were only failed jobs in the past 7 days | -| Sync Disabled | A connection was automatically disabled due to repeated failures. It will be disabled when it has failed 100 times consecutively or has been failing for 14 days in a row | +| Warning - Repeated Failures | A connection will be disabled soon due to repeated failures. It has failed 50 times consecutively or there were only failed jobs in the past 7 days | +| Sync Disabled - Repeated Failures | A connection was automatically disabled due to repeated failures. It will be disabled when it has failed 100 times consecutively or has been failing for 14 days in a row | +| Warning - Upgrade Required (email only) | A new connector version is available and requires manual upgrade | +| Sync Disabled - Upgrade Required (email only) | One or more connections were automatically disabled due to a connector upgrade deadline passing +| ## Configure Notification Settings diff --git a/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-workspace.md b/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-workspace.md index 1db3697191a5..40336d7b9273 100644 --- a/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-workspace.md +++ b/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-workspace.md @@ -80,6 +80,4 @@ To switch between workspaces: 1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click the current workspace name under the Airbyte logo in the navigation bar. -2. Click **View all workspaces**. - -3. Click the name of the workspace you want to switch to. +2. Search for the workspace or click the name of the workspace you want to switch to. diff --git a/docs/cloud/managing-airbyte-cloud/manage-connection-state.md b/docs/cloud/managing-airbyte-cloud/manage-connection-state.md index 929a56834534..321c3753e7b8 100644 --- a/docs/cloud/managing-airbyte-cloud/manage-connection-state.md +++ b/docs/cloud/managing-airbyte-cloud/manage-connection-state.md @@ -11,7 +11,7 @@ To review the connection state: **Connection State** displays. -To edit the connection state: +Editing the connection state allows the sync to start from any date in the past. If the state is edited, Airbyte will start syncing incrementally from the new date. This is helpful if you do not want to fully resync your data. To edit the connection state: :::warning Updates to connection state should be handled with extreme care. Updates may break your syncs, requiring a reset to fix. Make changes only as directed by the Airbyte team. diff --git a/docs/cloud/managing-airbyte-cloud/manage-credits.md b/docs/cloud/managing-airbyte-cloud/manage-credits.md index 040a083e58d5..7ed15c0ed76f 100644 --- a/docs/cloud/managing-airbyte-cloud/manage-credits.md +++ b/docs/cloud/managing-airbyte-cloud/manage-credits.md @@ -18,11 +18,11 @@ To buy credits: Purchase limits: * Minimum: 20 credits - * Maximum: 2,500 credits + * Maximum: 6,000 credits ::: - To buy more credits or a custom plan, reach out to [Sales](https://airbyte.com/talk-to-sales). + To buy more credits or discuss a custom plan, reach out to [Sales](https://airbyte.com/talk-to-sales). 5. Fill out the payment information. diff --git a/docs/cloud/managing-airbyte-cloud/manage-schema-changes.md b/docs/cloud/managing-airbyte-cloud/manage-schema-changes.md index c6113b461c65..1e76e5f6ff58 100644 --- a/docs/cloud/managing-airbyte-cloud/manage-schema-changes.md +++ b/docs/cloud/managing-airbyte-cloud/manage-schema-changes.md @@ -2,30 +2,38 @@ You can specify for each connection how Airbyte should handle any change of schema in the source. This process helps ensure accurate and efficient data syncs, minimizing errors and saving you time and effort in managing your data pipelines. -Airbyte checks for any changes in your source schema before syncing, at most once every 24 hours. +Airbyte checks for any changes in your source schema immediately before syncing, at most once every 24 hours. -Based on your configured settings for "Detect and propagate schema changes", Airbyte can automatically sync those changes or ignore them: -* **Propagate all changes** automatically propagates stream changes (additions or deletions) or column changes (additions or deletions) detected in the source -* **Propagate column changes only** automatically propagates column changes detected in the source -* **Ignore** any schema change, in which case the schema you’ve set up will not change even if the source schema changes until you approve the changes manually -* **Pause connection** disables the connection from syncing further once a change is detected +Based on your configured settings for **Detect and propagate schema changes**, Airbyte will automatically sync those changes or ignore them: -When a new column is detected and propagated, values for that column will be filled in for the updated rows. If you are missing values for rows not updated, a backfill can be done by completing a full refresh. +| Setting | Description | +|---------------------|---------------------------------------------------------------------------------------------------------------------| +| Propagate all changes | All new tables and column changes from the source will automatically be propagated and reflected in the destination. This includes stream changes (additions or deletions), column changes (additions or deletions) and data type changes +| Propagate column changes only (default) | Only column changes will be propagated +| Ignore | Schema changes will be detected, but not propagated. Syncs will continue running with the schema you've set up. To propagate the detected schema changes, you will need to approve the changes manually | +| Pause Connection | Connections will be automatically disabled as soon as any schema changes are detected | -When a column is deleted, the values for that column will stop updating for the updated rows and be filled with Null values. +When propagation is enabled, your data in the destination will automatically shift to bring in the new changes. -When a new stream is detected and propagated, the first sync will fill all data in as if it is a historical sync. When a stream is deleted from the source, the stream will stop updating, and we leave any existing data in the destination. The rest of the enabled streams will continue syncing. +| Type of Schema Change | Propagation Behavior | +|---------------------|---------------------------------------------------------------------------------------------------------------------| +| New Column | The new colummn will be created in the destination. Values for the column will be filled in for the updated rows. If you are missing values for rows not updated, a backfill can be done by completing a full resync. +| Removal of column | The old column will be removed from the destination. +| New stream | The first sync will create the new stream in the destination and fill all data in as if it is a historical sync. | +| Removal of stream | The stream will stop updating, and any existing data in the destination will remain. | +| Column data type changes | The data in the destination will remain the same. Any new or updated rows with incompatible data types will result in a row error in the raw Airbyte tables. You will need to refresh the schema and do a full resync to ensure the data types are consistent. -In all cases, if a breaking change is detected, the connection will be paused for manual review to prevent future syncs from failing. Breaking schema changes occur when: +In all cases, if a breaking schema change is detected, the connection will be paused immediately for manual review to prevent future syncs from failing. Breaking schema changes occur when: * An existing primary key is removed from the source * An existing cursor is removed from the source -See "Fix breaking schema changes" to understand how to resolve these types of changes. +To re-enable the streams, ensure the correct **Primary Key** and **Cursor** are selected for each stream and save the connection. ## Review non-breaking schema changes -To review non-breaking schema changes: -1. On the [Airbyte Cloud](http://cloud.airbyte.com/) dashboard, click **Connections** and select the connection with non-breaking changes (indicated by a **yellow exclamation mark** icon). +If the connection is set to **Ignore** any schema changes, Airbyte continues syncing according to your last saved schema. You need to manually approve any detected schema changes for the schema in the destination to change. + +1. On the [Airbyte Cloud](http://cloud.airbyte.com/) dashboard, click **Connections**. Select a connection and navigate to the **Replication** tab. If schema changes are detected, you'll see a blue "i" icon next to the Replication ab. 2. Click **Review changes**. @@ -35,41 +43,31 @@ To review non-breaking schema changes: 5. Scroll to the bottom of the page and click **Save changes**. -:::note - - By default, Airbyte ignores non-breaking changes and continues syncing. You can configure how Airbyte handles syncs when it detects non-breaking changes by [editing the stream configuration](https://docs.airbyte.com/cloud/managing-airbyte-cloud/edit-stream-configuration). - -::: - -## Resolve breaking changes +## Resolving breaking changes Breaking changes require your attention to resolve. They may immediately cause the connection to be disabled, or you can upgrade the connector manually within a time period once reviewing the changes. -A connection will automatically be disabled if: -* An existing primary key is removed -* An existing cursor field is removed +A connection will always automatically be disabled if an existing primary key or cursor field is removed. You must review and fix the changes before editing the connection or resuming syncs. -If the breaking change is due to a new version, the connection will alert you of a breaking change but continue to sync until the cutoff date. On the cutoff date, the connection will automatically be disabled on that date to prevent failure or unexpected behavior. These breaking changes include: +Breaking changes can also occur when a new version of the connector is released. In these cases, the connection will alert you of a breaking change but continue to sync until the cutoff date for upgrade. On the cutoff date, the connection will automatically be disabled on that date to prevent failure or unexpected behavior. It is **highly recommended** to upgrade before the cutoff date to ensure you continue syncing without interruption. + +A major version upgrade will include a breaking change if any of these apply: | Type of Change | Description | |------------------|---------------------------------------------------------------------------------------------------------------------| -| Spec Change | The configuration required by users of this connector has been changed and syncs will fail until users reconfigure or re-authenticate. | -| Schema Change | The type of property previously present within a record has changed +| Connector Spec Change | The configuration has been changed and syncs will fail until users reconfigure or re-authenticate. | +| Schema Change | The type of property previously present within a record has changed and a refresh of the source schema is required. | Stream or Property Removal | Data that was previously being synced is no longer going to be synced | -| Destination Format / Normalization Change | The way the destination writes the final data or how normalization cleans that data is changing in a way that requires a full refresh | +| Destination Format / Normalization Change | The way the destination writes the final data or how Airbyte cleans that data is changing in a way that requires a full refresh | | State Changes | The format of the source’s state has changed, and the full dataset will need to be re-synced | To review and fix breaking schema changes: -1. On the [Airbyte Cloud](http://cloud.airbyte.com/) dashboard, click **Connections** and select the connection with breaking changes (indicated by a **red exclamation mark** icon). +1. On the [Airbyte Cloud](http://cloud.airbyte.com/) dashboard, click **Connections** and select the connection with breaking changes. -2. Review the description of what has changed. The breaking change will require you to upgrade your source or destination to a new version. +2. Review the description of what has changed in the new version. The breaking change will require you to upgrade your source or destination to a new version by a specific cutoff date. 3. Update the source or destination to the new version to continue syncing. -:::note -If a connection’s source schema has breaking changes (an existing cursor or primary key is removed), it will stop syncing immediately. You must review and fix the changes before editing the connection or resuming syncs. -::: - ### Manually refresh the source schema In addition to Airbyte Cloud’s automatic schema change detection, you can manually refresh the source schema to stay up to date with changes in your schema. diff --git a/docs/cloud/managing-airbyte-cloud/review-sync-history.md b/docs/cloud/managing-airbyte-cloud/review-sync-history.md index b5a1f06ba903..0bb5cf2290f5 100644 --- a/docs/cloud/managing-airbyte-cloud/review-sync-history.md +++ b/docs/cloud/managing-airbyte-cloud/review-sync-history.md @@ -2,34 +2,19 @@ The job history displays information about synced data, such as the amount of data moved, the number of records read and committed, and the total sync time. Reviewing this summary can help you monitor the sync performance and identify any potential issues. -To review the sync history: -1. On the [Airbyte Cloud](http://cloud.airbyte.com/) dashboard, click **Connections**. - -2. Click a connection in the list to view its sync history. Sync History displays the sync status or [reset](https://docs.airbyte.com/operator-guides/reset/) status. The sync status is defined as: +To review the sync history, click a connection in the list to view its sync history. Sync History displays the sync status or [reset](https://docs.airbyte.com/operator-guides/reset/) status. The sync status is defined as: | Status | Description | |---------------------|---------------------------------------------------------------------------------------------------------------------| | Succeeded | 100% of the data has been extracted and loaded to the destination | -| Partially Succeeded | a subset of the data has been loaded to the destination -| Failed | none of the data has been loaded to the destination | -| Cancelled | the sync was cancelled manually before finishing | -| Running | the sync is currently running | - -:::note - -In the event of a failure, Airbyte will make several attempts to sync your data before waiting for the next sync to retry. The latest rules can be read about [here](../../understanding-airbyte/jobs.md#retry-rules). - -::: - -3. To view the full sync log, click the three grey dots next to any sync job. Select "View logs" to open the logs in the browser. - -4. To find a link to the job, click the three grey dots next to any sync job. Select "Copy link to job" to copy the link to your clipboard. - -5. To download a copy of the logs locally, click the three grey dots next to any sync job. Select "Donwload logs". +| Partially Succeeded | A subset of the data has been loaded to the destination +| Failed |Nnone of the data has been loaded to the destination | +| Cancelled | The sync was cancelled manually before finishing | +| Running | The sync is currently running | ## Sync summary -Each sync shows the time the sync was initiated and additional metadata. +Each sync shows the time the sync was initiated and additional metadata. This information can help in understanding sync performance over time. | Data | Description | |------------------------------------------|--------------------------------------------------------------------------------------| @@ -38,3 +23,11 @@ Each sync shows the time the sync was initiated and additional metadata. | x loaded records | Number of records the destination confirmed it received. | | xh xm xs | Total time (hours, minutes, seconds) for the sync to complete | + +:::note + +In the event of a failure, Airbyte will make several attempts to sync your data before waiting for the next sync to retry. The latest rules can be read about [here](../../understanding-airbyte/jobs.md#retry-rules). + +::: + +On this page, you can also view the complete logs and find any relevant errors, find a link to the job to share with Support, or download a copy of the logs locally. \ No newline at end of file diff --git a/docs/cloud/managing-airbyte-cloud/understand-airbyte-cloud-limits.md b/docs/cloud/managing-airbyte-cloud/understand-airbyte-cloud-limits.md index bbc2211fd2e6..9d8a429eab9e 100644 --- a/docs/cloud/managing-airbyte-cloud/understand-airbyte-cloud-limits.md +++ b/docs/cloud/managing-airbyte-cloud/understand-airbyte-cloud-limits.md @@ -10,7 +10,7 @@ Understanding the following limitations will help you more effectively manage Ai * Max number of streams that can be returned by a source in a discover call: 1K * Max number of streams that can be configured to sync in a single connection: 1K * Size of a single record: 20MB -* Shortest sync schedule: Every 60 min +* Shortest sync schedule: Every 60 min (Reach out to [Sales](https://airbyte.com/company/talk-to-sales) if you require replication more frequently than once per hour) * Schedule accuracy: +/- 30 min *Limits on workspaces, sources, and destinations do not apply to customers of [Powered by Airbyte](https://airbyte.com/solutions/powered-by-airbyte). To learn more [contact us](https://airbyte.com/talk-to-sales)! diff --git a/docs/integrations/destinations/firestore.md b/docs/integrations/destinations/firestore.md index c82a9f12068e..94a6002a70c4 100644 --- a/docs/integrations/destinations/firestore.md +++ b/docs/integrations/destinations/firestore.md @@ -1,6 +1,35 @@ # Firestore -The Firestore destination for Airbyte +This destination writes data to Google Firestore. + +Google Firestore, officially known as Cloud Firestore, is a flexible, scalable database for mobile, web, and server development from Firebase and Google Cloud. It is commonly used for developing applications as a NoSQL database that provides real-time data syncing across user devices. + +## Getting started + +### Requiremnets + +- An existing GCP project +- A role with permissions to create a Service Account Key in GCP + +### Step 1: Create a Service Account +1. Log in to the Google Cloud Console. Select the project where your Firestore database is located. +2. Navigate to "IAM & Admin" and select "Service Accounts". Create a Service Account and assign appropriate roles. Ensure “Cloud Datastore User” or “Firebase Rules System” are enabled. +3. Navigate to the service account and generate the JSON key. Download and copy the contents to the configuration. + +## Sync overview + +### Output schema + +Each stream will be output into a BigQuery table. + +#### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :----------------------------- | :------------------- | :---- | +| Full Refresh Sync | ✅ | | +| Incremental - Append Sync | ✅ | | +| Incremental - Append + Deduped | ✅ | | +| Namespaces | ✅ | | ## Changelog From 9dae0d3c2102a65ce258e3c7215b4da671d46d41 Mon Sep 17 00:00:00 2001 From: Natalie Kwong <38087517+nataliekwong@users.noreply.github.com> Date: Fri, 17 Nov 2023 14:29:53 -0800 Subject: [PATCH 124/124] Reinstate Pardot source connector doc (#32579) --- docs/integrations/sources/pardot.md | 55 +++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/docs/integrations/sources/pardot.md b/docs/integrations/sources/pardot.md index f8f304797a39..c4304a8abe0e 100644 --- a/docs/integrations/sources/pardot.md +++ b/docs/integrations/sources/pardot.md @@ -1,7 +1,62 @@ # Pardot +## Overview + The Airbyte Source for [Salesforce Pardot](https://www.pardot.com/) +The Pardot supports full refresh syncs + +### Output schema + +Several output streams are available from this source: + +* [Campaigns](https://developer.salesforce.com/docs/marketing/pardot/guide/campaigns-v4.html) +* [EmailClicks](https://developer.salesforce.com/docs/marketing/pardot/guide/batch-email-clicks-v4.html) +* [ListMembership](https://developer.salesforce.com/docs/marketing/pardot/guide/list-memberships-v4.html) +* [Lists](https://developer.salesforce.com/docs/marketing/pardot/guide/lists-v4.html) +* [ProspectAccounts](https://developer.salesforce.com/docs/marketing/pardot/guide/prospect-accounts-v4.html) +* [Prospects](https://developer.salesforce.com/docs/marketing/pardot/guide/prospects-v4.html) +* [Users](https://developer.salesforce.com/docs/marketing/pardot/guide/users-v4.html) +* [VisitorActivities](https://developer.salesforce.com/docs/marketing/pardot/guide/visitor-activities-v4.html) +* [Visitors](https://developer.salesforce.com/docs/marketing/pardot/guide/visitors-v4.html) +* [Visits](https://developer.salesforce.com/docs/marketing/pardot/guide/visits-v4.html) + +If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose) + +### Features + +| Feature | Supported? | +| :--- | :--- | +| Full Refresh Sync | Yes | +| Incremental Sync | No | +| SSL connection | No | +| Namespaces | No | + +### Performance considerations + +The Pardot connector should not run into Pardot API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. + +## Getting started + +### Requirements + +* Pardot Account +* Pardot Business Unit ID +* Client ID +* Client Secret +* Refresh Token +* Start Date +* Is Sandbox environment? + +### Setup guide + +* `pardot_business_unit_id`: Pardot Business ID, can be found at Setup > Pardot > Pardot Account Setup +* `client_id`: The Consumer Key that can be found when viewing your app in Salesforce +* `client_secret`: The Consumer Secret that can be found when viewing your app in Salesforce +* `refresh_token`: Salesforce Refresh Token used for Airbyte to access your Salesforce account. If you don't know what this is, follow [this guide](https://medium.com/@bpmmendis94/obtain-access-refresh-tokens-from-salesforce-rest-api-a324fe4ccd9b) to retrieve it. +* `start_date`: UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. Leave blank to skip this filter +* `is_sandbox`: Whether or not the the app is in a Salesforce sandbox. If you do not know what this, assume it is false. + ## Changelog | Version | Date | Pull Request | Subject |