Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

🐛 Source Slack: Fix reading threads issue #4860

Merged
merged 11 commits into from
Jul 23, 2021
Merged

🐛 Source Slack: Fix reading threads issue #4860

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
e62d296
Fixed reading threads issue
lazebnyi Jul 20, 2021
1e513fe
Updated to review
lazebnyi Jul 21, 2021
5bf5fc2
Added filtering for threads stream
lazebnyi Jul 21, 2021
3d96b9c
Added acceptance test for threads stream
lazebnyi Jul 21, 2021
3c9f50d
Merge branch 'master' into lazebnyi/4680-slack-reading-threads-issue
lazebnyi Jul 21, 2021
c2604d5
Added integration dummy test
lazebnyi Jul 22, 2021
076b68e
Added requirements file
lazebnyi Jul 22, 2021
5aa096c
Code foemated
lazebnyi Jul 22, 2021
bdf04b2
Deleted version of airbyte-cdk
lazebnyi Jul 22, 2021
02bcef1
Deleted validate_output_from_all_streams flag
lazebnyi Jul 23, 2021
9921714
Deleted condition status code in backoff
lazebnyi Jul 23, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...ain/resources/config/STANDARD_SOURCE_DEFINITION/c2281cee-86f9-4a86-bb48-d23286b4c7bd.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"sourceDefinitionId": "c2281cee-86f9-4a86-bb48-d23286b4c7bd",
"name": "Slack",
"dockerRepository": "airbyte/source-slack",
"dockerImageTag": "0.1.8",
"dockerImageTag": "0.1.9",
"documentationUrl": "https://hub.docker.com/repository/docker/airbyte/source-slack",
"icon": "slack.svg"
}
2 changes: 1 addition & 1 deletion airbyte-config/init/src/main/resources/seed/source_definitions.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@
- sourceDefinitionId: c2281cee-86f9-4a86-bb48-d23286b4c7bd
name: Slack
dockerRepository: airbyte/source-slack
dockerImageTag: 0.1.8
dockerImageTag: 0.1.9
documentationUrl: https://hub.docker.com/repository/docker/airbyte/source-slack
icon: slack.svg
- sourceDefinitionId: 6ff047c0-f5d5-4ce5-8c81-204a830fa7e1
Expand Down
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-slack/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ RUN pip install .
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]

LABEL io.airbyte.version=0.1.8
LABEL io.airbyte.version=0.1.9
LABEL io.airbyte.name=airbyte/source-slack
1 change: 1 addition & 0 deletions airbyte-integrations/connectors/source-slack/acceptance-test-config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ tests:
timeout_seconds: 3600
cursor_paths:
channel_messages: ["float_ts"]
threads: ["float_ts"]
full_refresh:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/full_refresh_catalog.json"
75 changes: 74 additions & 1 deletion airbyte-integrations/connectors/source-slack/integration_tests/configured_catalog.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
{
"comment": "deleted threads from configured catalog due to the timeout error in incremental stream",
"streams": [
{
"stream": {
Expand Down Expand Up @@ -223,6 +222,80 @@
"cursor_field": ["float_ts"],
"sync_mode": "incremental",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "threads",
"json_schema": {
"type": ["null", "object"],
"additionalProperties": false,
"properties": {
"channel_id": {
"type": ["null", "string"]
},
"client_msg_id": {
"type": ["null", "string"]
},
"type": {
"type": ["null", "string"]
},
"text": {
"type": ["null", "string"]
},
"user": {
"type": ["null", "string"]
},
"ts": {
"type": ["null", "string"]
},
"float_ts": {
"type": ["null", "number"]
},
"team": {
"type": ["null", "string"]
},
"blocks": {
"type": ["null", "array"],
"items": {
"type": ["null", "object"],
"additionalProperties": true,
"properties": {
"type": {
"type": ["null", "string"]
}
}
}
},
"thread_ts": {
"type": ["null", "string"]
},
"reply_count": {
"type": ["null", "integer"]
},
"reply_users_count": {
"type": ["null", "number"]
},
"latest_reply": {
"type": ["null", "string"]
},
"reply_users": {
"type": ["null", "array"],
"items": {
"type": "string"
}
},
"subscribed": {
"type": ["null", "boolean"]
}
}
},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["float_ts"]
},
"cursor_field": ["float_ts"],
"sync_mode": "incremental",
"destination_sync_mode": "append"
}
]
}
224 changes: 223 additions & 1 deletion airbyte-integrations/connectors/source-slack/sample_files/configured_catalog.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,228 @@
{
"comment": "deleted threads from configured catalog due to the timeout error in incremental stream",
"streams": [
{
"stream": {
"name": "channel_messages",
"json_schema": {
"additionalProperties": false,
"properties": {
"channel_id": {
"type": ["null", "string"]
},
"blocks": {
"items": {
"additionalProperties": true,
"properties": {
"type": {
"type": ["null", "string"]
}
},
"type": ["null", "object"]
},
"type": ["null", "array"]
},
"bot_id": {
"type": ["null", "string"]
},
"bot_profile": {
"additionalProperties": false,
"properties": {
"app_id": {
"type": ["null", "string"]
},
"deleted": {
"type": ["null", "boolean"]
},
"id": {
"type": ["null", "string"]
},
"name": {
"type": ["null", "string"]
},
"team_id": {
"type": ["null", "string"]
},
"updated": {
"type": ["null", "string"],
"format": "date-time"
}
},
"type": ["null", "object"]
},
"attachments": {
"items": {
"properties": {
"title": {
"type": ["null", "string"]
},
"id": {
"type": ["null", "integer"]
},
"color": {
"type": ["null", "string"]
},
"fallback": {
"type": ["null", "string"]
}
},
"type": ["null", "object"]
},
"type": ["null", "array"]
},
"client_msg_id": {
"type": ["null", "string"]
},
"display_as_bot": {
"type": ["null", "boolean"]
},
"file_id": {
"type": ["null", "string"]
},
"file_ids": {
"items": {
"type": ["null", "string"]
},
"type": ["null", "array"]
},
"icons": {
"additionalProperties": false,
"properties": {
"emoji": {
"type": ["null", "string"]
}
},
"type": ["null", "object"]
},
"inviter": {
"type": ["null", "string"]
},
"is_delayed_message": {
"type": ["null", "boolean"]
},
"is_intro": {
"type": ["null", "boolean"]
},
"is_starred": {
"type": ["null", "boolean"]
},
"last_read": {
"type": ["null", "string"],
"format": "date-time"
},
"latest_reply": {
"type": ["null", "string"]
},
"name": {
"type": ["null", "string"]
},
"old_name": {
"type": ["null", "string"]
},
"parent_user_id": {
"type": ["null", "string"]
},
"permalink": {
"format": "uri",
"type": ["null", "string"]
},
"pinned_to": {
"items": {
"type": ["null", "string"]
},
"type": ["null", "array"]
},
"purpose": {
"type": ["null", "string"]
},
"reactions": {
"items": {
"additionalProperties": true,
"properties": {
"count": {
"type": ["null", "integer"]
},
"name": {
"type": ["null", "string"]
},
"users": {
"items": {
"type": ["null", "string"]
},
"type": ["null", "array"]
}
},
"type": ["null", "object"]
},
"type": ["null", "array"]
},
"reply_count": {
"type": ["null", "integer"]
},
"reply_users": {
"items": {
"type": ["null", "string"]
},
"type": ["null", "array"]
},
"reply_users_count": {
"type": ["null", "integer"]
},
"source_team": {
"type": ["null", "string"]
},
"subscribed": {
"type": ["null", "boolean"]
},
"subtype": {
"type": ["null", "string"]
},
"team": {
"type": ["null", "string"]
},
"text": {
"type": ["null", "string"]
},
"thread_ts": {
"type": ["null", "string"]
},
"topic": {
"type": ["null", "string"]
},
"ts": {
"type": ["null", "string"]
},
"float_ts": {
"type": ["null", "number"]
},
"type": {
"type": ["null", "string"]
},
"unread_count": {
"type": ["null", "integer"]
},
"upload": {
"type": ["null", "boolean"]
},
"user": {
"type": ["null", "string"]
},
"user_team": {
"type": ["null", "string"]
},
"username": {
"type": ["null", "string"]
}
},
"type": ["null", "object"]
},
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": ["float_ts"]
},
"cursor_field": ["float_ts"],
"sync_mode": "incremental",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "threads",
Expand Down
Loading