Merge master

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.36.1-alpha
+current_version = 0.36.3-alpha
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.36.1-alpha
+VERSION=0.36.3-alpha
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data

.github/actions/build-and-push-branch/action.yml

@@ -10,44 +10,11 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: "Parse Input"
-      id: parse-input
-      shell: bash
-      run: |-
-        # if the *branch_version_tag* input param is not specified, then generate it as 'dev-<commit_hash>`
-        #
-        [[ "${{ inputs.branch_version_tag }}" != '' ]] && echo "::set-output name=branch_version_tag::${{ inputs.branch_version_tag }}" \
-          || { short_hash=$(git rev-parse --short HEAD); echo "::set-output name=branch_version_tag::dev-$short_hash"; }
-
-    - uses: actions/setup-java@v1
-      with:
-        java-version: "17"
-
-    - uses: actions/setup-node@v1
-      with:
-        node-version: "16.13.0"
-
-    - name: Set up CI Gradle Properties
-      run: |
-        mkdir -p ~/.gradle/
-        cat > ~/.gradle/gradle.properties <<EOF
-        org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
-          --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
-          --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
-          --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
-          --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
-        org.gradle.workers.max=8
-        org.gradle.vfs.watch=false
-        EOF
-      shell: bash
-
     - name: Build
-      run: VERSION=${{ steps.parse-input.outputs.branch_version_tag }} SUB_BUILD=PLATFORM ./gradlew build --scan
-      shell: bash
-
-    - name: Publish to Maven Local
-      run: VERSION=${{ steps.parse-input.outputs.branch_version_tag }} SUB_BUILD=PLATFORM ./gradlew publishToMavenLocal
-      shell: bash
+      id: build
+      uses: ./.github/actions/build-branch
+      with:
+        branch_version_tag: ${{ inputs.branch_version_tag }}
 
     - name: Login to Docker (on Master)
       uses: docker/login-action@v1
@@ -59,5 +26,5 @@ runs:
       run: |
         GIT_REVISION=$(git rev-parse HEAD)
         [ [ -z "$GIT_REVISION" ] ] && echo "Couldn't get the git revision..." && exit 1
-        VERSION=${{ steps.parse-input.outputs.branch_version_tag }} GIT_REVISION=$GIT_REVISION docker-compose -f docker-compose-cloud.build.yaml push
+        VERSION=${{ steps.build.outputs.branch_version_tag }} GIT_REVISION=$GIT_REVISION docker-compose -f docker-compose-cloud.build.yaml push
       shell: bash

.github/actions/build-branch/action.yml

@@ -0,0 +1,52 @@
+name: "Build OSS Branch"
+description: "Build jars and docker images tagged for a particular branch. Primarily used for running OSS branch code in Cloud."
+inputs:
+  branch_version_tag:
+    description: 'Used to tag jars and docker images with a branch-specific version (should use the form "dev-<commit_hash>" to pass AirbyteVersion validation)'
+    required: false
+outputs:
+  branch_version_tag:
+    description: 'Tag used for jars and docker images. Either user specified or auto generated as `dev-<commit_hash>`'
+    value: ${{ steps.parse-input.outputs.branch_version_tag }}
+runs:
+  using: "composite"
+  steps:
+    - name: "Parse Input"
+      id: parse-input
+      shell: bash
+      run: |-
+        # if the *branch_version_tag* input param is not specified, then generate it as 'dev-<commit_hash>`
+        #
+        [[ "${{ inputs.branch_version_tag }}" != '' ]] && echo "::set-output name=branch_version_tag::${{ inputs.branch_version_tag }}" \
+          || { short_hash=$(git rev-parse --short HEAD); echo "::set-output name=branch_version_tag::dev-$short_hash"; }
+
+    - uses: actions/setup-java@v1
+      with:
+        java-version: "17"
+
+    - uses: actions/setup-node@v1
+      with:
+        node-version: "16.13.0"
+
+    - name: Set up CI Gradle Properties
+      run: |
+        mkdir -p ~/.gradle/
+        cat > ~/.gradle/gradle.properties <<EOF
+        org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
+          --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
+          --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
+          --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
+          --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+        org.gradle.workers.max=8
+        org.gradle.vfs.watch=false
+        EOF
+      shell: bash
+
+    - name: Build
+      run: VERSION=${{ steps.parse-input.outputs.branch_version_tag }} SUB_BUILD=PLATFORM ./gradlew build --scan
+      shell: bash
+
+    - name: Publish to Maven Local
+      run: VERSION=${{ steps.parse-input.outputs.branch_version_tag }} SUB_BUILD=PLATFORM ./gradlew publishToMavenLocal
+      shell: bash
+

.github/workflows/gradle.yml

@@ -981,7 +981,7 @@ jobs:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
         with:
           args: >-
-            {\"channel\":\"C03AXC61YBZ\", \"blocks\":[
+            {\"channel\":\"C03BEADRPNY\", \"blocks\":[
             {\"type\":\"divider\"},
             {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" OSS Master Fixed! :white_check_mark: \n\n\"}},
             {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},

.github/workflows/notify-on-push-to-master.yml

@@ -0,0 +1,18 @@
+name: Notify Cloud of OSS Push to Master
+on:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+jobs:
+  repo-sync:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Repository Dispatch
+        uses: peter-evans/repository-dispatch@v2
+        with:
+          token: ${{ secrets.OCTAVIA_PAT }}
+          repository: airbytehq/airbyte-cloud
+          event-type: oss-push-to-master
+          client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}"}'

airbyte-api/src/main/openapi/config.yaml

@@ -533,7 +533,7 @@ paths:
         content:
           application/json:
             schema:
-              $ref: "#/components/schemas/SourceDefinitionIdRequestBody"
+              $ref: "#/components/schemas/SourceDefinitionIdWithWorkspaceId"
         required: true
       responses:
         "200":
@@ -1055,7 +1055,7 @@ paths:
         content:
           application/json:
             schema:
-              $ref: "#/components/schemas/DestinationDefinitionIdRequestBody"
+              $ref: "#/components/schemas/DestinationDefinitionIdWithWorkspaceId"
         required: true
       responses:
         "200":
@@ -2784,6 +2784,9 @@ components:
           $ref: "#/components/schemas/AirbyteCatalog"
         jobInfo:
           $ref: "#/components/schemas/SynchronousJobRead"
+        catalogId:
+          type: string
+          format: uuid
     SourceSearch:
       type: object
       properties:
@@ -3134,6 +3137,9 @@ components:
           $ref: "#/components/schemas/ConnectionStatus"
         resourceRequirements:
           $ref: "#/components/schemas/ResourceRequirements"
+        sourceCatalogId:
+          type: string
+          format: uuid
     WebBackendConnectionCreate:
       type: object
       required:
@@ -3175,6 +3181,9 @@ components:
           type: array
           items:
             $ref: "#/components/schemas/OperationCreate"
+        sourceCatalogId:
+          type: string
+          format: uuid
     ConnectionUpdate:
       type: object
       required:

airbyte-bootloader/Dockerfile

@@ -1,7 +1,7 @@
 ARG JDK_VERSION=17.0.1
 FROM openjdk:${JDK_VERSION}-slim
 
-ARG VERSION=0.36.1-alpha
+ARG VERSION=0.36.3-alpha
 
 ENV APPLICATION airbyte-bootloader
 ENV VERSION ${VERSION}

airbyte-cdk/python/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.1.55
+Add support for reading the spec from a YAML file (`spec.yaml`) 
+
 ## 0.1.54
 - Add ability to import `IncrementalMixin` from `airbyte_cdk.sources.streams`.
 - Bumped minimum supported Python version to 3.9.

airbyte-cdk/python/airbyte_cdk/connector.py

@@ -10,9 +10,18 @@
 from abc import ABC, abstractmethod
 from typing import Any, Mapping, Optional
 
+import yaml
 from airbyte_cdk.models import AirbyteConnectionStatus, ConnectorSpecification
 
 
+def load_optional_package_file(package: str, filename: str) -> Optional[bytes]:
+    """Gets a resource from a package, returning None if it does not exist"""
+    try:
+        return pkgutil.get_data(package, filename)
+    except FileNotFoundError:
+        return None
+
+
 class AirbyteSpec(object):
     @staticmethod
     def from_file(file_name: str):
@@ -51,12 +60,25 @@ def write_config(config: Mapping[str, Any], config_path: str):
     def spec(self, logger: logging.Logger) -> ConnectorSpecification:
         """
         Returns the spec for this integration. The spec is a JSON-Schema object describing the required configurations (e.g: username and password)
-        required to run this integration.
+        required to run this integration. By default, this will be loaded from a "spec.yaml" or a "spec.json" in the package root.
         """
-        raw_spec: Optional[bytes] = pkgutil.get_data(self.__class__.__module__.split(".")[0], "spec.json")
-        if not raw_spec:
-            raise ValueError("Unable to find spec.json.")
-        return ConnectorSpecification.parse_obj(json.loads(raw_spec))
+
+        package = self.__class__.__module__.split(".")[0]
+
+        yaml_spec = load_optional_package_file(package, "spec.yaml")
+        json_spec = load_optional_package_file(package, "spec.json")
+
+        if yaml_spec and json_spec:
+            raise RuntimeError("Found multiple spec files in the package. Only one of spec.yaml or spec.json should be provided.")
+
+        if yaml_spec:
+            spec_obj = yaml.load(yaml_spec, Loader=yaml.SafeLoader)
+        elif json_spec:
+            spec_obj = json.loads(json_spec)
+        else:
+            raise FileNotFoundError("Unable to find spec.yaml or spec.json in the package.")
+
+        return ConnectorSpecification.parse_obj(spec_obj)
 
     @abstractmethod
     def check(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus:

airbyte-cdk/python/setup.py

@@ -15,7 +15,7 @@
 
 setup(
     name="airbyte-cdk",
-    version="0.1.54",
+    version="0.1.55",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

airbyte-cdk/python/unit_tests/test_connector.py

@@ -5,14 +5,23 @@
 
 import json
 import logging
+import os
+import sys
 import tempfile
 from pathlib import Path
 from typing import Any, Mapping
 
 import pytest
+import yaml
 from airbyte_cdk import AirbyteSpec, Connector
 from airbyte_cdk.models import AirbyteConnectionStatus
 
+logger = logging.getLogger("airbyte")
+
+MODULE = sys.modules[__name__]
+MODULE_PATH = os.path.abspath(MODULE.__file__)
+SPEC_ROOT = os.path.dirname(MODULE_PATH)
+
 
 class TestAirbyteSpec:
     VALID_SPEC = {
@@ -71,3 +80,53 @@ def test_write_config(integration, mock_config):
     integration.write_config(mock_config, str(config_path))
     with open(config_path, "r") as actual:
         assert mock_config == json.loads(actual.read())
+
+
+class TestConnectorSpec:
+    CONNECTION_SPECIFICATION = {
+        "type": "object",
+        "required": ["api_token"],
+        "additionalProperties": False,
+        "properties": {"api_token": {"type": "string"}},
+    }
+
+    @pytest.fixture
+    def use_json_spec(self):
+        spec = {
+            "documentationUrl": "https://airbyte.com/#json",
+            "connectionSpecification": self.CONNECTION_SPECIFICATION,
+        }
+
+        json_path = os.path.join(SPEC_ROOT, "spec.json")
+        with open(json_path, "w") as f:
+            f.write(json.dumps(spec))
+        yield
+        os.remove(json_path)
+
+    @pytest.fixture
+    def use_yaml_spec(self):
+        spec = {"documentationUrl": "https://airbyte.com/#yaml", "connectionSpecification": self.CONNECTION_SPECIFICATION}
+
+        yaml_path = os.path.join(SPEC_ROOT, "spec.yaml")
+        with open(yaml_path, "w") as f:
+            f.write(yaml.dump(spec))
+        yield
+        os.remove(yaml_path)
+
+    def test_spec_from_json_file(self, integration, use_json_spec):
+        connector_spec = integration.spec(logger)
+        assert connector_spec.documentationUrl == "https://airbyte.com/#json"
+        assert connector_spec.connectionSpecification == self.CONNECTION_SPECIFICATION
+
+    def test_spec_from_yaml_file(self, integration, use_yaml_spec):
+        connector_spec = integration.spec(logger)
+        assert connector_spec.documentationUrl == "https://airbyte.com/#yaml"
+        assert connector_spec.connectionSpecification == self.CONNECTION_SPECIFICATION
+
+    def test_multiple_spec_files_raises_exception(self, integration, use_yaml_spec, use_json_spec):
+        with pytest.raises(RuntimeError, match="spec.yaml or spec.json"):
+            integration.spec(logger)
+
+    def test_no_spec_file_raises_exception(self, integration):
+        with pytest.raises(FileNotFoundError, match="Unable to find spec."):
+            integration.spec(logger)

airbyte-config/init/src/main/resources/seed/destination_definitions.yaml

@@ -83,7 +83,7 @@
 - name: Google Cloud Storage (GCS)
   destinationDefinitionId: ca8f6566-e555-4b40-943a-545bf123117a
   dockerRepository: airbyte/destination-gcs
-  dockerImageTag: 0.2.1
+  dockerImageTag: 0.2.4
   documentationUrl: https://docs.airbyte.io/integrations/destinations/gcs
   icon: googlecloudstorage.svg
   resourceRequirements:
@@ -167,7 +167,7 @@
 - name: Postgres
   destinationDefinitionId: 25c5221d-dce2-4163-ade9-739ef790f503
   dockerRepository: airbyte/destination-postgres
-  dockerImageTag: 0.3.17
+  dockerImageTag: 0.3.18
   documentationUrl: https://docs.airbyte.io/integrations/destinations/postgres
   icon: postgresql.svg
 - name: Pulsar
@@ -208,7 +208,7 @@
 - name: S3
   destinationDefinitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362
   dockerRepository: airbyte/destination-s3
-  dockerImageTag: 0.3.0
+  dockerImageTag: 0.3.3
   documentationUrl: https://docs.airbyte.io/integrations/destinations/s3
   icon: s3.svg
   resourceRequirements: