CDK: Autogenerate reference docs (#4759)

* CDK Autogenerated reference docs: base version

* Update docs config.
Add .readthedocs.yaml file.
Update build html files.

* Update .gitignore.
Remove sphinx build files.

* Add newline at the end of .gitignore

* Update setup.py requirements.
Update .readthedocs.yaml config.

* Update rst files.
Add Makefile rst config.

* Update CDK docstring format.
Change rst layout.
Update sphinx config.

* Add Sphinx docs.
Update index.rst.
Update abstract_source.py docstrings.

* Override master_doc and package templates.
Add docs schema enerator script.
Update sphinx docs.

* Add `Publishing to Read the Docs` section to sphinx-docs.md".
Replace sphinx-docs.md to `airbyte-cdk` module.

* Update sphinx-docs.md section name

* Bump airbyte-cdk version.
Update CHANGELOG.md.

Co-authored-by: ykurochkin <[email protected]>
Co-authored-by: Vadym Hevlich <[email protected]>

.gitignore

@@ -60,3 +60,6 @@ resources/examples/airflow/logs/*
 
 # Cloud Demo
 !airbyte-webapp/src/packages/cloud/data
+
+# Sphinx Docs
+_build

.readthedocs.yaml

@@ -0,0 +1,20 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  builder: html
+  configuration: airbyte-cdk/python/reference_docs/_source/conf.py
+
+# Set the version of Python and requirements required to build your docs
+python:
+  version: "3.7"
+  install:
+    - method: pip
+      path: airbyte-cdk/python
+      extra_requirements:
+        - sphinx-docs

airbyte-cdk/python/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.1.32
+- Added Sphinx docs `airbyte-cdk/python/reference_docs` module. 
+- Added module documents at `airbyte-cdk/python/sphinx-docs.md`.
+- Added Read the Docs publishing configuration at `.readthedocs.yaml`.
+
 ## 0.1.31
 Transforming Python log levels to Airbyte protocol log levels
 

airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py

@@ -39,11 +39,13 @@ class AbstractSource(Source, ABC):
     @abstractmethod
     def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]:
         """
-        :param config: The user-provided configuration as specified by the source's spec. This usually contains information required to check connection e.g. tokens, secrets and keys etc.
-        :return: A tuple of (boolean, error). If boolean is true, then the connection check is successful and we can connect to the underlying data
-        source using the provided configuration.
-        Otherwise, the input config cannot be used to connect to the underlying data source, and the "error" object should describe what went wrong.
-        The error object will be cast to string to display the problem to the user.
+        :param config: The user-provided configuration as specified by the source's spec.
+          This usually contains information required to check connection e.g. tokens, secrets and keys etc.
+        :return: A tuple of (boolean, error). If boolean is true, then the connection check is successful
+          and we can connect to the underlying data source using the provided configuration.
+          Otherwise, the input config cannot be used to connect to the underlying data source,
+          and the "error" object should describe what went wrong.
+          The error object will be cast to string to display the problem to the user.
         """
 
     @abstractmethod

airbyte-cdk/python/airbyte_cdk/sources/singer/singer_helpers.py

@@ -110,7 +110,7 @@ def singer_catalog_to_airbyte_catalog(
         """
         :param singer_catalog:
         :param sync_mode_overrides: A dict from stream name to the sync modes it should use. Each stream in this dict must exist in the Singer catalog,
-        but not every stream in the catalog should exist in this
+          but not every stream in the catalog should exist in this
         :param primary_key_overrides: A dict of stream name -> list of fields to be used as PKs.
         :return: Airbyte Catalog
         """

airbyte-cdk/python/airbyte_cdk/sources/streams/core.py

@@ -106,7 +106,7 @@ def source_defined_cursor(self) -> bool:
     def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]:
         """
         :return: string if single primary key, list of strings if composite primary key, list of list of strings if composite primary key consisting of nested fields.
-        If the stream has no primary keys, return None.
+          If the stream has no primary keys, return None.
         """
 
     def stream_slices(

airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py

@@ -165,7 +165,7 @@ def request_body_data(
 
         If returns a ready text that it will be sent as is.
         If returns a dict that it will be converted to a urlencoded form.
-           E.g. {"key1": "value1", "key2": "value2"} => "key1=value1&key2=value2"
+        E.g. {"key1": "value1", "key2": "value2"} => "key1=value1&key2=value2"
 
         At the same time only one of the 'request_body_data' and 'request_body_json' functions can be overridden.
         """

airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py

@@ -52,6 +52,7 @@ def get_schema(self, name: str) -> dict:
     def __resolve_schema_references(self, raw_schema: dict) -> dict:
         """
         Resolve links to external references and move it to local "definitions" map.
+
         :param raw_schema jsonschema to lookup for external links.
         :return JSON serializable object with references without external dependencies.
         """
@@ -78,6 +79,7 @@ def create_definitions(obj: dict, definitions: dict) -> Dict[str, Any]:
             """
             Scan resolved schema and compose definitions section, also convert
             jsonref.JsonRef object to JSON serializable dict.
+
             :param obj - jsonschema object with ref field resovled.
             :definitions - object for storing generated definitions.
             :return JSON serializable object with references without external dependencies.
@@ -110,6 +112,7 @@ def check_config_against_spec_or_exit(config: Mapping[str, Any], spec: Connector
     """
     Check config object against spec. In case of spec is invalid, throws
     an exception with validation error description.
+
     :param config - config loaded from file specified over command line
     :param spec - spec object generated by connector
     :param logger - Airbyte logger for reporting validation error
@@ -135,8 +138,10 @@ def split_config(config: Mapping[str, Any]) -> Tuple[dict, InternalConfig]:
     Break config map object into 2 instances: first is a dict with user defined
     configuration and second is internal config that contains private keys for
     acceptance test configuration.
+
     :param
      config - Dict object that has been loaded from config file.
+
     :return tuple of user defined config dict with filtered out internal
     parameters and SAT internal config object.
     """

airbyte-cdk/python/reference_docs/Makefile

@@ -0,0 +1,23 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = _source
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+rst:
+	sphinx-apidoc -f -F -o $(SOURCEDIR)/api ../airbyte_cdk

airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.destinations.rst

@@ -0,0 +1,19 @@
+
+Submodules
+----------
+
+airbyte\_cdk.destinations.destination module
+--------------------------------------------
+
+.. automodule:: airbyte_cdk.destinations.destination
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: airbyte_cdk.destinations
+   :members:
+   :undoc-members:
+   :show-inheritance:

airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.models.rst

@@ -0,0 +1,19 @@
+
+Submodules
+----------
+
+airbyte\_cdk.models.airbyte\_protocol module
+--------------------------------------------
+
+.. automodule:: airbyte_cdk.models.airbyte_protocol
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: airbyte_cdk.models
+   :members:
+   :undoc-members:
+   :show-inheritance:

airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.rst

@@ -0,0 +1,45 @@
+
+Subpackages
+-----------
+
+.. toctree::
+   :maxdepth: 4
+
+   airbyte_cdk.destinations
+   airbyte_cdk.models
+   airbyte_cdk.sources
+
+Submodules
+----------
+
+airbyte\_cdk.connector module
+-----------------------------
+
+.. automodule:: airbyte_cdk.connector
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+airbyte\_cdk.entrypoint module
+------------------------------
+
+.. automodule:: airbyte_cdk.entrypoint
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+airbyte\_cdk.logger module
+--------------------------
+
+.. automodule:: airbyte_cdk.logger
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: airbyte_cdk
+   :members:
+   :undoc-members:
+   :show-inheritance:

airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.deprecated.rst

@@ -0,0 +1,27 @@
+
+Submodules
+----------
+
+airbyte\_cdk.sources.deprecated.base\_source module
+---------------------------------------------------
+
+.. automodule:: airbyte_cdk.sources.deprecated.base_source
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+airbyte\_cdk.sources.deprecated.client module
+---------------------------------------------
+
+.. automodule:: airbyte_cdk.sources.deprecated.client
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: airbyte_cdk.sources.deprecated
+   :members:
+   :undoc-members:
+   :show-inheritance:

airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.rst

@@ -0,0 +1,38 @@
+
+Subpackages
+-----------
+
+.. toctree::
+   :maxdepth: 4
+
+   airbyte_cdk.sources.deprecated
+   airbyte_cdk.sources.singer
+   airbyte_cdk.sources.streams
+   airbyte_cdk.sources.utils
+
+Submodules
+----------
+
+airbyte\_cdk.sources.abstract\_source module
+--------------------------------------------
+
+.. automodule:: airbyte_cdk.sources.abstract_source
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+airbyte\_cdk.sources.source module
+----------------------------------
+
+.. automodule:: airbyte_cdk.sources.source
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: airbyte_cdk.sources
+   :members:
+   :undoc-members:
+   :show-inheritance:

airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.singer.rst

@@ -0,0 +1,27 @@
+
+Submodules
+----------
+
+airbyte\_cdk.sources.singer.singer\_helpers module
+--------------------------------------------------
+
+.. automodule:: airbyte_cdk.sources.singer.singer_helpers
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+airbyte\_cdk.sources.singer.source module
+-----------------------------------------
+
+.. automodule:: airbyte_cdk.sources.singer.source
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: airbyte_cdk.sources.singer
+   :members:
+   :undoc-members:
+   :show-inheritance:

airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.streams.http.auth.rst

@@ -0,0 +1,35 @@
+
+Submodules
+----------
+
+airbyte\_cdk.sources.streams.http.auth.core module
+--------------------------------------------------
+
+.. automodule:: airbyte_cdk.sources.streams.http.auth.core
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+airbyte\_cdk.sources.streams.http.auth.oauth module
+---------------------------------------------------
+
+.. automodule:: airbyte_cdk.sources.streams.http.auth.oauth
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+airbyte\_cdk.sources.streams.http.auth.token module
+---------------------------------------------------
+
+.. automodule:: airbyte_cdk.sources.streams.http.auth.token
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: airbyte_cdk.sources.streams.http.auth
+   :members:
+   :undoc-members:
+   :show-inheritance:

airbyte-cdk/python/reference_docs/_source/api/airbyte_cdk.sources.streams.http.requests_native_auth.rst

@@ -0,0 +1,27 @@
+
+Submodules
+----------
+
+airbyte\_cdk.sources.streams.http.requests\_native\_auth.oauth module
+---------------------------------------------------------------------
+
+.. automodule:: airbyte_cdk.sources.streams.http.requests_native_auth.oauth
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+airbyte\_cdk.sources.streams.http.requests\_native\_auth.token module
+---------------------------------------------------------------------
+
+.. automodule:: airbyte_cdk.sources.streams.http.requests_native_auth.token
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: airbyte_cdk.sources.streams.http.requests_native_auth
+   :members:
+   :undoc-members:
+   :show-inheritance: