diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..a7e2a00 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @grasskode @petrjanda diff --git a/docs/core_v1.md b/docs/core_v1.md deleted file mode 100644 index 154fe6e..0000000 --- a/docs/core_v1.md +++ /dev/null @@ -1,265 +0,0 @@ -# Protocol Documentation - - -## Table of Contents - -- [core/v1/entity_type.proto](#core_v1_entity_type-proto) - - [EntityType](#core-v1-EntityType) - -- [core/v1/platforms.proto](#core_v1_platforms-proto) - - [Platform](#core-v1-Platform) - -- [core/v1/entity.proto](#core_v1_entity-proto) - - [Asset](#core-v1-Asset) - - [Entity](#core-v1-Entity) - - [EntityRef](#core-v1-EntityRef) - - [Monitor](#core-v1-Monitor) - - [MonitorSegment](#core-v1-MonitorSegment) - -- [Scalar Value Types](#scalar-value-types) - - - - -

Top

- -## core/v1/entity_type.proto - - - - - - - -### EntityType -Type of Entity. -This enum lists all the types currently supported by Synq. - -| Name | Number | Description | -| ---- | ------ | ----------- | -| ENTITY_TYPE_UNSPECIFIED | 0 | | -| ENTITY_TYPE_FOLDER | 1 | | -| ENTITY_TYPE_BQ_PROJECT | 101 | | -| ENTITY_TYPE_BQ_DATASET | 102 | | -| ENTITY_TYPE_BQ_TABLE | 103 | | -| ENTITY_TYPE_BQ_VIEW | 105 | | -| ENTITY_TYPE_LOOKER_LOOK | 201 | | -| ENTITY_TYPE_LOOKER_EXPLORE | 203 | | -| ENTITY_TYPE_LOOKER_VIEW | 207 | | -| ENTITY_TYPE_LOOKER_DASHBOARD | 208 | | -| ENTITY_TYPE_DBT_MODEL | 301 | | -| ENTITY_TYPE_DBT_TEST | 302 | | -| ENTITY_TYPE_DBT_SOURCE | 303 | | -| ENTITY_TYPE_DBT_MACRO | 304 | | -| ENTITY_TYPE_DBT_PROJECT | 306 | | -| ENTITY_TYPE_DBT_METRIC | 307 | | -| ENTITY_TYPE_DBT_SNAPSHOT | 310 | | -| ENTITY_TYPE_DBT_SEED | 311 | | -| ENTITY_TYPE_DBT_ANALYSIS | 312 | | -| ENTITY_TYPE_DBT_EXPOSURE | 313 | | -| ENTITY_TYPE_DBT_GROUP | 314 | | -| ENTITY_TYPE_DBT_CLOUD_ACCOUNT | 351 | | -| ENTITY_TYPE_DBT_CLOUD_PROJECT | 352 | | -| ENTITY_TYPE_DBT_CLOUD_JOB | 353 | | -| ENTITY_TYPE_DBT_CLOUD_JOB_STEP | 354 | | -| ENTITY_TYPE_SNOWFLAKE_PROJECT | 501 | | -| ENTITY_TYPE_SNOWFLAKE_DATASET | 502 | | -| ENTITY_TYPE_SNOWFLAKE_TABLE | 503 | | -| ENTITY_TYPE_SNOWFLAKE_ACCOUNT | 505 | | -| ENTITY_TYPE_SNOWFLAKE_DATABASE | 506 | | -| ENTITY_TYPE_SNOWFLAKE_SCHEMA | 507 | | -| ENTITY_TYPE_SNOWFLAKE_VIEW | 508 | | -| ENTITY_TYPE_REDSHIFT_DATABASE | 801 | | -| ENTITY_TYPE_REDSHIFT_SCHEMA | 802 | | -| ENTITY_TYPE_REDSHIFT_TABLE | 803 | | -| ENTITY_TYPE_REDSHIFT_VIEW | 805 | | -| ENTITY_TYPE_TABLEAU_EMBEDDED | 1101 | | -| ENTITY_TYPE_TABLEAU_PUBLISHED | 1102 | | -| ENTITY_TYPE_TABLEAU_CUSTOM_SQL | 1103 | | -| ENTITY_TYPE_TABLEAU_TABLE | 1104 | | -| ENTITY_TYPE_TABLEAU_SHEET | 1105 | | -| ENTITY_TYPE_TABLEAU_DASHBOARD | 1106 | | -| ENTITY_TYPE_AIRFLOW_DAG | 1201 | | -| ENTITY_TYPE_AIRFLOW_TASK | 1202 | | -| ENTITY_TYPE_CLICKHOUSE_DATABASE | 1301 | | -| ENTITY_TYPE_CLICKHOUSE_SCHEMA | 1302 | | -| ENTITY_TYPE_CLICKHOUSE_TABLE | 1303 | | -| ENTITY_TYPE_CLICKHOUSE_VIEW | 1305 | | -| ENTITY_TYPE_ANOMALY_INTEGRATION | 1401 | | -| ENTITY_TYPE_ANOMALY_MONITOR | 1403 | | -| ENTITY_TYPE_ANOMALY_MONITOR_SEGMENT | 1404 | | -| ENTITY_TYPE_POSTGRES_DATABASE | 1601 | | -| ENTITY_TYPE_POSTGRES_SCHEMA | 1602 | | -| ENTITY_TYPE_POSTGRES_TABLE | 1603 | | -| ENTITY_TYPE_POSTGRES_VIEW | 1605 | | -| ENTITY_TYPE_MYSQL_DATABASE | 1701 | | -| ENTITY_TYPE_MYSQL_SCHEMA | 1702 | | -| ENTITY_TYPE_MYSQL_TABLE | 1703 | | -| ENTITY_TYPE_MYSQL_VIEW | 1705 | | - - - - - - - - - - - -

Top

- -## core/v1/platforms.proto - - - - - - - -### Platform -Platforms supported by Synq. - -| Name | Number | Description | -| ---- | ------ | ----------- | -| PLATFORM_UNSPECIFIED | 0 | | -| PLATFORM_BIGQUERY | 10 | | -| PLATFORM_LOOKER | 20 | | -| PLATFORM_DBT | 30 | | -| PLATFORM_DBT_CLOUD | 31 | | -| PLATFORM_DBT_SELF_HOSTED | 32 | | -| PLATFORM_SNOWFLAKE | 50 | | -| PLATFORM_REDSHIFT | 80 | | -| PLATFORM_TABLEAU | 110 | | -| PLATFORM_AIRFLOW | 120 | | -| PLATFORM_CLICKHOUSE | 130 | | -| PLATFORM_POSTGRES | 160 | | -| PLATFORM_MYSQL | 170 | | - - - - - - - - - - - -

Top

- -## core/v1/entity.proto - - - - - -### Asset -Entity: Asset. - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| asset_path | [string](#string) | | | -| platform | [Platform](#core-v1-Platform) | | | - - - - - - - - -### Entity -Entity in Synq. - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| asset | [Asset](#core-v1-Asset) | | | -| monitor | [Monitor](#core-v1-Monitor) | | | -| monitor_segment | [MonitorSegment](#core-v1-MonitorSegment) | | | - - - - - - - - -### EntityRef -Lightweight reference to an Entity - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| path | [string](#string) | | Unique path for entity. | -| type | [EntityType](#core-v1-EntityType) | | Type of entity. | -| name | [string](#string) | optional | Human friendly name. | - - - - - - - - -### Monitor -Entity: Monitor - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| monitor_path | [string](#string) | | | - - - - - - - - -### MonitorSegment -Entity: Monitor Segment - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| monitor_path | [string](#string) | | | -| metric_id | [string](#string) | | | -| segment | [string](#string) | | | - - - - - - - - - - - - - - - -## Scalar Value Types - -| .proto Type | Notes | C++ | Java | Python | Go | C# | PHP | Ruby | -| ----------- | ----- | --- | ---- | ------ | -- | -- | --- | ---- | -| double | | double | double | float | float64 | double | float | Float | -| float | | float | float | float | float32 | float | float | Float | -| int32 | Uses variable-length encoding. Inefficient for encoding negative numbers – if your field is likely to have negative values, use sint32 instead. | int32 | int | int | int32 | int | integer | Bignum or Fixnum (as required) | -| int64 | Uses variable-length encoding. Inefficient for encoding negative numbers – if your field is likely to have negative values, use sint64 instead. | int64 | long | int/long | int64 | long | integer/string | Bignum | -| uint32 | Uses variable-length encoding. | uint32 | int | int/long | uint32 | uint | integer | Bignum or Fixnum (as required) | -| uint64 | Uses variable-length encoding. | uint64 | long | int/long | uint64 | ulong | integer/string | Bignum or Fixnum (as required) | -| sint32 | Uses variable-length encoding. Signed int value. These more efficiently encode negative numbers than regular int32s. | int32 | int | int | int32 | int | integer | Bignum or Fixnum (as required) | -| sint64 | Uses variable-length encoding. Signed int value. These more efficiently encode negative numbers than regular int64s. | int64 | long | int/long | int64 | long | integer/string | Bignum | -| fixed32 | Always four bytes. More efficient than uint32 if values are often greater than 2^28. | uint32 | int | int | uint32 | uint | integer | Bignum or Fixnum (as required) | -| fixed64 | Always eight bytes. More efficient than uint64 if values are often greater than 2^56. | uint64 | long | int/long | uint64 | ulong | integer/string | Bignum | -| sfixed32 | Always four bytes. | int32 | int | int | int32 | int | integer | Bignum or Fixnum (as required) | -| sfixed64 | Always eight bytes. | int64 | long | int/long | int64 | long | integer/string | Bignum | -| bool | | bool | boolean | boolean | bool | bool | boolean | TrueClass/FalseClass | -| string | A string must always contain UTF-8 encoded or 7-bit ASCII text. | string | String | str/unicode | string | string | string | String (UTF-8) | -| bytes | May contain any arbitrary sequence of bytes. | string | ByteString | str | []byte | ByteString | string | String (ASCII-8BIT) | - diff --git a/docs/example_v1.md b/docs/example/v1.md similarity index 94% rename from docs/example_v1.md rename to docs/example/v1.md index 4b8209f..08b90b9 100644 --- a/docs/example_v1.md +++ b/docs/example/v1.md @@ -10,6 +10,7 @@ - [GetExampleRequest](#example-v1-GetExampleRequest) - [GetExampleResponse](#example-v1-GetExampleResponse) - [ListExamplesRequest](#example-v1-ListExamplesRequest) + - [ListExamplesResponse](#example-v1-ListExamplesResponse) - [RemoveExampleRequest](#example-v1-RemoveExampleRequest) - [RemoveExampleResponse](#example-v1-RemoveExampleResponse) - [UpdateExampleRequest](#example-v1-UpdateExampleRequest) @@ -122,6 +123,21 @@ ListExamplesRequest streams examples from the server. + + +### ListExamplesResponse + + + +| Field | Type | Label | Description | +| ----- | ---- | ----- | ----------- | +| example | [Example](#example-v1-Example) | | | + + + + + + ### RemoveExampleRequest @@ -160,6 +176,7 @@ UpdateExampleRequest contains updateable fields inside an example. | Field | Type | Label | Description | | ----- | ---- | ----- | ----------- | +| id | [string](#string) | | Id of the example to update | | text | [string](#string) | optional | Optional update of the example text. | | description | [string](#string) | optional | Optional update of the example description. | @@ -200,7 +217,7 @@ The service implements a basic CRUD API and contains server streaming example. | AddExample | [AddExampleRequest](#example-v1-AddExampleRequest) | [AddExampleResponse](#example-v1-AddExampleResponse) | AddExample adds the given example to the system. | | UpdateExample | [UpdateExampleRequest](#example-v1-UpdateExampleRequest) | [UpdateExampleResponse](#example-v1-UpdateExampleResponse) | UpdateExample updates the given example. Throws an error if the example was not found. | | GetExample | [GetExampleRequest](#example-v1-GetExampleRequest) | [GetExampleResponse](#example-v1-GetExampleResponse) | GetExample returns the requested example. | -| ListExamples | [ListExamplesRequest](#example-v1-ListExamplesRequest) | [ExampleResponse](#example-v1-ExampleResponse) stream | ListExamples streams the requested examples. | +| ListExamples | [ListExamplesRequest](#example-v1-ListExamplesRequest) | [ListExamplesResponse](#example-v1-ListExamplesResponse) stream | ListExamples streams the requested examples. | | RemoveExample | [RemoveExampleRequest](#example-v1-RemoveExampleRequest) | [RemoveExampleResponse](#example-v1-RemoveExampleResponse) | RemoveExample removes the provided example from the system. Fails silently if the example was not found. | diff --git a/docs/example_v2.md b/docs/example/v2.md similarity index 95% rename from docs/example_v2.md rename to docs/example/v2.md index 98c65ed..ecbd9c5 100644 --- a/docs/example_v2.md +++ b/docs/example/v2.md @@ -10,6 +10,7 @@ - [GetExampleRequest](#example-v2-GetExampleRequest) - [GetExampleResponse](#example-v2-GetExampleResponse) - [ListExamplesRequest](#example-v2-ListExamplesRequest) + - [ListExamplesResponse](#example-v2-ListExamplesResponse) - [RemoveExampleRequest](#example-v2-RemoveExampleRequest) - [RemoveExampleResponse](#example-v2-RemoveExampleResponse) - [UpdateExampleRequest](#example-v2-UpdateExampleRequest) @@ -123,6 +124,21 @@ ListExamplesRequest streams examples from the server. + + +### ListExamplesResponse + + + +| Field | Type | Label | Description | +| ----- | ---- | ----- | ----------- | +| example | [Example](#example-v2-Example) | | | + + + + + + ### RemoveExampleRequest @@ -202,7 +218,7 @@ The service implements a basic CRUD API and contains server streaming example. | AddExample | [AddExampleRequest](#example-v2-AddExampleRequest) | [AddExampleResponse](#example-v2-AddExampleResponse) | AddExample adds the given example to the system. | | UpdateExample | [UpdateExampleRequest](#example-v2-UpdateExampleRequest) | [UpdateExampleResponse](#example-v2-UpdateExampleResponse) | UpdateExample updates the given example. Throws an error if the example was not found. | | GetExample | [GetExampleRequest](#example-v2-GetExampleRequest) | [GetExampleResponse](#example-v2-GetExampleResponse) | GetExample returns the requested example. | -| ListExamples | [ListExamplesRequest](#example-v2-ListExamplesRequest) | [ExampleResponse](#example-v2-ExampleResponse) stream | ListExamples streams the requested examples. | +| ListExamples | [ListExamplesRequest](#example-v2-ListExamplesRequest) | [ListExamplesResponse](#example-v2-ListExamplesResponse) stream | ListExamples streams the requested examples. | | RemoveExample | [RemoveExampleRequest](#example-v2-RemoveExampleRequest) | [RemoveExampleResponse](#example-v2-RemoveExampleResponse) | RemoveExample removes the provided example from the system. Fails silently if the example was not found. | diff --git a/docs/schemas_v1.md b/docs/schemas_v1.md deleted file mode 100644 index 903d380..0000000 --- a/docs/schemas_v1.md +++ /dev/null @@ -1,322 +0,0 @@ -# Protocol Documentation - - -## Table of Contents - -- [schemas/v1/lineage_service.proto](#schemas_v1_lineage_service-proto) - - [EntitiesStartPoint](#schemas-v1-EntitiesStartPoint) - - [EntityColumnsStartPoint](#schemas-v1-EntityColumnsStartPoint) - - [GetLineageRequest](#schemas-v1-GetLineageRequest) - - [GetLineageResponse](#schemas-v1-GetLineageResponse) - - [GetLineageStartPoint](#schemas-v1-GetLineageStartPoint) - - - [LineageDirection](#schemas-v1-LineageDirection) - - - [LineageService](#schemas-v1-LineageService) - -- [schemas/v1/lineage.proto](#schemas_v1_lineage-proto) - - [CllDetails](#schemas-v1-CllDetails) - - [Column](#schemas-v1-Column) - - [ColumnDependency](#schemas-v1-ColumnDependency) - - [Lineage](#schemas-v1-Lineage) - - [LineageNode](#schemas-v1-LineageNode) - - [NodeDependency](#schemas-v1-NodeDependency) - - - [CllState](#schemas-v1-CllState) - - [NodePosition](#schemas-v1-NodePosition) - -- [Scalar Value Types](#scalar-value-types) - - - - -

Top

- -## schemas/v1/lineage_service.proto - - - - - -### EntitiesStartPoint - - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| entities | [core.v1.EntityRef](#core-v1-EntityRef) | repeated | | - - - - - - - - -### EntityColumnsStartPoint - - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| entitiy | [core.v1.EntityRef](#core-v1-EntityRef) | | | -| column_ids | [string](#string) | repeated | | - - - - - - - - -### GetLineageRequest - - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| lineage_direction | [LineageDirection](#schemas-v1-LineageDirection) | | | -| start_point | [GetLineageStartPoint](#schemas-v1-GetLineageStartPoint) | | | -| max_depth | [int32](#int32) | optional | | - - - - - - - - -### GetLineageResponse - - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| lineage | [Lineage](#schemas-v1-Lineage) | | | - - - - - - - - -### GetLineageStartPoint -Possible starting points to get lineage from. - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| entities | [EntitiesStartPoint](#schemas-v1-EntitiesStartPoint) | | | -| entity_columns | [EntityColumnsStartPoint](#schemas-v1-EntityColumnsStartPoint) | | | - - - - - - - - - - -### LineageDirection -Direction of the lineage to query. - -| Name | Number | Description | -| ---- | ------ | ----------- | -| LINEAGE_DIRECTION_UNSPECIFIED | 0 | | -| LINEAGE_DIRECTION_UPSTREAM | 1 | | -| LINEAGE_DIRECTION_DOWNSTREAM | 2 | | -| LINEAGE_DIRECTION_UPSTREAM_DOWNSTREAM | 3 | | - - - - - - - - - -### LineageService -LineageService allows you to fetch: -* Entity level lineage from a starting point of one or more entities. -* Column Level lineage from a starting point of multiple columns of a single entity. - -| Method Name | Request Type | Response Type | Description | -| ----------- | ------------ | ------------- | ------------| -| GetLineage | [GetLineageRequest](#schemas-v1-GetLineageRequest) | [GetLineageResponse](#schemas-v1-GetLineageResponse) | | - - - - - - -

Top

- -## schemas/v1/lineage.proto - - - - - -### CllDetails - - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| columns | [Column](#schemas-v1-Column) | repeated | Column details for CLL. | -| cll_state | [CllState](#schemas-v1-CllState) | | State of the CLL parse. UNSPECIFIED if CLL was not requested. | -| cll_messages | [string](#string) | repeated | Messages related to CLL. e.g. Description of parse errors, etc. | - - - - - - - - -### Column -Column in a table-like asset (used in CLL mode). - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| column_id | [string](#string) | | ID string for the column. This is the parsed column name. | -| name | [string](#string) | optional | Original column name as fetched from the table. | -| native_type | [string](#string) | optional | Column type as fetched from the table. | - - - - - - - - -### ColumnDependency -Indicates data flow between columns. -Source columns are used to compute value of target columns. - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| source_node_idx | [uint32](#uint32) | | Index of source node in the lineage nodes list. | -| source_node_column_id | [string](#string) | | | -| target_node_idx | [uint32](#uint32) | | Index of target node in the lineage nodes list. | -| target_node_column_id | [string](#string) | | | - - - - - - - - -### Lineage -Lineage defines the lineage of table-like entities. - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| nodes | [LineageNode](#schemas-v1-LineageNode) | repeated | Nodes in the lineage with their identities and columns. | -| node_dependencies | [NodeDependency](#schemas-v1-NodeDependency) | repeated | All edges in the lineage between nodes. This can be parsed to create a graph of all the nodes. | -| is_cll | [bool](#bool) | | Indicates whether the lineage was filtered for column level lineage (CLL). | -| column_dependencies | [ColumnDependency](#schemas-v1-ColumnDependency) | repeated | Dependencies between columns. Populated only for CLL. | - - - - - - - - -### LineageNode -Node in a lineage graph representing one or more entiities (e.g. database table). - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| entities | [core.v1.EntityRef](#core-v1-EntityRef) | repeated | All entities which have the same identity as this node. Must be at least one item. These are sorted by closeness to the type of the start point entities. e.g. if requesting lineage of a DBT source, first entity should be from DBT, similarly when viewing table it will be other tables. | -| position | [NodePosition](#schemas-v1-NodePosition) | | Position of the node in the lineage. | -| cll_details | [CllDetails](#schemas-v1-CllDetails) | optional | Populated only for Column Level Lineage (CLL). | - - - - - - - - -### NodeDependency -Indicates data flow between nodes. -Source nodes are used to compute value of target nodes. - - -| Field | Type | Label | Description | -| ----- | ---- | ----- | ----------- | -| source_node_idx | [uint32](#uint32) | | Index of source node in the lineage nodes list. | -| target_node_idx | [uint32](#uint32) | | Index of target node in the lineage nodes list. | - - - - - - - - - - -### CllState - - -| Name | Number | Description | -| ---- | ------ | ----------- | -| CLL_STATE_UNSPECIFIED | 0 | Unspecified state. | -| CLL_STATE_PARSE_FAILED | 1 | Parsing of the asset SQL failed. No upstream dependencies can be found. | -| CLL_STATE_EXTRACTION_FAILED | 2 | Extraction of the asset SQL failed. Some unsupported SQL features may be used. Some details might be missing. | -| CLL_STATE_RESOLUTION_FAILED | 3 | Not all columns or tables were found upstream, lineage is not complete. | -| CLL_STATE_OK | 10 | No known issues present. | - - - - - -### NodePosition - - -| Name | Number | Description | -| ---- | ------ | ----------- | -| NODE_POSITION_UNSPECIFIED | 0 | | -| NODE_POSITION_START_NODE | 1 | Node is one of the requested start point. | -| NODE_POSITION_UPSTREAM | 2 | Node is upstream of the requested start point. | -| NODE_POSITION_DOWNSTREAM | 3 | Node is downstream of the requested start point. | - - - - - - - - - - -## Scalar Value Types - -| .proto Type | Notes | C++ | Java | Python | Go | C# | PHP | Ruby | -| ----------- | ----- | --- | ---- | ------ | -- | -- | --- | ---- | -| double | | double | double | float | float64 | double | float | Float | -| float | | float | float | float | float32 | float | float | Float | -| int32 | Uses variable-length encoding. Inefficient for encoding negative numbers – if your field is likely to have negative values, use sint32 instead. | int32 | int | int | int32 | int | integer | Bignum or Fixnum (as required) | -| int64 | Uses variable-length encoding. Inefficient for encoding negative numbers – if your field is likely to have negative values, use sint64 instead. | int64 | long | int/long | int64 | long | integer/string | Bignum | -| uint32 | Uses variable-length encoding. | uint32 | int | int/long | uint32 | uint | integer | Bignum or Fixnum (as required) | -| uint64 | Uses variable-length encoding. | uint64 | long | int/long | uint64 | ulong | integer/string | Bignum or Fixnum (as required) | -| sint32 | Uses variable-length encoding. Signed int value. These more efficiently encode negative numbers than regular int32s. | int32 | int | int | int32 | int | integer | Bignum or Fixnum (as required) | -| sint64 | Uses variable-length encoding. Signed int value. These more efficiently encode negative numbers than regular int64s. | int64 | long | int/long | int64 | long | integer/string | Bignum | -| fixed32 | Always four bytes. More efficient than uint32 if values are often greater than 2^28. | uint32 | int | int | uint32 | uint | integer | Bignum or Fixnum (as required) | -| fixed64 | Always eight bytes. More efficient than uint64 if values are often greater than 2^56. | uint64 | long | int/long | uint64 | ulong | integer/string | Bignum | -| sfixed32 | Always four bytes. | int32 | int | int | int32 | int | integer | Bignum or Fixnum (as required) | -| sfixed64 | Always eight bytes. | int64 | long | int/long | int64 | long | integer/string | Bignum | -| bool | | bool | boolean | boolean | bool | bool | boolean | TrueClass/FalseClass | -| string | A string must always contain UTF-8 encoded or 7-bit ASCII text. | string | String | str/unicode | string | string | string | String (UTF-8) | -| bytes | May contain any arbitrary sequence of bytes. | string | ByteString | str | []byte | ByteString | string | String (ASCII-8BIT) | - diff --git a/protos/core/v1/entity.proto b/protos/core/v1/entity.proto deleted file mode 100644 index 2d0a4ee..0000000 --- a/protos/core/v1/entity.proto +++ /dev/null @@ -1,42 +0,0 @@ -syntax = "proto3"; - -package core.v1; - -import "core/v1/entity_type.proto"; -import "core/v1/platforms.proto"; - -option go_package = "github.com/getsynq/api/core/v1"; - -// Lightweight reference to an Entity -message EntityRef { - string path = 1; // Unique path for entity. - EntityType type = 2; // Type of entity. - optional string name = 3; // Human friendly name. -} - -// Entity in Synq. -message Entity { - oneof data { - Asset asset = 1; - Monitor monitor = 2; - MonitorSegment monitor_segment = 3; - } -} - -// Entity: Asset. -message Asset { - string asset_path = 1; - Platform platform = 2; -} - -// Entity: Monitor -message Monitor { - string monitor_path = 1; -} - -// Entity: Monitor Segment -message MonitorSegment { - string monitor_path = 1; - string metric_id = 2; - string segment = 3; -} diff --git a/protos/core/v1/entity_type.proto b/protos/core/v1/entity_type.proto deleted file mode 100644 index 041d24d..0000000 --- a/protos/core/v1/entity_type.proto +++ /dev/null @@ -1,103 +0,0 @@ -syntax = "proto3"; - -package core.v1; - -option go_package = "github.com/getsynq/api/core/v1"; - -// Type of Entity. -// This enum lists all the types currently supported by Synq. -enum EntityType { - ENTITY_TYPE_UNSPECIFIED = 0; - ENTITY_TYPE_FOLDER = 1; - - // BIGQUERY 1XX - - ENTITY_TYPE_BQ_PROJECT = 101; - ENTITY_TYPE_BQ_DATASET = 102; - ENTITY_TYPE_BQ_TABLE = 103; - ENTITY_TYPE_BQ_VIEW = 105; - - // LOOKER 2XX - - ENTITY_TYPE_LOOKER_LOOK = 201; - ENTITY_TYPE_LOOKER_EXPLORE = 203; - ENTITY_TYPE_LOOKER_VIEW = 207; - ENTITY_TYPE_LOOKER_DASHBOARD = 208; - - // DBT 3XX - - ENTITY_TYPE_DBT_MODEL = 301; - ENTITY_TYPE_DBT_TEST = 302; - ENTITY_TYPE_DBT_SOURCE = 303; - ENTITY_TYPE_DBT_MACRO = 304; - ENTITY_TYPE_DBT_PROJECT = 306; - ENTITY_TYPE_DBT_METRIC = 307; - ENTITY_TYPE_DBT_SNAPSHOT = 310; - ENTITY_TYPE_DBT_SEED = 311; - ENTITY_TYPE_DBT_ANALYSIS = 312; - ENTITY_TYPE_DBT_EXPOSURE = 313; - ENTITY_TYPE_DBT_GROUP = 314; - - // DBT 35X - - ENTITY_TYPE_DBT_CLOUD_ACCOUNT = 351; - ENTITY_TYPE_DBT_CLOUD_PROJECT = 352; - ENTITY_TYPE_DBT_CLOUD_JOB = 353; - ENTITY_TYPE_DBT_CLOUD_JOB_STEP = 354; - - // SNOWFLAKE 5XX - - ENTITY_TYPE_SNOWFLAKE_PROJECT = 501; - ENTITY_TYPE_SNOWFLAKE_DATASET = 502; - ENTITY_TYPE_SNOWFLAKE_TABLE = 503; - ENTITY_TYPE_SNOWFLAKE_ACCOUNT = 505; - ENTITY_TYPE_SNOWFLAKE_DATABASE = 506; - ENTITY_TYPE_SNOWFLAKE_SCHEMA = 507; - ENTITY_TYPE_SNOWFLAKE_VIEW = 508; - - // REDSHIFT 8XX - - ENTITY_TYPE_REDSHIFT_DATABASE = 801; - ENTITY_TYPE_REDSHIFT_SCHEMA = 802; - ENTITY_TYPE_REDSHIFT_TABLE = 803; - ENTITY_TYPE_REDSHIFT_VIEW = 805; - - // TABLEAU 11xx - - ENTITY_TYPE_TABLEAU_EMBEDDED = 1101; - ENTITY_TYPE_TABLEAU_PUBLISHED = 1102; - ENTITY_TYPE_TABLEAU_CUSTOM_SQL = 1103; - ENTITY_TYPE_TABLEAU_TABLE = 1104; - ENTITY_TYPE_TABLEAU_SHEET = 1105; - ENTITY_TYPE_TABLEAU_DASHBOARD = 1106; - - // AIRFLOW 12xx - - ENTITY_TYPE_AIRFLOW_DAG = 1201; - ENTITY_TYPE_AIRFLOW_TASK = 1202; - - // CLICKHOUSE 13XX - - ENTITY_TYPE_CLICKHOUSE_DATABASE = 1301; - ENTITY_TYPE_CLICKHOUSE_SCHEMA = 1302; - ENTITY_TYPE_CLICKHOUSE_TABLE = 1303; - ENTITY_TYPE_CLICKHOUSE_VIEW = 1305; - - ENTITY_TYPE_ANOMALY_INTEGRATION = 1401; - ENTITY_TYPE_ANOMALY_MONITOR = 1403; - ENTITY_TYPE_ANOMALY_MONITOR_SEGMENT = 1404; - - // POSTGRES 16XX - - ENTITY_TYPE_POSTGRES_DATABASE = 1601; - ENTITY_TYPE_POSTGRES_SCHEMA = 1602; - ENTITY_TYPE_POSTGRES_TABLE = 1603; - ENTITY_TYPE_POSTGRES_VIEW = 1605; - - // MYSQL 17XX - - ENTITY_TYPE_MYSQL_DATABASE = 1701; - ENTITY_TYPE_MYSQL_SCHEMA = 1702; - ENTITY_TYPE_MYSQL_TABLE = 1703; - ENTITY_TYPE_MYSQL_VIEW = 1705; -} diff --git a/protos/core/v1/platforms.proto b/protos/core/v1/platforms.proto deleted file mode 100644 index 31c97e0..0000000 --- a/protos/core/v1/platforms.proto +++ /dev/null @@ -1,23 +0,0 @@ -syntax = "proto3"; - -package core.v1; - -option go_package = "github.com/getsynq/api/core/v1"; - -// Platforms supported by Synq. -enum Platform { - reserved 60, 70, 90, 140, 150, 1000; - PLATFORM_UNSPECIFIED = 0; - PLATFORM_BIGQUERY = 10; - PLATFORM_LOOKER = 20; - PLATFORM_DBT = 30; - PLATFORM_DBT_CLOUD = 31; - PLATFORM_DBT_SELF_HOSTED = 32; - PLATFORM_SNOWFLAKE = 50; - PLATFORM_REDSHIFT = 80; - PLATFORM_TABLEAU = 110; - PLATFORM_AIRFLOW = 120; - PLATFORM_CLICKHOUSE = 130; - PLATFORM_POSTGRES = 160; - PLATFORM_MYSQL = 170; -} diff --git a/protos/example/v1/example_service.proto b/protos/example/v1/example_service.proto index 1a53505..c9a31df 100644 --- a/protos/example/v1/example_service.proto +++ b/protos/example/v1/example_service.proto @@ -21,7 +21,7 @@ service ExampleService { rpc GetExample(GetExampleRequest) returns (GetExampleResponse) {} // ListExamples streams the requested examples. - rpc ListExamples(ListExamplesRequest) returns (stream ExampleResponse) {} + rpc ListExamples(ListExamplesRequest) returns (stream ListExamplesResponse) {} // RemoveExample removes the provided example from the system. // Fails silently if the example was not found. @@ -46,8 +46,9 @@ message AddExampleResponse { * UpdateExampleRequest contains updateable fields inside an example. */ message UpdateExampleRequest { - optional string text = 1; /// Optional update of the example text. - optional string description = 2; /// Optional update of the example description. + string id = 1; /// Id of the example to update + optional string text = 2; /// Optional update of the example text. + optional string description = 3; /// Optional update of the example description. } // Response with updated example. @@ -73,6 +74,10 @@ message ExampleResponse { Example example = 1; } +message ListExamplesResponse { + Example example = 1; +} + // RemoveExampleRequest removes an example by its ID. message RemoveExampleRequest { string id = 1; diff --git a/protos/example/v2/example_service.proto b/protos/example/v2/example_service.proto index 7766ffb..e3981a1 100644 --- a/protos/example/v2/example_service.proto +++ b/protos/example/v2/example_service.proto @@ -21,7 +21,7 @@ service ExampleService { rpc GetExample(GetExampleRequest) returns (GetExampleResponse) {} // ListExamples streams the requested examples. - rpc ListExamples(ListExamplesRequest) returns (stream ExampleResponse) {} + rpc ListExamples(ListExamplesRequest) returns (stream ListExamplesResponse) {} // RemoveExample removes the provided example from the system. // Fails silently if the example was not found. @@ -71,10 +71,15 @@ message ListExamplesRequest { repeated string ids = 1; } +message ListExamplesResponse { + Example example = 1; +} + message ExampleResponse { Example example = 1; } + // RemoveExampleRequest removes an example by its ID. message RemoveExampleRequest { string id = 1; diff --git a/protos/schemas/v1/lineage.proto b/protos/schemas/v1/lineage.proto deleted file mode 100644 index b6c0cfd..0000000 --- a/protos/schemas/v1/lineage.proto +++ /dev/null @@ -1,86 +0,0 @@ -syntax = "proto3"; - -package schemas.v1; - -import "core/v1/entity.proto"; - -option go_package = "github.com/getsynq/api/schemas/v1"; - -// Lineage defines the lineage of table-like entities. -message Lineage { - // Nodes in the lineage with their identities and columns. - repeated LineageNode nodes = 1; - // All edges in the lineage between nodes. - // This can be parsed to create a graph of all the nodes. - repeated NodeDependency node_dependencies = 2; - - // Indicates whether the lineage was filtered for column level lineage (CLL). - bool is_cll = 3; - // Dependencies between columns. Populated only for CLL. - repeated ColumnDependency column_dependencies = 4; -} - -// Indicates data flow between nodes. -// Source nodes are used to compute value of target nodes. -message NodeDependency { - uint32 source_node_idx = 1; // Index of source node in the lineage nodes list. - uint32 target_node_idx = 2; // Index of target node in the lineage nodes list. -} - -// Indicates data flow between columns. -// Source columns are used to compute value of target columns. -message ColumnDependency { - uint32 source_node_idx = 1; // Index of source node in the lineage nodes list. - string source_node_column_id = 2; - uint32 target_node_idx = 3; // Index of target node in the lineage nodes list. - string target_node_column_id = 4; -} - -enum NodePosition { - NODE_POSITION_UNSPECIFIED = 0; - NODE_POSITION_START_NODE = 1; // Node is one of the requested start point. - NODE_POSITION_UPSTREAM = 2; // Node is upstream of the requested start point. - NODE_POSITION_DOWNSTREAM = 3; // Node is downstream of the requested start point. -} - -// Node in a lineage graph representing one or more entiities (e.g. database table). -message LineageNode { - // All entities which have the same identity as this node. Must be at least one item. - // These are sorted by closeness to the type of the start point entities. - // e.g. if requesting lineage of a DBT source, first entity should be from DBT, similarly when viewing table it will be other tables. - repeated core.v1.EntityRef entities = 1; - // Position of the node in the lineage. - NodePosition position = 2; - // Populated only for Column Level Lineage (CLL). - optional CllDetails cll_details = 3; -} - -message CllDetails { - // Column details for CLL. - repeated Column columns = 1; - // State of the CLL parse. UNSPECIFIED if CLL was not requested. - CllState cll_state = 2; - // Messages related to CLL. - // e.g. Description of parse errors, etc. - repeated string cll_messages = 3; -} - -// Column in a table-like asset (used in CLL mode). -message Column { - string column_id = 1; // ID string for the column. This is the parsed column name. - optional string name = 2; // Original column name as fetched from the table. - optional string native_type = 3; // Column type as fetched from the table. -} - -enum CllState { - // Unspecified state. - CLL_STATE_UNSPECIFIED = 0; - // Parsing of the asset SQL failed. No upstream dependencies can be found. - CLL_STATE_PARSE_FAILED = 1; - // Extraction of the asset SQL failed. Some unsupported SQL features may be used. Some details might be missing. - CLL_STATE_EXTRACTION_FAILED = 2; - // Not all columns or tables were found upstream, lineage is not complete. - CLL_STATE_RESOLUTION_FAILED = 3; - // No known issues present. - CLL_STATE_OK = 10; -} diff --git a/protos/schemas/v1/lineage_service.proto b/protos/schemas/v1/lineage_service.proto deleted file mode 100644 index 4a8b516..0000000 --- a/protos/schemas/v1/lineage_service.proto +++ /dev/null @@ -1,51 +0,0 @@ -syntax = "proto3"; - -package schemas.v1; - -import "schemas/v1/lineage.proto"; - -option go_package = "github.com/getsynq/api/schemas/v1"; - -import "core/v1/entity.proto"; - -// LineageService allows you to fetch: -// * Entity level lineage from a starting point of one or more entities. -// * Column Level lineage from a starting point of multiple columns of a single entity. -service LineageService { - rpc GetLineage(GetLineageRequest) returns (GetLineageResponse) {} -} - -message GetLineageRequest { - LineageDirection lineage_direction = 1; - GetLineageStartPoint start_point = 2; - optional int32 max_depth = 3; -} - -message GetLineageResponse { - Lineage lineage = 1; -} - -// Direction of the lineage to query. -enum LineageDirection { - LINEAGE_DIRECTION_UNSPECIFIED = 0; - LINEAGE_DIRECTION_UPSTREAM = 1; - LINEAGE_DIRECTION_DOWNSTREAM = 2; - LINEAGE_DIRECTION_UPSTREAM_DOWNSTREAM = 3; -} - -// Possible starting points to get lineage from. -message GetLineageStartPoint { - oneof from { - EntitiesStartPoint entities = 1; - EntityColumnsStartPoint entity_columns = 2; - } -} - -message EntitiesStartPoint { - repeated core.v1.EntityRef entities = 1; -} - -message EntityColumnsStartPoint { - core.v1.EntityRef entitiy = 1; - repeated string column_ids = 2; -} diff --git a/scripts/copy_protos.sh b/scripts/copy_protos.sh new file mode 100644 index 0000000..83c9335 --- /dev/null +++ b/scripts/copy_protos.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +if [ "$#" -ne 2 ]; then + echo "Need source and destination" +fi +source=$1 +destination=$2 + +set -e + +rm -rf $destination +mkdir -p $destination + +proto_files=`find $source -name *.proto -type f` +declare -a proto_dirs + +# find unique proto directories in source +for pf in $proto_files; do + pd=`dirname "${pf}"` + if ! [[ $(echo ${proto_dirs[@]} | fgrep -w $pd) ]]; then + proto_dirs+=("$pd") + fi +done + +# copy proto directories to same file path in destination +for pd in "${proto_dirs[@]}"; do + dest="${pd/$source/$destination}" + # ensure dest exists + mkdir -p $dest + echo "$pd -> $dest" + rsync -avW --include='*.proto' --no-compress "${pd}/" "${dest}/" --delete +done + +set +e +exit 0 diff --git a/scripts/generate_docs.sh b/scripts/generate_docs.sh new file mode 100644 index 0000000..d0c352d --- /dev/null +++ b/scripts/generate_docs.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +PROTOS_DIR="." +DOCS_DIR="./tmp" + +set -e + +VALID_ARGS=$(getopt -o p:d: --long protos:,docs: -- "$@") +if [[ $? -ne 0 ]]; then + exit 1; +fi + +eval set -- "$VALID_ARGS" +while [ : ]; do + case "$1" in + -p | --protos) + PROTOS_DIR=$2 + shift 2 + ;; + -d | --docs) + DOCS_DIR=$2 + shift 2 + ;; + --) shift; + break + ;; + esac +done + +set -e + +echo "Clearing docs directory -> $DOCS_DIR" +rm -rf $DOCS_DIR +mkdir -p $DOCS_DIR + +echo "Generating docs..." +module_dirs=`find $PROTOS_DIR -maxdepth 1 -mindepth 1 -type d` +for module_dir in $module_dirs; do + echo "processing $module_dir" + module=`basename $module_dir` + mkdir "${DOCS_DIR}/${module}" + version_dirs=`find $module_dir -maxdepth 1 -mindepth 1 -type d` + for version_dir in $version_dirs; do + version=`basename $version_dir` + proto_files=`find $version_dir -name *.proto -type f` + if [[ "${proto_files}" != "" ]]; then + protoc --proto_path=${PROTOS_DIR} \ + --doc_out="${DOCS_DIR}/${module}/" \ + --doc_opt=markdown,${version}.md \ + ${proto_files} + fi + done + if [ -z "$(ls -A ${DOCS_DIR}/${module})" ]; then + rm -r "${DOCS_DIR}/${module}" + fi +done + +set +e +exit 0 diff --git a/scripts/test_compile.sh b/scripts/test_compile.sh new file mode 100644 index 0000000..be5c4cb --- /dev/null +++ b/scripts/test_compile.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +PROTOS_DIR="." +GEN_DIR="./tmp" +DIRTY=false + +set -e + +VALID_ARGS=$(getopt -o dp:g: --long dirty,protos:,gen: -- "$@") +if [[ $? -ne 0 ]]; then + exit 1; +fi + +eval set -- "$VALID_ARGS" +while [ : ]; do + case "$1" in + -d | --dirty) + DIRTY=true + shift + ;; + -p | --protos) + PROTOS_DIR=$2 + shift 2 + ;; + -g | --gen) + GEN_DIR=$2 + shift 2 + ;; + --) shift; + break + ;; + esac +done + +echo "Clearing generated directory -> $GEN_DIR" +rm -rf $GEN_DIR +mkdir -p $GEN_DIR +DOCS_DIR="$GEN_DIR/_docs" +mkdir -p $DOCS_DIR + +echo "Compiling protos from -> $PROTOS_DIR" +module_dirs=`find $PROTOS_DIR -maxdepth 1 -mindepth 1 -type d` +for module_dir in $module_dirs; do + echo "processing $module_dir" + module=`basename $module_dir` + mkdir "${DOCS_DIR}/${module}" + version_dirs=`find $module_dir -maxdepth 1 -mindepth 1 -type d` + for version_dir in $version_dirs; do + version=`basename $version_dir` + proto_files=`find $version_dir -name *.proto -type f` + if [[ "${proto_files}" != "" ]]; then + protoc --proto_path=${PROTOS_DIR} \ + --go_out=${GEN_DIR} \ + --go-grpc_out=${GEN_DIR} \ + --doc_out=${DOCS_DIR}/${module} \ + --doc_opt=markdown,${version}.md \ + ${proto_files} + fi + done + if [ -z "$(ls -A ${DOCS_DIR}/${module})" ]; then + rm -r "${DOCS_DIR}/${module}" + fi +done + +if [ "$DIRTY" = false ] ; then + echo 'Removing generated directory.' + rm -rf $GEN_DIR +fi + +set +e +exit 0