From 97e3c6d2077c6c9628fe75507dee6e0b27c7b9b9 Mon Sep 17 00:00:00 2001 From: Lukasz Stefaniak Date: Fri, 23 Aug 2024 13:19:03 +0200 Subject: [PATCH] Publish latest changes --- .../datachecks/sqltests/v1/sql_tests.proto | 35 +++++ .../sqltests/v1/sql_tests_service.proto | 86 +++++++++++++ .../entities/custom/features/v1/code.proto | 32 +++++ .../features/v1/git_file_reference.proto | 34 +++++ .../entities/custom/features/v1/schema.proto | 88 +++++++++++++ .../custom/features/v1/sql_definition.proto | 41 ++++++ .../v1/checks_relationships_service.proto | 41 ++++++ .../entities/custom/v1/entities_service.proto | 63 +++++++++ .../entities/custom/v1/features_service.proto | 106 ++++++++++++++++ .../entities/custom/v1/groups_service.proto | 87 +++++++++++++ .../custom}/v1/relationships_service.proto | 35 +++-- .../entities/custom/v1/types_service.proto | 60 +++++++++ .../entities/v1/entities_service.proto | 52 ++++++++ .../v1/entity_executions_service.proto | 90 +++++++++++++ .../{ => entities}/lineage/v1/lineage.proto | 6 +- .../lineage/v1/lineage_service.proto | 12 +- .../status/v1/entity_incidents.proto | 4 +- .../status/v1/entity_incidents_service.proto | 16 +-- .../status/v1/entity_issues.proto | 4 +- .../status/v1/entity_issues_service.proto | 16 +-- protos/synq/entities/v1/annotation.proto | 34 +++++ .../synq/entities/v1/entities_service.proto | 96 -------------- .../v1/{entities.proto => entity.proto} | 37 +++++- .../{identifiers.proto => identifier.proto} | 9 +- protos/synq/entities/v1/type.proto | 52 ++++++++ protos/synq/ingest/dbt/v1/dbt_service.proto | 47 +++++++ .../ingest/sqlmesh/v1/sqlmesh_service.proto | 97 ++++++++++++++ protos/synq/issues/actor/v1/actor.proto | 43 +++++++ .../issues/commands/v1/issues_command.proto | 20 +++ .../issues/issues/v1/issues_service.proto | 120 ++++++++++++++++++ protos/synq/lineage/v1/relationships.proto | 13 -- protos/synq/platforms/v1/data_platforms.proto | 71 +++++++++++ protos/synq/v1/scope_authorization.proto | 57 +++++++-- protos/synq/webhooks/v1/callback.proto | 15 +++ protos/synq/webhooks/v1/event.proto | 46 +++++++ 35 files changed, 1492 insertions(+), 173 deletions(-) create mode 100644 protos/synq/datachecks/sqltests/v1/sql_tests.proto create mode 100644 protos/synq/datachecks/sqltests/v1/sql_tests_service.proto create mode 100644 protos/synq/entities/custom/features/v1/code.proto create mode 100644 protos/synq/entities/custom/features/v1/git_file_reference.proto create mode 100644 protos/synq/entities/custom/features/v1/schema.proto create mode 100644 protos/synq/entities/custom/features/v1/sql_definition.proto create mode 100644 protos/synq/entities/custom/v1/checks_relationships_service.proto create mode 100644 protos/synq/entities/custom/v1/entities_service.proto create mode 100644 protos/synq/entities/custom/v1/features_service.proto create mode 100644 protos/synq/entities/custom/v1/groups_service.proto rename protos/synq/{lineage => entities/custom}/v1/relationships_service.proto (57%) create mode 100644 protos/synq/entities/custom/v1/types_service.proto create mode 100644 protos/synq/entities/entities/v1/entities_service.proto create mode 100644 protos/synq/entities/executions/v1/entity_executions_service.proto rename protos/synq/{ => entities}/lineage/v1/lineage.proto (95%) rename protos/synq/{ => entities}/lineage/v1/lineage_service.proto (84%) rename protos/synq/{ => entities}/status/v1/entity_incidents.proto (87%) rename protos/synq/{ => entities}/status/v1/entity_incidents_service.proto (86%) rename protos/synq/{ => entities}/status/v1/entity_issues.proto (65%) rename protos/synq/{ => entities}/status/v1/entity_issues_service.proto (89%) create mode 100644 protos/synq/entities/v1/annotation.proto delete mode 100644 protos/synq/entities/v1/entities_service.proto rename protos/synq/entities/v1/{entities.proto => entity.proto} (61%) rename protos/synq/entities/v1/{identifiers.proto => identifier.proto} (98%) create mode 100644 protos/synq/entities/v1/type.proto create mode 100644 protos/synq/ingest/dbt/v1/dbt_service.proto create mode 100644 protos/synq/ingest/sqlmesh/v1/sqlmesh_service.proto create mode 100644 protos/synq/issues/actor/v1/actor.proto create mode 100644 protos/synq/issues/commands/v1/issues_command.proto create mode 100644 protos/synq/issues/issues/v1/issues_service.proto delete mode 100644 protos/synq/lineage/v1/relationships.proto create mode 100644 protos/synq/platforms/v1/data_platforms.proto create mode 100644 protos/synq/webhooks/v1/callback.proto create mode 100644 protos/synq/webhooks/v1/event.proto diff --git a/protos/synq/datachecks/sqltests/v1/sql_tests.proto b/protos/synq/datachecks/sqltests/v1/sql_tests.proto new file mode 100644 index 0000000..72ffa9c --- /dev/null +++ b/protos/synq/datachecks/sqltests/v1/sql_tests.proto @@ -0,0 +1,35 @@ +syntax = "proto3"; + +package synq.datachecks.sqltests.v1; + +import "buf/validate/validate.proto"; +import "synq/entities/v1/annotation.proto"; +import "synq/platforms/v1/data_platforms.proto"; + +option go_package = "github.com/getsynq/api/datachecks/sqltests/v1"; + +// The SqlTest is a SQL test that is executed on a synq entity. +message SqlTest { + // Identifier for the data platform which the SqlTest would be executed on. + platforms.v1.DataPlatformIdentifier platform = 1 [(buf.validate.field).required = true]; + + // Unique resource identifier for the SqlTest. This is externally maintained and can + // be used to fetch/update/delete this test. + string id = 2 [(buf.validate.field).required = true]; + + // Human friendly name. + string name = 3 [(buf.validate.field).required = true]; + + // A valid SQL expression which is the test. + string sql_expression = 4 [(buf.validate.field).required = true]; + + // [Recurrence rule](https://icalendar.org/iCalendar-RFC-5545/3-8-5-3-recurrence-rule.html) for the execution schedule of the SqlTest. + string recurrence_rule = 5; + + // Annotations for the given SqlTest.These help filter the SqlTests for later + // operations like listing by API, selection in UI, analytics, etc. + repeated entities.v1.Annotation annotations = 6; + + // Boolean flag to to toggle saving of failure runs. + bool save_failures = 7; +} diff --git a/protos/synq/datachecks/sqltests/v1/sql_tests_service.proto b/protos/synq/datachecks/sqltests/v1/sql_tests_service.proto new file mode 100644 index 0000000..3531e29 --- /dev/null +++ b/protos/synq/datachecks/sqltests/v1/sql_tests_service.proto @@ -0,0 +1,86 @@ +syntax = "proto3"; + +package synq.datachecks.sqltests.v1; + +import "synq/datachecks/sqltests/v1/sql_tests.proto"; +import "synq/entities/v1/annotation.proto"; +import "synq/v1/scope_authorization.proto"; + +option go_package = "github.com/getsynq/api/datachecks/sqltests/v1"; + +// SqlTestsService is a service for managing SqlTests. +service SqlTestsService { + // Upsert SqlTests based on a unique ID. + rpc BatchUpsertSqlTests(BatchUpsertSqlTestsRequest) returns (BatchUpsertSqlTestsResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_DATACHECKS_SQLTESTS_EDIT] + }; + } + + // List SqlTests for given annotations. + rpc ListSqlTests(ListSqlTestsRequest) returns (ListSqlTestsResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_DATACHECKS_SQLTESTS_READ] + }; + } + + // Get SqlTests by their IDs. + rpc BatchGetSqlTests(BatchGetSqlTestsRequest) returns (BatchGetSqlTestsResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_DATACHECKS_SQLTESTS_READ] + }; + } + + // Delete SqlTests by their IDs. + rpc BatchDeleteSqlTests(BatchDeleteSqlTestsRequest) returns (BatchDeleteSqlTestsResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_DATACHECKS_SQLTESTS_EDIT] + }; + } +} + +message BatchUpsertSqlTestsRequest { + // List of SqlTests to upsert. + // The upsert is performed based on the unique path provided for each SqlTest. + repeated SqlTest sql_tests = 1; +} + +message BatchUpsertSqlTestsResponse { + // IDs for which SqlTests were created without error. + repeated string created_ids = 1; + // IDs for which SqlTests were updated without error. + repeated string updated_ids = 2; + + // Details of errors encountered during the upsert operation. + message UpsertError { + string id = 1; + string reason = 2; + } + // Errors raised while upsert. This list will be empty if there were no errors. + repeated UpsertError errors = 3; +} + +message ListSqlTestsRequest { + // Optional annotations to fetch SqlTests for. + repeated entities.v1.Annotation annotations = 1; +} + +message ListSqlTestsResponse { + repeated SqlTest sql_tests = 1; +} + +message BatchGetSqlTestsRequest { + // IDs of the SqlTests to fetch. + repeated string ids = 1; +} + +message BatchGetSqlTestsResponse { + map sql_tests = 1; +} + +message BatchDeleteSqlTestsRequest { + // IDs of the SqlTests to delete. + repeated string ids = 1; +} + +message BatchDeleteSqlTestsResponse {} diff --git a/protos/synq/entities/custom/features/v1/code.proto b/protos/synq/entities/custom/features/v1/code.proto new file mode 100644 index 0000000..1a173d7 --- /dev/null +++ b/protos/synq/entities/custom/features/v1/code.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; + +package synq.entities.custom.features.v1; + +import "buf/validate/validate.proto"; + +option go_package = "github.com/getsynq/api/entities/custom/features/v1"; + +enum CodeType { + CODE_TYPE_UNSPECIFIED = 0; + CODE_TYPE_SQL = 1; + CODE_TYPE_PYTHON = 2; + CODE_TYPE_JSON = 3; +} + +// Code feature represents a code snippet associated with the entity. +// If support for code parsing and dependency extraction is needed use [SqlDefinition](sql_definition.proto). +message Code { + // Might be e.g. name of the file or a function. + string name = 1 [(buf.validate.field) = { + required: false, + string: {max_len: 50} + }]; + + // Type of the code. + CodeType code_type = 2; + + // Content of the code. Displayed in the UI. + string content = 3 [(buf.validate.field) = { + string: {max_len: 100000} + }]; +} diff --git a/protos/synq/entities/custom/features/v1/git_file_reference.proto b/protos/synq/entities/custom/features/v1/git_file_reference.proto new file mode 100644 index 0000000..dfb9f04 --- /dev/null +++ b/protos/synq/entities/custom/features/v1/git_file_reference.proto @@ -0,0 +1,34 @@ +syntax = "proto3"; + +package synq.entities.custom.features.v1; + +import "buf/validate/validate.proto"; + +option go_package = "github.com/getsynq/api/entities/custom/features/v1"; + +// GitFileReference represents a reference to a file in a git repository. +message GitFileReference { + // URL of the git repository. Preferably SSH clone URL. + string repository_url = 1 [(buf.validate.field) = { + required: true, + string: { + min_len: 1, + max_len: 255, + } + }]; + + // Name of the branch in the git repository. + string branch_name = 2 [(buf.validate.field) = { + required: true, + string: {max_bytes: 244} + }]; + + // Path to the file in the git repository. + string file_path = 3 [(buf.validate.field) = { + required: true, + string: { + min_len: 1, + max_len: 255, + } + }]; +} diff --git a/protos/synq/entities/custom/features/v1/schema.proto b/protos/synq/entities/custom/features/v1/schema.proto new file mode 100644 index 0000000..7d8fc95 --- /dev/null +++ b/protos/synq/entities/custom/features/v1/schema.proto @@ -0,0 +1,88 @@ +syntax = "proto3"; + +package synq.entities.custom.features.v1; + +import "buf/validate/validate.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/getsynq/api/entities/custom/features/v1"; + +message SchemaColumnField { + // Human readable name of the column as present in dbt or data warehouse. + string name = 1 [(buf.validate.field) = { + required: true, + string: { + min_len: 1, + max_len: 255 + } + }]; + // Native data type of the column as present in data warehouse. + string native_type = 2 [(buf.validate.field) = { + string: {max_len: 255} + }]; + // Description of the column + string description = 3 [(buf.validate.field) = { + string: {max_len: 1000} + }]; + // Ordinal position of the column in the struct, starting from 1 + int32 ordinal_position = 4 [(buf.validate.field) = { + int32: { + gte: 0, + lte: 1001 + } + }]; + // Indicates that the field type could be used as a struct/json in a data warehouse + bool is_struct = 5; + // Indicates that the field is a repeated field in a data warehouse (e.g. array) + bool is_repeated = 6; + // Fields inside of the struct/record like field + repeated SchemaColumnField fields = 7 [(buf.validate.field) = { + repeated: {max_items: 100} + }]; +} + +message SchemaColumn { + // Human readable name of the column as present in dbt or data warehouse. + string name = 1 [(buf.validate.field) = { + required: true, + string: { + min_len: 1, + max_len: 255 + } + }]; + // Native data type of the column as present in data warehouse. + string native_type = 2 [(buf.validate.field) = { + string: {max_len: 255} + }]; + // Description of the column + string description = 3 [(buf.validate.field) = { + string: {max_len: 1000} + }]; + // Ordinal position of the column in the table, starting from 1 + int32 ordinal_position = 4 [(buf.validate.field) = { + int32: { + gte: 0, + lte: 1001 + } + }]; + // Indicates that the column type could be used as a struct/json in a data warehouse + bool is_struct = 5; + // Indicates that the column is a repeated field in a data warehouse (e.g. array) + bool is_repeated = 6; + // Fields inside of the struct/record like column + repeated SchemaColumnField fields = 7 [(buf.validate.field) = { + repeated: {max_items: 100} + }]; +} + +// Schema represents a schema of a table like entity +message Schema { + // Time when the schema was defined, will default to `now` if not set. + google.protobuf.Timestamp state_at = 1; + // Columns of the schema + // Columns are ordered by ordinal_position, it has to be defined for all or none of the columns, + // when not defined, ordinal positions are attached according to the order of columns in the list. + repeated SchemaColumn columns = 2 [(buf.validate.field) = { + repeated: {max_items: 1000} + }]; +} diff --git a/protos/synq/entities/custom/features/v1/sql_definition.proto b/protos/synq/entities/custom/features/v1/sql_definition.proto new file mode 100644 index 0000000..046b3a5 --- /dev/null +++ b/protos/synq/entities/custom/features/v1/sql_definition.proto @@ -0,0 +1,41 @@ +syntax = "proto3"; + +package synq.entities.custom.features.v1; + +import "buf/validate/validate.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/getsynq/api/entities/custom/features/v1"; + +enum SqlDialect { + SQL_DIALECT_UNSPECIFIED = 0; + SQL_DIALECT_BIGQUERY = 1; + SQL_DIALECT_CLICKHOUSE = 2; + SQL_DIALECT_DATABRICKS = 3; + SQL_DIALECT_MYSQL = 4; + SQL_DIALECT_POSTGRESQL = 5; + SQL_DIALECT_REDSHIFT = 6; + SQL_DIALECT_SNOWFLAKE = 7; +} + +// SqlDefinition is a feature that allows to define lineage via SQL for a custom entity. +// Used to generate dependencies and other metadata. If parsed successfully, +// it will provide a code and column level lineage. Can be used to define simplified transformation +// of data which component performs using e.g. `INSERT INTO foo SELECT * FROM bar`. +message SqlDefinition { + //Time when the sql was defined, will default to `now` if not set. + google.protobuf.Timestamp state_at = 1; + + // SQL dialect used in the SQL. + SqlDialect dialect = 2 [(buf.validate.field) = { + enum: {defined_only: true} + }]; + + // Final form of the SQL as executed in the database/data warehouse. Must be a valid SQL in the selected dialect. + string sql = 3 [(buf.validate.field) = { + string: { + min_len: 0, + max_len: 1000000, + } + }]; +} diff --git a/protos/synq/entities/custom/v1/checks_relationships_service.proto b/protos/synq/entities/custom/v1/checks_relationships_service.proto new file mode 100644 index 0000000..686a364 --- /dev/null +++ b/protos/synq/entities/custom/v1/checks_relationships_service.proto @@ -0,0 +1,41 @@ +syntax = "proto3"; + +package synq.entities.custom.v1; + +import "buf/validate/validate.proto"; +import "synq/entities/v1/identifier.proto"; +import "synq/v1/scope_authorization.proto"; + +option go_package = "github.com/getsynq/api/entities/custom/v1"; + +service ChecksRelationshipsService { + rpc UpsertCheckRelationships(UpsertCheckRelationshipsRequest) returns (UpsertCheckRelationshipsResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_ENTITY_EDIT] + }; + } + + rpc DeleteCheckRelationships(DeleteCheckRelationshipsRequest) returns (DeleteCheckRelationshipsResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_ENTITY_EDIT] + }; + } +} + +message UpsertCheckRelationshipsRequest { + repeated CheckRelationship check_relationships = 1 [(buf.validate.field).required = true]; +} + +message UpsertCheckRelationshipsResponse {} + +message DeleteCheckRelationshipsRequest { + repeated CheckRelationship check_relationships = 1 [(buf.validate.field).required = true]; +} + +message DeleteCheckRelationshipsResponse {} + +message CheckRelationship { + synq.entities.v1.Identifier check = 1 [(buf.validate.field).required = true]; + synq.entities.v1.Identifier checked = 2 [(buf.validate.field).required = true]; + repeated string checked_columns = 3; +} diff --git a/protos/synq/entities/custom/v1/entities_service.proto b/protos/synq/entities/custom/v1/entities_service.proto new file mode 100644 index 0000000..8a61f30 --- /dev/null +++ b/protos/synq/entities/custom/v1/entities_service.proto @@ -0,0 +1,63 @@ +syntax = "proto3"; + +package synq.entities.custom.v1; + +import "buf/validate/validate.proto"; +import "synq/entities/v1/entity.proto"; +import "synq/entities/v1/identifier.proto"; +import "synq/v1/scope_authorization.proto"; + +option go_package = "github.com/getsynq/api/entities/custom/v1"; + +// custom.EntitiesService is a service for managing custom entities. Entities can represent +// various data platform concepts such as services, consumers, applications or data pipelines +// that are not natively available in Synq. +// +// Entities are identified by a unique identifier and can be created, updated, read and deleted. +// +service EntitiesService { + // Create or update an entity. If the entity does not exist, it will be created, otherwise it will be updated. Entities are identified and deduplicated by their Identifier in a scope of a given customer workspace. + rpc UpsertEntity(UpsertEntityRequest) returns (UpsertEntityResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_ENTITY_EDIT] + }; + } + + // Delete an entity by its identifier. If the entity does not exist, it will be a no-op. + rpc DeleteEntity(DeleteEntityRequest) returns (DeleteEntityResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_ENTITY_EDIT] + }; + } + + // List all entities. + rpc ListEntities(ListEntitiesRequest) returns (ListEntitiesResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_ENTITY_READ] + }; + } +} + +// UpsertEntityRequest is the request message for the UpsertEntity method. +message UpsertEntityRequest { + // The entity to create or update. + synq.entities.v1.Entity entity = 1 [(buf.validate.field).required = true]; +} + +// UpsertEntityResponse is the response message for the UpsertEntity method. +message UpsertEntityResponse {} + +// DeleteEntityRequest is the request message for the DeleteEntity method. +message DeleteEntityRequest { + // Identifier of the entity to delete. + synq.entities.v1.Identifier id = 1 [(buf.validate.field).required = true]; +} + +// DeleteEntityResponse is the response message for the DeleteEntity method. +message DeleteEntityResponse {} + +message ListEntitiesRequest {} + +message ListEntitiesResponse { + repeated synq.entities.v1.Entity entities = 1; +} diff --git a/protos/synq/entities/custom/v1/features_service.proto b/protos/synq/entities/custom/v1/features_service.proto new file mode 100644 index 0000000..fb0fa7e --- /dev/null +++ b/protos/synq/entities/custom/v1/features_service.proto @@ -0,0 +1,106 @@ +syntax = "proto3"; + +package synq.entities.custom.v1; + +import "buf/validate/validate.proto"; +import "google/protobuf/timestamp.proto"; +import "synq/entities/custom/features/v1/code.proto"; +import "synq/entities/custom/features/v1/git_file_reference.proto"; +import "synq/entities/custom/features/v1/schema.proto"; +import "synq/entities/custom/features/v1/sql_definition.proto"; +import "synq/entities/v1/identifier.proto"; +import "synq/v1/scope_authorization.proto"; + +option go_package = "github.com/getsynq/api/entities/custom/v1"; + +message Feature { + // Immutable workspace identifier that this entity belongs to. + string workspace = 1; + // The identifier of the entity to create or update feature for. + synq.entities.v1.Identifier entity_id = 2 [(buf.validate.field).required = true]; + // Identifier of the feature. Can be any string, e.g. "sql" for SQL feature. Use same identifier to update the feature. + // Do not use random identifier as it will create a new feature on every update. + string feature_id = 3 [(buf.validate.field) = { + required: true, + string: { + min_len: 1, + max_len: 255, + } + }]; + // Feature to define for the entity. Depending on the feature type, only one feature can be defined per entity. + oneof feature { + option (buf.validate.oneof).required = true; + // Only one feature per entity. + synq.entities.custom.features.v1.SqlDefinition sql_definition = 10; + // Multiple features per entity. + synq.entities.custom.features.v1.GitFileReference git_file_reference = 11; + // Multiple features per entity. + synq.entities.custom.features.v1.Code code = 12; + // Only one feature per entity. + synq.entities.custom.features.v1.Schema schema = 13; + } + + // Timestamp when the entity was created. + google.protobuf.Timestamp created_at = 5; + + // Timestamp when the entity was last updated. + google.protobuf.Timestamp updated_at = 6; + + // Timestamp when the entity was deleted. If this is set, the entity is considered deleted. + google.protobuf.Timestamp deleted_at = 7; +} + +service FeaturesService { + // Create or update an entity. If the entity does not exist, it will be created, otherwise it will be updated. Entities are identified and deduplicated by their Identifier in a scope of a given customer workspace. + rpc UpsertEntityFeature(UpsertEntityFeatureRequest) returns (UpsertEntityFeatureResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_ENTITY_EDIT] + }; + } + + // Delete an entity by its identifier. If the entity does not exist, it will be a no-op. + rpc DeleteEntityFeature(DeleteEntityFeatureRequest) returns (DeleteEntityFeatureResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_ENTITY_EDIT] + }; + } + + // List all entities. + rpc ListEntityFeatures(ListEntityFeaturesRequest) returns (ListEntityFeaturesResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_ENTITY_READ] + }; + } +} + +message UpsertEntityFeatureRequest { + // The feature to create or update. + Feature feature = 1 [(buf.validate.field).required = true]; +} + +message UpsertEntityFeatureResponse {} + +message DeleteEntityFeatureRequest { + // Identifier of the entity to delete feature from. + synq.entities.v1.Identifier entity_id = 1 [(buf.validate.field).required = true]; + // Identifier of the feature to delete. + string feature_id = 2 [(buf.validate.field) = { + required: true, + string: { + min_len: 1, + max_len: 255, + } + }]; +} + +message DeleteEntityFeatureResponse {} + +message ListEntityFeaturesRequest { + // Identifier of the entity to get features for. + synq.entities.v1.Identifier entity_id = 1 [(buf.validate.field).required = true]; +} + +message ListEntityFeaturesResponse { + // List of features for the entity. + repeated Feature features = 1; +} diff --git a/protos/synq/entities/custom/v1/groups_service.proto b/protos/synq/entities/custom/v1/groups_service.proto new file mode 100644 index 0000000..695ebdc --- /dev/null +++ b/protos/synq/entities/custom/v1/groups_service.proto @@ -0,0 +1,87 @@ +syntax = "proto3"; + +package synq.entities.custom.v1; + +import "buf/validate/validate.proto"; +import "google/protobuf/timestamp.proto"; +import "synq/entities/v1/identifier.proto"; +import "synq/v1/scope_authorization.proto"; + +option go_package = "github.com/getsynq/api/entities/custom/v1"; + +// GroupsService helps with management of life-cycle of Entities, more specifically their deletion. +// A Group is a identifyable collection of entities such as a specific test suite to manage custom +// tests, infrastructure project to manage custom infrastructure resources, etc. + +// It eliminates the need to keep state on client side to remember which assets were already created +// and which should be deleted. The server will keep track of the current state of the group and client +// can always send the intended new state. The server will calculate the diff and entities that are +// no longer present in the group will be removed. +// +// Example: +// 1. group has entities A, B, C at time t1 +// 2. client sends group with entities B, C, D at time t2 +// 3. server will remove entity A from the system and update the current state of the group to B, C, D +// +// The service is designed to be idempotent and can be called multiple times with the same state without +// causing any side effects. +service GroupsService { + // ListEntityGroups lists all groups. + rpc ListEntityGroups(ListEntityGroupsRequest) returns (ListEntityGroupsResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_ENTITY_READ] + }; + } + + // UpsertEntitiesGroup updates current definition of group to the server. As part of the upsert operation + // the difference of entity ids is calculated between past state and the current state and entities that are + // no longer part of the group are removed. + rpc UpsertEntitiesGroup(UpsertEntitiesGroupRequest) returns (UpsertEntitiesGroupResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_ENTITY_EDIT] + }; + } + + // DeleteEntitiesGroup deletes a group by its identifier. If the group does not exist, it will be a no-op. + rpc DeleteEntitiesGroup(DeleteEntitiesGroupRequest) returns (DeleteEntitiesGroupResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_ENTITY_EDIT] + }; + } +} + +message ListEntityGroupsRequest {} + +message ListEntityGroupsResponse { + repeated Group entities_groups = 1; +} + +message UpsertEntitiesGroupRequest { + Group group = 1 [(buf.validate.field).required = true]; + bool dry_run = 3; +} + +message UpsertEntitiesGroupResponse { + repeated synq.entities.v1.Identifier deleted_ids = 1; +} + +message DeleteEntitiesGroupRequest { + string group_id = 1 [(buf.validate.field).required = true]; +} + +message DeleteEntitiesGroupResponse {} + +// Group is a collection of entities. +message Group { + // Identifier of the group. + string group_id = 1 [(buf.validate.field).required = true]; + + // Identifiers of entities that belong to the group. + repeated synq.entities.v1.Identifier entity_ids = 2 [(buf.validate.field).required = true]; + + // Timestamp when the group was created. + google.protobuf.Timestamp created_at = 3; + + // Timestamp when the group was last updated. + google.protobuf.Timestamp updated_at = 4; +} diff --git a/protos/synq/lineage/v1/relationships_service.proto b/protos/synq/entities/custom/v1/relationships_service.proto similarity index 57% rename from protos/synq/lineage/v1/relationships_service.proto rename to protos/synq/entities/custom/v1/relationships_service.proto index 9b0afce..387da69 100644 --- a/protos/synq/lineage/v1/relationships_service.proto +++ b/protos/synq/entities/custom/v1/relationships_service.proto @@ -1,12 +1,12 @@ syntax = "proto3"; -package synq.lineage.v1; +package synq.entities.custom.v1; import "buf/validate/validate.proto"; -import "synq/lineage/v1/relationships.proto"; +import "synq/entities/v1/identifier.proto"; import "synq/v1/scope_authorization.proto"; -option go_package = "github.com/getsynq/api/lineage/v1"; +option go_package = "github.com/getsynq/api/entities/custom/v1"; // RelationshipsService allow management of relationships between entities. Relationships can // be created, updated, and deleted between 2 custom entities, or between a custom entity and Synq native entity.enum @@ -15,18 +15,21 @@ service RelationshipsService { // Create or update relationships between entities. If the relationship already exists, it will be updated, otherwise it will be created. rpc UpsertRelationships(UpsertRelationshipsRequest) returns (UpsertRelationshipsResponse) { option (synq.v1.scope_authorization) = { - scopes: [ - SCOPE_LINEAGE_EDIT - ] + scopes: [SCOPE_LINEAGE_EDIT] }; } // Delete relationships between entities. If the relationship does not exist, it will be ignored. rpc DeleteRelationships(DeleteRelationshipsRequest) returns (DeleteRelationshipsResponse) { option (synq.v1.scope_authorization) = { - scopes: [ - SCOPE_LINEAGE_EDIT - ] + scopes: [SCOPE_LINEAGE_EDIT] + }; + } + + // Delete relationships between entities. If the relationship does not exist, it will be ignored. + rpc ListRelationships(ListRelationshipsRequest) returns (ListRelationshipsResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_LINEAGE_READ] }; } } @@ -42,3 +45,17 @@ message DeleteRelationshipsRequest { } message DeleteRelationshipsResponse {} + +message Relationship { + synq.entities.v1.Identifier upstream = 1 [(buf.validate.field).required = true]; + synq.entities.v1.Identifier downstream = 2 [(buf.validate.field).required = true]; +} + +message ListRelationshipsRequest { + // Return relationships for the given entity_id. Either upstream or downstream. + repeated synq.entities.v1.Identifier ids = 1 [(buf.validate.field).required = true]; +} + +message ListRelationshipsResponse { + repeated Relationship relationships = 1; +} diff --git a/protos/synq/entities/custom/v1/types_service.proto b/protos/synq/entities/custom/v1/types_service.proto new file mode 100644 index 0000000..82416aa --- /dev/null +++ b/protos/synq/entities/custom/v1/types_service.proto @@ -0,0 +1,60 @@ +syntax = "proto3"; + +package synq.entities.custom.v1; + +import "buf/validate/validate.proto"; +import "synq/entities/v1/type.proto"; +import "synq/v1/scope_authorization.proto"; + +option go_package = "github.com/getsynq/api/entities/custom/v1"; + +// TypesService is a service for managing custom entity types. +service TypesService { + // Create or update an entity. If the entity does not exist, it will be created, otherwise it will be updated. Entities are identified and deduplicated by their Identifier in a scope of a given customer workspace. + rpc UpsertType(UpsertTypeRequest) returns (UpsertTypeResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_ENTITY_TYPE_EDIT] + }; + } + + // Delete an entity by its identifier. If the entity does not exist, it will be a no-op. + rpc DeleteType(DeleteTypeRequest) returns (DeleteTypeResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_ENTITY_TYPE_EDIT] + }; + } + + // List all entities. + rpc ListTypes(ListTypesRequest) returns (ListTypesResponse) { + option (synq.v1.scope_authorization) = { + scopes: [ + SCOPE_ENTITY_READ, + SCOPE_ENTITY_TYPE_READ + ] + }; + } +} + +message UpsertTypeRequest { + // Entity to create or update. + synq.entities.v1.Type type = 1 [(buf.validate.field).required = true]; +} +message UpsertTypeResponse {} + +message DeleteTypeRequest { + // Identifier of the custom entity type to delete. + int32 type_id = 2 [(buf.validate.field) = { + required: true, + int32: { + gte: 1, + lte: 1000 + } + }]; +} +message DeleteTypeResponse {} + +message ListTypesRequest {} +message ListTypesResponse { + // List of entities. + repeated synq.entities.v1.Type types = 1; +} diff --git a/protos/synq/entities/entities/v1/entities_service.proto b/protos/synq/entities/entities/v1/entities_service.proto new file mode 100644 index 0000000..9480faf --- /dev/null +++ b/protos/synq/entities/entities/v1/entities_service.proto @@ -0,0 +1,52 @@ +syntax = "proto3"; + +package synq.entities.entities.v1; + +import "buf/validate/validate.proto"; +import "synq/entities/v1/entity.proto"; +import "synq/entities/v1/identifier.proto"; +import "synq/v1/scope_authorization.proto"; + +option go_package = "github.com/getsynq/api/entities/entities/v1"; + +// EntitiesService is a service for retriving any entity. +// +service EntitiesService { + // Get specific entity by its identifier. + rpc GetEntity(GetEntityRequest) returns (GetEntityResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_ENTITY_READ] + }; + } + + // Get multiple entities by their identifiers. + rpc BatchGetEntities(BatchGetEntitiesRequest) returns (BatchGetEntitiesResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_ENTITY_READ] + }; + } +} + +// GetEntityRequest is the request message for the GetEntity method. +message GetEntityRequest { + // Identifier of the entity to get. + synq.entities.v1.Identifier id = 1 [(buf.validate.field).required = true]; +} + +// GetEntityResponse is the response message for the GetEntity method. +message GetEntityResponse { + // The entity that was retrieved. + synq.entities.v1.Entity entity = 1; +} + +// BatchGetEntitiesRequest is the request message for the BatchGetEntities method. +message BatchGetEntitiesRequest { + // Identifiers of the entities to get. + repeated synq.entities.v1.Identifier ids = 1 [(buf.validate.field).required = true]; +} + +// BatchGetEntitiesResponse is the response message for the BatchGetEntities method. +message BatchGetEntitiesResponse { + // The entities that were retrieved. + repeated synq.entities.v1.Entity entities = 1; +} diff --git a/protos/synq/entities/executions/v1/entity_executions_service.proto b/protos/synq/entities/executions/v1/entity_executions_service.proto new file mode 100644 index 0000000..78ee3e8 --- /dev/null +++ b/protos/synq/entities/executions/v1/entity_executions_service.proto @@ -0,0 +1,90 @@ +syntax = "proto3"; + +package synq.entities.executions.v1; + +import "buf/validate/validate.proto"; +import "google/protobuf/timestamp.proto"; +import "synq/entities/v1/annotation.proto"; +import "synq/entities/v1/identifier.proto"; +import "synq/v1/scope_authorization.proto"; + +option go_package = "github.com/getsynq/api/entities/executions/v1"; + +service EntityExecutionsService { + // Execution affects status of the custom entity + rpc UpsertExecution(UpsertExecutionRequest) returns (UpsertExecutionResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_EXECUTION_EDIT] + }; + } + + // LogEntry is a log message for the custom entity + rpc UpsertLogEntry(UpsertLogEntryRequest) returns (UpsertLogEntryResponse) { + option (synq.v1.scope_authorization) = { + scopes: [ + SCOPE_EXECUTION_EDIT, + SCOPE_EXECUTION_LOG_EDIT + ] + }; + } +} + +message UpsertExecutionRequest { + Execution execution = 1 [(buf.validate.field).required = true]; +} + +message UpsertExecutionResponse {} + +message Execution { + synq.entities.v1.Identifier id = 1 [(buf.validate.field).required = true]; + ExecutionStatus status = 2 [(buf.validate.field).required = true]; + string message = 3; + google.protobuf.Timestamp created_at = 4 [(buf.validate.field) = { + timestamp: { + lt_now: true, + gt: {seconds: 1717707428} + }, + required: true + }]; + google.protobuf.Timestamp started_at = 5; + google.protobuf.Timestamp finished_at = 6; + + repeated entities.v1.Annotation annotations = 7; + repeated ExecutionExtra extras = 8; +} + +message ExecutionExtra { + oneof extra { + string executed_sql = 1; + } +} + +enum ExecutionStatus { + EXECUTION_STATUS_UNSPECIFIED = 0; + EXECUTION_STATUS_OK = 1; + EXECUTION_STATUS_WARN = 2; + EXECUTION_STATUS_ERROR = 3; + EXECUTION_STATUS_CRITICAL = 4; +} + +message UpsertLogEntryRequest { + LogEntry log_entry = 1 [(buf.validate.field).required = true]; +} + +message UpsertLogEntryResponse {} + +message LogEntry { + synq.entities.v1.Identifier id = 1 [(buf.validate.field).required = true]; + string message = 2; + google.protobuf.Timestamp created_at = 3 [(buf.validate.field) = { + timestamp: { + lt_now: true, + gt: {seconds: 1717707428} + }, + required: true + }]; + google.protobuf.Timestamp started_at = 4; + google.protobuf.Timestamp finished_at = 5; + + repeated entities.v1.Annotation annotations = 6; +} diff --git a/protos/synq/lineage/v1/lineage.proto b/protos/synq/entities/lineage/v1/lineage.proto similarity index 95% rename from protos/synq/lineage/v1/lineage.proto rename to protos/synq/entities/lineage/v1/lineage.proto index 4133053..a686cd5 100644 --- a/protos/synq/lineage/v1/lineage.proto +++ b/protos/synq/entities/lineage/v1/lineage.proto @@ -1,10 +1,10 @@ syntax = "proto3"; -package synq.lineage.v1; +package synq.entities.lineage.v1; -import "synq/entities/v1/identifiers.proto"; +import "synq/entities/v1/identifier.proto"; -option go_package = "github.com/getsynq/api/lineage/v1"; +option go_package = "github.com/getsynq/api/entities/lineage/v1"; // Lineage defines the lineage of table-like entities. message Lineage { diff --git a/protos/synq/lineage/v1/lineage_service.proto b/protos/synq/entities/lineage/v1/lineage_service.proto similarity index 84% rename from protos/synq/lineage/v1/lineage_service.proto rename to protos/synq/entities/lineage/v1/lineage_service.proto index b4aa442..0925d3a 100644 --- a/protos/synq/lineage/v1/lineage_service.proto +++ b/protos/synq/entities/lineage/v1/lineage_service.proto @@ -1,12 +1,12 @@ syntax = "proto3"; -package synq.lineage.v1; +package synq.entities.lineage.v1; -import "synq/entities/v1/identifiers.proto"; -import "synq/lineage/v1/lineage.proto"; +import "synq/entities/lineage/v1/lineage.proto"; +import "synq/entities/v1/identifier.proto"; import "synq/v1/scope_authorization.proto"; -option go_package = "github.com/getsynq/api/lineage/v1"; +option go_package = "github.com/getsynq/api/entities/lineage/v1"; // LineageService allows you to fetch: // * Entity level lineage from a starting point of one or more entities. @@ -14,9 +14,7 @@ option go_package = "github.com/getsynq/api/lineage/v1"; service LineageService { rpc GetLineage(GetLineageRequest) returns (GetLineageResponse) { option (synq.v1.scope_authorization) = { - scopes: [ - SCOPE_LINEAGE_READ - ] + scopes: [SCOPE_LINEAGE_READ] }; } } diff --git a/protos/synq/status/v1/entity_incidents.proto b/protos/synq/entities/status/v1/entity_incidents.proto similarity index 87% rename from protos/synq/status/v1/entity_incidents.proto rename to protos/synq/entities/status/v1/entity_incidents.proto index cfb8cef..7543b8f 100644 --- a/protos/synq/status/v1/entity_incidents.proto +++ b/protos/synq/entities/status/v1/entity_incidents.proto @@ -1,10 +1,10 @@ syntax = "proto3"; -package synq.status.v1; +package synq.entities.status.v1; import "google/protobuf/timestamp.proto"; -option go_package = "github.com/getsynq/api/status/v1"; +option go_package = "github.com/getsynq/api/entities/status/v1"; message EntityIncident { // Unique identifier of the incident. diff --git a/protos/synq/status/v1/entity_incidents_service.proto b/protos/synq/entities/status/v1/entity_incidents_service.proto similarity index 86% rename from protos/synq/status/v1/entity_incidents_service.proto rename to protos/synq/entities/status/v1/entity_incidents_service.proto index 79d830f..cf2e3d5 100644 --- a/protos/synq/status/v1/entity_incidents_service.proto +++ b/protos/synq/entities/status/v1/entity_incidents_service.proto @@ -1,13 +1,13 @@ syntax = "proto3"; -package synq.status.v1; +package synq.entities.status.v1; import "buf/validate/validate.proto"; -import "synq/entities/v1/identifiers.proto"; -import "synq/status/v1/entity_incidents.proto"; +import "synq/entities/status/v1/entity_incidents.proto"; +import "synq/entities/v1/identifier.proto"; import "synq/v1/scope_authorization.proto"; -option go_package = "github.com/getsynq/api/status/v1"; +option go_package = "github.com/getsynq/api/entities/status/v1"; // EntityIncidentsService is the service which retrieves entity status. // @@ -15,18 +15,14 @@ service EntityIncidentsService { // Get specific entity status by its identifier. rpc GetIncidents(GetIncidentsRequest) returns (GetIncidentsResponse) { option (synq.v1.scope_authorization) = { - scopes: [ - SCOPE_STATUS_READ - ] + scopes: [SCOPE_STATUS_READ] }; } // Get multiple entity statuses by their identifiers. rpc BatchGetIncidents(BatchGetIncidentsRequest) returns (BatchGetIncidentsResponse) { option (synq.v1.scope_authorization) = { - scopes: [ - SCOPE_STATUS_READ - ] + scopes: [SCOPE_STATUS_READ] }; } } diff --git a/protos/synq/status/v1/entity_issues.proto b/protos/synq/entities/status/v1/entity_issues.proto similarity index 65% rename from protos/synq/status/v1/entity_issues.proto rename to protos/synq/entities/status/v1/entity_issues.proto index 91ec829..3a3b075 100644 --- a/protos/synq/status/v1/entity_issues.proto +++ b/protos/synq/entities/status/v1/entity_issues.proto @@ -1,8 +1,8 @@ syntax = "proto3"; -package synq.status.v1; +package synq.entities.status.v1; -option go_package = "github.com/getsynq/api/status/v1"; +option go_package = "github.com/getsynq/api/entities/status/v1"; enum IssueStatus { ISSUE_STATUS_UNSPECIFIED = 0; diff --git a/protos/synq/status/v1/entity_issues_service.proto b/protos/synq/entities/status/v1/entity_issues_service.proto similarity index 89% rename from protos/synq/status/v1/entity_issues_service.proto rename to protos/synq/entities/status/v1/entity_issues_service.proto index 25a8894..dbb7bd5 100644 --- a/protos/synq/status/v1/entity_issues_service.proto +++ b/protos/synq/entities/status/v1/entity_issues_service.proto @@ -1,13 +1,13 @@ syntax = "proto3"; -package synq.status.v1; +package synq.entities.status.v1; import "buf/validate/validate.proto"; -import "synq/entities/v1/identifiers.proto"; -import "synq/status/v1/entity_issues.proto"; +import "synq/entities/status/v1/entity_issues.proto"; +import "synq/entities/v1/identifier.proto"; import "synq/v1/scope_authorization.proto"; -option go_package = "github.com/getsynq/api/status/v1"; +option go_package = "github.com/getsynq/api/entities/status/v1"; // EntityIssuesService is the service which retrieves entity issues status. // @@ -15,18 +15,14 @@ service EntityIssuesService { // Get specific entity status by its identifier. rpc GetIssuesStatus(GetIssuesStatusRequest) returns (GetIssuesStatusResponse) { option (synq.v1.scope_authorization) = { - scopes: [ - SCOPE_STATUS_READ - ] + scopes: [SCOPE_STATUS_READ] }; } // Get multiple entity statuses by their identifiers. rpc BatchGetIssuesStatus(BatchGetIssuesStatusRequest) returns (BatchGetIssuesStatusResponse) { option (synq.v1.scope_authorization) = { - scopes: [ - SCOPE_STATUS_READ - ] + scopes: [SCOPE_STATUS_READ] }; } } diff --git a/protos/synq/entities/v1/annotation.proto b/protos/synq/entities/v1/annotation.proto new file mode 100644 index 0000000..8a955f8 --- /dev/null +++ b/protos/synq/entities/v1/annotation.proto @@ -0,0 +1,34 @@ +syntax = "proto3"; + +package synq.entities.v1; + +import "buf/validate/validate.proto"; + +option go_package = "github.com/getsynq/api/entities/v1"; + +// Annotations can be used to annotate any entity with a key:value pair. +// These annotations can be used for filtering and searching entities. +message Annotation { + // String key for the annotation. + string name = 1 [(buf.validate.field) = { + required: true, + string: { + min_len: 1, + max_len: 100 + } + }]; + + // Optional list of values that the annotation can carry. + repeated string values = 2 [(buf.validate.field) = { + repeated: { + min_items: 0, + max_items: 10, + items: { + string: { + min_len: 1, + max_len: 100 + } + } + } + }]; +} diff --git a/protos/synq/entities/v1/entities_service.proto b/protos/synq/entities/v1/entities_service.proto deleted file mode 100644 index 23b0346..0000000 --- a/protos/synq/entities/v1/entities_service.proto +++ /dev/null @@ -1,96 +0,0 @@ -syntax = "proto3"; - -package synq.entities.v1; - -import "buf/validate/validate.proto"; -import "synq/entities/v1/entities.proto"; -import "synq/entities/v1/identifiers.proto"; -import "synq/v1/scope_authorization.proto"; - -option go_package = "github.com/getsynq/api/entities/v1"; - -// EntitiesService is a service for managing custom entities. Entities can represent -// various data platform concepts such as services, consumers, applications or data pipelines -// that are not natively available in Synq. -// -// Entities are identified by a unique identifier and can be created, updated, read and deleted. -// -service EntitiesService { - // Get specific entity by its identifier. - rpc GetEntity(GetEntityRequest) returns (GetEntityResponse) { - option (synq.v1.scope_authorization) = { - scopes: [ - SCOPE_ENTITY_READ - ] - }; - } - - // Get multiple entities by their identifiers. - rpc BatchGetEntities(BatchGetEntitiesRequest) returns (BatchGetEntitiesResponse) { - option (synq.v1.scope_authorization) = { - scopes: [ - SCOPE_ENTITY_READ - ] - }; - } - - // Create or update an entity. If the entity does not exist, it will be created, otherwise it will be updated. Entities are identified and deduplicated by their Identifier in a scope of a given customer workspace. - rpc UpsertEntity(UpsertEntityRequest) returns (UpsertEntityResponse) { - option (synq.v1.scope_authorization) = { - scopes: [ - SCOPE_ENTITY_EDIT - ] - }; - } - - // Delete an entity by its identifier. If the entity does not exist, it will be a no-op. - rpc DeleteEntity(DeleteEntityRequest) returns (DeleteEntityResponse) { - option (synq.v1.scope_authorization) = { - scopes: [ - SCOPE_ENTITY_EDIT - ] - }; - } -} - -// GetEntityRequest is the request message for the GetEntity method. -message GetEntityRequest { - // Identifier of the entity to get. - Identifier id = 1 [(buf.validate.field).required = true]; -} - -// GetEntityResponse is the response message for the GetEntity method. -message GetEntityResponse { - // The entity that was retrieved. - Entity entity = 1; -} - -// BatchGetEntitiesRequest is the request message for the BatchGetEntities method. -message BatchGetEntitiesRequest { - // Identifiers of the entities to get. - repeated Identifier ids = 1 [(buf.validate.field).required = true]; -} - -// BatchGetEntitiesResponse is the response message for the BatchGetEntities method. -message BatchGetEntitiesResponse { - // The entities that were retrieved. - repeated Entity entities = 1; -} - -// UpsertEntityRequest is the request message for the UpsertEntity method. -message UpsertEntityRequest { - // The entity to create or update. - entities.v1.Entity entity = 1 [(buf.validate.field).required = true]; -} - -// UpsertEntityResponse is the response message for the UpsertEntity method. -message UpsertEntityResponse {} - -// DeleteEntityRequest is the request message for the DeleteEntity method. -message DeleteEntityRequest { - // Identifier of the entity to delete. - Identifier id = 1 [(buf.validate.field).required = true]; -} - -// DeleteEntityResponse is the response message for the DeleteEntity method. -message DeleteEntityResponse {} diff --git a/protos/synq/entities/v1/entities.proto b/protos/synq/entities/v1/entity.proto similarity index 61% rename from protos/synq/entities/v1/entities.proto rename to protos/synq/entities/v1/entity.proto index 9e520e7..ae87167 100644 --- a/protos/synq/entities/v1/entities.proto +++ b/protos/synq/entities/v1/entity.proto @@ -4,7 +4,8 @@ package synq.entities.v1; import "buf/validate/validate.proto"; import "google/protobuf/timestamp.proto"; -import "synq/entities/v1/identifiers.proto"; +import "synq/entities/v1/annotation.proto"; +import "synq/entities/v1/identifier.proto"; option go_package = "github.com/getsynq/api/entities/v1"; @@ -17,11 +18,30 @@ message Entity { // Unique identifier for this entity. Identifier id = 2 [(buf.validate.field).required = true]; + // Id of custom type or default type when 0 + int32 type_id = 11 [(buf.validate.field) = { + int32: { + gte: 0, + lte: 1000 + } + }]; + // Name of the entity that is displayed to the user. - string name = 3 [(buf.validate.field).required = true]; + string name = 3 [(buf.validate.field) = { + required: true, + string: { + min_len: 1, + max_len: 100 + } + }]; - // Description of the entity that is displayed in relevant contexts. - string description = 4; + // Description of the entity that is displayed in relevant contexts. Can contain Markdown. + string description = 4 [(buf.validate.field) = { + string: { + min_len: 0, + max_len: 10000 + } + }]; // Timestamp when the entity was created. google.protobuf.Timestamp created_at = 5; @@ -36,4 +56,13 @@ message Entity { string synq_path = 8; // Return only value. Synq URL to the entity. This is a unique URL that can be used to access the entity in the Synq's UI. string synq_catalog_url = 9; + + // Annotations attached to the entity. Annotations are key-value pairs that can be used to store metadata about the entity. + repeated entities.v1.Annotation annotations = 10 [(buf.validate.field) = { + repeated: { + min_items: 0, + max_items: 20, + items: {required: true} + } + }]; } diff --git a/protos/synq/entities/v1/identifiers.proto b/protos/synq/entities/v1/identifier.proto similarity index 98% rename from protos/synq/entities/v1/identifiers.proto rename to protos/synq/entities/v1/identifier.proto index 6bc18b1..1fa631f 100644 --- a/protos/synq/entities/v1/identifiers.proto +++ b/protos/synq/entities/v1/identifier.proto @@ -164,6 +164,7 @@ message ClickhouseTableIdentifier { } message DatabricksTableIdentifier { + // URL of Databricks workspace string workspace = 1 [(buf.validate.field).required = true]; // Databricks catalog @@ -211,10 +212,10 @@ message CustomIdentifier { message DataproductIdentifier { // Dataproduct id that identifies the Dataproduct - string id = 1 [ - (buf.validate.field).required = true, - (buf.validate.field).string.uuid = true - ]; + string id = 1 [(buf.validate.field) = { + required: true, + string: {uuid: true} + }]; } // diff --git a/protos/synq/entities/v1/type.proto b/protos/synq/entities/v1/type.proto new file mode 100644 index 0000000..adb4ea0 --- /dev/null +++ b/protos/synq/entities/v1/type.proto @@ -0,0 +1,52 @@ +syntax = "proto3"; + +package synq.entities.v1; + +import "buf/validate/validate.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/getsynq/api/entities/v1"; + +// Custom entity type that is used to categorize custom entities. +message Type { + // Immutable workspace identifier that this custom entity type belongs to. + string workspace = 1; + + int32 type_id = 2 [(buf.validate.field) = { + required: true, + int32: { + gte: 1, + lte: 1000 + } + }]; + + // Name of the custom entity type that is displayed to the user. + string name = 3 [(buf.validate.field) = { + required: true, + string: { + min_len: 1, + max_len: 20 + } + }]; + + // SVG icon of the custom entity type that is displayed to the user. + bytes svg_icon = 4 [(buf.validate.field) = { + required: true, + bytes: { + min_len: 1, + max_len: 1048576 + } + }]; + + // SYNQ identifier used for CDN purposes, read-only. + string svg_icon_id = 5; + + // Timestamp when the custom entity type was created. + google.protobuf.Timestamp created_at = 6; + + // Timestamp when the custom entity type was last updated. + google.protobuf.Timestamp updated_at = 7; + + // Timestamp when the custom entity type was deleted. If this is set, the custom entity type is considered deleted. + google.protobuf.Timestamp deleted_at = 8; +} diff --git a/protos/synq/ingest/dbt/v1/dbt_service.proto b/protos/synq/ingest/dbt/v1/dbt_service.proto new file mode 100644 index 0000000..44d2897 --- /dev/null +++ b/protos/synq/ingest/dbt/v1/dbt_service.proto @@ -0,0 +1,47 @@ +syntax = "proto3"; + +package synq.ingest.dbt.v1; + +import "synq/v1/scope_authorization.proto"; + +option go_package = "github.com/getsynq/api/ingest/dbt/v1"; + +//https://docs.getdbt.com/reference/artifacts/dbt-artifacts +message DbtArtifact { + oneof artifact { + bytes manifest_json = 1; + bytes run_results_json = 2; + bytes catalog_json = 3; + bytes sources_json = 4; + bytes semantic_manifest_json = 5; + } +} + +message IngestInvocationRequest { + // Arguments passed to the dbt executable + repeated string args = 1; + // Exit code of the dbt command + int32 exit_code = 2; + // Standard output log of the dbt command + bytes std_out = 3; + // Standard error log of the dbt command + bytes std_err = 4; + // Environment variables collected during execution + map environment_vars = 5; + // Artifacts generated by the dbt command + repeated DbtArtifact artifacts = 6; + // Version of the uploader tool + string uploader_version = 7; + // Build time of the uploader tool + string uploader_build_time = 8; +} + +message IngestInvocationResponse {} + +service DbtService { + rpc IngestInvocation(IngestInvocationRequest) returns (IngestInvocationResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_INGEST_DBT] + }; + } +} diff --git a/protos/synq/ingest/sqlmesh/v1/sqlmesh_service.proto b/protos/synq/ingest/sqlmesh/v1/sqlmesh_service.proto new file mode 100644 index 0000000..da30c3a --- /dev/null +++ b/protos/synq/ingest/sqlmesh/v1/sqlmesh_service.proto @@ -0,0 +1,97 @@ +syntax = "proto3"; + +package synq.ingest.sqlmesh.v1; + +import "buf/validate/validate.proto"; +import "google/protobuf/timestamp.proto"; +import "synq/v1/scope_authorization.proto"; + +option go_package = "github.com/getsynq/api/ingest/sqlmesh/v1"; + +message GitContext { + string clone_url = 1; + string branch = 2; + string commit_sha = 3; +} + +message IngestMetadataRequest { + // Data from /api/meta + bytes api_meta = 1; + // Data from /api/models + bytes models = 2; + // Data from /api/models/{model_id} + map model_details = 3; + // Data from /api/lineage/{model_id} + map model_lineage = 4; + // Data from /api/files + bytes files = 5; + // Data from /api/file/{file_path} + map file_content = 10; + // Data from /api/environments + bytes environments = 6; + // Version of the uploader tool + string uploader_version = 7; + // Build time of the uploader tool + string uploader_build_time = 8; + // Time at which state was collected + google.protobuf.Timestamp state_at = 9 [(buf.validate.field) = { + timestamp: { + lt_now: true, + gt: {seconds: 1717707428} + }, + required: true + }]; + // Git context of the project + GitContext git_context = 11; +} +message IngestMetadataResponse {} + +message IngestExecutionRequest { + // Command passed to the sqlmesh tool + repeated string command = 1; + // Exit code of the sqlmesh command + int32 exit_code = 2; + // Standard output log of the SqlMesh command + bytes std_out = 3; + // Standard error log of the SqlMesh command + bytes std_err = 4; + // Environment variables collected during execution + map environment_vars = 5; + // Version of the uploader tool + string uploader_version = 7; + // Build time of the uploader tool + string uploader_build_time = 8; + // Time at which the execution started + google.protobuf.Timestamp started_at = 9 [(buf.validate.field) = { + timestamp: { + lt_now: true, + gt: {seconds: 1717707428} + }, + required: true + }]; + // Time at which the execution finished + google.protobuf.Timestamp finished_at = 10 [(buf.validate.field).timestamp.lt_now = true]; + // Git context of the project + GitContext git_context = 11; + + option (buf.validate.message).cel = { + id: "ingest_execution_request.finished_at", + message: "Finished at must be greater than or equal to started at.", + expression: "this.finished_at >= this.started_at" + }; +} + +message IngestExecutionResponse {} + +service SqlMeshService { + rpc IngestMetadata(IngestMetadataRequest) returns (IngestMetadataResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_INGEST_SQLMESH] + }; + } + rpc IngestExecution(IngestExecutionRequest) returns (IngestExecutionResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_INGEST_SQLMESH] + }; + } +} diff --git a/protos/synq/issues/actor/v1/actor.proto b/protos/synq/issues/actor/v1/actor.proto new file mode 100644 index 0000000..80cce2d --- /dev/null +++ b/protos/synq/issues/actor/v1/actor.proto @@ -0,0 +1,43 @@ +syntax = "proto3"; + +package synq.issues.actor.v1; + +import "buf/validate/validate.proto"; + +option go_package = "github.com/getsynq/api/issues/actor/v1"; + +message Actor { + string name = 1 [(buf.validate.field) = { + string: {min_len: 1} + required: true; + }]; + oneof user { + SlackUser slack = 2; + EmailUser email = 3; + PagerdutyUser pagerduty = 4; + } +} + +message SlackUser { + string user_id = 1 [(buf.validate.field) = { + string: {min_len: 1} + required: true; + }]; +} + +message EmailUser { + string user_email = 1 [(buf.validate.field) = { + string: { + min_len: 1; + email: true; + } + required: true; + }]; +} + +message PagerdutyUser { + string user_id = 1 [(buf.validate.field) = { + string: {min_len: 1} + required: true; + }]; +} diff --git a/protos/synq/issues/commands/v1/issues_command.proto b/protos/synq/issues/commands/v1/issues_command.proto new file mode 100644 index 0000000..329a0eb --- /dev/null +++ b/protos/synq/issues/commands/v1/issues_command.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; + +package synq.issues.commands.v1; + +import "buf/validate/validate.proto"; +import "synq/issues/issues/v1/issues_service.proto"; + +option go_package = "github.com/getsynq/api/issues/commands/v1"; + +// Not to be used directly. Use the IssuesService instead when calling via API. +message IssuesCommand { + string workspace = 1 [(buf.validate.field) = {required: true}]; + oneof action { + synq.issues.issues.v1.MarkInvestigatingRequest mark_investigating = 10; + synq.issues.issues.v1.MarkFixedRequest mark_fixed = 11; + synq.issues.issues.v1.MarkExpectedRequest mark_expected = 12; + synq.issues.issues.v1.MarkNoActionNeededRequest mark_no_action_needed = 13; + synq.issues.issues.v1.PostCommentRequest post_comment = 14; + } +} diff --git a/protos/synq/issues/issues/v1/issues_service.proto b/protos/synq/issues/issues/v1/issues_service.proto new file mode 100644 index 0000000..72d7eef --- /dev/null +++ b/protos/synq/issues/issues/v1/issues_service.proto @@ -0,0 +1,120 @@ +syntax = "proto3"; + +package synq.issues.issues.v1; + +import "buf/validate/validate.proto"; +import "synq/issues/actor/v1/actor.proto"; +import "synq/v1/scope_authorization.proto"; + +option go_package = "github.com/getsynq/api/issues/issues/v1"; + +// IssuesService is a service for managing Issues. +service IssuesService { + // Mark issue as being investigated. + rpc MarkInvestigating(MarkInvestigatingRequest) returns (MarkInvestigatingResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_ISSUES_EDIT] + }; + } + + // Mark issue as fixed. + rpc MarkFixed(MarkFixedRequest) returns (MarkFixedResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_ISSUES_EDIT] + }; + } + + // Mark issue as expected. + rpc MarkExpected(MarkExpectedRequest) returns (MarkExpectedResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_ISSUES_EDIT] + }; + } + + // Mark issue as no action needed. + rpc MarkNoActionNeeded(MarkNoActionNeededRequest) returns (MarkNoActionNeededResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_ISSUES_EDIT] + }; + } + + // Post a comment on an issue. + rpc PostComment(PostCommentRequest) returns (PostCommentResponse) { + option (synq.v1.scope_authorization) = { + scopes: [SCOPE_ISSUES_EDIT] + }; + } +} + +message MarkInvestigatingRequest { + // ID of the issue to mark as investigating. + string issue_id = 1 [(buf.validate.field) = { + string: {uuid: true} + required: true; + }]; + + // Actor marking the issue as investigating. + synq.issues.actor.v1.Actor actor = 2 [(buf.validate.field) = {required: true}]; +} + +message MarkInvestigatingResponse {} + +message MarkFixedRequest { + // ID of the issue to mark as fixed. + string issue_id = 1 [(buf.validate.field) = { + string: {uuid: true} + required: true; + }]; + + // Actor marking the issue as fixed. + synq.issues.actor.v1.Actor actor = 2 [(buf.validate.field) = {required: true}]; +} + +message MarkFixedResponse {} + +message MarkExpectedRequest { + // ID of the issue to mark as expected. + string issue_id = 1 [(buf.validate.field) = { + string: {uuid: true} + required: true; + }]; + + // Actor marking the issue as expected. + synq.issues.actor.v1.Actor actor = 2 [(buf.validate.field) = {required: true}]; +} + +message MarkExpectedResponse {} + +message MarkNoActionNeededRequest { + // ID of the issue to mark as no action needed. + string issue_id = 1 [(buf.validate.field) = { + string: {uuid: true} + required: true; + }]; + + // Actor marking the issue as no action needed. + synq.issues.actor.v1.Actor actor = 2 [(buf.validate.field) = {required: true}]; +} + +message MarkNoActionNeededResponse {} + +message PostCommentRequest { + // ID of the issue to post a comment on. + string issue_id = 1 [(buf.validate.field) = { + string: {uuid: true} + required: true; + }]; + + // Actor posting the comment. + synq.issues.actor.v1.Actor actor = 2 [(buf.validate.field) = {required: true}]; + + // Comment to post. + string comment = 3 [(buf.validate.field) = {required: true}]; +} + +message PostCommentResponse { + string comment_id = 1 [(buf.validate.field) = { + string: {uuid: true} + required: true; + }]; +} diff --git a/protos/synq/lineage/v1/relationships.proto b/protos/synq/lineage/v1/relationships.proto deleted file mode 100644 index 416675b..0000000 --- a/protos/synq/lineage/v1/relationships.proto +++ /dev/null @@ -1,13 +0,0 @@ -syntax = "proto3"; - -package synq.lineage.v1; - -import "buf/validate/validate.proto"; -import "synq/entities/v1/identifiers.proto"; - -option go_package = "github.com/getsynq/api/lineage/v1"; - -message Relationship { - entities.v1.Identifier upstream = 1 [(buf.validate.field).required = true]; - entities.v1.Identifier downstream = 2 [(buf.validate.field).required = true]; -} diff --git a/protos/synq/platforms/v1/data_platforms.proto b/protos/synq/platforms/v1/data_platforms.proto new file mode 100644 index 0000000..155b705 --- /dev/null +++ b/protos/synq/platforms/v1/data_platforms.proto @@ -0,0 +1,71 @@ +syntax = "proto3"; + +package synq.platforms.v1; + +import "buf/validate/validate.proto"; + +option go_package = "github.com/getsynq/api/platforms/v1"; + +// DataPlatformIdentifier is a unique reference to a data platform integrated with Synq. +message DataPlatformIdentifier { + oneof id { + option (buf.validate.oneof).required = true; + UnknownDataPlatform unknown = 1; + BigqueryIdentifier bigquery = 2; + ClickhouseIdentifier clickhouse = 3; + SnowflakeIdentifier snowflake = 4; + RedshiftIdentifier redshift = 5; + PostgresIdentifier postgres = 6; + MysqlIdentifier mysql = 7; + DatabricksIdentifier databricks = 8; + } +} + +message UnknownDataPlatform {} + +message BigqueryIdentifier { + // BigQuery project + string project = 1 [(buf.validate.field).required = true]; +} + +message ClickhouseIdentifier { + // Clickhouse host inclusive of port + string host = 1 [(buf.validate.field).required = true]; + + // Clickhouse database + string schema = 2 [(buf.validate.field).required = true]; +} + +message SnowflakeIdentifier { + // Snowflake account + string account = 1 [(buf.validate.field).required = true]; + + // Snowflake database + string database = 2 [(buf.validate.field).required = true]; +} + +message RedshiftIdentifier { + // Redshift cluster + string cluster = 1 [(buf.validate.field).required = true]; + + // Redshift database + string database = 2 [(buf.validate.field).required = true]; +} + +message PostgresIdentifier { + // Postgres host inclusive of port + string host = 1 [(buf.validate.field).required = true]; + + // Postgres database + string database = 2 [(buf.validate.field).required = true]; +} + +message MysqlIdentifier { + // Mysql host inclusive of port + string host = 1 [(buf.validate.field).required = true]; +} + +message DatabricksIdentifier { + // URl of the databricks workspace + string workspace = 1 [(buf.validate.field).required = true]; +} diff --git a/protos/synq/v1/scope_authorization.proto b/protos/synq/v1/scope_authorization.proto index 3e4ad45..dd334e2 100644 --- a/protos/synq/v1/scope_authorization.proto +++ b/protos/synq/v1/scope_authorization.proto @@ -1,3 +1,7 @@ +// Generated by dev-tools; DO NOT EDIT. +// Modify or add scopes in proto/core/permissions/v1/scopes.proto +// Modify template at dev-tools/contracts/templates/scope_authorization.proto.tpl + syntax = "proto3"; package synq.v1; @@ -9,19 +13,46 @@ extend google.protobuf.MethodOptions { ScopeAuthorization scope_authorization = 50010; } +message ScopeAuthorization { + repeated Scope scopes = 1; +} + enum Scope { + SCOPE_UNSPECIFIED = 0; - - SCOPE_ENTITY_READ = 1; - SCOPE_ENTITY_EDIT = 2; - - SCOPE_LINEAGE_READ = 10; - SCOPE_LINEAGE_EDIT = 11; - - SCOPE_STATUS_READ = 20; + + SCOPE_ISSUES_EDIT = 10; + + SCOPE_ISSUES_READ = 11; + + SCOPE_ENTITY_EDIT = 30; + + SCOPE_ENTITY_READ = 31; + + SCOPE_ENTITY_TYPE_EDIT = 32; + + SCOPE_ENTITY_TYPE_READ = 33; + + SCOPE_LINEAGE_EDIT = 34; + + SCOPE_LINEAGE_READ = 35; + + SCOPE_STATUS_READ = 40; + + SCOPE_INGEST_DBT = 41; + + SCOPE_INGEST_SQLMESH = 42; + + SCOPE_DATACHECKS_SQLTESTS_EDIT = 50; + + SCOPE_DATACHECKS_SQLTESTS_READ = 51; + + SCOPE_EXECUTION_EDIT = 60; + + SCOPE_EXECUTION_LOG_EDIT = 61; + + SCOPE_EXECUTION_READ = 62; + + SCOPE_HOOK = 70; + } - - -message ScopeAuthorization { - repeated Scope scopes = 1; -} \ No newline at end of file diff --git a/protos/synq/webhooks/v1/callback.proto b/protos/synq/webhooks/v1/callback.proto new file mode 100644 index 0000000..924d4de --- /dev/null +++ b/protos/synq/webhooks/v1/callback.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package synq.webhooks.v1; + +import "synq/issues/commands/v1/issues_command.proto"; + +option go_package = "github.com/getsynq/api/webhooks/v1"; + +message Callback { + string url = 1; + string action_name = 2; + oneof action { + synq.issues.commands.v1.IssuesCommand issues_command = 10; + } +} diff --git a/protos/synq/webhooks/v1/event.proto b/protos/synq/webhooks/v1/event.proto new file mode 100644 index 0000000..c3ebf61 --- /dev/null +++ b/protos/synq/webhooks/v1/event.proto @@ -0,0 +1,46 @@ +syntax = "proto3"; + +package synq.webhooks.v1; + +import "google/protobuf/timestamp.proto"; +import "synq/webhooks/v1/callback.proto"; + +option go_package = "github.com/getsynq/api/webhooks/v1"; + +message Event { + string workspace = 1; + string event_id = 2; + google.protobuf.Timestamp event_time = 3; + + oneof data { + Ping ping = 10; + IssueCreated issue_created = 11; + IssueUpdated issue_updated = 12; + IssueClosed issue_closed = 13; + } + + repeated Callback callbacks = 40; +} + +// Test event sent during a webhook setup. +message Ping { + string message = 1; +} + +message Issue { + string issue_id = 1; + string group_id = 2; + string message = 3; +} + +message IssueCreated { + Issue issue = 1; +} + +message IssueUpdated { + Issue issue = 1; +} + +message IssueClosed { + Issue issue = 1; +}