Publish latest changes

protos/synq/datachecks/sqltests/v1/sql_tests.proto

@@ -0,0 +1,35 @@
+syntax = "proto3";
+
+package synq.datachecks.sqltests.v1;
+
+import "buf/validate/validate.proto";
+import "synq/entities/v1/annotation.proto";
+import "synq/platforms/v1/data_platforms.proto";
+
+option go_package = "github.com/getsynq/api/datachecks/sqltests/v1";
+
+// The SqlTest is a SQL test that is executed on a synq entity.
+message SqlTest {
+  // Identifier for the data platform which the SqlTest would be executed on.
+  platforms.v1.DataPlatformIdentifier platform = 1 [(buf.validate.field).required = true];
+
+  // Unique resource identifier for the SqlTest. This is externally maintained and can
+  // be used to fetch/update/delete this test.
+  string id = 2 [(buf.validate.field).required = true];
+
+  // Human friendly name.
+  string name = 3 [(buf.validate.field).required = true];
+
+  // A valid SQL expression which is the test.
+  string sql_expression = 4 [(buf.validate.field).required = true];
+
+  // [Recurrence rule](https://icalendar.org/iCalendar-RFC-5545/3-8-5-3-recurrence-rule.html) for the execution schedule of the SqlTest.
+  string recurrence_rule = 5;
+
+  // Annotations for the given SqlTest.These help filter the SqlTests for later
+  // operations like listing by API, selection in UI, analytics, etc.
+  repeated entities.v1.Annotation annotations = 6;
+
+  // Boolean flag to to toggle saving of failure runs.
+  bool save_failures = 7;
+}

protos/synq/datachecks/sqltests/v1/sql_tests_service.proto

@@ -0,0 +1,86 @@
+syntax = "proto3";
+
+package synq.datachecks.sqltests.v1;
+
+import "synq/datachecks/sqltests/v1/sql_tests.proto";
+import "synq/entities/v1/annotation.proto";
+import "synq/v1/scope_authorization.proto";
+
+option go_package = "github.com/getsynq/api/datachecks/sqltests/v1";
+
+// SqlTestsService is a service for managing SqlTests.
+service SqlTestsService {
+  // Upsert SqlTests based on a unique ID.
+  rpc BatchUpsertSqlTests(BatchUpsertSqlTestsRequest) returns (BatchUpsertSqlTestsResponse) {
+    option (synq.v1.scope_authorization) = {
+      scopes: [SCOPE_DATACHECKS_SQLTESTS_EDIT]
+    };
+  }
+
+  // List SqlTests for given annotations.
+  rpc ListSqlTests(ListSqlTestsRequest) returns (ListSqlTestsResponse) {
+    option (synq.v1.scope_authorization) = {
+      scopes: [SCOPE_DATACHECKS_SQLTESTS_READ]
+    };
+  }
+
+  // Get SqlTests by their IDs.
+  rpc BatchGetSqlTests(BatchGetSqlTestsRequest) returns (BatchGetSqlTestsResponse) {
+    option (synq.v1.scope_authorization) = {
+      scopes: [SCOPE_DATACHECKS_SQLTESTS_READ]
+    };
+  }
+
+  // Delete SqlTests by their IDs.
+  rpc BatchDeleteSqlTests(BatchDeleteSqlTestsRequest) returns (BatchDeleteSqlTestsResponse) {
+    option (synq.v1.scope_authorization) = {
+      scopes: [SCOPE_DATACHECKS_SQLTESTS_EDIT]
+    };
+  }
+}
+
+message BatchUpsertSqlTestsRequest {
+  // List of SqlTests to upsert.
+  // The upsert is performed based on the unique path provided for each SqlTest.
+  repeated SqlTest sql_tests = 1;
+}
+
+message BatchUpsertSqlTestsResponse {
+  // IDs for which SqlTests were created without error.
+  repeated string created_ids = 1;
+  // IDs for which SqlTests were updated without error.
+  repeated string updated_ids = 2;
+
+  // Details of errors encountered during the upsert operation.
+  message UpsertError {
+    string id = 1;
+    string reason = 2;
+  }
+  // Errors raised while upsert. This list will be empty if there were no errors.
+  repeated UpsertError errors = 3;
+}
+
+message ListSqlTestsRequest {
+  // Optional annotations to fetch SqlTests for.
+  repeated entities.v1.Annotation annotations = 1;
+}
+
+message ListSqlTestsResponse {
+  repeated SqlTest sql_tests = 1;
+}
+
+message BatchGetSqlTestsRequest {
+  // IDs of the SqlTests to fetch.
+  repeated string ids = 1;
+}
+
+message BatchGetSqlTestsResponse {
+  map<string, SqlTest> sql_tests = 1;
+}
+
+message BatchDeleteSqlTestsRequest {
+  // IDs of the SqlTests to delete.
+  repeated string ids = 1;
+}
+
+message BatchDeleteSqlTestsResponse {}

protos/synq/entities/custom/features/v1/code.proto

@@ -0,0 +1,32 @@
+syntax = "proto3";
+
+package synq.entities.custom.features.v1;
+
+import "buf/validate/validate.proto";
+
+option go_package = "github.com/getsynq/api/entities/custom/features/v1";
+
+enum CodeType {
+  CODE_TYPE_UNSPECIFIED = 0;
+  CODE_TYPE_SQL = 1;
+  CODE_TYPE_PYTHON = 2;
+  CODE_TYPE_JSON = 3;
+}
+
+// Code feature represents a code snippet associated with the entity.
+// If support for code parsing and dependency extraction is needed use [SqlDefinition](sql_definition.proto).
+message Code {
+  // Might be e.g. name of the file or a function.
+  string name = 1 [(buf.validate.field) = {
+    required: false,
+    string: {max_len: 50}
+  }];
+
+  // Type of the code.
+  CodeType code_type = 2;
+
+  // Content of the code. Displayed in the UI.
+  string content = 3 [(buf.validate.field) = {
+    string: {max_len: 100000}
+  }];
+}

protos/synq/entities/custom/features/v1/git_file_reference.proto

@@ -0,0 +1,34 @@
+syntax = "proto3";
+
+package synq.entities.custom.features.v1;
+
+import "buf/validate/validate.proto";
+
+option go_package = "github.com/getsynq/api/entities/custom/features/v1";
+
+// GitFileReference represents a reference to a file in a git repository.
+message GitFileReference {
+  // URL of the git repository. Preferably SSH clone URL.
+  string repository_url = 1 [(buf.validate.field) = {
+    required: true,
+    string: {
+      min_len: 1,
+      max_len: 255,
+    }
+  }];
+
+  // Name of the branch in the git repository.
+  string branch_name = 2 [(buf.validate.field) = {
+    required: true,
+    string: {max_bytes: 244}
+  }];
+
+  // Path to the file in the git repository.
+  string file_path = 3 [(buf.validate.field) = {
+    required: true,
+    string: {
+      min_len: 1,
+      max_len: 255,
+    }
+  }];
+}

protos/synq/entities/custom/features/v1/schema.proto

@@ -0,0 +1,88 @@
+syntax = "proto3";
+
+package synq.entities.custom.features.v1;
+
+import "buf/validate/validate.proto";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/getsynq/api/entities/custom/features/v1";
+
+message SchemaColumnField {
+  // Human readable name of the column as present in dbt or data warehouse.
+  string name = 1 [(buf.validate.field) = {
+    required: true,
+    string: {
+      min_len: 1,
+      max_len: 255
+    }
+  }];
+  // Native data type of the column as present in data warehouse.
+  string native_type = 2 [(buf.validate.field) = {
+    string: {max_len: 255}
+  }];
+  // Description of the column
+  string description = 3 [(buf.validate.field) = {
+    string: {max_len: 1000}
+  }];
+  // Ordinal position of the column in the struct, starting from 1
+  int32 ordinal_position = 4 [(buf.validate.field) = {
+    int32: {
+      gte: 0,
+      lte: 1001
+    }
+  }];
+  // Indicates that the field type could be used as a struct/json in a data warehouse
+  bool is_struct = 5;
+  // Indicates that the field is a repeated field in a data warehouse (e.g. array)
+  bool is_repeated = 6;
+  // Fields inside of the struct/record like field
+  repeated SchemaColumnField fields = 7 [(buf.validate.field) = {
+    repeated: {max_items: 100}
+  }];
+}
+
+message SchemaColumn {
+  // Human readable name of the column as present in dbt or data warehouse.
+  string name = 1 [(buf.validate.field) = {
+    required: true,
+    string: {
+      min_len: 1,
+      max_len: 255
+    }
+  }];
+  // Native data type of the column as present in data warehouse.
+  string native_type = 2 [(buf.validate.field) = {
+    string: {max_len: 255}
+  }];
+  // Description of the column
+  string description = 3 [(buf.validate.field) = {
+    string: {max_len: 1000}
+  }];
+  // Ordinal position of the column in the table, starting from 1
+  int32 ordinal_position = 4 [(buf.validate.field) = {
+    int32: {
+      gte: 0,
+      lte: 1001
+    }
+  }];
+  // Indicates that the column type could be used as a struct/json in a data warehouse
+  bool is_struct = 5;
+  // Indicates that the column is a repeated field in a data warehouse (e.g. array)
+  bool is_repeated = 6;
+  // Fields inside of the struct/record like column
+  repeated SchemaColumnField fields = 7 [(buf.validate.field) = {
+    repeated: {max_items: 100}
+  }];
+}
+
+// Schema represents a schema of a table like entity
+message Schema {
+  // Time when the schema was defined, will default to `now` if not set.
+  google.protobuf.Timestamp state_at = 1;
+  // Columns of the schema
+  // Columns are ordered by ordinal_position, it has to be defined for all or none of the columns,
+  // when not defined, ordinal positions are attached according to the order of columns in the list.
+  repeated SchemaColumn columns = 2 [(buf.validate.field) = {
+    repeated: {max_items: 1000}
+  }];
+}

protos/synq/entities/custom/features/v1/sql_definition.proto

@@ -0,0 +1,41 @@
+syntax = "proto3";
+
+package synq.entities.custom.features.v1;
+
+import "buf/validate/validate.proto";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/getsynq/api/entities/custom/features/v1";
+
+enum SqlDialect {
+  SQL_DIALECT_UNSPECIFIED = 0;
+  SQL_DIALECT_BIGQUERY = 1;
+  SQL_DIALECT_CLICKHOUSE = 2;
+  SQL_DIALECT_DATABRICKS = 3;
+  SQL_DIALECT_MYSQL = 4;
+  SQL_DIALECT_POSTGRESQL = 5;
+  SQL_DIALECT_REDSHIFT = 6;
+  SQL_DIALECT_SNOWFLAKE = 7;
+}
+
+// SqlDefinition is a feature that allows to define lineage via SQL for a custom entity.
+// Used to generate dependencies and other metadata. If parsed successfully,
+// it will provide a code and column level lineage. Can be used to define simplified transformation
+// of data which component performs using e.g. `INSERT INTO foo SELECT * FROM bar`.
+message SqlDefinition {
+  //Time when the sql was defined, will default to `now` if not set.
+  google.protobuf.Timestamp state_at = 1;
+
+  // SQL dialect used in the SQL.
+  SqlDialect dialect = 2 [(buf.validate.field) = {
+    enum: {defined_only: true}
+  }];
+
+  // Final form of the SQL as executed in the database/data warehouse. Must be a valid SQL in the selected dialect.
+  string sql = 3 [(buf.validate.field) = {
+    string: {
+      min_len: 0,
+      max_len: 1000000,
+    }
+  }];
+}

protos/synq/entities/custom/v1/checks_relationships_service.proto

@@ -0,0 +1,41 @@
+syntax = "proto3";
+
+package synq.entities.custom.v1;
+
+import "buf/validate/validate.proto";
+import "synq/entities/v1/identifier.proto";
+import "synq/v1/scope_authorization.proto";
+
+option go_package = "github.com/getsynq/api/entities/custom/v1";
+
+service ChecksRelationshipsService {
+  rpc UpsertCheckRelationships(UpsertCheckRelationshipsRequest) returns (UpsertCheckRelationshipsResponse) {
+    option (synq.v1.scope_authorization) = {
+      scopes: [SCOPE_ENTITY_EDIT]
+    };
+  }
+
+  rpc DeleteCheckRelationships(DeleteCheckRelationshipsRequest) returns (DeleteCheckRelationshipsResponse) {
+    option (synq.v1.scope_authorization) = {
+      scopes: [SCOPE_ENTITY_EDIT]
+    };
+  }
+}
+
+message UpsertCheckRelationshipsRequest {
+  repeated CheckRelationship check_relationships = 1 [(buf.validate.field).required = true];
+}
+
+message UpsertCheckRelationshipsResponse {}
+
+message DeleteCheckRelationshipsRequest {
+  repeated CheckRelationship check_relationships = 1 [(buf.validate.field).required = true];
+}
+
+message DeleteCheckRelationshipsResponse {}
+
+message CheckRelationship {
+  synq.entities.v1.Identifier check = 1 [(buf.validate.field).required = true];
+  synq.entities.v1.Identifier checked = 2 [(buf.validate.field).required = true];
+  repeated string checked_columns = 3;
+}