diff --git a/apis/docs/v1/docs.md b/apis/docs/v1/docs.md index 19fa20d7db..8c7c76e29e 100644 --- a/apis/docs/v1/docs.md +++ b/apis/docs/v1/docs.md @@ -108,6 +108,7 @@ - [Upsert.MultiRequest](#payload-v1-Upsert-MultiRequest) - [Upsert.ObjectRequest](#payload-v1-Upsert-ObjectRequest) - [Upsert.Request](#payload-v1-Upsert-Request) + - [Remove.Timestamp.Operator](#payload-v1-Remove-Timestamp-Operator) - [Search.AggregationAlgorithm](#payload-v1-Search-AggregationAlgorithm) @@ -971,6 +972,8 @@ Represent the remove request. ### Remove.Timestamp +Represent the timestamp comparison. + | Field | Type | Label | Description | | --------- | ------------------------------------------------------------------ | ----- | ------------------------- | | timestamp | [int64](#int64) | | The timestamp. | @@ -982,9 +985,9 @@ Represent the remove request. Represent the remove request based on timestamp. -| Field | Type | Label | Description | -| ---------- | ------------------------------------------------ | -------- | ------------------------------------------------------------------ | -| timestamps | [Remove.Timestamp](#payload-v1-Remove-Timestamp) | repeated | Represent the multiple remove request contents based on timestamp. | +| Field | Type | Label | Description | +| ---------- | ------------------------------------------------ | -------- | ------------------------------------------------------------------------------------------ | +| timestamps | [Remove.Timestamp](#payload-v1-Remove-Timestamp) | repeated | The timestamp comparison list. If more than one is specified, the `AND` search is applied. | @@ -1236,14 +1239,14 @@ Represent the upsert request. Operator is enum of each conditional operator. -| Name | Number | Description | -| ---- | ------ | ----------- | -| Eq | 0 | | -| Ne | 1 | | -| Ge | 2 | | -| Gt | 3 | | -| Le | 4 | | -| Lt | 5 | | +| Name | Number | Description | +| ---- | ------ | ----------------------------------------------------------------------------- | +| Eq | 0 | The timestamp is equal to the specified value in the request. | +| Ne | 1 | The timestamp is not equal to the specified value in the request. | +| Ge | 2 | The timestamp is greater than or equal to the specified value in the request. | +| Gt | 3 | The timestamp is greater than the specified value in the request. | +| Le | 4 | The timestamp is less than or equal to the specified value in the request. | +| Lt | 5 | The timestamp is less than the specified value in the request. | diff --git a/apis/grpc/v1/payload/payload.pb.go b/apis/grpc/v1/payload/payload.pb.go index de2e288108..1bf72db76b 100644 --- a/apis/grpc/v1/payload/payload.pb.go +++ b/apis/grpc/v1/payload/payload.pb.go @@ -100,11 +100,17 @@ func (Search_AggregationAlgorithm) EnumDescriptor() ([]byte, []int) { type Remove_Timestamp_Operator int32 const ( + // The timestamp is equal to the specified value in the request. Remove_Timestamp_Eq Remove_Timestamp_Operator = 0 + // The timestamp is not equal to the specified value in the request. Remove_Timestamp_Ne Remove_Timestamp_Operator = 1 + // The timestamp is greater than or equal to the specified value in the request. Remove_Timestamp_Ge Remove_Timestamp_Operator = 2 + // The timestamp is greater than the specified value in the request. Remove_Timestamp_Gt Remove_Timestamp_Operator = 3 + // The timestamp is less than or equal to the specified value in the request. Remove_Timestamp_Le Remove_Timestamp_Operator = 4 + // The timestamp is less than the specified value in the request. Remove_Timestamp_Lt Remove_Timestamp_Operator = 5 ) @@ -2336,7 +2342,7 @@ type Remove_TimestampRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Represent the multiple remove request contents based on timestamp. + // The timestamp comparison list. If more than one is specified, the `AND` search is applied. Timestamps []*Remove_Timestamp `protobuf:"bytes,1,rep,name=timestamps,proto3" json:"timestamps,omitempty"` } @@ -2379,6 +2385,7 @@ func (x *Remove_TimestampRequest) GetTimestamps() []*Remove_Timestamp { return nil } +// Represent the timestamp comparison. type Remove_Timestamp struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache diff --git a/apis/proto/v1/payload/payload.proto b/apis/proto/v1/payload/payload.proto index 7ad7fd30fe..dfc009b318 100644 --- a/apis/proto/v1/payload/payload.proto +++ b/apis/proto/v1/payload/payload.proto @@ -300,18 +300,25 @@ message Remove { // Represent the remove request based on timestamp. message TimestampRequest { - // Represent the multiple remove request contents based on timestamp. + // The timestamp comparison list. If more than one is specified, the `AND` search is applied. repeated Timestamp timestamps = 1; } + // Represent the timestamp comparison. message Timestamp { // Operator is enum of each conditional operator. enum Operator { + // The timestamp is equal to the specified value in the request. Eq = 0; + // The timestamp is not equal to the specified value in the request. Ne = 1; + // The timestamp is greater than or equal to the specified value in the request. Ge = 2; + // The timestamp is greater than the specified value in the request. Gt = 3; + // The timestamp is less than or equal to the specified value in the request. Le = 4; + // The timestamp is less than the specified value in the request. Lt = 5; } // The timestamp. diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/remove.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/remove.swagger.json index 350b138215..ba4c96ce97 100644 --- a/apis/swagger/v1/vald/apis/proto/v1/vald/remove.swagger.json +++ b/apis/swagger/v1/vald/apis/proto/v1/vald/remove.swagger.json @@ -145,7 +145,7 @@ "items": { "$ref": "#/definitions/v1RemoveTimestamp" }, - "description": "Represent the multiple remove request contents based on timestamp." + "description": "The timestamp comparison list. If more than one is specified, the `AND` search is applied." } }, "description": "Represent the remove request based on timestamp." @@ -154,7 +154,7 @@ "type": "string", "enum": ["Eq", "Ne", "Ge", "Gt", "Le", "Lt"], "default": "Eq", - "description": "Operator is enum of each conditional operator." + "description": "Operator is enum of each conditional operator.\n\n - Eq: The timestamp is equal to the specified value in the request.\n - Ne: The timestamp is not equal to the specified value in the request.\n - Ge: The timestamp is greater than or equal to the specified value in the request.\n - Gt: The timestamp is greater than the specified value in the request.\n - Le: The timestamp is less than or equal to the specified value in the request.\n - Lt: The timestamp is less than the specified value in the request." }, "protobufAny": { "type": "object", @@ -314,7 +314,8 @@ "$ref": "#/definitions/TimestampOperator", "description": "The conditional operator." } - } + }, + "description": "Represent the timestamp comparison." } } } diff --git a/docs/api/remove.md b/docs/api/remove.md index 0fe159d293..2d2e303b7f 100644 --- a/docs/api/remove.md +++ b/docs/api/remove.md @@ -9,6 +9,8 @@ service Remove { rpc Remove(payload.v1.Remove.Request) returns (payload.v1.Object.Location) {} + rpc RemoveByTimestamp(payload.v1.Remove.TimestampRequest) returns (payload.v1.Object.Locations) {} + rpc StreamRemove(stream payload.v1.Remove.Request) returns (stream payload.v1.Object.StreamLocation) {} @@ -114,6 +116,126 @@ Here are some common reasons and how to resolve each error. | NOT_FOUND | Requested ID is NOT inserted. | Send a request with an ID that is already inserted. | | INTERNAL | Target Vald cluster or network route has some critical error. | Check target Vald cluster first and check network route including ingress as second. | +## RemoveByTimestamp RPC + +RemoveByTimestamp RPC is the method to remove vectors based on timestamp. + +### Input + +- the scheme of `payload.v1.Remove.TimestampRequest` + + ```rpc + message Remove { + message TimestampRequest { + repeated Timestamp timestamps = 1; + } + + message Timestamp { + enum Operator { + Eq = 0; + Ne = 1; + Ge = 2; + Gt = 3; + Le = 4; + Lt = 5; + } + int64 timestamp = 1; + Operator operator = 2; + } + } + + message Object { + message ID { + string id = 1 [ (validate.rules).string.min_len = 1 ]; + } + } + ``` + + - Remove.TimestampRequest + + | field | type | label | required | description | + | :--------: | :--------------- | :-------------------------------- | :------: | :-------------------------------------------------------------------------------------------- | + | timestamps | Remove.Timestamp | repeated(Array[Remove.Timestamp]) | \* | The timestamp comparison list.
If more than one is specified, the `AND` search is applied. | + + - Remove.Timestamp + + | field | type | label | required | description | + | :-------: | :------------------------ | :---- | :------: | :------------------------------------------------ | + | timestamp | int64 | | \* | The timestamp. | + | operator | Remove.Timestamp.Operator | | | The conditionl operator. (default value is `Eq`). | + + - Remove.Timestamp.Operator + + | value | description | + | :---: | :--------------------- | + | Eq | Equal. | + | Ne | Not Equal. | + | Ge | Greater than or Equal. | + | Gt | Greater than. | + | Le | Less than or Equal. | + | Lt | Less than. | + +
+ In the TimestampRequest message, the 'timestamps' field is repeated, allowing the inclusion of multiple Timestamp.
+ When multiple Timestamps are provided, it results in an `AND` condition, enabling the realization of deletions with specified ranges.
+ This design allows for versatile deletion operations, facilitating tasks such as removing data within a specific time range. +
+ +### Output + +- the scheme of `payload.v1.Object.Locations`. + + ```rpc + message Object { + message Locations { repeated Location locations = 1; } + + message Location { + string name = 1; + string uuid = 2; + repeated string ips = 3; + } + } + ``` + + - Object.Locations + + | field | type | label | description | + | :------: | :-------------- | :------------------------------- | :----------------------------- | + | location | Object.Location | repeated(Array[Object.Location]) | The list of `Object.Location`. | + + - Object.Location + + | field | type | label | description | + | :---: | :----- | :---------------------- | :-------------------------------------------------------------------- | + | name | string | | The name of vald agent pod where the request vector is removed. | + | uuid | string | | The ID of the removed vector. It is the same as an `Object.ID`. | + | ips | string | repeated(Array[string]) | The IP list of `vald-agent` pods where the request vector is removed. | + +### Status Code + +| code | name | +| :--: | :---------------- | +| 0 | OK | +| 1 | CANCELLED | +| 4 | DEADLINE_EXCEEDED | +| 5 | NOT_FOUND | +| 13 | INTERNAL | + +Please refer to [Response Status Code](./status.md) for more details. + +### Troubleshooting + +The request process may not be completed when the response code is NOT `0 (OK)`. + +Here are some common reasons and how to resolve each error. + +| name | common reason | how to resolve | +| :---------------- | :---------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------- | +| CANCELLED | Executed cancel() of rpc from client/server-side or network problems between client and server. | Check the code, especially around timeout and connection management, and fix if needed. | +| DEADLINE_EXCEEDED | The RPC timeout setting is too short on the client/server side. | Check the gRPC timeout setting on both the client and server sides and fix it if needed. | +| NOT_FOUND | No vectors in the system match the specified timestamp conditions. | Check whether vectors matching the specified timestamp conditions exist in the system, and fix conditions if needed. | +| INTERNAL | Target Vald cluster or network route has some critical error. | Check target Vald cluster first and check network route including ingress as second. | + ## StreamRemove RPC StreamRemove RPC is the method to remove multiple vectors using the [bidirectional streaming RPC](https://grpc.io/docs/what-is-grpc/core-concepts/#bidirectional-streaming-rpc).