buildfarm.proto

syntax = "proto3";

package build.buildfarm.v1test;

import "google/api/annotations.proto";
import "build/bazel/remote/execution/v2/remote_execution.proto";
import "google/longrunning/operations.proto";
import "google/protobuf/duration.proto";
import "google/protobuf/timestamp.proto";
import "google/rpc/status.proto";

option go_package = "buildfarm";
option java_package = "build.buildfarm.v1test";
option java_multiple_files = true;
option java_outer_classname = "OperationQueueProto";

// The Admin API is used to perform administrative functions on Buildfarm
// infrastructure
service Admin {
  rpc TerminateHost(TerminateHostRequest) returns (google.rpc.Status) {
    option (google.api.http) = {
      get: "/v1test/admin:terminateHost"
    };
  }

  rpc StopContainer(StopContainerRequest) returns (google.rpc.Status) {
    option (google.api.http) = {
      get: "/v1test/admin:stopContainer"
    };
  }

  rpc GetHosts(GetHostsRequest) returns (GetHostsResult) {
    option (google.api.http) = {
      get: "/v1test/admin:getHosts"
    };
  }

  rpc GetClientStartTime(GetClientStartTimeRequest)
      returns (GetClientStartTimeResult) {
    option (google.api.http) = {
      get: "/v1test/admin:getClientStartTime"
    };
  }

  rpc ScaleCluster(ScaleClusterRequest) returns (google.rpc.Status) {
    option (google.api.http) = {
      get: "/v1test/admin:scaleCluster"
    };
  }

  rpc ReindexCas(ReindexCasRequest) returns (ReindexCasRequestResults) {
    option (google.api.http) = {
      get: "/v1test/admin:reindexCas"
    };
  }

  rpc ShutDownWorkerGracefully(ShutDownWorkerGracefullyRequest)
      returns (ShutDownWorkerGracefullyRequestResults) {
    option (google.api.http) = {
      get: "/v1test/admin:deregisterWorker"
    };
  }

  rpc DisableScaleInProtection(DisableScaleInProtectionRequest)
      returns (DisableScaleInProtectionRequestResults) {
    option (google.api.http) = {
      get: "/v1test/admin:disableScaleInProtection"
    };
  }
}

message ShutDownWorkerGracefullyRequest {
  string instance_name = 1;

  string worker_name = 2;
}

message ShutDownWorkerGracefullyRequestResults {}

message DisableScaleInProtectionRequest {
  string instance_name = 1;
}

message DisableScaleInProtectionRequestResults {}

message ReindexCasRequest {
  string instance_name = 1;

  // format: ip:port
  string host_id = 2;
}

message ReindexCasRequestResults {
  int64 removedHosts = 1;

  int64 removedKeys = 2;

  int64 totalKeys = 3;
}

message TerminateHostRequest {
  string instance_name = 1;

  string host_id = 2;
}

message StopContainerRequest {
  string instance_name = 1;

  string host_id = 2;

  string container_name = 3;
}

message GetHostsRequest {
  string filter = 1;

  int32 age_in_minutes = 2;

  string status = 3;
}

message GetHostsResult {
  int64 num_hosts = 1;

  repeated Host hosts = 2;
}

message Host {
  int64 host_num = 1;

  string host_id = 2;

  string ip_address = 3;

  google.protobuf.Timestamp launch_time = 4;

  string state = 5;

  string dns_name = 6;

  int64 uptime_minutes = 7;

  int64 num_cores = 8;

  string type = 9;

  string lifecycle = 10;
}

message GetClientStartTimeRequest {
  string instance_name = 1;
  repeated string host_name = 2;
}

message GetClientStartTimeResult {
  repeated GetClientStartTime client_start_time = 1;
}

message GetClientStartTime {
  string instance_name = 1;
  google.protobuf.Timestamp client_start_time = 2;
}

message ScaleClusterRequest {
  string scale_group_name = 1;

  int32 min_hosts = 2;

  int32 max_hosts = 3;

  int32 target_hosts = 4;

  int32 target_reserved_hosts_percent = 5;
}

// The OperationQueue API is used internally to communicate with Workers
service OperationQueue {
  rpc Take(TakeOperationRequest) returns (QueueEntry) {
    option (google.api.http) = {
      get: "/v1test/{instance_name=**}/operation:take"
    };
  }

  rpc Put(google.longrunning.Operation) returns (google.rpc.Status) {
    option (google.api.http) = {
      post: "/v1test/{instance_name=**}/operation:put"
      body: "*"
    };
  }

  rpc Poll(PollOperationRequest) returns (google.rpc.Status) {
    option (google.api.http) = {
      get: "/v1test/{instance_name=**}/operation:poll"
    };
  }

  rpc Status(BackplaneStatusRequest) returns (BackplaneStatus) {
    option (google.api.http) = {
      get: "/v1test/{instance_name=**}/operation:status"
    };
  }
}

message BackplaneStatusRequest {
  string instance_name = 1;
}

message TakeOperationRequest {
  // The instance of the execution system to operate against. A server may
  // support multiple instances of the execution system (with their own workers,
  // storage, caches, etc.). The server MAY require use of this field to select
  // between them in an implementation-defined fashion, otherwise it can be
  // omitted.
  string instance_name = 1;

  // The platform features available for the execution environment. The server
  // MAY choose to execute the action on any worker satisfying the requirements,
  // so the client SHOULD ensure that running the action on any such worker will
  // have the same result.
  build.bazel.remote.execution.v2.Platform platform = 5;
}

message PollOperationRequest {
  // The operation name in question
  string operation_name = 2;

  // The current state of the worker
  build.bazel.remote.execution.v2.ExecutionStage.Value stage = 3;
}

// Publish GRPC metrics to promethesus for each server endpoint. This is used
// for SLAs at API boundaries. You may want something similar at the proxy level
// to see frontend/backend performance.
message GrpcPrometheusMetrics {
  // Whether to enable this metric gathering at all. If disabled, there should
  // be no related metrics registered in prometheus.
  bool enabled = 1;

  // These metrics are more expensive to calculate. If disabled, you only
  // receive core metrics. Core metrics include send/receive totals tagged
  // with return codes.  No latencies.
  bool provide_latency_histograms = 2;
}

message AwsSecret {

  string access_key_id = 1;
  string secret_key = 2;
}

message S3BucketConfig {

  string region = 1;

  string name = 2;

  int32 max_connections = 3;

  string secret_name = 4;
}

enum WorkerType {
  NONE = 0;

  EXECUTE = 1;

  STORAGE = 2;
}

message ShardWorker {
  string endpoint = 1;

  int64 expire_at = 2;

  int32 worker_type = 3;

  int64 first_registered_at = 4;
}

message WorkerChange {
  message Add {
    google.protobuf.Timestamp effectiveAt = 1;

    int32 worker_type = 2;
  }

  message Remove {
    string source = 1;

    string reason = 2;
  }

  string name = 1;

  google.protobuf.Timestamp effectiveAt = 2;

  oneof type {
    Add add = 3;

    Remove remove = 4;
  }
}

message OperationChange {
  message Reset {
    google.protobuf.Timestamp expiresAt = 1;

    google.longrunning.Operation operation = 2;
  }

  message Expire {
    bool force = 1;
  }

  google.protobuf.Timestamp effectiveAt = 1;

  string source = 2;

  oneof type {
    Reset reset = 3;

    Expire expire = 4;
  }
}

message DispatchedOperation {
  QueueEntry queue_entry = 1;

  int64 requeue_at = 2;
}

message ProfiledQueuedOperationMetadata {
  QueuedOperation queued_operation = 1;

  QueuedOperationMetadata queued_operation_metadata = 2;

  google.protobuf.Duration transformed_in = 3;

  google.protobuf.Duration validated_in = 4;

  google.protobuf.Duration uploaded_in = 5;
}

message ExecuteEntry {
  string operation_name = 1;

  Digest action_digest = 2;

  bool skip_cache_lookup = 3;

  build.bazel.remote.execution.v2.RequestMetadata request_metadata = 4;

  build.bazel.remote.execution.v2.ExecutionPolicy execution_policy = 5;

  build.bazel.remote.execution.v2.ResultsCachePolicy results_cache_policy = 6;

  string stdout_stream_name = 7;

  string stderr_stream_name = 8;

  google.protobuf.Timestamp queued_timestamp = 9;
}

message QueueEntry {
  ExecuteEntry execute_entry = 1;

  Digest queued_operation_digest = 2;

  build.bazel.remote.execution.v2.Platform platform = 3;

  int32 requeue_attempts = 4;
}

message QueueStatus {
  int64 size = 1;

  repeated int64 internal_sizes = 2;

  string name = 3;
}

message OperationQueueStatus {
  repeated QueueStatus provisions = 1;

  int64 size = 2;
}

message LabeledCount {
  string name = 1;

  int64 size = 2;
}

message DispatchedOperationsStatus {

  // the total number of currently dispatched operations
  int64 size = 1;

  // how many of the current dispatched operations are 'build' related
  int64 build_action_amount = 2;

  // how many of the current dispatched operations are 'test' related
  int64 test_action_amount = 3;

  // how many of the current dispatched operations cannot be fetched and
  // determine the type
  int64 unknown_action_amount = 4;

  // count dispatched operations by the queue they originated in
  // for example, "how many cpu vs gpu operations are running?"
  repeated LabeledCount from_queues = 5;

  // count dispatched operations by the tool they originated from
  // for example, "Is anyone using a build tool or bazel version not supported
  // by the main repo?"
  repeated LabeledCount tools = 6;

  // count the kinds of action mnemonics occurring.
  // for example, CppCompile or GoLink.
  repeated LabeledCount action_mnemonics = 7;

  // count the program names being run for the command
  // for example, clang, bash.
  repeated LabeledCount command_tools = 8;

  // count the unique targets being resolved
  repeated LabeledCount target_ids = 9;

  // count the unique configurations being used
  repeated LabeledCount config_ids = 10;

  // platform properties used by the dispatched operations
  repeated LabeledCount platform_properties = 11;

  // how many unique clients are currently being served.
  // This is based on the uniqueness of tool_id for currently dispatched
  // operations.
  int64 unique_clients_amount = 12;

  // how many of the dispatched operations have been requeued.
  // many requeued operations could be a sign of a problem.
  int64 requeued_operations_amount = 13;
}

message BackplaneStatus {
  QueueStatus prequeue = 1;

  OperationQueueStatus operation_queue = 2;

  DispatchedOperationsStatus dispatched_operations = 9;

  // Maintained for backward compatibility.
  repeated string active_workers = 4;

  repeated string active_storage_workers = 12;

  repeated string active_execute_workers = 13;

  int64 cas_lookup_size = 5;

  int64 action_cache_size = 6;

  int64 blocked_actions_size = 7;

  int64 blocked_invocations_size = 8;

  int64 fetch_time_ms = 10;

  int64 dispatched_size = 11;
}

message QueuedOperationMetadata {
  build.bazel.remote.execution.v2.ExecuteOperationMetadata
      execute_operation_metadata = 1;

  Digest queued_operation_digest = 2;

  build.bazel.remote.execution.v2.RequestMetadata request_metadata = 3;
}

// A `Tree` contains all the
// [Directory][build.bazel.remote.execution.v2.Directory] protos in a
// single directory Merkle tree, compressed into one message, with a map
// to index the directories.
message Tree {
  // The digest of the root directory in the tree.
  Digest rootDigest = 1;

  map<string, build.bazel.remote.execution.v2.Directory> directories = 2;
}

message QueuedOperation {
  build.bazel.remote.execution.v2.Action action = 1;

  build.bazel.remote.execution.v2.Command command = 2;

  reserved 4;

  Tree tree = 5;
}

message OperationRequestMetadata {
  string operation_name = 1;

  bool done = 2;

  build.bazel.remote.execution.v2.RequestMetadata request_metadata = 3;

  build.bazel.remote.execution.v2.ExecuteOperationMetadata
      execute_operation_metadata = 4;

  build.bazel.remote.execution.v2.ExecuteResponse execute_response = 5;

  string cluster_id = 6;
}

message InstanceEndpoint {
  // target suitable for grpc channel creation: host:port is common
  string target = 1;

  // instance to be used
  string instance_name = 2;

  // deadline for requests in seconds
  int32 deadline_after_seconds = 3;
}

message ExecutionWrapper {
  string path = 1;

  repeated string arguments = 2;
}

// selectable controls for executions
// a universal policy can be specified with an empty name
message ExecutionPolicy {
  string name = 1;

  oneof policy {
    ExecutionWrapper wrapper = 2;
  }
}

message TreeIteratorToken {
  repeated build.bazel.remote.execution.v2.Digest directories = 1;
}

message OperationIteratorToken {
  string operation_name = 1;
}

message BlobWriteKey {
  Digest digest = 1;

  string identifier = 2;

  build.bazel.remote.execution.v2.Compressor.Value compressor = 3;
}

message OperationTimesBetweenStages {
  google.protobuf.Duration queued_to_match = 1;

  google.protobuf.Duration match_to_input_fetch_start = 2;

  google.protobuf.Duration input_fetch_start_to_complete = 3;

  google.protobuf.Duration input_fetch_complete_to_execution_start = 4;

  google.protobuf.Duration execution_start_to_complete = 5;

  google.protobuf.Duration execution_complete_to_output_upload_start = 6;

  google.protobuf.Duration output_upload_start_to_complete = 7;

  google.protobuf.Duration period = 8;

  int32 operation_count = 9;
}

message StageInformation {
  // name of the stage this message represent, i.e. InputFetchStage
  string name = 1;

  // number of slots configured for this stage
  int32 slots_configured = 2;

  // number of slots used for this stage
  int32 slots_used = 3;

  repeated string operation_names = 4;
}

message WorkerProfileMessage {
  // number of Entry in CAS
  int64 cas_entry_count = 1;

  // current size of the CAS as sum of entries
  int64 cas_size = 12;

  // configured maximum size of the CAS
  int64 cas_max_size = 13;

  // configured maximum entry size of the CAS
  int64 cas_max_entry_size = 14;

  // number of unreferenced Entry in CAS
  int64 cas_unreferenced_entry_count = 11;

  // number of DirectoryEntry in CAS
  int64 cas_directory_entry_count = 2;

  // number of evicted Entries since last profile request
  int32 cas_evicted_entry_count = 5;

  // the total size of evicted Entries since last profile request
  int64 cas_evicted_entry_size = 6;

  // used for removed fields representing composed used/configured slots
  reserved 7 to 8;

  repeated StageInformation stages = 9;

  repeated OperationTimesBetweenStages times = 10;
}

message WorkerProfileRequest {}

message WorkerListRequest {}

message WorkerListMessage {
  repeated string workers = 1;
}

service WorkerProfile {
  rpc GetWorkerProfile(WorkerProfileRequest) returns (WorkerProfileMessage) {}

  rpc GetWorkerList(WorkerListRequest) returns (WorkerListMessage) {}
}

message PrepareWorkerForGracefulShutDownRequest {
  string instance_name = 1;

  string worker_name = 2;
}

message PrepareWorkerForGracefulShutDownRequestResults {}

service ShutDownWorker {
  rpc PrepareWorkerForGracefulShutdown(PrepareWorkerForGracefulShutDownRequest)
      returns (PrepareWorkerForGracefulShutDownRequestResults) {}
}

message FetchQualifiers {
  Digest digest = 1;

  string canonical_id = 2;

  map<string, string> headers = 3;
}

message Digest {
  string hash = 1;

  int64 size = 2;

  build.bazel.remote.execution.v2.DigestFunction.Value digest_function = 3;
}