Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: [storagetransfer] add HDFS configuration #5619

Merged
merged 2 commits into from
Aug 15, 2024
Merged

feat: [storagetransfer] add HDFS configuration #5619

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
261034f
feat: add HDFS configuration
gcf-owl-bot[bot] Aug 15, 2024
50594e7
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Aug 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 81 additions & 42 deletions packages/google-storagetransfer/protos/google/storagetransfer/v1/transfer_types.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,21 @@ message GcsData {
// The root path value must meet
// [Object Name Requirements](/storage/docs/naming#objectnames).
string path = 3;

// Preview. Enables the transfer of managed folders between Cloud Storage
// buckets. Set this option on the gcs_data_source.
//
// If set to true:
//
// - Managed folders in the source bucket are transferred to the
// destination bucket.
// - Managed folders in the destination bucket are overwritten. Other
// OVERWRITE options are not supported.
//
// See
// [Transfer Cloud Storage managed
// folders](/storage-transfer/docs/managed-folders).
bool managed_folder_transfer_enabled = 4;
}

// An AwsS3Data resource can be a data source, but not a data sink.
Expand Down Expand Up @@ -242,26 +257,46 @@ message AwsS3Data {
// this project.
string role_arn = 4;

// Optional. The CloudFront distribution domain name pointing to this bucket,
// to use when fetching.
//
// See
// [Transfer from S3 via
// CloudFront](https://cloud.google.com/storage-transfer/docs/s3-cloudfront)
// for more information.
//
// Format: `https://{id}.cloudfront.net` or any valid custom domain. Must
// begin with `https://`.
string cloudfront_domain = 6 [(google.api.field_behavior) = OPTIONAL];

// Optional. The Resource name of a secret in Secret Manager.
//
// The Azure SAS token must be stored in Secret Manager in JSON format:
// <pre>{
// "sas_token" : "<var>SAS_TOKEN</var>"
// }</pre>
// AWS credentials must be stored in Secret Manager in JSON format:
//
// {
// "access_key_id": "ACCESS_KEY_ID",
// "secret_access_key": "SECRET_ACCESS_KEY"
// }
//
// [GoogleServiceAccount][google.storagetransfer.v1.GoogleServiceAccount] must
// be granted `roles/secretmanager.secretAccessor` for the resource.
//
// See [Configure access to a source: Microsoft Azure Blob Storage]
// (https://cloud.google.com/storage-transfer/docs/source-microsoft-azure#secret_manager)
// See [Configure access to a source: Amazon S3]
// (https://cloud.google.com/storage-transfer/docs/source-amazon-s3#secret_manager)
// for more information.
//
// If `credentials_secret` is specified, do not specify [azure_credentials][].
//
// This feature is in
// [preview](https://cloud.google.com/terms/service-terms#1).
// If `credentials_secret` is specified, do not specify
// [role_arn][google.storagetransfer.v1.AwsS3Data.role_arn] or
// [aws_access_key][google.storagetransfer.v1.AwsS3Data.aws_access_key].
//
// Format: `projects/{project_number}/secrets/{secret_name}`
string credentials_secret = 7 [(google.api.field_behavior) = OPTIONAL];

oneof private_network {
// Egress bytes over a Google-managed private network.
// This network is shared between other users of Storage Transfer Service.
bool managed_private_network = 8;
}
}

// An AzureBlobStorageData resource can be a data source, but not a data sink.
Expand Down Expand Up @@ -298,9 +333,11 @@ message AzureBlobStorageData {
// Optional. The Resource name of a secret in Secret Manager.
//
// The Azure SAS token must be stored in Secret Manager in JSON format:
// <pre>{
// "sas_token" : "<var>SAS_TOKEN</var>"
// }</pre>
//
// {
// "sas_token" : "SAS_TOKEN"
// }
//
// [GoogleServiceAccount][google.storagetransfer.v1.GoogleServiceAccount] must
// be granted `roles/secretmanager.secretAccessor` for the resource.
//
Expand All @@ -311,9 +348,6 @@ message AzureBlobStorageData {
// If `credentials_secret` is specified, do not specify
// [azure_credentials][google.storagetransfer.v1.AzureBlobStorageData.azure_credentials].
//
// This feature is in
// [preview](https://cloud.google.com/terms/service-terms#1).
//
// Format: `projects/{project_number}/secrets/{secret_name}`
string credentials_secret = 7 [(google.api.field_behavior) = OPTIONAL];
}
Expand Down Expand Up @@ -371,6 +405,15 @@ message PosixFilesystem {
string root_directory = 1;
}

// An HdfsData resource specifies a path within an HDFS entity (e.g. a cluster).
// All cluster-specific settings, such as namenodes and ports, are configured on
// the transfer agents servicing requests, so HdfsData only contains the root
// path to the data in our transfer.
message HdfsData {
// Root path to transfer files.
string path = 1;
}

// An AwsS3CompatibleData resource.
message AwsS3CompatibleData {
// Required. Specifies the name of the bucket.
Expand Down Expand Up @@ -473,7 +516,7 @@ message S3CompatibleMetadata {
ListApi list_api = 4;
}

// Represents an On-Premises Agent pool.
// Represents an agent pool.
message AgentPool {
option (google.api.resource) = {
type: "storagetransfer.googleapis.com/agentPools"
Expand All @@ -485,8 +528,8 @@ message AgentPool {
// Default value. This value is unused.
STATE_UNSPECIFIED = 0;

// This is an initialization state. During this stage, the resources such as
// Pub/Sub topics are allocated for the AgentPool.
// This is an initialization state. During this stage, resources are
// allocated for the AgentPool.
CREATING = 1;

// Determines that the AgentPool is created for use. At this state, Agents
Expand Down Expand Up @@ -602,6 +645,9 @@ message TransferSpec {

// An AWS S3 compatible data source.
AwsS3CompatibleData aws_s3_compatible_data_source = 19;

// An HDFS cluster data source.
HdfsData hdfs_data_source = 20;
}

oneof intermediate_data_location {
Expand Down Expand Up @@ -768,10 +814,10 @@ message MetadataOptions {
// Do not preserve the `timeCreated` metadata from the source object.
TIME_CREATED_SKIP = 1;

// Preserves the source object's `timeCreated` metadata in the `customTime`
// field in the destination object. Note that any value stored in the
// source object's `customTime` field will not be propagated to the
// destination object.
// Preserves the source object's `timeCreated` or `lastModified` metadata in
// the `customTime` field in the destination object. Note that any value
// stored in the source object's `customTime` field will not be propagated
// to the destination object.
TIME_CREATED_PRESERVE_AS_CUSTOM_TIME = 2;
}

Expand Down Expand Up @@ -818,9 +864,10 @@ message MetadataOptions {
KmsKey kms_key = 8;

// Specifies how each object's `timeCreated` metadata is preserved for
// transfers between Google Cloud Storage buckets. If unspecified, the
// default behavior is the same as
// transfers. If unspecified, the default behavior is the same as
// [TIME_CREATED_SKIP][google.storagetransfer.v1.MetadataOptions.TimeCreated.TIME_CREATED_SKIP].
// This behavior is supported for transfers to Cloud Storage buckets from
// Cloud Storage, Amazon S3, S3-compatible storage, and Azure sources.
TimeCreated time_created = 9;
}

Expand Down Expand Up @@ -986,8 +1033,7 @@ message TransferJob {
// Transfer specification.
TransferSpec transfer_spec = 4;

// Notification configuration. This is not supported for transfers involving
// PosixFilesystem.
// Notification configuration.
NotificationConfig notification_config = 11;

// Logging configuration.
Expand Down Expand Up @@ -1219,16 +1265,10 @@ message NotificationConfig {

// Specifies the logging behavior for transfer operations.
//
// For cloud-to-cloud transfers, logs are sent to Cloud Logging. See
// Logs can be sent to Cloud Logging for all transfer types. See
// [Read transfer
// logs](https://cloud.google.com/storage-transfer/docs/read-transfer-logs) for
// details.
//
// For transfers to or from a POSIX file system, logs are stored in the
// Cloud Storage bucket that is the source or sink of the transfer.
// See [Managing Transfer for on-premises jobs]
// (https://cloud.google.com/storage-transfer/docs/managing-on-prem-jobs#viewing-logs)
// for details.
message LoggingConfig {
// Loggable actions.
enum LoggableAction {
Expand Down Expand Up @@ -1260,19 +1300,18 @@ message LoggingConfig {
}

// Specifies the actions to be logged. If empty, no logs are generated.
// Not supported for transfers with PosixFilesystem data sources; use
// [enable_onprem_gcs_transfer_logs][google.storagetransfer.v1.LoggingConfig.enable_onprem_gcs_transfer_logs]
// instead.
repeated LoggableAction log_actions = 1;

// States in which `log_actions` are logged. If empty, no logs are generated.
// Not supported for transfers with PosixFilesystem data sources; use
// [enable_onprem_gcs_transfer_logs][google.storagetransfer.v1.LoggingConfig.enable_onprem_gcs_transfer_logs]
// instead.
repeated LoggableActionState log_action_states = 2;

// For transfers with a PosixFilesystem source, this option enables the Cloud
// Storage transfer logs for this transfer.
// For PosixFilesystem transfers, enables
// [file system transfer
// logs](https://cloud.google.com/storage-transfer/docs/on-prem-transfer-log-format)
// instead of, or in addition to, Cloud Logging.
//
// This option ignores [LoggableAction] and [LoggableActionState]. If these
// are set, Cloud Logging will also be enabled for this transfer.
bool enable_onprem_gcs_transfer_logs = 3;
}

Expand Down
Loading
Loading