From 209bc39a65ac21a7fda754af448a3cb05b4268c6 Mon Sep 17 00:00:00 2001 From: Dave Supplee Date: Fri, 30 Mar 2018 10:08:03 -0400 Subject: [PATCH 1/4] Introduce BigQuery Regionalization Support --- BigQuery/src/BigQueryClient.php | 54 ++- .../ServiceDefinition/bigquery-v2.json | 441 ++++++++++++++++-- BigQuery/src/CopyJobConfiguration.php | 6 +- BigQuery/src/Dataset.php | 17 +- BigQuery/src/ExtractJobConfiguration.php | 6 +- BigQuery/src/Job.php | 12 +- BigQuery/src/JobConfigurationTrait.php | 35 +- BigQuery/src/LoadJobConfiguration.php | 6 +- BigQuery/src/QueryJobConfiguration.php | 12 +- BigQuery/src/QueryResults.php | 7 +- BigQuery/src/Table.php | 26 +- BigQuery/tests/Snippet/BigQueryClientTest.php | 16 +- .../Snippet/CopyJobConfigurationTest.php | 3 +- .../Snippet/ExtractJobConfigurationTest.php | 3 +- .../Snippet/LoadJobConfigurationTest.php | 3 +- .../Snippet/QueryJobConfigurationTest.php | 3 +- BigQuery/tests/System/BigQueryTestCase.php | 18 +- BigQuery/tests/System/RegionalizationTest.php | 207 ++++++++ BigQuery/tests/Unit/BigQueryClientTest.php | 174 +++++-- .../tests/Unit/CopyJobConfigurationTest.php | 3 +- .../Unit/ExtractJobConfigurationTest.php | 3 +- .../tests/Unit/JobConfigurationTraitTest.php | 38 +- BigQuery/tests/Unit/JobTest.php | 23 +- .../tests/Unit/LoadJobConfigurationTest.php | 3 +- .../tests/Unit/QueryJobConfigurationTest.php | 3 +- Core/src/ServiceBuilder.php | 6 + phpunit-snippets.xml.dist | 3 + 27 files changed, 1002 insertions(+), 129 deletions(-) create mode 100644 BigQuery/tests/System/RegionalizationTest.php diff --git a/BigQuery/src/BigQueryClient.php b/BigQuery/src/BigQueryClient.php index 5c7318ddeabe..cc0a34dcd0c1 100644 --- a/BigQuery/src/BigQueryClient.php +++ b/BigQuery/src/BigQueryClient.php @@ -61,6 +61,11 @@ class BigQueryClient */ protected $connection; + /** + * @var string The default geographic location. + */ + private $location; + /** * @var ValueMapper Maps values between PHP and BigQuery. */ @@ -97,10 +102,17 @@ class BigQueryClient * @type bool $returnInt64AsObject If true, 64 bit integers will be * returned as a {@see Google\Cloud\Core\Int64} object for 32 bit * platform compatibility. **Defaults to** false. + * @type string $location If provided, determines the default geographic + * location used when creating datasets and managing jobs. Please + * note: This is only required for jobs started outside of the US + * and EU regions. Also, if location metadata has already been + * fetched over the network it will take precedent over this + * setting. * } */ public function __construct(array $config = []) { + $this->location = $this->pluck('location', $config, false); $this->setHttpRetryCodes([502]); $this->setHttpRetryMessages([ 'rateLimitExceeded', @@ -164,6 +176,13 @@ public function __construct(array $config = []) * ); * ``` * + * ``` + * // Set a region to run the job in. + * $queryJobConfig = $bigQuery->query( + * 'SELECT name FROM `my_project.users_dataset.users` LIMIT 100' + * )->location('asia-northeast1'); + * ``` + * * @param string $query A BigQuery SQL query. * @param array $options [optional] Please see the * [API documentation for Job configuration](https://goo.gl/vSTbGp) @@ -175,7 +194,8 @@ public function query($query, array $options = []) return (new QueryJobConfiguration( $this->mapper, $this->projectId, - $options + $options, + $this->location ))->query($query); } @@ -368,11 +388,28 @@ public function startQuery(JobConfigurationInterface $query, array $options = [] * ``` * * @param string $id The id of the already run or running job to request. + * @param array $options [optional] { + * Configuration options. + * + * @type string $location The geographic location of the job. Required + * for jobs started outside of the US and EU regions. + * **Defaults to** a location specified in the client + * configuration. + * } * @return Job */ - public function job($id) + public function job($id, array $options = []) { - return new Job($this->connection, $id, $this->projectId, $this->mapper); + return new Job( + $this->connection, + $id, + $this->projectId, + $this->mapper, + [], + isset($options['location']) + ? $options['location'] + : $this->location + ); } /** @@ -451,7 +488,9 @@ public function dataset($id) $this->connection, $id, $this->projectId, - $this->mapper + $this->mapper, + [], + $this->location ); } @@ -511,7 +550,8 @@ function (array $dataset) { * Creates a dataset. * * Please note that by default the library will not attempt to retry this - * call on your behalf. + * call on your behalf. Additionally, if a default location is provided in + * the client configuration it will be used when creating the dataset. * * Example: * ``` @@ -537,6 +577,10 @@ public function createDataset($id, array $options = []) unset($options['metadata']); } + if (!isset($options['location']) && $this->location) { + $options['location'] = $this->location; + } + $response = $this->connection->insertDataset( [ 'projectId' => $this->projectId, diff --git a/BigQuery/src/Connection/ServiceDefinition/bigquery-v2.json b/BigQuery/src/Connection/ServiceDefinition/bigquery-v2.json index af979c11d68f..e7fc75715f71 100644 --- a/BigQuery/src/Connection/ServiceDefinition/bigquery-v2.json +++ b/BigQuery/src/Connection/ServiceDefinition/bigquery-v2.json @@ -1,11 +1,11 @@ { "kind": "discovery#restDescription", - "etag": "\"tbys6C40o18GZwyMen5GMkdK-3s/38ec4LBichi45rs7Z88p5kE7xpM\"", + "etag": "\"-iA1DTNe4s-I6JZXPt1t1Ypy8IU/77sYFu0di9EzzwuBGmUFES4JWbE\"", "discoveryVersion": "v1", "id": "bigquery:v2", "name": "bigquery", "version": "v2", - "revision": "20161029", + "revision": "20180311", "title": "BigQuery API", "description": "A data platform for customers to create, manage, share and query data.", "ownerDomain": "google.com", @@ -20,7 +20,7 @@ "basePath": "/bigquery/v2/", "rootUrl": "https://www.googleapis.com/", "servicePath": "bigquery/v2/", - "batchPath": "batch", + "batchPath": "batch/bigquery/v2", "parameters": { "alt": { "type": "string", @@ -281,7 +281,7 @@ }, "labels": { "type": "object", - "description": "[Experimental] The labels associated with this dataset. You can use these to organize and group your datasets. You can set this property when inserting or updating a dataset. See Labeling Datasets for more information.", + "description": "The labels associated with this dataset. You can use these to organize and group your datasets. You can set this property when inserting or updating a dataset. See Labeling Datasets for more information.", "additionalProperties": { "type": "string" } @@ -293,7 +293,7 @@ }, "location": { "type": "string", - "description": "[Experimental] The geographic location where the dataset should reside. Possible values include EU and US. The default value is US." + "description": "The geographic location where the dataset should reside. Possible values include EU and US. The default value is US." }, "selfLink": { "type": "string", @@ -330,10 +330,14 @@ }, "labels": { "type": "object", - "description": "[Experimental] The labels associated with this dataset. You can use these to organize and group your datasets.", + "description": "The labels associated with this dataset. You can use these to organize and group your datasets.", "additionalProperties": { "type": "string" } + }, + "location": { + "type": "string", + "description": "[Experimental] The geographic location where the data resides." } } } @@ -377,6 +381,30 @@ } } }, + "DestinationTableProperties": { + "id": "DestinationTableProperties", + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "[Optional] The description for the destination table. This will only be used if the destination table is newly created. If the table already exists and a value different than the current description is provided, the job will fail." + }, + "friendlyName": { + "type": "string", + "description": "[Optional] The friendly name for the destination table. This will only be used if the destination table is newly created. If the table already exists and a value different than the current friendly name is provided, the job will fail." + } + } + }, + "EncryptionConfiguration": { + "id": "EncryptionConfiguration", + "type": "object", + "properties": { + "kmsKeyName": { + "type": "string", + "description": "[Optional] Describes the Cloud KMS encryption key that will be used to protect destination BigQuery table. The BigQuery Service Account associated with your project requires access to this encryption key." + } + } + }, "ErrorProto": { "id": "ErrorProto", "type": "object", @@ -403,6 +431,21 @@ "id": "ExplainQueryStage", "type": "object", "properties": { + "completedParallelInputs": { + "type": "string", + "description": "Number of parallel input segments completed.", + "format": "int64" + }, + "computeMsAvg": { + "type": "string", + "description": "Milliseconds the average shard spent on CPU-bound tasks.", + "format": "int64" + }, + "computeMsMax": { + "type": "string", + "description": "Milliseconds the slowest shard spent on CPU-bound tasks.", + "format": "int64" + }, "computeRatioAvg": { "type": "number", "description": "Relative amount of time the average shard spent on CPU-bound tasks.", @@ -413,15 +456,43 @@ "description": "Relative amount of time the slowest shard spent on CPU-bound tasks.", "format": "double" }, + "endMs": { + "type": "string", + "description": "Stage end time in milliseconds.", + "format": "int64" + }, "id": { "type": "string", "description": "Unique ID for stage within plan.", "format": "int64" }, + "inputStages": { + "type": "array", + "description": "IDs for stages that are inputs to this stage.", + "items": { + "type": "string", + "format": "int64" + } + }, "name": { "type": "string", "description": "Human-readable name for stage." }, + "parallelInputs": { + "type": "string", + "description": "Number of parallel input segments to be processed.", + "format": "int64" + }, + "readMsAvg": { + "type": "string", + "description": "Milliseconds the average shard spent reading input.", + "format": "int64" + }, + "readMsMax": { + "type": "string", + "description": "Milliseconds the slowest shard spent reading input.", + "format": "int64" + }, "readRatioAvg": { "type": "number", "description": "Relative amount of time the average shard spent reading input.", @@ -442,6 +513,25 @@ "description": "Number of records written by the stage.", "format": "int64" }, + "shuffleOutputBytes": { + "type": "string", + "description": "Total number of bytes written to shuffle.", + "format": "int64" + }, + "shuffleOutputBytesSpilled": { + "type": "string", + "description": "Total number of bytes written to shuffle and spilled to disk.", + "format": "int64" + }, + "startMs": { + "type": "string", + "description": "Stage start time in milliseconds.", + "format": "int64" + }, + "status": { + "type": "string", + "description": "Current status for the stage." + }, "steps": { "type": "array", "description": "List of operations within the stage in dependency order (approximately chronological).", @@ -449,6 +539,16 @@ "$ref": "ExplainQueryStep" } }, + "waitMsAvg": { + "type": "string", + "description": "Milliseconds the average shard spent waiting to be scheduled.", + "format": "int64" + }, + "waitMsMax": { + "type": "string", + "description": "Milliseconds the slowest shard spent waiting to be scheduled.", + "format": "int64" + }, "waitRatioAvg": { "type": "number", "description": "Relative amount of time the average shard spent waiting to be scheduled.", @@ -459,6 +559,16 @@ "description": "Relative amount of time the slowest shard spent waiting to be scheduled.", "format": "double" }, + "writeMsAvg": { + "type": "string", + "description": "Milliseconds the average shard spent on writing output.", + "format": "int64" + }, + "writeMsMax": { + "type": "string", + "description": "Milliseconds the slowest shard spent on writing output.", + "format": "int64" + }, "writeRatioAvg": { "type": "number", "description": "Relative amount of time the average shard spent on writing output.", @@ -494,7 +604,7 @@ "properties": { "autodetect": { "type": "boolean", - "description": "[Experimental] Try to detect schema and format options automatically. Any option specified explicitly will be honored." + "description": "Try to detect schema and format options automatically. Any option specified explicitly will be honored." }, "bigtableOptions": { "$ref": "BigtableOptions", @@ -527,11 +637,11 @@ }, "sourceFormat": { "type": "string", - "description": "[Required] The data format. For CSV files, specify \"CSV\". For Google sheets, specify \"GOOGLE_SHEETS\". For newline-delimited JSON, specify \"NEWLINE_DELIMITED_JSON\". For Avro files, specify \"AVRO\". For Google Cloud Datastore backups, specify \"DATASTORE_BACKUP\". [Experimental] For Google Cloud Bigtable, specify \"BIGTABLE\". Please note that reading from Google Cloud Bigtable is experimental and has to be enabled for your project. Please contact Google Cloud Support to enable this for your project." + "description": "[Required] The data format. For CSV files, specify \"CSV\". For Google sheets, specify \"GOOGLE_SHEETS\". For newline-delimited JSON, specify \"NEWLINE_DELIMITED_JSON\". For Avro files, specify \"AVRO\". For Google Cloud Datastore backups, specify \"DATASTORE_BACKUP\". [Beta] For Google Cloud Bigtable, specify \"BIGTABLE\"." }, "sourceUris": { "type": "array", - "description": "[Required] The fully-qualified URIs that point to your data in Google Cloud. For Google Cloud Storage URIs: Each URI can contain one '*' wildcard character and it must come after the 'bucket' name. Size limits related to load jobs apply to external data sources. For Google Cloud Bigtable URIs: Exactly one URI can be specified and it has be a fully specified and valid HTTPS URL for a Google Cloud Bigtable table. For Google Cloud Datastore backups, exactly one URI can be specified, and it must end with '.backup_info'. Also, the '*' wildcard character is not allowed.", + "description": "[Required] The fully-qualified URIs that point to your data in Google Cloud. For Google Cloud Storage URIs: Each URI can contain one '*' wildcard character and it must come after the 'bucket' name. Size limits related to load jobs apply to external data sources. For Google Cloud Bigtable URIs: Exactly one URI can be specified and it has be a fully specified and valid HTTPS URL for a Google Cloud Bigtable table. For Google Cloud Datastore backups, exactly one URI can be specified. Also, the '*' wildcard character is not allowed.", "items": { "type": "string" } @@ -548,7 +658,7 @@ }, "errors": { "type": "array", - "description": "[Output-only] All errors and warnings encountered during the running of the job. Errors here do not necessarily mean that the job has completed or was unsuccessful.", + "description": "[Output-only] The first errors or warnings encountered during the running of the job. The final message includes the number of errors that caused the process to stop. Errors here do not necessarily mean that the job has completed or was unsuccessful.", "items": { "$ref": "ErrorProto" } @@ -572,7 +682,7 @@ }, "numDmlAffectedRows": { "type": "string", - "description": "[Output-only, Experimental] The number of rows affected by a DML statement. Present only for DML statements INSERT, UPDATE or DELETE.", + "description": "[Output-only] The number of rows affected by a DML statement. Present only for DML statements INSERT, UPDATE or DELETE.", "format": "int64" }, "pageToken": { @@ -602,6 +712,21 @@ } } }, + "GetServiceAccountResponse": { + "id": "GetServiceAccountResponse", + "type": "object", + "properties": { + "email": { + "type": "string", + "description": "The service account email address." + }, + "kind": { + "type": "string", + "description": "The resource type of the response.", + "default": "bigquery#getServiceAccountResponse" + } + } + }, "GoogleSheetsOptions": { "id": "GoogleSheetsOptions", "type": "object", @@ -687,9 +812,14 @@ "$ref": "JobConfigurationExtract", "description": "[Pick one] Configures an extract job." }, + "jobTimeoutMs": { + "type": "string", + "description": "[Optional] Job timeout in milliseconds. If this time limit is exceeded, BigQuery may attempt to terminate the job.", + "format": "int64" + }, "labels": { "type": "object", - "description": "[Experimental] The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only contain letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and must be unique within a dataset. Both keys and values are additionally constrained to be <= 128 bytes in size.", + "description": "The labels associated with this job. You can use these to organize and group your jobs. Label keys and values can be no longer than 63 characters, can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key.", "additionalProperties": { "type": "string" } @@ -710,7 +840,7 @@ "properties": { "compression": { "type": "string", - "description": "[Optional] The compression type to use for exported files. Possible values include GZIP and NONE. The default value is NONE." + "description": "[Optional] The compression type to use for exported files. Possible values include GZIP, DEFLATE, SNAPPY, and NONE. The default value is NONE. DEFLATE and SNAPPY are only supported for Avro." }, "destinationFormat": { "type": "string", @@ -756,16 +886,24 @@ }, "autodetect": { "type": "boolean", - "description": "[Experimental] Indicates if we should automatically infer the options and schema for CSV and JSON sources." + "description": "Indicates if we should automatically infer the options and schema for CSV and JSON sources." }, "createDisposition": { "type": "string", "description": "[Optional] Specifies whether the job is allowed to create new tables. The following values are supported: CREATE_IF_NEEDED: If the table does not exist, BigQuery creates the table. CREATE_NEVER: The table must already exist. If it does not, a 'notFound' error is returned in the job result. The default value is CREATE_IF_NEEDED. Creation, truncation and append actions occur as one atomic update upon job completion." }, + "destinationEncryptionConfiguration": { + "$ref": "EncryptionConfiguration", + "description": "Custom encryption configuration (e.g., Cloud KMS keys)." + }, "destinationTable": { "$ref": "TableReference", "description": "[Required] The destination table to load the data into." }, + "destinationTableProperties": { + "$ref": "DestinationTableProperties", + "description": "[Experimental] [Optional] Properties with which to create the destination table if it is new." + }, "encoding": { "type": "string", "description": "[Optional] The character encoding of the data. The supported values are UTF-8 or ISO-8859-1. The default value is UTF-8. BigQuery decodes the data after the raw, binary data has been split using the values of the quote and fieldDelimiter properties." @@ -783,9 +921,13 @@ "description": "[Optional] The maximum number of bad records that BigQuery can ignore when running the job. If the number of bad records exceeds this value, an invalid error is returned in the job result. The default value is 0, which requires that all records are valid.", "format": "int32" }, + "nullMarker": { + "type": "string", + "description": "[Optional] Specifies a string that represents a null value in a CSV file. For example, if you specify \"\\N\", BigQuery interprets \"\\N\" as a null value when loading a CSV file. The default value is the empty string. If you set this property to a custom value, BigQuery throws an error if an empty string is present for all data types except for STRING and BYTE. For STRING and BYTE columns, BigQuery interprets the empty string as an empty value." + }, "projectionFields": { "type": "array", - "description": "[Experimental] If sourceFormat is set to \"DATASTORE_BACKUP\", indicates which entity properties to load into BigQuery from a Cloud Datastore backup. Property names are case sensitive and must be top-level properties. If no properties are specified, BigQuery loads all properties. If any named property isn't found in the Cloud Datastore backup, an invalid error is returned in the job result.", + "description": "If sourceFormat is set to \"DATASTORE_BACKUP\", indicates which entity properties to load into BigQuery from a Cloud Datastore backup. Property names are case sensitive and must be top-level properties. If no properties are specified, BigQuery loads all properties. If any named property isn't found in the Cloud Datastore backup, an invalid error is returned in the job result.", "items": { "type": "string" } @@ -810,7 +952,7 @@ }, "schemaUpdateOptions": { "type": "array", - "description": "[Experimental] Allows the schema of the desitination table to be updated as a side effect of the load job. Schema update options are supported in two cases: when writeDisposition is WRITE_APPEND; when writeDisposition is WRITE_TRUNCATE and the destination table is a partition of a table, specified by partition decorators. For normal tables, WRITE_TRUNCATE will always overwrite the schema. One or more of the following values are specified: ALLOW_FIELD_ADDITION: allow adding a nullable field to the schema. ALLOW_FIELD_RELAXATION: allow relaxing a required field in the original schema to nullable.", + "description": "Allows the schema of the destination table to be updated as a side effect of the load job if a schema is autodetected or supplied in the job configuration. Schema update options are supported in two cases: when writeDisposition is WRITE_APPEND; when writeDisposition is WRITE_TRUNCATE and the destination table is a partition of a table, specified by partition decorators. For normal tables, WRITE_TRUNCATE will always overwrite the schema. One or more of the following values are specified: ALLOW_FIELD_ADDITION: allow adding a nullable field to the schema. ALLOW_FIELD_RELAXATION: allow relaxing a required field in the original schema to nullable.", "items": { "type": "string" } @@ -822,15 +964,19 @@ }, "sourceFormat": { "type": "string", - "description": "[Optional] The format of the data files. For CSV files, specify \"CSV\". For datastore backups, specify \"DATASTORE_BACKUP\". For newline-delimited JSON, specify \"NEWLINE_DELIMITED_JSON\". For Avro, specify \"AVRO\". The default value is CSV." + "description": "[Optional] The format of the data files. For CSV files, specify \"CSV\". For datastore backups, specify \"DATASTORE_BACKUP\". For newline-delimited JSON, specify \"NEWLINE_DELIMITED_JSON\". For Avro, specify \"AVRO\". For parquet, specify \"PARQUET\". For orc, specify \"ORC\". The default value is CSV." }, "sourceUris": { "type": "array", - "description": "[Required] The fully-qualified URIs that point to your data in Google Cloud Storage. Each URI can contain one '*' wildcard character and it must come after the 'bucket' name.", + "description": "[Required] The fully-qualified URIs that point to your data in Google Cloud. For Google Cloud Storage URIs: Each URI can contain one '*' wildcard character and it must come after the 'bucket' name. Size limits related to load jobs apply to external data sources. For Google Cloud Bigtable URIs: Exactly one URI can be specified and it has be a fully specified and valid HTTPS URL for a Google Cloud Bigtable table. For Google Cloud Datastore backups: Exactly one URI can be specified. Also, the '*' wildcard character is not allowed.", "items": { "type": "string" } }, + "timePartitioning": { + "$ref": "TimePartitioning", + "description": "If specified, configures time-based partitioning for the destination table." + }, "writeDisposition": { "type": "string", "description": "[Optional] Specifies the action that occurs if the destination table already exists. The following values are supported: WRITE_TRUNCATE: If the table already exists, BigQuery overwrites the table data. WRITE_APPEND: If the table already exists, BigQuery appends the data to the table. WRITE_EMPTY: If the table already exists and contains data, a 'duplicate' error is returned in the job result. The default value is WRITE_APPEND. Each action is atomic and only occurs if BigQuery is able to complete the job successfully. Creation, truncation and append actions occur as one atomic update upon job completion." @@ -843,7 +989,8 @@ "properties": { "allowLargeResults": { "type": "boolean", - "description": "If true, allows the query to produce arbitrarily large result tables at a slight cost in performance. Requires destinationTable to be set." + "description": "[Optional] If true and query uses legacy SQL dialect, allows the query to produce arbitrarily large result tables at a slight cost in performance. Requires destinationTable to be set. For standard SQL queries, this flag is ignored and large results are always allowed. However, you must still set destinationTable when result size exceeds the allowed maximum response size.", + "default": "false" }, "createDisposition": { "type": "string", @@ -853,13 +1000,17 @@ "$ref": "DatasetReference", "description": "[Optional] Specifies the default dataset to use for unqualified table names in the query." }, + "destinationEncryptionConfiguration": { + "$ref": "EncryptionConfiguration", + "description": "Custom encryption configuration (e.g., Cloud KMS keys)." + }, "destinationTable": { "$ref": "TableReference", - "description": "[Optional] Describes the table where the query results should be stored. If not present, a new table will be created to store the results." + "description": "[Optional] Describes the table where the query results should be stored. If not present, a new table will be created to store the results. This property must be set for large results that exceed the maximum response size." }, "flattenResults": { "type": "boolean", - "description": "[Optional] Flattens all nested and repeated fields in the query results. The default value is true. allowLargeResults must be true if this is set to false.", + "description": "[Optional] If true and query uses legacy SQL dialect, flattens all nested and repeated fields in the query results. allowLargeResults must be true if this is set to false. For standard SQL queries, this flag is ignored and results are never flattened.", "default": "true" }, "maximumBillingTier": { @@ -875,7 +1026,7 @@ }, "parameterMode": { "type": "string", - "description": "[Experimental] Standard SQL only. Whether to use positional (?) or named (@myparam) query parameters in this query." + "description": "Standard SQL only. Set to POSITIONAL to use positional (?) query parameters or to NAMED to use named (@myparam) query parameters in this query." }, "preserveNulls": { "type": "boolean", @@ -887,7 +1038,7 @@ }, "query": { "type": "string", - "description": "[Required] BigQuery SQL query to execute." + "description": "[Required] SQL query text to execute. The useLegacySql field can be used to indicate whether the query uses legacy SQL or standard SQL." }, "queryParameters": { "type": "array", @@ -898,7 +1049,7 @@ }, "schemaUpdateOptions": { "type": "array", - "description": "[Experimental] Allows the schema of the destination table to be updated as a side effect of the query job. Schema update options are supported in two cases: when writeDisposition is WRITE_APPEND; when writeDisposition is WRITE_TRUNCATE and the destination table is a partition of a table, specified by partition decorators. For normal tables, WRITE_TRUNCATE will always overwrite the schema. One or more of the following values are specified: ALLOW_FIELD_ADDITION: allow adding a nullable field to the schema. ALLOW_FIELD_RELAXATION: allow relaxing a required field in the original schema to nullable.", + "description": "Allows the schema of the destination table to be updated as a side effect of the query job. Schema update options are supported in two cases: when writeDisposition is WRITE_APPEND; when writeDisposition is WRITE_TRUNCATE and the destination table is a partition of a table, specified by partition decorators. For normal tables, WRITE_TRUNCATE will always overwrite the schema. One or more of the following values are specified: ALLOW_FIELD_ADDITION: allow adding a nullable field to the schema. ALLOW_FIELD_RELAXATION: allow relaxing a required field in the original schema to nullable.", "items": { "type": "string" } @@ -910,9 +1061,13 @@ "$ref": "ExternalDataConfiguration" } }, + "timePartitioning": { + "$ref": "TimePartitioning", + "description": "If specified, configures time-based partitioning for the destination table." + }, "useLegacySql": { "type": "boolean", - "description": "Specifies whether to use BigQuery's legacy SQL dialect for this query. The default value is true. If set to false, the query will use BigQuery's standard SQL: https://cloud.google.com/bigquery/sql-reference/ When useLegacySql is set to false, the values of allowLargeResults and flattenResults are ignored; query will be run as if allowLargeResults is true and flattenResults is false." + "description": "Specifies whether to use BigQuery's legacy SQL dialect for this query. The default value is true. If set to false, the query will use BigQuery's standard SQL: https://cloud.google.com/bigquery/sql-reference/ When useLegacySql is set to false, the value of flattenResults is ignored; query will be run as if flattenResults is false." }, "useQueryCache": { "type": "boolean", @@ -921,14 +1076,14 @@ }, "userDefinedFunctionResources": { "type": "array", - "description": "[Experimental] Describes user-defined function resources used in the query.", + "description": "Describes user-defined function resources used in the query.", "items": { "$ref": "UserDefinedFunctionResource" } }, "writeDisposition": { "type": "string", - "description": "[Optional] Specifies the action that occurs if the destination table already exists. The following values are supported: WRITE_TRUNCATE: If the table already exists, BigQuery overwrites the table data. WRITE_APPEND: If the table already exists, BigQuery appends the data to the table. WRITE_EMPTY: If the table already exists and contains data, a 'duplicate' error is returned in the job result. The default value is WRITE_EMPTY. Each action is atomic and only occurs if BigQuery is able to complete the job successfully. Creation, truncation and append actions occur as one atomic update upon job completion." + "description": "[Optional] Specifies the action that occurs if the destination table already exists. The following values are supported: WRITE_TRUNCATE: If the table already exists, BigQuery overwrites the table data and uses the schema from the query result. WRITE_APPEND: If the table already exists, BigQuery appends the data to the table. WRITE_EMPTY: If the table already exists and contains data, a 'duplicate' error is returned in the job result. The default value is WRITE_EMPTY. Each action is atomic and only occurs if BigQuery is able to complete the job successfully. Creation, truncation and append actions occur as one atomic update upon job completion." } } }, @@ -940,6 +1095,10 @@ "type": "string", "description": "[Optional] Specifies whether the job is allowed to create new tables. The following values are supported: CREATE_IF_NEEDED: If the table does not exist, BigQuery creates the table. CREATE_NEVER: The table must already exist. If it does not, a 'notFound' error is returned in the job result. The default value is CREATE_IF_NEEDED. Creation, truncation and append actions occur as one atomic update upon job completion." }, + "destinationEncryptionConfiguration": { + "$ref": "EncryptionConfiguration", + "description": "Custom encryption configuration (e.g., Cloud KMS keys)." + }, "destinationTable": { "$ref": "TableReference", "description": "[Required] The destination table" @@ -1039,6 +1198,10 @@ ] } }, + "location": { + "type": "string", + "description": "[Experimental] The geographic location of the job. Required except for US and EU." + }, "projectId": { "type": "string", "description": "[Required] The ID of the project containing this job.", @@ -1054,6 +1217,11 @@ "id": "JobStatistics", "type": "object", "properties": { + "completionRatio": { + "type": "number", + "description": "[Experimental] [Output-only] Job progress (0.0 -> 1.0) for LOAD and EXTRACT jobs.", + "format": "double" + }, "creationTime": { "type": "string", "description": "[Output-only] Creation time of this job, in milliseconds since the epoch. This field will be present on all jobs.", @@ -1101,28 +1269,52 @@ "type": "boolean", "description": "[Output-only] Whether the query result was fetched from the query cache." }, + "ddlOperationPerformed": { + "type": "string", + "description": "[Output-only, Experimental] The DDL operation performed, possibly dependent on the pre-existence of the DDL target. Possible values (new values might be added in the future): \"CREATE\": The query created the DDL target. \"SKIP\": No-op. Example cases: the query is CREATE TABLE IF NOT EXISTS while the table already exists, or the query is DROP TABLE IF EXISTS while the table does not exist. \"REPLACE\": The query replaced the DDL target. Example case: the query is CREATE OR REPLACE TABLE, and the table already exists. \"DROP\": The query deleted the DDL target." + }, + "ddlTargetTable": { + "$ref": "TableReference", + "description": "[Output-only, Experimental] The DDL target table. Present only for CREATE/DROP TABLE/VIEW queries." + }, + "estimatedBytesProcessed": { + "type": "string", + "description": "[Output-only] The original estimate of bytes processed for the job.", + "format": "int64" + }, "numDmlAffectedRows": { "type": "string", - "description": "[Output-only, Experimental] The number of rows affected by a DML statement. Present only for DML statements INSERT, UPDATE or DELETE.", + "description": "[Output-only] The number of rows affected by a DML statement. Present only for DML statements INSERT, UPDATE or DELETE.", "format": "int64" }, "queryPlan": { "type": "array", - "description": "[Output-only, Experimental] Describes execution plan for the query.", + "description": "[Output-only] Describes execution plan for the query.", "items": { "$ref": "ExplainQueryStage" } }, "referencedTables": { "type": "array", - "description": "[Output-only, Experimental] Referenced tables for the job. Queries that reference more than 50 tables will not have a complete list.", + "description": "[Output-only] Referenced tables for the job. Queries that reference more than 50 tables will not have a complete list.", "items": { "$ref": "TableReference" } }, "schema": { "$ref": "TableSchema", - "description": "[Output-only, Experimental] The schema of the results. Present only for successful dry run of non-legacy SQL queries." + "description": "[Output-only] The schema of the results. Present only for successful dry run of non-legacy SQL queries." + }, + "statementType": { + "type": "string", + "description": "[Output-only, Experimental] The type of query statement, if valid. Possible values (new values might be added in the future): \"SELECT\": SELECT query. \"INSERT\": INSERT query; see https://cloud.google.com/bigquery/docs/reference/standard-sql/data-manipulation-language \"UPDATE\": UPDATE query; see https://cloud.google.com/bigquery/docs/reference/standard-sql/data-manipulation-language \"DELETE\": DELETE query; see https://cloud.google.com/bigquery/docs/reference/standard-sql/data-manipulation-language \"CREATE_TABLE\": CREATE [OR REPLACE] TABLE without AS SELECT. \"CREATE_TABLE_AS_SELECT\": CREATE [OR REPLACE] TABLE ... AS SELECT ... \"DROP_TABLE\": DROP TABLE query. \"CREATE_VIEW\": CREATE [OR REPLACE] VIEW ... AS SELECT ... \"DROP_VIEW\": DROP VIEW query." + }, + "timeline": { + "type": "array", + "description": "[Output-only] [Experimental] Describes a timeline of job execution.", + "items": { + "$ref": "QueryTimelineSample" + } }, "totalBytesBilled": { "type": "string", @@ -1134,6 +1326,16 @@ "description": "[Output-only] Total bytes processed for the job.", "format": "int64" }, + "totalPartitionsProcessed": { + "type": "string", + "description": "[Output-only] Total number of partitions processed from all partitioned tables referenced in the job.", + "format": "int64" + }, + "totalSlotMs": { + "type": "string", + "description": "[Output-only] Slot-milliseconds for the job.", + "format": "int64" + }, "undeclaredQueryParameters": { "type": "array", "description": "[Output-only, Experimental] Standard SQL only: list of undeclared query parameters detected during a dry run validation.", @@ -1147,6 +1349,11 @@ "id": "JobStatistics3", "type": "object", "properties": { + "badRecords": { + "type": "string", + "description": "[Output-only] The number of bad records encountered. Note that if the job has failed because of more bad records encountered than the maximum allowed in the load job configuration, then this number can be less than the total number of bad records present in the input data.", + "format": "int64" + }, "inputFileBytes": { "type": "string", "description": "[Output-only] Number of bytes of source data in a load job.", @@ -1193,7 +1400,7 @@ }, "errors": { "type": "array", - "description": "[Output-only] All errors encountered during the running of the job. Errors here do not necessarily mean that the job has completed or was unsuccessful.", + "description": "[Output-only] The first errors encountered during the running of the job. The final message includes the number of errors that caused the process to stop. Errors here do not necessarily mean that the job has completed or was unsuccessful.", "items": { "$ref": "ErrorProto" } @@ -1375,6 +1582,10 @@ "description": "The resource type of the request.", "default": "bigquery#queryRequest" }, + "location": { + "type": "string", + "description": "[Experimental] The geographic location where the job should run. Required except for US and EU." + }, "maxResults": { "type": "integer", "description": "[Optional] The maximum number of rows of data to return per page of results. Setting this flag to a small value such as 1000 and then paging through results might improve reliability when the query result set is large. In addition to this limit, responses are also limited to 10 MB. By default, there is no maximum row count, and only the byte limit applies.", @@ -1382,7 +1593,7 @@ }, "parameterMode": { "type": "string", - "description": "[Experimental] Standard SQL only. Whether to use positional (?) or named (@myparam) query parameters in this query." + "description": "Standard SQL only. Set to POSITIONAL to use positional (?) query parameters or to NAMED to use named (@myparam) query parameters in this query." }, "preserveNulls": { "type": "boolean", @@ -1399,7 +1610,7 @@ }, "queryParameters": { "type": "array", - "description": "[Experimental] Query parameters for Standard SQL queries.", + "description": "Query parameters for Standard SQL queries.", "items": { "$ref": "QueryParameter" } @@ -1411,7 +1622,7 @@ }, "useLegacySql": { "type": "boolean", - "description": "Specifies whether to use BigQuery's legacy SQL dialect for this query. The default value is true. If set to false, the query will use BigQuery's standard SQL: https://cloud.google.com/bigquery/sql-reference/ When useLegacySql is set to false, the values of allowLargeResults and flattenResults are ignored; query will be run as if allowLargeResults is true and flattenResults is false.", + "description": "Specifies whether to use BigQuery's legacy SQL dialect for this query. The default value is true. If set to false, the query will use BigQuery's standard SQL: https://cloud.google.com/bigquery/sql-reference/ When useLegacySql is set to false, the value of flattenResults is ignored; query will be run as if flattenResults is false.", "default": "true" }, "useQueryCache": { @@ -1431,7 +1642,7 @@ }, "errors": { "type": "array", - "description": "[Output-only] All errors and warnings encountered during the running of the job. Errors here do not necessarily mean that the job has completed or was unsuccessful.", + "description": "[Output-only] The first errors or warnings encountered during the running of the job. The final message includes the number of errors that caused the process to stop. Errors here do not necessarily mean that the job has completed or was unsuccessful.", "items": { "$ref": "ErrorProto" } @@ -1451,7 +1662,7 @@ }, "numDmlAffectedRows": { "type": "string", - "description": "[Output-only, Experimental] The number of rows affected by a DML statement. Present only for DML statements INSERT, UPDATE or DELETE.", + "description": "[Output-only] The number of rows affected by a DML statement. Present only for DML statements INSERT, UPDATE or DELETE.", "format": "int64" }, "pageToken": { @@ -1481,6 +1692,37 @@ } } }, + "QueryTimelineSample": { + "id": "QueryTimelineSample", + "type": "object", + "properties": { + "activeInputs": { + "type": "string", + "description": "Total number of active workers. This does not correspond directly to slot usage. This is the largest value observed since the last sample.", + "format": "int64" + }, + "completedInputs": { + "type": "string", + "description": "Total parallel units of work completed by this query.", + "format": "int64" + }, + "elapsedMs": { + "type": "string", + "description": "Milliseconds elapsed since the start of query execution.", + "format": "int64" + }, + "pendingInputs": { + "type": "string", + "description": "Total parallel units of work remaining for the active stages.", + "format": "int64" + }, + "totalSlotMs": { + "type": "string", + "description": "Cumulative slot-ms consumed by the query.", + "format": "int64" + } + } + }, "Streamingbuffer": { "id": "Streamingbuffer", "type": "object", @@ -1515,13 +1757,17 @@ "type": "string", "description": "[Optional] A user-friendly description of this table." }, + "encryptionConfiguration": { + "$ref": "EncryptionConfiguration", + "description": "Custom encryption configuration (e.g., Cloud KMS keys)." + }, "etag": { "type": "string", "description": "[Output-only] A hash of this resource." }, "expirationTime": { "type": "string", - "description": "[Optional] The time when this table expires, in milliseconds since the epoch. If not present, the table will persist indefinitely. Expired tables will be deleted and their storage reclaimed.", + "description": "[Optional] The time when this table expires, in milliseconds since the epoch. If not present, the table will persist indefinitely. Expired tables will be deleted and their storage reclaimed. The defaultTableExpirationMs property of the encapsulating dataset can be used to set a default expirationTime on newly created tables.", "format": "int64" }, "externalDataConfiguration": { @@ -1541,6 +1787,13 @@ "description": "[Output-only] The type of the resource.", "default": "bigquery#table" }, + "labels": { + "type": "object", + "description": "The labels associated with this table. You can use these to organize and group your tables. Label keys and values can be no longer than 63 characters, can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. Label values are optional. Label keys must start with a letter and each label in the list must have a different key.", + "additionalProperties": { + "type": "string" + } + }, "lastModifiedTime": { "type": "string", "description": "[Output-only] The time when this table was last modified, in milliseconds since the epoch.", @@ -1583,7 +1836,7 @@ }, "timePartitioning": { "$ref": "TimePartitioning", - "description": "[Experimental] If specified, configures time-based partitioning for this table." + "description": "If specified, configures time-based partitioning for this table." }, "type": { "type": "string", @@ -1713,7 +1966,7 @@ "properties": { "description": { "type": "string", - "description": "[Optional] The field description. The maximum length is 16K characters." + "description": "[Optional] The field description. The maximum length is 1,024 characters." }, "fields": { "type": "array", @@ -1732,7 +1985,7 @@ }, "type": { "type": "string", - "description": "[Required] The field data type. Possible values include STRING, BYTES, INTEGER, FLOAT, BOOLEAN, TIMESTAMP, DATE, TIME, DATETIME, or RECORD (where RECORD indicates that the field contains a nested schema)." + "description": "[Required] The field data type. Possible values include STRING, BYTES, INTEGER, INT64 (same as INTEGER), FLOAT, FLOAT64 (same as FLOAT), BOOLEAN, BOOL (same as BOOLEAN), TIMESTAMP, DATE, TIME, DATETIME, RECORD (where RECORD indicates that the field contains a nested schema) or STRUCT (same as RECORD)." } } }, @@ -1759,6 +2012,16 @@ "items": { "type": "object", "properties": { + "creationTime": { + "type": "string", + "description": "The time when this table was created, in milliseconds since the epoch.", + "format": "int64" + }, + "expirationTime": { + "type": "string", + "description": "[Optional] The time when this table expires, in milliseconds since the epoch. If not present, the table will persist indefinitely. Expired tables will be deleted and their storage reclaimed.", + "format": "int64" + }, "friendlyName": { "type": "string", "description": "The user-friendly name for this table." @@ -1772,13 +2035,34 @@ "description": "The resource type.", "default": "bigquery#table" }, + "labels": { + "type": "object", + "description": "The labels associated with this table. You can use these to organize and group your tables.", + "additionalProperties": { + "type": "string" + } + }, "tableReference": { "$ref": "TableReference", "description": "A reference uniquely identifying the table." }, + "timePartitioning": { + "$ref": "TimePartitioning", + "description": "The time-based partitioning for this table." + }, "type": { "type": "string", "description": "The type of table. Possible values are: TABLE, VIEW." + }, + "view": { + "type": "object", + "description": "Additional details for a view.", + "properties": { + "useLegacySql": { + "type": "boolean", + "description": "True if view is defined in legacy SQL dialect, false if in standard SQL." + } + } } } } @@ -1858,9 +2142,18 @@ "description": "[Optional] Number of milliseconds for which to keep the storage for a partition.", "format": "int64" }, + "field": { + "type": "string", + "description": "[Experimental] [Optional] If not set, the table is partitioned by pseudo column '_PARTITIONTIME'; if set, the table is partitioned by this field. The field must be a top-level TIMESTAMP or DATE field. Its mode must be NULLABLE or REQUIRED." + }, + "requirePartitionFilter": { + "type": "boolean", + "description": "[Experimental] [Optional] If set to true, queries over this table require a partition filter that can be used for partition elimination to be specified.", + "default": "false" + }, "type": { "type": "string", - "description": "[Required] The only type supported is DAY, which will generate one partition per day based on data loading time." + "description": "[Required] The only type supported is DAY, which will generate one partition per day." } } }, @@ -1892,7 +2185,7 @@ }, "userDefinedFunctionResources": { "type": "array", - "description": "[Experimental] Describes user-defined function resources used in the query.", + "description": "Describes user-defined function resources used in the query.", "items": { "$ref": "UserDefinedFunctionResource" } @@ -2125,6 +2418,11 @@ "required": true, "location": "path" }, + "location": { + "type": "string", + "description": "[Experimental] The geographic location of the job. Required except for US and EU.", + "location": "query" + }, "projectId": { "type": "string", "description": "[Required] Project ID of the job to cancel", @@ -2156,6 +2454,11 @@ "required": true, "location": "path" }, + "location": { + "type": "string", + "description": "[Experimental] The geographic location of the job. Required except for US and EU.", + "location": "query" + }, "projectId": { "type": "string", "description": "[Required] Project ID of the requested job", @@ -2188,6 +2491,11 @@ "required": true, "location": "path" }, + "location": { + "type": "string", + "description": "[Experimental] The geographic location where the job should run. Required except for US and EU.", + "location": "query" + }, "maxResults": { "type": "integer", "description": "Maximum number of results to read", @@ -2288,12 +2596,24 @@ "description": "Whether to display jobs owned by all users in the project. Default false", "location": "query" }, + "maxCreationTime": { + "type": "string", + "description": "Max value for job creation time, in milliseconds since the POSIX epoch. If set, only jobs created before or at this timestamp are returned", + "format": "uint64", + "location": "query" + }, "maxResults": { "type": "integer", "description": "Maximum number of results to return", "format": "uint32", "location": "query" }, + "minCreationTime": { + "type": "string", + "description": "Min value for job creation time, in milliseconds since the POSIX epoch. If set, only jobs created after or at this timestamp are returned", + "format": "uint64", + "location": "query" + }, "pageToken": { "type": "string", "description": "Page token, returned by a previous call, to request the next page of results", @@ -2379,6 +2699,31 @@ }, "projects": { "methods": { + "getServiceAccount": { + "id": "bigquery.projects.getServiceAccount", + "path": "projects/{projectId}/serviceAccount", + "httpMethod": "GET", + "description": "Returns the email address of the service account for your project used for interactions with Google Cloud KMS.", + "parameters": { + "projectId": { + "type": "string", + "description": "Project ID for which the service account is requested.", + "required": true, + "location": "path" + } + }, + "parameterOrder": [ + "projectId" + ], + "response": { + "$ref": "GetServiceAccountResponse" + }, + "scopes": [ + "https://www.googleapis.com/auth/bigquery", + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloud-platform.read-only" + ] + }, "list": { "id": "bigquery.projects.list", "path": "projects", @@ -2481,6 +2826,11 @@ "required": true, "location": "path" }, + "selectedFields": { + "type": "string", + "description": "List of fields to return (comma-separated). If unspecified, all fields are returned", + "location": "query" + }, "startIndex": { "type": "string", "description": "Zero-based index of the starting row to read", @@ -2565,6 +2915,11 @@ "required": true, "location": "path" }, + "selectedFields": { + "type": "string", + "description": "List of fields to return (comma-separated). If unspecified, all fields are returned", + "location": "query" + }, "tableId": { "type": "string", "description": "Table ID of the requested table", diff --git a/BigQuery/src/CopyJobConfiguration.php b/BigQuery/src/CopyJobConfiguration.php index 29d92aabd270..d373a00f1849 100644 --- a/BigQuery/src/CopyJobConfiguration.php +++ b/BigQuery/src/CopyJobConfiguration.php @@ -42,10 +42,12 @@ class CopyJobConfiguration implements JobConfigurationInterface /** * @param string $projectId The project's ID. * @param array $config A set of configuration options for a job. + * @param string|null $location The geographic location in which the job is + * executed. */ - public function __construct($projectId, array $config) + public function __construct($projectId, array $config, $location) { - $this->jobConfigurationProperties($projectId, $config); + $this->jobConfigurationProperties($projectId, $config, $location); } /** diff --git a/BigQuery/src/Dataset.php b/BigQuery/src/Dataset.php index 864febdc767d..83a12eac7f53 100644 --- a/BigQuery/src/Dataset.php +++ b/BigQuery/src/Dataset.php @@ -48,6 +48,11 @@ class Dataset */ private $info; + /** + * @var string|null A default geographic location. + */ + private $location; + /** * @var ValueMapper Maps values between PHP and BigQuery. */ @@ -59,13 +64,16 @@ class Dataset * @param string $id The dataset's ID. * @param string $projectId The project's ID. * @param array $info [optional] The dataset's metadata. + * @param string|null $location [optional] A default geographic location, + * used when no dataset metadata exists. */ public function __construct( ConnectionInterface $connection, $id, $projectId, ValueMapper $mapper, - array $info = [] + array $info = [], + $location = null ) { $this->connection = $connection; $this->info = $info; @@ -74,6 +82,7 @@ public function __construct( 'datasetId' => $id, 'projectId' => $projectId ]; + $this->location = $location; } /** @@ -187,7 +196,11 @@ public function table($id) $id, $this->identity['datasetId'], $this->identity['projectId'], - $this->mapper + $this->mapper, + [], + isset($this->info['location']) + ? $this->info['location'] + : $this->location ); } diff --git a/BigQuery/src/ExtractJobConfiguration.php b/BigQuery/src/ExtractJobConfiguration.php index 06801edb5b7b..16b2210887cb 100644 --- a/BigQuery/src/ExtractJobConfiguration.php +++ b/BigQuery/src/ExtractJobConfiguration.php @@ -39,10 +39,12 @@ class ExtractJobConfiguration implements JobConfigurationInterface /** * @param string $projectId The project's ID. * @param array $config A set of configuration options for a job. + * @param string|null $location The geographic location in which the job is + * executed. */ - public function __construct($projectId, array $config) + public function __construct($projectId, array $config, $location) { - $this->jobConfigurationProperties($projectId, $config); + $this->jobConfigurationProperties($projectId, $config, $location); } /** diff --git a/BigQuery/src/Job.php b/BigQuery/src/Job.php index 69d09c507a9e..0b7b8a085d41 100644 --- a/BigQuery/src/Job.php +++ b/BigQuery/src/Job.php @@ -45,7 +45,7 @@ class Job private $identity; /** - * @var array The job's metadata + * @var array The job's metadata. */ private $info; @@ -61,19 +61,25 @@ class Job * @param string $projectId The project's ID. * @param ValueMapper $mapper Maps values between PHP and BigQuery. * @param array $info [optional] The job's metadata. + * @param string|null $location [optional] A default geographic location, + * used when no job metadata exists. */ public function __construct( ConnectionInterface $connection, $id, $projectId, ValueMapper $mapper, - array $info = [] + array $info = [], + $location = null ) { $this->connection = $connection; $this->info = $info; $this->identity = [ 'jobId' => $id, - 'projectId' => $projectId + 'projectId' => $projectId, + 'location' => isset($info['jobReference']['location']) + ? $info['jobReference']['location'] + : $location ]; $this->mapper = $mapper; } diff --git a/BigQuery/src/JobConfigurationTrait.php b/BigQuery/src/JobConfigurationTrait.php index 9070633cc107..b7638c6605ef 100644 --- a/BigQuery/src/JobConfigurationTrait.php +++ b/BigQuery/src/JobConfigurationTrait.php @@ -43,14 +43,23 @@ trait JobConfigurationTrait * @access private * @param string $projectId The project's ID. * @param array $config A set of configuration options for a job. + * @param string|null $location The geographic location in which the job is + * executed. */ - public function jobConfigurationProperties($projectId, array $config) - { + public function jobConfigurationProperties( + $projectId, + array $config, + $location + ) { $this->config = array_replace_recursive([ 'projectId' => $projectId, 'jobReference' => ['projectId' => $projectId] ], $config); + if (!isset($this->config['jobReference']['location']) && $location) { + $this->config['jobReference']['location'] = $location; + } + if (!isset($this->config['jobReference']['jobId'])) { $this->config['jobReference']['jobId'] = $this->generateJobId(); } @@ -60,7 +69,7 @@ public function jobConfigurationProperties($projectId, array $config) * Specifies the default dataset to use for unqualified table names in the * query. * - * @param bool $dryRun + * @param bool $dryRun Whether or not to execute as a dry run. * @return JobConfigInterface */ public function dryRun($dryRun) @@ -88,7 +97,7 @@ public function jobIdPrefix($jobIdPrefix) * Specifies the default dataset to use for unqualified table names in the * query. * - * @param array $labels + * @param array $labels The labels to apply. * @return JobConfigInterface */ public function labels(array $labels) @@ -98,6 +107,24 @@ public function labels(array $labels) return $this; } + /** + * Specifies the geographic location of the job. Required for jobs started + * outside of the US and EU regions. + * + * @param string $location The geographic location of the job. + * **Defaults to** a location specified in the client configuration, + * if provided, or through location metadata fetched by a network + * request + * (by calling {@see Google\Cloud\BigQuery\Table::reload()}, for example). + * @return JobConfigInterface + */ + public function location($location) + { + $this->config['jobReference']['location'] = $location; + + return $this; + } + /** * Returns the job config as an array. * diff --git a/BigQuery/src/LoadJobConfiguration.php b/BigQuery/src/LoadJobConfiguration.php index 7e38b2c8a5f0..dee70fb47544 100644 --- a/BigQuery/src/LoadJobConfiguration.php +++ b/BigQuery/src/LoadJobConfiguration.php @@ -39,10 +39,12 @@ class LoadJobConfiguration implements JobConfigurationInterface /** * @param string $projectId The project's ID. * @param array $config A set of configuration options for a job. + * @param string|null $location The geographic location in which the job is + * executed. */ - public function __construct($projectId, array $config) + public function __construct($projectId, array $config, $location) { - $this->jobConfigurationProperties($projectId, $config); + $this->jobConfigurationProperties($projectId, $config, $location); } /** diff --git a/BigQuery/src/QueryJobConfiguration.php b/BigQuery/src/QueryJobConfiguration.php index f41a8ee16f17..4c0af07971c2 100644 --- a/BigQuery/src/QueryJobConfiguration.php +++ b/BigQuery/src/QueryJobConfiguration.php @@ -43,11 +43,17 @@ class QueryJobConfiguration implements JobConfigurationInterface * @param ValueMapper $mapper Maps values between PHP and BigQuery. * @param string $projectId The project's ID. * @param array $config A set of configuration options for a job. + * @param string|null $location The geographic location in which the job is + * executed. */ - public function __construct(ValueMapper $mapper, $projectId, array $config) - { + public function __construct( + ValueMapper $mapper, + $projectId, + array $config, + $location + ) { $this->mapper = $mapper; - $this->jobConfigurationProperties($projectId, $config); + $this->jobConfigurationProperties($projectId, $config, $location); if (!isset($this->config['configuration']['query']['useLegacySql'])) { $this->config['configuration']['query']['useLegacySql'] = false; diff --git a/BigQuery/src/QueryResults.php b/BigQuery/src/QueryResults.php index f8cdcccce79c..b41b6f50b01c 100644 --- a/BigQuery/src/QueryResults.php +++ b/BigQuery/src/QueryResults.php @@ -88,12 +88,15 @@ public function __construct( ) { $this->connection = $connection; $this->info = $info; + $this->job = $job; $this->identity = [ 'jobId' => $jobId, - 'projectId' => $projectId + 'projectId' => $projectId, + 'location' => isset($info['jobReference']['location']) + ? $info['jobReference']['location'] + : $job->identity()['location'] ]; $this->mapper = $mapper; - $this->job = $job; $this->queryResultsOptions = $queryResultsOptions; } diff --git a/BigQuery/src/Table.php b/BigQuery/src/Table.php index edb8d3b5f778..1f34cd855f3c 100644 --- a/BigQuery/src/Table.php +++ b/BigQuery/src/Table.php @@ -58,6 +58,11 @@ class Table */ private $info; + /** + * @var string|null A default geographic location. + */ + private $location; + /** * @var ValueMapper Maps values between PHP and BigQuery. */ @@ -71,6 +76,8 @@ class Table * @param string $projectId The project's id. * @param ValueMapper $mapper Maps values between PHP and BigQuery. * @param array $info [optional] The table's metadata. + * @param string|null $location [optional] A default geographic location, + * used when no table metadata exists. */ public function __construct( ConnectionInterface $connection, @@ -78,7 +85,8 @@ public function __construct( $datasetId, $projectId, ValueMapper $mapper, - array $info = [] + array $info = [], + $location = null ) { $this->connection = $connection; $this->info = $info; @@ -88,6 +96,7 @@ public function __construct( 'datasetId' => $datasetId, 'projectId' => $projectId ]; + $this->location = $location; $this->setHttpRetryCodes([502]); $this->setHttpRetryMessages([ 'rateLimitExceeded', @@ -341,7 +350,10 @@ public function copy(Table $destination, array $options = []) { return (new CopyJobConfiguration( $this->identity['projectId'], - $options + $options, + isset($this->info['location']) + ? $this->info['location'] + : $this->location )) ->destinationTable($destination) ->sourceTable($this); @@ -380,7 +392,10 @@ public function extract($destination, array $options = []) return (new ExtractJobConfiguration( $this->identity['projectId'], - $options + $options, + isset($this->info['location']) + ? $this->info['location'] + : $this->location )) ->destinationUris([$destination]) ->sourceTable($this); @@ -411,7 +426,10 @@ public function load($data, array $options = []) { $config = (new LoadJobConfiguration( $this->identity['projectId'], - $options + $options, + isset($this->info['location']) + ? $this->info['location'] + : $this->location )) ->destinationTable($this); diff --git a/BigQuery/tests/Snippet/BigQueryClientTest.php b/BigQuery/tests/Snippet/BigQueryClientTest.php index 4a422551bb44..4df12305e52f 100644 --- a/BigQuery/tests/Snippet/BigQueryClientTest.php +++ b/BigQuery/tests/Snippet/BigQueryClientTest.php @@ -122,6 +122,19 @@ public function testQueryWithArrayOfOptions() $this->assertEquals(self::CREATE_DISPOSITION, $array['configuration']['query']['createDisposition']); } + public function testQueryWithLocation() + { + $snippet = $this->snippetFromMethod(BigQueryClient::class, 'query', 3); + $snippet->addLocal('bigQuery', $this->client); + $config = $snippet->invoke('queryJobConfig') + ->returnVal(); + $array = $config->toArray(); + + $this->assertInstanceOf(QueryJobConfiguration::class, $config); + $this->assertEquals('SELECT name FROM `my_project.users_dataset.users` LIMIT 100', $array['configuration']['query']['query']); + $this->assertEquals('asia-northeast1', $array['jobReference']['location']); + } + public function testRunQuery() { $snippet = $this->snippetFromMethod(BigQueryClient::class, 'runQuery'); @@ -408,7 +421,8 @@ public function query($query, array $options = []) return (new QueryJobConfigurationStub( new ValueMapper(false), BigQueryClientTest::PROJECT_ID, - $options + $options, + null ))->query($query); } } diff --git a/BigQuery/tests/Snippet/CopyJobConfigurationTest.php b/BigQuery/tests/Snippet/CopyJobConfigurationTest.php index 33e6d3753bbb..dee8508dfe08 100644 --- a/BigQuery/tests/Snippet/CopyJobConfigurationTest.php +++ b/BigQuery/tests/Snippet/CopyJobConfigurationTest.php @@ -37,7 +37,8 @@ public function setUp() { $this->config = new CopyJobConfiguration( self::PROJECT_ID, - ['jobReference' => ['jobId' => self::JOB_ID]] + ['jobReference' => ['jobId' => self::JOB_ID]], + null ); } diff --git a/BigQuery/tests/Snippet/ExtractJobConfigurationTest.php b/BigQuery/tests/Snippet/ExtractJobConfigurationTest.php index 663d59a75a83..36228a85a9ca 100644 --- a/BigQuery/tests/Snippet/ExtractJobConfigurationTest.php +++ b/BigQuery/tests/Snippet/ExtractJobConfigurationTest.php @@ -38,7 +38,8 @@ public function setUp() { $this->config = new ExtractJobConfiguration( self::PROJECT_ID, - ['jobReference' => ['jobId' => self::JOB_ID]] + ['jobReference' => ['jobId' => self::JOB_ID]], + null ); } diff --git a/BigQuery/tests/Snippet/LoadJobConfigurationTest.php b/BigQuery/tests/Snippet/LoadJobConfigurationTest.php index 28a92c1dbed2..16350828e723 100644 --- a/BigQuery/tests/Snippet/LoadJobConfigurationTest.php +++ b/BigQuery/tests/Snippet/LoadJobConfigurationTest.php @@ -37,7 +37,8 @@ public function setUp() { $this->config = new LoadJobConfiguration( self::PROJECT_ID, - ['jobReference' => ['jobId' => self::JOB_ID]] + ['jobReference' => ['jobId' => self::JOB_ID]], + null ); } diff --git a/BigQuery/tests/Snippet/QueryJobConfigurationTest.php b/BigQuery/tests/Snippet/QueryJobConfigurationTest.php index 20048fcad3ea..8649bd28745e 100644 --- a/BigQuery/tests/Snippet/QueryJobConfigurationTest.php +++ b/BigQuery/tests/Snippet/QueryJobConfigurationTest.php @@ -39,7 +39,8 @@ public function setUp() $this->config = new QueryJobConfiguration( new ValueMapper(false), self::PROJECT_ID, - ['jobReference' => ['jobId' => self::JOB_ID]] + ['jobReference' => ['jobId' => self::JOB_ID]], + null ); } diff --git a/BigQuery/tests/System/BigQueryTestCase.php b/BigQuery/tests/System/BigQueryTestCase.php index 09d6721e1269..f3ef8e43bd3a 100644 --- a/BigQuery/tests/System/BigQueryTestCase.php +++ b/BigQuery/tests/System/BigQueryTestCase.php @@ -18,6 +18,7 @@ namespace Google\Cloud\BigQuery\Tests\System; use Google\Cloud\BigQuery\BigQueryClient; +use Google\Cloud\BigQuery\Dataset; use Google\Cloud\Storage\StorageClient; use Google\Cloud\Core\Testing\System\SystemTestCase; @@ -38,7 +39,6 @@ public static function setUpBeforeClass() } $keyFilePath = getenv('GOOGLE_CLOUD_PHP_TESTS_KEY_PATH'); - $schema = json_decode(file_get_contents(__DIR__ . '/data/table-schema.json'), true); $storage = new StorageClient([ 'keyFilePath' => $keyFilePath @@ -50,14 +50,20 @@ public static function setUpBeforeClass() 'keyFilePath' => $keyFilePath ]); self::$dataset = self::createDataset(self::$client, uniqid(self::TESTING_PREFIX)); - self::$table = self::$dataset->createTable(uniqid(self::TESTING_PREFIX), [ - 'schema' => [ - 'fields' => $schema - ] - ]); + self::$table = self::createTable(self::$dataset, uniqid(self::TESTING_PREFIX)); self::$hasSetUp = true; } + + protected static function createTable(Dataset $dataset, $name, array $options = []) + { + if (!isset($options['schema'])) { + $options['schema']['fields'] = json_decode( + file_get_contents(__DIR__ . '/data/table-schema.json'), true + ); + } + return $dataset->createTable(uniqid(self::TESTING_PREFIX), $options); + } } diff --git a/BigQuery/tests/System/RegionalizationTest.php b/BigQuery/tests/System/RegionalizationTest.php new file mode 100644 index 000000000000..0f407113c682 --- /dev/null +++ b/BigQuery/tests/System/RegionalizationTest.php @@ -0,0 +1,207 @@ + self::REGION_ASIA] + ); + self::$tableAsia = self::createTable( + self::$datasetAsia, + uniqid(self::TESTING_PREFIX) + ); + } + + /** + * @dataProvider locationDataProvider + */ + public function testRunQuery($location) + { + $caught = false; + $query = self::$client->query( + sprintf( + self::QUERY_TEMPLATE, + self::$datasetAsia->id(), + self::$tableAsia->id() + ) + )->location($location); + + try { + self::$client->runQuery($query); + } catch (NotFoundException $e) { + $caught = true; + } + + $this->assertExceptionShouldBeThrown($location, $caught); + } + + /** + * @dataProvider locationDataProvider + */ + public function testRunCopyJob($location) + { + $caught = false; + $copyTable = self::$dataset->table(self::TESTING_PREFIX); + $copy = self::$tableAsia->copy($copyTable) + ->location($location); + + try { + self::$tableAsia->startJob($copy); + self::$deletionQueue->add($copyTable); + } catch (NotFoundException $e) { + $caught = true; + } catch (ServiceException $e) { + // Swallow this, as the important bit is whether or not + // we get a 404. + } + + $this->assertExceptionShouldBeThrown($location, $caught); + } + + /** + * @dataProvider locationDataProvider + */ + public function testRunExtractJob($location) + { + $caught = false; + $extract = self::$tableAsia->extract( + self::$bucket->object(self::TESTING_PREFIX) + )->location($location); + + try { + self::$tableAsia->startJob($extract); + } catch (NotFoundException $e) { + $caught = true; + } + + $this->assertExceptionShouldBeThrown($location, $caught); + } + + /** + * @dataProvider locationDataProvider + */ + public function testRunLoadJob($location) + { + $caught = false; + $load = self::$tableAsia->load( + file_get_contents(__DIR__ . '/data/table-data.json') + ) + ->sourceFormat('NEWLINE_DELIMITED_JSON') + ->location($location); + + try { + self::$tableAsia->startJob($load); + }catch (ServiceException $e) { + $code = $e->getCode(); + if ($code === 403 || $code === 404) { + $caught = true; + } + } + + $this->assertExceptionShouldBeThrown($location, $caught); + } + + /** + * @dataProvider locationDataProvider + */ + public function testGetsJob($location) + { + $caught = false; + $query = self::$client->query( + sprintf( + self::QUERY_TEMPLATE, + self::$datasetAsia->id(), + self::$tableAsia->id() + ) + )->location(self::REGION_ASIA); + $job = self::$client->startQuery($query); + + try { + self::$client->job($job->id(), [ + 'location' => $location + ])->reload(); + }catch (NotFoundException $e) { + $caught = true; + } + + $this->assertExceptionShouldBeThrown($location, $caught); + } + + /** + * @dataProvider locationDataProvider + */ + public function testCancelsJob($location) + { + $caught = false; + $query = self::$client->query( + sprintf( + self::QUERY_TEMPLATE, + self::$datasetAsia->id(), + self::$tableAsia->id() + ) + )->location(self::REGION_ASIA); + $job = self::$client->startQuery($query); + + try { + self::$client->job($job->id(), [ + 'location' => $location + ])->cancel(); + }catch (NotFoundException $e) { + $caught = true; + } + + $this->assertExceptionShouldBeThrown($location, $caught); + } + + public function locationDataProvider() + { + return [ + [self::REGION_ASIA], + [self::REGION_US] + ]; + } + + private function assertExceptionShouldBeThrown($location, $caught) + { + if ($location === self::REGION_ASIA) { + $this->assertFalse($caught); + } else { + $this->assertTrue($caught); + } + } +} diff --git a/BigQuery/tests/Unit/BigQueryClientTest.php b/BigQuery/tests/Unit/BigQueryClientTest.php index 74c60961ac6b..9d1e57ff2ec5 100644 --- a/BigQuery/tests/Unit/BigQueryClientTest.php +++ b/BigQuery/tests/Unit/BigQueryClientTest.php @@ -28,6 +28,7 @@ use Google\Cloud\BigQuery\Time; use Google\Cloud\BigQuery\Timestamp; use Google\Cloud\Core\Iterator\ItemIterator; +use Google\Cloud\Core\Testing\TestHelpers; use Prophecy\Argument; use PHPUnit\Framework\TestCase; @@ -39,27 +40,51 @@ class BigQueryClientTest extends TestCase const JOB_ID = 'myJobId'; const PROJECT_ID = 'myProjectId'; const DATASET_ID = 'myDatasetId'; + const TABLE_ID = 'myTableId'; const QUERY_STRING = 'someQuery'; + const LOCATION = 'asia-northeast1'; public $connection; - public $client; public function setUp() { $this->connection = $this->prophesize(ConnectionInterface::class); - $this->client = \Google\Cloud\Core\Testing\TestHelpers::stub(BigQueryClient::class, ['options' => ['projectId' => self::PROJECT_ID]]); + } + + public function getClient($options = []) + { + return TestHelpers::stub( + BigQueryClient::class, + [ + 'options' => [ + 'projectId' => self::PROJECT_ID + ] + $options + ] + ); } public function testQueryConfig() { - $query = $this->client->queryConfig(self::QUERY_STRING); + $query = $this->getClient()->queryConfig(self::QUERY_STRING); $this->assertInstanceOf(QueryJobConfiguration::class, $query); } + public function testQueryUsesDefaultLocation() + { + $client = $this->getClient(['location' => self::LOCATION]); + $query = $client->queryConfig(self::QUERY_STRING); + + $this->assertEquals( + self::LOCATION, + $query->toArray()['jobReference']['location'] + ); + } + public function testRunsQuery() { - $query = $this->client->query(self::QUERY_STRING, [ + $client = $this->getClient(); + $query = $client->query(self::QUERY_STRING, [ 'jobReference' => ['jobId' => self::JOB_ID] ]); $this->connection->insertJob([ @@ -87,8 +112,8 @@ public function testRunsQuery() 'jobComplete' => true ]) ->shouldBeCalledTimes(1); - $this->client->___setProperty('connection', $this->connection->reveal()); - $queryResults = $this->client->runQuery($query); + $client->___setProperty('connection', $this->connection->reveal()); + $queryResults = $client->runQuery($query); $this->assertInstanceOf(QueryResults::class, $queryResults); $this->assertEquals(self::JOB_ID, $queryResults->identity()['jobId']); @@ -96,7 +121,8 @@ public function testRunsQuery() public function testRunsQueryWithRetry() { - $query = $this->client->query(self::QUERY_STRING, [ + $client = $this->getClient(); + $query = $client->query(self::QUERY_STRING, [ 'jobReference' => ['jobId' => self::JOB_ID] ]); $this->connection->insertJob([ @@ -128,8 +154,8 @@ public function testRunsQueryWithRetry() ]) ->shouldBeCalledTimes(1); - $this->client->___setProperty('connection', $this->connection->reveal()); - $queryResults = $this->client->runQuery($query); + $client->___setProperty('connection', $this->connection->reveal()); + $queryResults = $client->runQuery($query); $this->assertInstanceOf(QueryResults::class, $queryResults); $this->assertEquals(self::JOB_ID, $queryResults->identity()['jobId']); @@ -137,7 +163,8 @@ public function testRunsQueryWithRetry() public function testStartQuery() { - $query = $this->client->query(self::QUERY_STRING, [ + $client = $this->getClient(); + $query = $client->query(self::QUERY_STRING, [ 'jobReference' => ['jobId' => self::JOB_ID] ]); $this->connection->insertJob([ @@ -158,8 +185,8 @@ public function testStartQuery() ]) ->shouldBeCalledTimes(1); - $this->client->___setProperty('connection', $this->connection->reveal()); - $job = $this->client->startQuery($query); + $client->___setProperty('connection', $this->connection->reveal()); + $job = $client->startQuery($query); $this->assertInstanceOf(Job::class, $job); $this->assertEquals(self::JOB_ID, $job->id()); @@ -167,24 +194,27 @@ public function testStartQuery() public function testGetsJob() { - $this->client->___setProperty('connection', $this->connection->reveal()); - $this->assertInstanceOf(Job::class, $this->client->job(self::JOB_ID)); + $client = $this->getClient(); + $client->___setProperty('connection', $this->connection->reveal()); + $this->assertInstanceOf(Job::class, $client->job(self::JOB_ID)); } public function testGetsJobsWithNoResults() { + $client = $this->getClient(); $this->connection->listJobs(['projectId' => self::PROJECT_ID]) ->willReturn([]) ->shouldBeCalledTimes(1); - $this->client->___setProperty('connection', $this->connection->reveal()); - $jobs = iterator_to_array($this->client->jobs()); + $client->___setProperty('connection', $this->connection->reveal()); + $jobs = iterator_to_array($client->jobs()); $this->assertEmpty($jobs); } public function testGetsJobsWithoutToken() { + $client = $this->getClient(); $this->connection->listJobs(['projectId' => self::PROJECT_ID]) ->willReturn([ 'jobs' => [ @@ -193,14 +223,15 @@ public function testGetsJobsWithoutToken() ]) ->shouldBeCalledTimes(1); - $this->client->___setProperty('connection', $this->connection->reveal()); - $jobs = iterator_to_array($this->client->jobs()); + $client->___setProperty('connection', $this->connection->reveal()); + $jobs = iterator_to_array($client->jobs()); $this->assertEquals(self::JOB_ID, $jobs[0]->id()); } public function testGetsJobsWithToken() { + $client = $this->getClient(); $token = 'token'; $this->connection->listJobs(['projectId' => self::PROJECT_ID]) ->willReturn([ @@ -219,32 +250,35 @@ public function testGetsJobsWithToken() ] ])->shouldBeCalledTimes(1); - $this->client->___setProperty('connection', $this->connection->reveal()); - $job = iterator_to_array($this->client->jobs()); + $client->___setProperty('connection', $this->connection->reveal()); + $job = iterator_to_array($client->jobs()); $this->assertEquals(self::JOB_ID, $job[1]->id()); } public function testGetsDataset() { - $this->client->___setProperty('connection', $this->connection->reveal()); - $this->assertInstanceOf(Dataset::class, $this->client->dataset(self::DATASET_ID)); + $client = $this->getClient(); + $client->___setProperty('connection', $this->connection->reveal()); + $this->assertInstanceOf(Dataset::class, $client->dataset(self::DATASET_ID)); } public function testGetsDatasetsWithNoResults() { + $client = $this->getClient(); $this->connection->listDatasets(Argument::any()) ->willReturn([]) ->shouldBeCalledTimes(1); - $this->client->___setProperty('connection', $this->connection->reveal()); - $datasets = iterator_to_array($this->client->datasets()); + $client->___setProperty('connection', $this->connection->reveal()); + $datasets = iterator_to_array($client->datasets()); $this->assertEmpty($datasets); } public function testGetsDatasetsWithoutToken() { + $client = $this->getClient(); $this->connection->listDatasets(Argument::any()) ->willReturn([ 'datasets' => [ @@ -253,14 +287,15 @@ public function testGetsDatasetsWithoutToken() ]) ->shouldBeCalledTimes(1); - $this->client->___setProperty('connection', $this->connection->reveal()); - $datasets = iterator_to_array($this->client->datasets()); + $client->___setProperty('connection', $this->connection->reveal()); + $datasets = iterator_to_array($client->datasets()); $this->assertEquals(self::DATASET_ID, $datasets[0]->id()); } public function testGetsDatasetsWithToken() { + $client = $this->getClient(); $this->connection->listDatasets(Argument::any()) ->willReturn( [ @@ -277,14 +312,15 @@ public function testGetsDatasetsWithToken() ) ->shouldBeCalledTimes(2); - $this->client->___setProperty('connection', $this->connection->reveal()); - $dataset = iterator_to_array($this->client->datasets()); + $client->___setProperty('connection', $this->connection->reveal()); + $dataset = iterator_to_array($client->datasets()); $this->assertEquals(self::DATASET_ID, $dataset[1]->id()); } public function testCreatesDataset() { + $client = $this->getClient(); $this->connection->insertDataset(Argument::any()) ->willReturn([ 'datasetReference' => [ @@ -292,9 +328,38 @@ public function testCreatesDataset() ] ]) ->shouldBeCalledTimes(1); - $this->client->___setProperty('connection', $this->connection->reveal()); + $client->___setProperty('connection', $this->connection->reveal()); - $dataset = $this->client->createDataset(self::DATASET_ID, [ + $dataset = $client->createDataset(self::DATASET_ID, [ + 'metadata' => [ + 'friendlyName' => 'A dataset.' + ] + ]); + + $this->assertInstanceOf(Dataset::class, $dataset); + } + + public function testCreatesDatasetWithDefaultLocation() + { + $client = $this->getClient(['location' => self::LOCATION]); + $this->connection->insertDataset([ + 'friendlyName' => 'A dataset.', + 'location' => self::LOCATION, + 'projectId' => self::PROJECT_ID, + 'datasetReference' => [ + 'datasetId' => self::DATASET_ID + ], + 'retries' => 0 + ]) + ->willReturn([ + 'datasetReference' => [ + 'datasetId' => self::DATASET_ID + ] + ]) + ->shouldBeCalledTimes(1); + $client->___setProperty('connection', $this->connection->reveal()); + + $dataset = $client->createDataset(self::DATASET_ID, [ 'metadata' => [ 'friendlyName' => 'A dataset.' ] @@ -305,29 +370,64 @@ public function testCreatesDataset() public function testGetsBytes() { - $bytes = $this->client->bytes('1234'); + $bytes = $this->getClient()->bytes('1234'); $this->assertInstanceOf(Bytes::class, $bytes); } public function testGetsDate() { - $bytes = $this->client->date(new \DateTime()); + $date = $this->getClient()->date(new \DateTime()); - $this->assertInstanceOf(Date::class, $bytes); + $this->assertInstanceOf(Date::class, $date); } public function testGetsTime() { - $bytes = $this->client->time(new \DateTime()); + $time = $this->getClient()->time(new \DateTime()); - $this->assertInstanceOf(Time::class, $bytes); + $this->assertInstanceOf(Time::class, $time); } public function testGetsTimestamp() { - $bytes = $this->client->timestamp(new \DateTime()); + $timestamp = $this->getClient()->timestamp(new \DateTime()); + + $this->assertInstanceOf(Timestamp::class, $timestamp); + } + + public function testDefaultLocationPropagatesToTable() + { + $client = $this->getClient(['location' => self::LOCATION]); + $table = $client->dataset(self::DATASET_ID) + ->table(self::TABLE_ID); + + $this->assertEquals( + self::LOCATION, + $table->load('1234')->toArray()['jobReference']['location'] + ); + } - $this->assertInstanceOf(Timestamp::class, $bytes); + public function testDefaultLocationPropagatesToJob() + { + $client = $this->getClient(['location' => self::LOCATION]); + + $this->assertEquals( + self::LOCATION, + $client->job(self::JOB_ID)->identity()['location'] + ); + } + + public function testExplicitLocationPropagatesToJob() + { + $client = $this->getClient(); + + $this->assertEquals( + self::LOCATION, + $client->job( + self::JOB_ID, + ['location' => self::LOCATION] + )->identity()['location'] + ); } } diff --git a/BigQuery/tests/Unit/CopyJobConfigurationTest.php b/BigQuery/tests/Unit/CopyJobConfigurationTest.php index dea77fb18a9f..640f7233fdda 100644 --- a/BigQuery/tests/Unit/CopyJobConfigurationTest.php +++ b/BigQuery/tests/Unit/CopyJobConfigurationTest.php @@ -54,7 +54,8 @@ public function setUp() ]; $this->config = new CopyJobConfiguration( self::PROJECT_ID, - ['jobReference' => ['jobId' => self::JOB_ID]] + ['jobReference' => ['jobId' => self::JOB_ID]], + null ); } diff --git a/BigQuery/tests/Unit/ExtractJobConfigurationTest.php b/BigQuery/tests/Unit/ExtractJobConfigurationTest.php index 744ed6903b90..4f7b714f68f5 100644 --- a/BigQuery/tests/Unit/ExtractJobConfigurationTest.php +++ b/BigQuery/tests/Unit/ExtractJobConfigurationTest.php @@ -54,7 +54,8 @@ public function setUp() ]; $this->config = new ExtractJobConfiguration( self::PROJECT_ID, - ['jobReference' => ['jobId' => self::JOB_ID]] + ['jobReference' => ['jobId' => self::JOB_ID]], + null ); } diff --git a/BigQuery/tests/Unit/JobConfigurationTraitTest.php b/BigQuery/tests/Unit/JobConfigurationTraitTest.php index d059d7bcfc8d..7b5ca7e74dff 100644 --- a/BigQuery/tests/Unit/JobConfigurationTraitTest.php +++ b/BigQuery/tests/Unit/JobConfigurationTraitTest.php @@ -28,6 +28,7 @@ class JobConfigurationTraitTest extends TestCase { const PROJECT_ID = 'project-id'; const JOB_ID = '1234'; + const LOCATION = 'asia-northeast1'; private $trait; @@ -40,7 +41,8 @@ public function testJobConfigurationProperties() { $this->trait->call('jobConfigurationProperties', [ self::PROJECT_ID, - ['jobReference' => ['jobId' => self::JOB_ID]] + ['jobReference' => ['jobId' => self::JOB_ID]], + null ]); $this->assertEquals([ @@ -52,11 +54,30 @@ public function testJobConfigurationProperties() ], $this->trait->call('toArray')); } + public function testJobConfigurationPropertiesSetsDefaultLocationWhenOneIsProvided() + { + $this->trait->call('jobConfigurationProperties', [ + self::PROJECT_ID, + ['jobReference' => ['jobId' => self::JOB_ID]], + self::LOCATION + ]); + + $this->assertEquals([ + 'projectId' => self::PROJECT_ID, + 'jobReference' => [ + 'jobId' => self::JOB_ID, + 'projectId' => self::PROJECT_ID, + 'location' => self::LOCATION + ] + ], $this->trait->call('toArray')); + } + public function testJobConfigurationPropertiesSetsJobIDWhenNotProvided() { $this->trait->call('jobConfigurationProperties', [ self::PROJECT_ID, - [] + [], + null ]); $jobId = $this->trait->call('toArray')['jobReference']['jobId']; @@ -80,7 +101,8 @@ public function testJobIdPrefix() $jobIdPrefix = 'prefix'; $this->trait->call('jobConfigurationProperties', [ self::PROJECT_ID, - ['jobReference' => ['jobId' => self::JOB_ID]] + ['jobReference' => ['jobId' => self::JOB_ID]], + null ]); $this->trait->call('jobIdPrefix', [$jobIdPrefix]); @@ -101,6 +123,16 @@ public function testLabels() ); } + public function testLocation() + { + $this->trait->call('location', [self::LOCATION]); + + $this->assertEquals( + self::LOCATION, + $this->trait->call('toArray')['jobReference']['location'] + ); + } + public function testGenerateJobId() { $uuid = $this->trait->call('generateJobId'); diff --git a/BigQuery/tests/Unit/JobTest.php b/BigQuery/tests/Unit/JobTest.php index 41c25974445e..a80d43bf97fd 100644 --- a/BigQuery/tests/Unit/JobTest.php +++ b/BigQuery/tests/Unit/JobTest.php @@ -31,6 +31,7 @@ class JobTest extends TestCase { public $connection; + public $location = 'asia-northeast1'; public $projectId = 'myProjectId'; public $jobId = 'myJobId'; public $jobInfo = ['status' => ['state' => 'DONE']]; @@ -40,10 +41,10 @@ public function setUp() $this->connection = $this->prophesize(ConnectionInterface::class); } - public function getJob($connection, array $data = []) + public function getJob($connection, array $data = [], $location = null) { $mapper = $this->prophesize(ValueMapper::class); - return new Job($connection->reveal(), $this->jobId, $this->projectId, $mapper->reveal(), $data); + return new Job($connection->reveal(), $this->jobId, $this->projectId, $mapper->reveal(), $data, $location); } public function testDoesExistTrue() @@ -152,4 +153,22 @@ public function testGetsIdentity() $this->assertEquals($this->jobId, $job->identity()['jobId']); $this->assertEquals($this->projectId, $job->identity()['projectId']); } + + public function testDefaultLocationSurfaced() + { + $job = $this->getJob($this->connection, [], $this->location); + + $this->assertEquals($this->location, $job->identity()['location']); + } + + public function testMetadataLocationSurfaced() + { + $job = $this->getJob($this->connection, [ + 'jobReference' => [ + 'location' => $this->location + ] + ]); + + $this->assertEquals($this->location, $job->identity()['location']); + } } diff --git a/BigQuery/tests/Unit/LoadJobConfigurationTest.php b/BigQuery/tests/Unit/LoadJobConfigurationTest.php index a0cc427e15ee..53e1a5d945fe 100644 --- a/BigQuery/tests/Unit/LoadJobConfigurationTest.php +++ b/BigQuery/tests/Unit/LoadJobConfigurationTest.php @@ -53,7 +53,8 @@ public function setUp() ]; $this->config = new LoadJobConfiguration( self::PROJECT_ID, - ['jobReference' => ['jobId' => self::JOB_ID]] + ['jobReference' => ['jobId' => self::JOB_ID]], + null ); } diff --git a/BigQuery/tests/Unit/QueryJobConfigurationTest.php b/BigQuery/tests/Unit/QueryJobConfigurationTest.php index 6f1a16cd4139..08fba2edf763 100644 --- a/BigQuery/tests/Unit/QueryJobConfigurationTest.php +++ b/BigQuery/tests/Unit/QueryJobConfigurationTest.php @@ -62,7 +62,8 @@ public function setUp() $this->config = new QueryJobConfiguration( new ValueMapper(false), self::PROJECT_ID, - ['jobReference' => ['jobId' => self::JOB_ID]] + ['jobReference' => ['jobId' => self::JOB_ID]], + null ); } diff --git a/Core/src/ServiceBuilder.php b/Core/src/ServiceBuilder.php index e4ce6a2a57c6..a031371f7276 100644 --- a/Core/src/ServiceBuilder.php +++ b/Core/src/ServiceBuilder.php @@ -114,6 +114,12 @@ public function __construct(array $config = []) * @type bool $returnInt64AsObject If true, 64 bit integers will be * returned as a {@see Google\Cloud\Core\Int64} object for 32 bit * platform compatibility. **Defaults to** false. + * @type string $location If provided, determines the default geographic + * location used when creating datasets and managing jobs. Please + * note: This is only required for jobs started outside of the US + * and EU regions. Also, if location metadata has already been + * fetched over the network it will take precedent over this + * setting. * } * @return BigQueryClient */ diff --git a/phpunit-snippets.xml.dist b/phpunit-snippets.xml.dist index 656a09e9e834..5d8acdcddf34 100644 --- a/phpunit-snippets.xml.dist +++ b/phpunit-snippets.xml.dist @@ -22,4 +22,7 @@ + + + From 636046b2e0601ccdb265417ef23bbb83807f3814 Mon Sep 17 00:00:00 2001 From: Dave Supplee Date: Fri, 30 Mar 2018 11:30:10 -0400 Subject: [PATCH 2/4] clarify documentation --- BigQuery/src/BigQueryClient.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/BigQuery/src/BigQueryClient.php b/BigQuery/src/BigQueryClient.php index cc0a34dcd0c1..c573618a3a6b 100644 --- a/BigQuery/src/BigQueryClient.php +++ b/BigQuery/src/BigQueryClient.php @@ -107,7 +107,8 @@ class BigQueryClient * note: This is only required for jobs started outside of the US * and EU regions. Also, if location metadata has already been * fetched over the network it will take precedent over this - * setting. + * setting (by calling + * {@see Google\Cloud\BigQuery\Table::reload()}, for example). * } */ public function __construct(array $config = []) From 2c47689c1da4c16c914773fbaa125c242396cc02 Mon Sep 17 00:00:00 2001 From: Dave Supplee Date: Fri, 30 Mar 2018 11:35:39 -0400 Subject: [PATCH 3/4] inverse conditions --- BigQuery/src/BigQueryClient.php | 2 +- BigQuery/src/JobConfigurationTrait.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BigQuery/src/BigQueryClient.php b/BigQuery/src/BigQueryClient.php index c573618a3a6b..efd20350c648 100644 --- a/BigQuery/src/BigQueryClient.php +++ b/BigQuery/src/BigQueryClient.php @@ -578,7 +578,7 @@ public function createDataset($id, array $options = []) unset($options['metadata']); } - if (!isset($options['location']) && $this->location) { + if ($this->location && !isset($options['location'])) { $options['location'] = $this->location; } diff --git a/BigQuery/src/JobConfigurationTrait.php b/BigQuery/src/JobConfigurationTrait.php index b7638c6605ef..df624b154d31 100644 --- a/BigQuery/src/JobConfigurationTrait.php +++ b/BigQuery/src/JobConfigurationTrait.php @@ -56,7 +56,7 @@ public function jobConfigurationProperties( 'jobReference' => ['projectId' => $projectId] ], $config); - if (!isset($this->config['jobReference']['location']) && $location) { + if ($location && !isset($this->config['jobReference']['location'])) { $this->config['jobReference']['location'] = $location; } From 778a1cb12d8e7949760b489ba4f8c9fef32dc570 Mon Sep 17 00:00:00 2001 From: Dave Supplee Date: Fri, 30 Mar 2018 17:05:04 -0400 Subject: [PATCH 4/4] address feedback --- BigQuery/tests/System/BigQueryTestCase.php | 5 +- BigQuery/tests/System/RegionalizationTest.php | 263 ++++++++++-------- 2 files changed, 156 insertions(+), 112 deletions(-) diff --git a/BigQuery/tests/System/BigQueryTestCase.php b/BigQuery/tests/System/BigQueryTestCase.php index f3ef8e43bd3a..d503310dc197 100644 --- a/BigQuery/tests/System/BigQueryTestCase.php +++ b/BigQuery/tests/System/BigQueryTestCase.php @@ -27,6 +27,7 @@ class BigQueryTestCase extends SystemTestCase const TESTING_PREFIX = 'gcloud_testing_'; protected static $bucket; + protected static $storageClient; protected static $client; protected static $dataset; protected static $table; @@ -40,11 +41,11 @@ public static function setUpBeforeClass() $keyFilePath = getenv('GOOGLE_CLOUD_PHP_TESTS_KEY_PATH'); - $storage = new StorageClient([ + self::$storageClient = new StorageClient([ 'keyFilePath' => $keyFilePath ]); - self::$bucket = self::createBucket($storage, uniqid(self::TESTING_PREFIX)); + self::$bucket = self::createBucket(self::$storageClient, uniqid(self::TESTING_PREFIX)); self::$client = new BigQueryClient([ 'keyFilePath' => $keyFilePath diff --git a/BigQuery/tests/System/RegionalizationTest.php b/BigQuery/tests/System/RegionalizationTest.php index 0f407113c682..a8998dbfcc2a 100644 --- a/BigQuery/tests/System/RegionalizationTest.php +++ b/BigQuery/tests/System/RegionalizationTest.php @@ -26,12 +26,13 @@ */ class RegionalizationTest extends BigQueryTestCase { - const REGION_ASIA = 'asia-northeast1'; - const REGION_US = 'US'; + const LOCATION_ASIA = 'asia-northeast1'; + const LOCATION_US = 'US'; const QUERY_TEMPLATE = 'SELECT 1 FROM `%s.%s`'; private static $datasetAsia; private static $tableAsia; + private static $bucketAsia; public static function setUpBeforeClass() { @@ -39,169 +40,211 @@ public static function setUpBeforeClass() self::$datasetAsia = self::createDataset( self::$client, uniqid(self::TESTING_PREFIX), - ['location' => self::REGION_ASIA] + ['location' => self::LOCATION_ASIA] ); self::$tableAsia = self::createTable( self::$datasetAsia, uniqid(self::TESTING_PREFIX) ); + self::$bucketAsia = self::createBucket( + self::$storageClient, + uniqid(self::TESTING_PREFIX), + ['location' => self::LOCATION_ASIA] + ); + } + + public function testCopyJobSucceedsInAsia() + { + $targetTable = self::$datasetAsia + ->table(uniqid(self::TESTING_PREFIX)); + $copyConfig = self::$tableAsia->copy($targetTable) + ->location(self::LOCATION_ASIA); + $results = self::$tableAsia->runJob($copyConfig); + + $this->assertArrayNotHasKey( + 'errorResult', + $results->info()['status'] + ); + $this->assertEquals( + self::$tableAsia->info()['location'], + $targetTable->reload()['location'] + ); } /** - * @dataProvider locationDataProvider + * @expectedException Google\Cloud\Core\Exception\NotFoundException */ - public function testRunQuery($location) + public function testCopyJobThrowsNotFoundExceptionInUS() { - $caught = false; - $query = self::$client->query( - sprintf( - self::QUERY_TEMPLATE, - self::$datasetAsia->id(), - self::$tableAsia->id() - ) - )->location($location); - - try { - self::$client->runQuery($query); - } catch (NotFoundException $e) { - $caught = true; - } + $targetTable = self::$datasetAsia + ->table(uniqid(self::TESTING_PREFIX)); + $copyConfig = self::$tableAsia->copy($targetTable) + ->location(self::LOCATION_US); + self::$tableAsia->runJob($copyConfig); + } - $this->assertExceptionShouldBeThrown($location, $caught); + public function testExtractJobSucceedsInAsia() + { + $object = self::$bucketAsia->object(uniqid(self::TESTING_PREFIX)); + $extractConfig = self::$tableAsia->extract($object) + ->destinationFormat('NEWLINE_DELIMITED_JSON') + ->location(self::LOCATION_ASIA); + $results = self::$tableAsia->runJob($extractConfig); + + $this->assertArrayNotHasKey( + 'errorResult', + $results->info()['status'] + ); + $this->assertTrue($object->exists()); } /** - * @dataProvider locationDataProvider + * @expectedException Google\Cloud\Core\Exception\NotFoundException */ - public function testRunCopyJob($location) + public function testExtractJobThrowsNotFoundExceptionInUS() { - $caught = false; - $copyTable = self::$dataset->table(self::TESTING_PREFIX); - $copy = self::$tableAsia->copy($copyTable) - ->location($location); - - try { - self::$tableAsia->startJob($copy); - self::$deletionQueue->add($copyTable); - } catch (NotFoundException $e) { - $caught = true; - } catch (ServiceException $e) { - // Swallow this, as the important bit is whether or not - // we get a 404. - } - - $this->assertExceptionShouldBeThrown($location, $caught); + $object = self::$bucketAsia->object(uniqid(self::TESTING_PREFIX)); + $extractConfig = self::$tableAsia->extract($object) + ->destinationFormat('NEWLINE_DELIMITED_JSON') + ->location(self::LOCATION_US); + self::$tableAsia->runJob($extractConfig); } - /** - * @dataProvider locationDataProvider - */ - public function testRunExtractJob($location) + public function testLoadJobSucceedsInAsia() { - $caught = false; - $extract = self::$tableAsia->extract( - self::$bucket->object(self::TESTING_PREFIX) - )->location($location); - - try { - self::$tableAsia->startJob($extract); - } catch (NotFoundException $e) { - $caught = true; - } - - $this->assertExceptionShouldBeThrown($location, $caught); + $loadConfig = self::$tableAsia->load( + file_get_contents(__DIR__ . '/data/table-data.json') + ) + ->sourceFormat('NEWLINE_DELIMITED_JSON') + ->location(self::LOCATION_ASIA); + $results = self::$tableAsia->runJob($loadConfig); + + $this->assertArrayNotHasKey( + 'errorResult', + $results->info()['status'] + ); + $this->assertEquals(3, (int) self::$tableAsia->reload()['numRows']); } /** - * @dataProvider locationDataProvider + * @expectedException Google\Cloud\Core\Exception\NotFoundException */ - public function testRunLoadJob($location) + public function testLoadJobThrowsNotFoundExceptionInUS() { - $caught = false; - $load = self::$tableAsia->load( + $loadConfig = self::$tableAsia->load( file_get_contents(__DIR__ . '/data/table-data.json') ) ->sourceFormat('NEWLINE_DELIMITED_JSON') - ->location($location); - - try { - self::$tableAsia->startJob($load); - }catch (ServiceException $e) { - $code = $e->getCode(); - if ($code === 403 || $code === 404) { - $caught = true; - } - } - - $this->assertExceptionShouldBeThrown($location, $caught); + ->createDisposition('CREATE_NEVER') + ->location(self::LOCATION_US); + self::$tableAsia->runJob($loadConfig); + } + + public function testRunQuerySucceedsInAsia() + { + $queryConfig = self::$client->query( + sprintf( + self::QUERY_TEMPLATE, + self::$datasetAsia->id(), + self::$tableAsia->id() + ) + )->location(self::LOCATION_ASIA); + $results = self::$client->runQuery($queryConfig); + + $this->assertArrayNotHasKey( + 'errors', + $results->info() + ); + $this->assertEquals(3, (int) $results->info()['totalRows']); } /** - * @dataProvider locationDataProvider + * @expectedException Google\Cloud\Core\Exception\NotFoundException */ - public function testGetsJob($location) + public function testRunQueryThrowsNotFoundExceptionInUS() { - $caught = false; - $query = self::$client->query( + $queryConfig = self::$client->query( sprintf( self::QUERY_TEMPLATE, self::$datasetAsia->id(), self::$tableAsia->id() ) - )->location(self::REGION_ASIA); - $job = self::$client->startQuery($query); + )->location(self::LOCATION_US); + self::$client->runQuery($queryConfig); + } - try { - self::$client->job($job->id(), [ - 'location' => $location - ])->reload(); - }catch (NotFoundException $e) { - $caught = true; - } + public function testGetJobSucceedsInAsia() + { + $queryConfig = self::$client->query( + sprintf( + self::QUERY_TEMPLATE, + self::$datasetAsia->id(), + self::$tableAsia->id() + ) + )->location(self::LOCATION_ASIA); + $job = self::$client->startQuery($queryConfig); - $this->assertExceptionShouldBeThrown($location, $caught); + $this->assertEquals( + self::LOCATION_ASIA, + self::$client->job($job->id(), [ + 'location' => self::LOCATION_ASIA + ])->reload()['jobReference']['location'] + ); } /** - * @dataProvider locationDataProvider + * @expectedException Google\Cloud\Core\Exception\NotFoundException */ - public function testCancelsJob($location) + public function testGetJobThrowsNotFoundExceptionInUS() { - $caught = false; - $query = self::$client->query( + $queryConfig = self::$client->query( sprintf( self::QUERY_TEMPLATE, self::$datasetAsia->id(), self::$tableAsia->id() ) - )->location(self::REGION_ASIA); - $job = self::$client->startQuery($query); + )->location(self::LOCATION_ASIA); + $job = self::$client->startQuery($queryConfig); - try { - self::$client->job($job->id(), [ - 'location' => $location - ])->cancel(); - }catch (NotFoundException $e) { - $caught = true; - } - - $this->assertExceptionShouldBeThrown($location, $caught); + self::$client->job($job->id(), [ + 'location' => self::LOCATION_US + ])->reload(); } - public function locationDataProvider() + public function testCancelJobSucceedsInAsia() { - return [ - [self::REGION_ASIA], - [self::REGION_US] - ]; + $queryConfig = self::$client->query( + sprintf( + self::QUERY_TEMPLATE, + self::$datasetAsia->id(), + self::$tableAsia->id() + ) + )->location(self::LOCATION_ASIA); + $job = self::$client->startQuery($queryConfig); + + $this->assertNull( + self::$client->job($job->id(), [ + 'location' => self::LOCATION_ASIA + ])->cancel() + ); } - private function assertExceptionShouldBeThrown($location, $caught) + /** + * @expectedException Google\Cloud\Core\Exception\NotFoundException + */ + public function testCancelJobThrowsNotFoundExceptionInUS() { - if ($location === self::REGION_ASIA) { - $this->assertFalse($caught); - } else { - $this->assertTrue($caught); - } + $queryConfig = self::$client->query( + sprintf( + self::QUERY_TEMPLATE, + self::$datasetAsia->id(), + self::$tableAsia->id() + ) + )->location(self::LOCATION_ASIA); + $job = self::$client->startQuery($queryConfig); + + self::$client->job($job->id(), [ + 'location' => self::LOCATION_US + ])->cancel(); } }