feat: add universe domain support to core, bigquery, storage, and pubsub

BigQuery/src/Connection/Rest.php

@@ -17,6 +17,7 @@
 
 namespace Google\Cloud\BigQuery\Connection;
 
+use Google\Auth\GetUniverseDomainInterface;
 use Google\Cloud\BigQuery\BigQueryClient;
 use Google\Cloud\BigQuery\Connection\ConnectionInterface;
 use Google\Cloud\Core\RequestBuilder;
@@ -43,8 +44,13 @@ class Rest implements ConnectionInterface
      */
     const BASE_URI = 'https://www.googleapis.com/bigquery/v2/';
 
+    /**
+     * @deprecated
+     */
     const DEFAULT_API_ENDPOINT = 'https://bigquery.googleapis.com';
 
+    private const DEFAULT_API_ENDPOINT_TEMPLATE = 'https://bigquery.UNIVERSE_DOMAIN';
+
     /**
      * @deprecated
      */
@@ -65,10 +71,11 @@ public function __construct(array $config = [])
         $config += [
             'serviceDefinitionPath' => __DIR__ . '/ServiceDefinition/bigquery-v2.json',
             'componentVersion' => BigQueryClient::VERSION,
-            'apiEndpoint' => self::DEFAULT_API_ENDPOINT
+            'apiEndpoint' => null,
+            'universeDomain' => GetUniverseDomainInterface::DEFAULT_UNIVERSE_DOMAIN,
         ];
 
-        $apiEndpoint = $this->getApiEndpoint(self::DEFAULT_API_ENDPOINT, $config);
+        $apiEndpoint = $this->getApiEndpoint(null, $config, self::DEFAULT_API_ENDPOINT_TEMPLATE);
 
         $this->setRequestWrapper(new RequestWrapper($config));
         $this->setRequestBuilder(new RequestBuilder(

Core/composer.json

@@ -6,7 +6,7 @@
     "require": {
         "php": ">=7.4",
         "rize/uri-template": "~0.3",
-        "google/auth": "^1.25",
+        "google/auth": "^1.33",
         "guzzlehttp/guzzle": "^6.5.8|^7.4.4",
         "guzzlehttp/promises": "^1.4||^2.0",
         "guzzlehttp/psr7": "^1.7|^2.0",

Core/src/GrpcTrait.php

@@ -17,6 +17,7 @@
 
 namespace Google\Cloud\Core;
 
+use Google\Auth\GetUniverseDomainInterface;
 use Google\ApiCore\CredentialsWrapper;
 use Google\Cloud\Core\ArrayTrait;
 use Google\Cloud\Core\Duration;
@@ -94,10 +95,14 @@ public function send(callable $request, array $args, $whitelisted = false)
      *
      * @param string $version
      * @param callable|null $authHttpHandler
+     * @param string|null $universeDomain
      * @return array
      */
-    private function getGaxConfig($version, callable $authHttpHandler = null)
-    {
+    private function getGaxConfig(
+        $version,
+        callable $authHttpHandler = null,
+        string $universeDomain = null
+    ) {
         $config = [
             'libName' => 'gccl',
             'libVersion' => $version,
@@ -110,7 +115,8 @@ private function getGaxConfig($version, callable $authHttpHandler = null)
         if (class_exists(CredentialsWrapper::class)) {
             $config['credentials'] = new CredentialsWrapper(
                 $this->requestWrapper->getCredentialsFetcher(),
-                $authHttpHandler
+                $authHttpHandler,
+                $universeDomain ?: GetUniverseDomainInterface::DEFAULT_UNIVERSE_DOMAIN
             );
         } else {
             $config += [

Core/src/RequestWrapper.php

@@ -20,11 +20,13 @@
 use Google\Auth\FetchAuthTokenCache;
 use Google\Auth\FetchAuthTokenInterface;
 use Google\Auth\GetQuotaProjectInterface;
+use Google\Auth\GetUniverseDomainInterface;
 use Google\Auth\HttpHandler\Guzzle6HttpHandler;
 use Google\Auth\HttpHandler\HttpHandlerFactory;
 use Google\Auth\UpdateMetadataInterface;
 use Google\Cloud\Core\Exception\ServiceException;
 use Google\Cloud\Core\RequestWrapperTrait;
+use Google\Cloud\Core\Exception\GoogleException;
 use GuzzleHttp\Exception\RequestException;
 use GuzzleHttp\Promise\PromiseInterface;
 use GuzzleHttp\Psr7\Utils;
@@ -94,6 +96,16 @@ class RequestWrapper
      */
     private $calcDelayFunction;
 
+    /**
+     * @var string The universe domain to verify against the credentials.
+     */
+    private string $universeDomain;
+
+    /**
+     * @var bool Ensure we only check the universe domain once.
+     */
+    private bool $hasCheckedUniverse = false;
+
     /**
      * @param array $config [optional] {
      *     Configuration options. Please see
@@ -125,6 +137,7 @@ class RequestWrapper
      *     @type callable $restCalcDelayFunction Sets the conditions for
      *           determining how long to wait between attempts to retry. Function
      *           signature should match: `function (int $attempt) : int`.
+     *     @type string $universeDomain The expected universe of the credentials. Defaults to "googleapis.com".
      * }
      */
     public function __construct(array $config = [])
@@ -140,7 +153,8 @@ public function __construct(array $config = [])
             'componentVersion' => null,
             'restRetryFunction' => null,
             'restDelayFunction' => null,
-            'restCalcDelayFunction' => null
+            'restCalcDelayFunction' => null,
+            'universeDomain' => GetUniverseDomainInterface::DEFAULT_UNIVERSE_DOMAIN,
         ];
 
         $this->componentVersion = $config['componentVersion'];
@@ -155,6 +169,7 @@ public function __construct(array $config = [])
         $this->httpHandler = $config['httpHandler'] ?: HttpHandlerFactory::build();
         $this->authHttpHandler = $config['authHttpHandler'] ?: $this->httpHandler;
         $this->asyncHttpHandler = $config['asyncHttpHandler'] ?: $this->buildDefaultAsyncHandler();
+        $this->universeDomain = $config['universeDomain'];
 
         if ($this->credentialsFetcher instanceof AnonymousCredentials) {
             $this->shouldSignRequest = false;
@@ -313,9 +328,14 @@ private function applyHeaders(RequestInterface $request, array $options = [])
             $quotaProject = $this->quotaProject;
 
             if ($this->accessToken) {
+                // if an access token is provided, check the universe domain against "googleapis.com"
+                $this->checkUniverseDomain(null);
                 $request = $request->withHeader('authorization', 'Bearer ' . $this->accessToken);
             } else {
+                // if a credentials fetcher is provided, check the universe domain against the
+                // credential's universe domain
                 $credentialsFetcher = $this->getCredentialsFetcher();
+                $this->checkUniverseDomain($credentialsFetcher);
                 $request = $this->addAuthHeaders($request, $credentialsFetcher);
 
                 if ($credentialsFetcher instanceof GetQuotaProjectInterface) {
@@ -326,6 +346,9 @@ private function applyHeaders(RequestInterface $request, array $options = [])
             if ($quotaProject) {
                 $request = $request->withHeader('X-Goog-User-Project', $quotaProject);
             }
+        } else {
+            // If we are not signing the request, check the universe domain against "googleapis.com"
+            $this->checkUniverseDomain(null);
         }
 
         return $request;
@@ -484,4 +507,36 @@ private function buildDefaultAsyncHandler()
             ? [$this->httpHandler, 'async']
             : [HttpHandlerFactory::build(), 'async'];
     }
+
+    /**
+     * Verify that the expected universe domain matches the universe domain from the credentials.
+     */
+    private function checkUniverseDomain(FetchAuthTokenInterface $credentialsFetcher = null)
+    {
+        if (false === $this->hasCheckedUniverse) {
+            if ($this->universeDomain === '') {
+                throw new GoogleException('The universe domain cannot be empty.');
+            }
+            if (is_null($credentialsFetcher)) {
+                if ($this->universeDomain !== GetUniverseDomainInterface::DEFAULT_UNIVERSE_DOMAIN) {
+                    throw new GoogleException(sprintf(
+                        'The accessToken option is not supported outside of the default universe domain (%s).',
+                        GetUniverseDomainInterface::DEFAULT_UNIVERSE_DOMAIN
+                    ));
+                }
+            } else {
+                $credentialsUniverse = $credentialsFetcher instanceof GetUniverseDomainInterface
+                    ? $credentialsFetcher->getUniverseDomain()
+                    : GetUniverseDomainInterface::DEFAULT_UNIVERSE_DOMAIN;
+                if ($credentialsUniverse !== $this->universeDomain) {
+                    throw new GoogleException(sprintf(
+                        'The configured universe domain (%s) does not match the credential universe domain (%s)',
+                        $this->universeDomain,
+                        $credentialsUniverse
+                    ));
+                }
+            }
+            $this->hasCheckedUniverse = true;
+        }
+    }
 }

Core/src/RequestWrapperTrait.php

@@ -73,6 +73,9 @@ trait RequestWrapperTrait
      */
     private $quotaProject;
 
+    private string $universeDomain;
+    private bool $hasCheckedUniverse = false;
+
     /**
      * Sets common defaults between request wrappers.
      *

Core/src/RestTrait.php

@@ -19,6 +19,7 @@
 
 use Google\Cloud\Core\Exception\NotFoundException;
 use Google\Cloud\Core\Exception\ServiceException;
+use UnexpectedValueException;
 
 /**
  * Provides shared functionality for REST service implementations.
@@ -118,20 +119,45 @@ public function send($resource, $method, array $options = [], $whitelisted = fal
      *
      * @param string $default
      * @param array $config
+     * @param string $apiEndpointTemplate
      * @return string
      */
-    private function getApiEndpoint($default, array $config)
+    private function getApiEndpoint($default, array $config, string $apiEndpointTemplate = null)
     {
-        $res = $config['apiEndpoint'] ?? $default;
+        // If the $default parameter is provided, or the user has set an "apiEndoint" config option,
+        // fall back to the previous behavior.
+        if ($res = $config['apiEndpoint'] ?? $default) {
+            if (substr($res, -1) !== '/') {
+                $res = $res . '/';
+            }
+
+            if (strpos($res, '//') === false) {
+                $res = 'https://' . $res;
+            }
+
+            return $res;
+        }
 
-        if (substr($res, -1) !== '/') {
-            $res = $res . '/';
+        // One of the $default or the $template must always be set
+        if (!$apiEndpointTemplate) {
+            throw new UnexpectedValueException(
+                'An API endpoint template must be provided if no "apiEndpoint" or default endpoint is set.'
+            );
         }
 
-        if (strpos($res, '//') === false) {
-            $res = 'https://' . $res;
+        if (!isset($config['universeDomain'])) {
+            throw new UnexpectedValueException(
+                'The "universeDomain" config value must be set to use the default API endpoint template.'
+            );
         }
 
-        return $res;
+        $apiEndpoint = str_replace(
+            'UNIVERSE_DOMAIN',
+            $config['universeDomain'],
+            $apiEndpointTemplate
+        );
+
+        // Preserve the behavior of guaranteeing a trailing "/"
+        return $apiEndpoint . (substr($apiEndpoint, -1) !== '/' ? '/' : '');
     }
 }

Core/tests/Unit/RequestWrapperTest.php

@@ -21,6 +21,7 @@
 use Google\Auth\Credentials\ServiceAccountCredentials;
 use Google\Auth\FetchAuthTokenCache;
 use Google\Auth\FetchAuthTokenInterface;
+use Google\Auth\GetUniverseDomainInterface;
 use Google\Auth\UpdateMetadataInterface;
 use Google\Cloud\Core\AnonymousCredentials;
 use Google\Cloud\Core\Exception\BadRequestException;
@@ -776,6 +777,98 @@ public function testFetchingCredentialAsAuthHeaderWithOverlappingHeaders()
         );
     }
 
+    /**
+     * @dataProvider provideCheckUniverseDomainFails
+     */
+    public function testCheckUniverseDomainFails(
+        ?string $universeDomain,
+        ?string $credentialsUniverse,
+        string $message = null
+    ) {
+        $this->expectException(GoogleException::class);
+        $this->expectExceptionMessage($message ?: sprintf(
+            'The configured universe domain (%s) does not match the credential universe domain (%s)',
+            is_null($universeDomain) ? GetUniverseDomainInterface::DEFAULT_UNIVERSE_DOMAIN : $universeDomain,
+            is_null($credentialsUniverse) ? GetUniverseDomainInterface::DEFAULT_UNIVERSE_DOMAIN : $credentialsUniverse,
+        ));
+        $fetcher = $this->prophesize(FetchAuthTokenInterface::class);
+        // When the $credentialsUniverse is null, the fetcher doesn't implement GetUniverseDomainInterface
+        if (!is_null($credentialsUniverse)) {
+            $fetcher->willImplement(GetUniverseDomainInterface::class);
+            $fetcher->getUniverseDomain()->willReturn($credentialsUniverse);
+        }
+        $fetcher->getLastReceivedToken()->willReturn(null);
+
+        $config = ['credentialsFetcher' => $fetcher->reveal()];
+        // A null value here represents not passing in a universeDomain
+        if (!is_null($universeDomain)) {
+            $config['universeDomain'] = $universeDomain;
+        }
+        $requestWrapper = new RequestWrapper($config);
+        // Send a fake request
+        $requestWrapper->send(new Request('GET', 'http://www.example.com'));
+    }
+
+    public function provideCheckUniverseDomainFails()
+    {
+        return [
+            ['foo.com', 'googleapis.com'],
+            ['googleapis.com', 'foo.com'],
+            ['googleapis.com', ''],
+            ['', 'googleapis.com', 'The universe domain cannot be empty'],
+            [null, 'foo.com'], // null in RequestWrapper will default to "googleapis.com"
+            ['foo.com', null], // Credentials not implementing GetUniverseDomainInterface will default to "googleapis.com"
+        ];
+    }
+
+    /**
+     * @dataProvider provideCheckUniverseDomainPasses
+     */
+    public function testCheckUniverseDomainPasses(?string $universeDomain, ?string $credentialsUniverse)
+    {
+        $fetcher = $this->prophesize(FetchAuthTokenInterface::class);
+        // When the $credentialsUniverse is null, the fetcher doesn't implement GetUniverseDomainInterface
+        if (!is_null($credentialsUniverse)) {
+            $fetcher->willImplement(GetUniverseDomainInterface::class);
+            $fetcher->getUniverseDomain()->shouldBeCalledOnce()->willReturn($credentialsUniverse);
+        }
+        $fetcher->getLastReceivedToken()->willReturn(null);
+        $fetcher->fetchAuthToken(Argument::any())->willReturn(['access_token' => 'abc']);
+        $fetcher->getCacheKey()
+            ->shouldBeCalledTimes(2)
+            ->willReturn(null);
+
+        $called = false;
+        $config = [
+            'credentialsFetcher' => $fetcher->reveal(),
+            'httpHandler' => function (Request $request) use (&$called) {
+                $headers = $request->getHeaders();
+                $this->assertArrayHasKey('authorization', $headers);
+                $this->assertEquals('Bearer abc', $headers['authorization'][0]);
+                $called = true;
+            }
+        ];
+        // A null value here represents not passing in a universeDomain
+        if (!is_null($universeDomain)) {
+            $config['universeDomain'] = $universeDomain;
+        }
+        $requestWrapper = new RequestWrapper($config);
+
+        // send a fake request
+        $requestWrapper->send(new Request('GET', 'http://www.example.com'));
+        $this->assertTrue($called);
+    }
+
+    public function provideCheckUniverseDomainPasses()
+    {
+        return [
+            [null, 'googleapis.com'], // null will default to "googleapis.com"
+            ['foo.com', 'foo.com'],
+            ['googleapis.com', 'googleapis.com'],
+            ['googleapis.com', null],
+        ];
+    }
+
     private function prophesizeUpdateMetadataFetcher($credentialsFetcher)
     {
         // We have to mock this message because RequestWrapper wraps the credentials using the