feat: Base TPC Support

package.json

@@ -82,7 +82,7 @@
     "ent": "^2.2.0",
     "fast-xml-parser": "^4.3.0",
     "gaxios": "^6.0.2",
-    "google-auth-library": "^9.0.0",
+    "google-auth-library": "^9.5.0",
     "mime": "^3.0.0",
     "mime-types": "^2.0.8",
     "p-limit": "^3.0.1",

src/bucket.ts

@@ -1975,7 +1975,7 @@ class Bucket extends ServiceObject<Bucket, BucketMetadata> {
       body.topic = 'projects/{{projectId}}/topics/' + body.topic;
     }
 
-    body.topic = '//pubsub.googleapis.com/' + body.topic;
+    body.topic = `//pubsub.${this.storage.universeDomain}/` + body.topic;
 
     if (!body.payloadFormat) {
       body.payloadFormat = 'JSON_API_V1';
@@ -3143,7 +3143,12 @@ class Bucket extends ServiceObject<Bucket, BucketMetadata> {
     } as SignerGetSignedUrlConfig;
 
     if (!this.signer) {
-      this.signer = new URLSigner(this.storage.authClient, this);
+      this.signer = new URLSigner(
+        this.storage.authClient,
+        this,
+        undefined,
+        this.storage.universeDomain
+      );
     }
 
     this.signer

src/file.ts

@@ -108,6 +108,11 @@ export interface GenerateSignedPostPolicyV2Options {
   successRedirect?: string;
   successStatus?: string;
   contentLengthRange?: {min?: number; max?: number};
+  /**
+   * @example
+   * 'https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/'
+   */
+  signingEndpoint?: string;
 }
 
 export interface PolicyFields {
@@ -120,6 +125,11 @@ export interface GenerateSignedPostPolicyV4Options {
   virtualHostedStyle?: boolean;
   conditions?: object[];
   fields?: PolicyFields;
+  /**
+   * @example
+   * 'https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/'
+   */
+  signingEndpoint?: string;
 }
 
 export interface GenerateSignedPostPolicyV4Callback {
@@ -302,7 +312,7 @@ export enum ActionToHTTPMethod {
 }
 
 /**
- * @private
+ * @deprecated - no longer used
  */
 export const STORAGE_POST_POLICY_BASE_URL = 'https://storage.googleapis.com';
 
@@ -1760,6 +1770,7 @@ class File extends ServiceObject<File, FileMetadata> {
         userProject: options.userProject || this.userProject,
         retryOptions: retryOptions,
         params: options?.preconditionOpts || this.instancePreconditionOpts,
+        universeDomain: this.bucket.storage.universeDomain,
         [GCCL_GCS_CMD_KEY]: options[GCCL_GCS_CMD_KEY],
       },
       callback!
@@ -2580,7 +2591,7 @@ class File extends ServiceObject<File, FileMetadata> {
     const policyString = JSON.stringify(policy);
     const policyBase64 = Buffer.from(policyString).toString('base64');
 
-    this.storage.authClient.sign(policyBase64).then(
+    this.storage.authClient.sign(policyBase64, options.signingEndpoint).then(
       signature => {
         callback(null, {
           string: policyString,
@@ -2763,18 +2774,23 @@ class File extends ServiceObject<File, FileMetadata> {
       const policyBase64 = Buffer.from(policyString).toString('base64');
 
       try {
-        const signature = await this.storage.authClient.sign(policyBase64);
+        const signature = await this.storage.authClient.sign(
+          policyBase64,
+          options.signingEndpoint
+        );
         const signatureHex = Buffer.from(signature, 'base64').toString('hex');
+        const universe = this.parent.storage.universeDomain;
         fields['policy'] = policyBase64;
         fields['x-goog-signature'] = signatureHex;
 
         let url: string;
+
         if (options.virtualHostedStyle) {
-          url = `https://${this.bucket.name}.storage.googleapis.com/`;
+          url = `https://${this.bucket.name}.storage.${universe}/`;
         } else if (options.bucketBoundHostname) {
           url = `${options.bucketBoundHostname}/`;
         } else {
-          url = `${STORAGE_POST_POLICY_BASE_URL}/${this.bucket.name}/`;
+          url = `https://storage.${universe}/${this.bucket.name}/`;
         }
 
         return {
@@ -2828,8 +2844,8 @@ class File extends ServiceObject<File, FileMetadata> {
    * @param {string} [config.version='v2'] The signing version to use, either
    *     'v2' or 'v4'.
    * @param {boolean} [config.virtualHostedStyle=false] Use virtual hosted-style
-   *     URLs ('https://mybucket.storage.googleapis.com/...') instead of path-style
-   *     ('https://storage.googleapis.com/mybucket/...'). Virtual hosted-style URLs
+   *     URLs (e.g. 'https://mybucket.storage.googleapis.com/...') instead of path-style
+   *     (e.g. 'https://storage.googleapis.com/mybucket/...'). Virtual hosted-style URLs
    *     should generally be preferred instaed of path-style URL.
    *     Currently defaults to `false` for path-style, although this may change in a
    *     future major-version release.
@@ -3012,7 +3028,12 @@ class File extends ServiceObject<File, FileMetadata> {
     }
 
     if (!this.signer) {
-      this.signer = new URLSigner(this.storage.authClient, this.bucket, this);
+      this.signer = new URLSigner(
+        this.storage.authClient,
+        this.bucket,
+        this,
+        this.storage.universeDomain
+      );
     }
 
     this.signer
@@ -4004,6 +4025,7 @@ class File extends ServiceObject<File, FileMetadata> {
       params: options?.preconditionOpts || this.instancePreconditionOpts,
       chunkSize: options?.chunkSize,
       highWaterMark: options?.highWaterMark,
+      universeDomain: this.bucket.storage.universeDomain,
       [GCCL_GCS_CMD_KEY]: options[GCCL_GCS_CMD_KEY],
     });
 

src/nodejs-common/service.ts

@@ -13,7 +13,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import {AuthClient, GoogleAuth, GoogleAuthOptions} from 'google-auth-library';
+import {
+  AuthClient,
+  DEFAULT_UNIVERSE,
+  GoogleAuth,
+  GoogleAuthOptions,
+} from 'google-auth-library';
 import * as r from 'teeny-request';
 import * as uuid from 'uuid';
 
@@ -84,9 +89,9 @@ export class Service {
   providedUserAgent?: string;
   makeAuthenticatedRequest: MakeAuthenticatedRequest;
   authClient: GoogleAuth<AuthClient>;
-  private getCredentials: {};
-  readonly apiEndpoint: string;
+  apiEndpoint: string;
   timeout?: number;
+  universeDomain: string;
 
   /**
    * Service is a base class, meant to be inherited from by a "service," like
@@ -115,22 +120,22 @@ export class Service {
     this.projectId = options.projectId || DEFAULT_PROJECT_ID_TOKEN;
     this.projectIdRequired = config.projectIdRequired !== false;
     this.providedUserAgent = options.userAgent;
+    this.universeDomain = options.universeDomain || DEFAULT_UNIVERSE;
 
-    const reqCfg = {
+    this.makeAuthenticatedRequest = util.makeAuthenticatedRequestFactory({
       ...config,
       projectIdRequired: this.projectIdRequired,
       projectId: this.projectId,
       authClient: options.authClient || config.authClient,
       credentials: options.credentials,
       keyFile: options.keyFilename,
       email: options.email,
-      token: options.token,
-    };
-
-    this.makeAuthenticatedRequest =
-      util.makeAuthenticatedRequestFactory(reqCfg);
+      clientOptions: {
+        universeDomain: options.universeDomain,
+        ...options.clientOptions,
+      },
+    });
     this.authClient = this.makeAuthenticatedRequest.authClient;
-    this.getCredentials = this.makeAuthenticatedRequest.getCredentials;
 
     const isCloudFunctionEnv = !!process.env.FUNCTION_NAME;
 

src/nodejs-common/util.ts

@@ -179,7 +179,7 @@ export interface MakeAuthenticatedRequestFactoryConfig
 
   /**
    * A pre-instantiated `AuthClient` or `GoogleAuth` client that should be used.
-   * A new will be created if this is not set.
+   * A new client will be created if this is not set.
    */
   authClient?: AuthClient | GoogleAuth;
 
@@ -638,13 +638,12 @@ export class Util {
       // Use an existing `GoogleAuth`
       authClient = googleAutoAuthConfig.authClient;
     } else {
-      // Pass an `AuthClient` to `GoogleAuth`, if available
-      const config = {
+      // Pass an `AuthClient` & `clientOptions` to `GoogleAuth`, if available
+      authClient = new GoogleAuth({
         ...googleAutoAuthConfig,
         authClient: googleAutoAuthConfig.authClient,
-      };
-
-      authClient = new GoogleAuth(config);
+        clientOptions: googleAutoAuthConfig.clientOptions,
+      });
     }
 
     /**

src/resumable-upload.ts

@@ -21,7 +21,11 @@ import {
   GaxiosError,
 } from 'gaxios';
 import * as gaxios from 'gaxios';
-import {GoogleAuth, GoogleAuthOptions} from 'google-auth-library';
+import {
+  DEFAULT_UNIVERSE,
+  GoogleAuth,
+  GoogleAuthOptions,
+} from 'google-auth-library';
 import {Readable, Writable, WritableOptions} from 'stream';
 import AsyncRetry from 'async-retry';
 import {RetryOptions, PreconditionOptions} from './storage.js';
@@ -39,7 +43,6 @@ import {getPackageJSON} from './package-json-helper.cjs';
 
 const NOT_FOUND_STATUS_CODE = 404;
 const RESUMABLE_INCOMPLETE_STATUS_CODE = 308;
-const DEFAULT_API_ENDPOINT_REGEX = /.*\.googleapis\.com/;
 const packageJson = getPackageJSON();
 
 export const PROTOCOL_REGEX = /^(\w*):\/\//;
@@ -75,9 +78,10 @@ export interface UploadConfig extends Pick<WritableOptions, 'highWaterMark'> {
   /**
    * The API endpoint used for the request.
    * Defaults to `storage.googleapis.com`.
+   *
    * **Warning**:
-   * If this value does not match the pattern *.googleapis.com,
-   * an emulator context will be assumed and authentication will be bypassed.
+   * If this value does not match the current GCP universe an emulator context
+   * will be assumed and authentication will be bypassed.
    */
   apiEndpoint?: string;
 
@@ -209,6 +213,11 @@ export interface UploadConfig extends Pick<WritableOptions, 'highWaterMark'> {
    */
   public?: boolean;
 
+  /**
+   * The service domain for a given Cloud universe.
+   */
+  universeDomain?: string;
+
   /**
    * If you already have a resumable URI from a previously-created resumable
    * upload, just pass it in here and we'll use that.
@@ -356,10 +365,34 @@ export class Upload extends Writable {
     ];
     this.authClient = cfg.authClient || new GoogleAuth(cfg.authConfig);
 
-    this.apiEndpoint = 'https://storage.googleapis.com';
-    if (cfg.apiEndpoint) {
+    const universe = cfg.universeDomain || DEFAULT_UNIVERSE;
+
+    this.apiEndpoint = `https://storage.${universe}`;
+    if (cfg.apiEndpoint && cfg.apiEndpoint !== this.apiEndpoint) {
       this.apiEndpoint = this.sanitizeEndpoint(cfg.apiEndpoint);
-      if (!DEFAULT_API_ENDPOINT_REGEX.test(cfg.apiEndpoint)) {
+
+      const hostname = new URL(this.apiEndpoint).hostname;
+
+      // check if it is a domain of a known universe
+      const isDomain = hostname === universe;
+      const isDefaultUniverseDomain = hostname === DEFAULT_UNIVERSE;
+
+      // check if it is a subdomain of a known universe
+      // by checking a last (universe's length + 1) of a hostname
+      const isSubDomainOfUniverse =
+        hostname.slice(-(universe.length + 1)) === `.${universe}`;
+      const isSubDomainOfDefaultUniverse =
+        hostname.slice(-(DEFAULT_UNIVERSE.length + 1)) ===
+        `.${DEFAULT_UNIVERSE}`;
+
+      if (
+        !isDomain &&
+        !isDefaultUniverseDomain &&
+        !isSubDomainOfUniverse &&
+        !isSubDomainOfDefaultUniverse
+      ) {
+        // a custom, non-universe domain,
+        // use gaxios
         this.authClient = gaxios;
       }
     }