feat(storage): retry SignBlob call for URL signing

.gitignore

@@ -17,3 +17,4 @@ keys/
 .testing
 .split
 __pycache__
+.vscode/

Storage/src/SigningHelper.php

@@ -23,6 +23,10 @@
 use Google\Cloud\Core\JsonTrait;
 use Google\Cloud\Core\Timestamp;
 use Google\Cloud\Storage\Connection\ConnectionInterface;
+use Google\Cloud\Core\Exception\ServiceException;
+use Google\Cloud\Storage\Connection\RetryTrait;
+use Google\Cloud\Storage\StorageClient;
+use Ramsey\Uuid\Uuid;
 
 /**
  * Provides common methods for signing storage URLs.
@@ -33,6 +37,7 @@ class SigningHelper
 {
     use ArrayTrait;
     use JsonTrait;
+    use RetryTrait;
 
     const DEFAULT_URL_SIGNING_VERSION = 'v2';
     const DEFAULT_DOWNLOAD_HOST = 'storage.googleapis.com';
@@ -50,7 +55,7 @@ public static function getHelper()
     {
         static $helper;
         if (!$helper) {
-            $helper = new static;
+            $helper = new static();
         }
 
         return $helper;
@@ -169,7 +174,7 @@ public function v2Sign(ConnectionInterface $connection, $expires, $resource, $ge
 
         $signedHeaders = [];
         foreach ($headers as $name => $value) {
-            $signedHeaders[] = $name .':'. $value;
+            $signedHeaders[] = $name . ':' . $value;
         }
 
         // Push the headers onto the end of the signing string.
@@ -181,9 +186,12 @@ public function v2Sign(ConnectionInterface $connection, $expires, $resource, $ge
 
         $stringToSign = $this->createV2CanonicalRequest($toSign);
 
-        $signature = $credentials->signBlob($stringToSign, [
-            'forceOpenssl' => $options['forceOpenssl']
-        ]);
+        // Use exponential backOff
+        $signature = $this->retrySignBlob(function () use ($credentials, $stringToSign, $options) {
+            return $credentials->signBlob($stringToSign, [
+                'forceOpenssl' => $options['forceOpenssl']
+            ]);
+        });
 
         // Start with user-provided query params and add required parameters.
         $params = $options['queryParams'];
@@ -337,9 +345,15 @@ public function v4Sign(ConnectionInterface $connection, $expires, $resource, $ge
             $requestHash
         ]);
 
-        $signature = bin2hex(base64_decode($credentials->signBlob($stringToSign, [
-            'forceOpenssl' => $options['forceOpenssl']
-        ])));
+        $signature = bin2hex(base64_decode($this->retrySignBlob(function () use (
+            $credentials,
+            $stringToSign,
+            $options
+        ) {
+            return $credentials->signBlob($stringToSign, [
+                'forceOpenssl' => $options['forceOpenssl']
+            ]);
+        })));
 
         // Construct the modified resource name. If a custom hostname is provided,
         // this will remove the bucket name from the resource.
@@ -677,8 +691,8 @@ private function normalizeOptions(array $options)
                 if (!$options['timestamp']) {
                     throw new \InvalidArgumentException(
                         'Given timestamp string is in an invalid format. Provide timestamp formatted as follows: `' .
-                        \DateTime::RFC3339 .
-                        '`. Note that timestamps MUST be in UTC.'
+                            \DateTime::RFC3339 .
+                            '`. Note that timestamps MUST be in UTC.'
                     );
                 }
             }
@@ -882,4 +896,47 @@ private function buildQueryString(array $input)
 
         return implode('&', $q);
     }
+
+    /**
+     *   Retry logic for signBlob using RetryTrait.
+     *
+     * @param callable $signBlobFn  A callable that perform the actual signBlob operation.
+     * @param string $resourceName The resource name for logging or retry strategy determination.
+     * @param array $args Arguments for the operations, include preconditions
+     * @return string The signature genarated by signBlob.
+     * @throws ServiceException If non-retryable error occur.
+     * @throws \RuntimeException If retries are exhausted.
+     */
+    private function retrySignBlob(callable $signBlobFn, string $resourceName = 'signBlob', array $args = [])
+    {
+        $attempts = 0;
+        $maxRetries = 5;
+        $invocationId = Uuid::uuid4()->toString();
+        $args['retryStrategy'] = StorageClient::RETRY_ALWAYS;
+
+        // Generate a retry decider function using the RetryTrait logic.
+        $retryDecider = $this->getRestRetryFunction($resourceName, 'execute', $args);
+
+        while ($attempts < $maxRetries) {
+            $attempts++;
+            try {
+                // Attach retry headers
+                $headers = self::getRetryHeaders($invocationId, $attempts);
+                $args['headers'] = array_merge($args['headers'] ?? [], $headers);
+
+                // Attempt the operation
+                return $signBlobFn();
+            } catch (\Exception $exception) {
+                if (!$retryDecider($exception)) {
+                    // Non-retryable error
+                    throw $exception;
+                }
+            }
+        }
+
+        throw new \RuntimeException(sprintf(
+            'Failed to sign message after `%s` attempts.',
+            $attempts
+        ));
+    }
 }

Storage/tests/Unit/SigningHelperTest.php

@@ -1,4 +1,5 @@
 <?php
+
 /**
  * Copyright 2019 Google LLC
  *
@@ -19,6 +20,7 @@
 
 use Google\Auth\Credentials\ServiceAccountCredentials;
 use Google\Auth\SignBlobInterface;
+use Google\Cloud\Core\Exception\ServiceException;
 use Google\Cloud\Core\RequestWrapper;
 use Google\Cloud\Core\Testing\TestHelpers;
 use Google\Cloud\Core\Timestamp;
@@ -398,7 +400,7 @@ public function testV4SignCanonicalRequestSaveAsName()
 
         $this->helper->createV4CanonicalRequest = function ($request) use ($saveAsName) {
             parse_str($request[2], $query);
-            $expectedDisposition = 'attachment; filename="' . $saveAsName .'"';
+            $expectedDisposition = 'attachment; filename="' . $saveAsName . '"';
             $this->assertEquals($expectedDisposition, $query['response-content-disposition']);
         };
 
@@ -417,7 +419,7 @@ public function testV4SignInvalidExpiration()
     {
         $this->expectException(\InvalidArgumentException::class);
 
-        $expires = (new \DateTime)->modify('+20 days');
+        $expires = (new \DateTime())->modify('+20 days');
         $this->helper->v4Sign(
             $this->mockConnection($this->createCredentialsMock()->reveal()),
             $expires,
@@ -453,7 +455,7 @@ public function testExpirations($expiration, $expected)
 
     public function expirations()
     {
-        $tenMins = (new \DateTimeImmutable)->modify('+10 minutes');
+        $tenMins = (new \DateTimeImmutable())->modify('+10 minutes');
 
         return [
             [
@@ -789,6 +791,68 @@ private function mockConnection($credentials = null, $rw = null)
 
         return $conn->reveal();
     }
+
+    public function testRetrySignBlobSuccessFirstAttempt()
+    {
+        $signBlobFn = function () {
+            return 'signature';
+        };
+
+        $res = $this->helper->proxyPrivateMethodCall('retrySignBlob', [
+            $signBlobFn
+        ]);
+
+        $this->assertEquals('signature', $res);
+    }
+
+    public function testRetrySignBlobSuccessAfterRetries()
+    {
+        $attempts = 0;
+        $signBlobFn = function () use (&$attempts) {
+            $attempts++;
+            if ($attempts < 5) {
+                throw new ServiceException('Transient error', 503);
+            }
+            return 'signature';
+        };
+
+        $res = $this->helper->proxyPrivateMethodCall('retrySignBlob', [
+            $signBlobFn
+        ]);
+
+        $this->assertEquals('signature', $res);
+        $this->assertEquals(5, $attempts);
+    }
+
+    public function testRetrySignBlobNonRetryableError()
+    {
+        $this->expectException(ServiceException::class);
+        $this->getExpectedExceptionMessage('Non-retryable error');
+
+        $signBlobFn = function () {
+            throw new ServiceException('Non-retryable error', 400);
+        };
+
+        $res = $this->helper->proxyPrivateMethodCall('retrySignBlob', [
+            $signBlobFn
+        ]);
+
+        $this->assertEquals('Non-retryable error', $res);
+    }
+
+    public function testRetrySignBlobRetriesExhausted()
+    {
+        $this->expectException(\RuntimeException::class);
+        $this->getExpectedExceptionMessage('Failed to sign message after maximum attempts.');
+
+        $signBlobFn = function () {
+            throw new ServiceException('Transient error', 503);
+        };
+
+        $res = $this->helper->proxyPrivateMethodCall('retrySignBlob', [
+            $signBlobFn
+        ]);
+    }
 }
 
 //@codingStandardsIgnoreStart
@@ -803,20 +867,20 @@ private function createV4CanonicalRequest(array $request)
         $callPrivate = $this->callPrivate('createV4CanonicalRequest', [$request]);
         return $this->createV4CanonicalRequest
             ? call_user_func($this->createV4CanonicalRequest, $request)
-            : \Closure::bind($callPrivate, null, new SigningHelper);
+            : \Closure::bind($callPrivate, null, new SigningHelper());
     }
 
     private function createV2CanonicalRequest(array $request)
     {
         $callPrivate = $this->callPrivate('createV2CanonicalRequest', [$request]);
         return $this->createV2CanonicalRequest
             ? call_user_func($this->createV2CanonicalRequest, $request)
-            : \Closure::bind($callPrivate, null, new SigningHelper);
+            : \Closure::bind($callPrivate, null, new SigningHelper());
     }
 
     public function proxyPrivateMethodCall($method, array $args)
     {
-        $parent = new SigningHelper;
+        $parent = new SigningHelper();
         $cb = function () use ($method) {
             return call_user_func_array([$this, $method], func_get_args()[0]);
         };