feat: add retry strategy to storage client (#1396)

This commit adds a new FixedTimeoutRetryStrategy, which implements retries for cases where the request timed out for the first time in our SDKs. It is implemented such that this strategy could be used by any of our clients, but it is currently only used by the StorageClient (for which it is the default).



---------

Co-authored-by: Michael Landis <[email protected]>
Co-authored-by: Chris Price <[email protected]>

examples/nodejs/storage/basic.ts

@@ -0,0 +1,65 @@
+import {
+  CreateStoreResponse,
+  CredentialProvider,
+  PreviewStorageClient,
+  StorageConfigurations,
+  StorageGetResponse,
+  StoragePutResponse,
+} from '@gomomento/sdk';
+
+async function main() {
+  const storageClient = new PreviewStorageClient({
+    configuration: StorageConfigurations.Laptop.latest(),
+    credentialProvider: CredentialProvider.fromEnvironmentVariable('MOMENTO_API_KEY'),
+  });
+
+  const storeName = 'my-store';
+  const createStoreResponse = await storageClient.createStore(storeName);
+  switch (createStoreResponse.type) {
+    case CreateStoreResponse.AlreadyExists:
+      console.log(`Store '${storeName}' already exists`);
+      break;
+    case CreateStoreResponse.Success:
+      console.log(`Store '${storeName}' created`);
+      break;
+    case CreateStoreResponse.Error:
+      throw new Error(
+        `An error occurred while attempting to create store '${storeName}': ${createStoreResponse.errorCode()}: ${createStoreResponse.toString()}`
+      );
+  }
+
+  const putResponse = await storageClient.putString(storeName, 'test-key', 'test-value');
+  switch (putResponse.type) {
+    case StoragePutResponse.Success:
+      console.log("Key 'test-key' stored successfully");
+      break;
+    case StoragePutResponse.Error:
+      throw new Error(
+        `An error occurred while attempting to store key 'test-key' in store '${storeName}': ${putResponse.errorCode()}: ${putResponse.toString()}`
+      );
+  }
+
+  const getResponse = await storageClient.get(storeName, 'test-key');
+  // simplified style; assume the value was found, and that it was a string
+  console.log(`string hit: ${getResponse.value()!.string()!}`);
+
+  // pattern-matching style; safer for production code
+  switch (getResponse.type) {
+    case StorageGetResponse.Found:
+      // if you know the value is a string:
+      console.log(`Retrieved value for key 'test-key': ${getResponse.value().string()!}`);
+      break;
+    case StorageGetResponse.NotFound:
+      console.log(`Key 'test-key' was not found in store '${storeName}'`);
+      break;
+    case StorageGetResponse.Error:
+      throw new Error(
+        `An error occurred while attempting to get key 'test-key' from store '${storeName}': ${getResponse.errorCode()}: ${getResponse.toString()}`
+      );
+  }
+}
+
+main().catch(e => {
+  console.error('An error occurred! ', e);
+  throw e;
+});

examples/nodejs/storage/package.json

@@ -6,8 +6,9 @@
   "scripts": {
     "prebuild": "eslint . --ext .ts",
     "build": "tsc",
+    "basic": "tsc && node dist/basic.js",
     "doc-examples": "tsc && node dist/doc-examples-js-apis.js",
-    "validate-examples": "tsc && node dist/doc-examples-js-apis.js && node dist/cheat-sheet-main.js",
+    "validate-examples": "tsc && node dist/doc-example-files/doc-examples-js-apis.js && node dist/doc-example-files/cheat-sheet-main.js",
     "test": "jest",
     "lint": "eslint . --ext .ts",
     "format": "eslint . --ext .ts --fix"

packages/client-sdk-nodejs/src/auth-client-props.ts

@@ -1,11 +1,17 @@
 import {CredentialProvider} from '.';
+import {AuthClientConfiguration} from './config/auth-client-configuration';
 
 export interface AuthClientProps {
   /**
    * controls how the client will get authentication information for connecting to the Momento service
    */
   credentialProvider: CredentialProvider;
 
+  /**
+   * Controls the configuration settings for the auth client, such as logging configuration.
+   */
+  configuration?: AuthClientConfiguration;
+
   /**
    * Configures whether the client should return a Momento Error object or throw an exception when an
    * error occurs. By default, this is set to false, and the client will return a Momento Error object on errors. Set it

packages/client-sdk-nodejs/src/config/auth-client-configuration.ts

@@ -0,0 +1,33 @@
+import {MomentoLoggerFactory} from '@gomomento/sdk-core';
+
+export interface AuthClientConfigurationProps {
+  /**
+   * Configures logging verbosity and format
+   */
+  loggerFactory: MomentoLoggerFactory;
+}
+
+/**
+ * Configuration options for Momento CacheClient.
+ *
+ * @export
+ * @interface Configuration
+ */
+export interface AuthConfiguration {
+  /**
+   * @returns {MomentoLoggerFactory} the current configuration options for logging verbosity and format
+   */
+  getLoggerFactory(): MomentoLoggerFactory;
+}
+
+export class AuthClientConfiguration implements AuthConfiguration {
+  private readonly loggerFactory: MomentoLoggerFactory;
+
+  constructor(props: AuthClientConfigurationProps) {
+    this.loggerFactory = props.loggerFactory;
+  }
+
+  getLoggerFactory(): MomentoLoggerFactory {
+    return this.loggerFactory;
+  }
+}

packages/client-sdk-nodejs/src/config/auth-client-configurations.ts

@@ -0,0 +1,28 @@
+import {AuthClientConfiguration} from './auth-client-configuration';
+import {MomentoLoggerFactory} from '@gomomento/sdk-core';
+import {DefaultMomentoLoggerFactory} from './logging/default-momento-logger';
+
+const defaultLoggerFactory: MomentoLoggerFactory =
+  new DefaultMomentoLoggerFactory();
+
+/**
+ * Laptop config provides defaults suitable for a medium-to-high-latency dev environment.  Permissive timeouts, retries, and
+ * relaxed latency and throughput targets.
+ * @export
+ * @class Laptop
+ */
+export class Default extends AuthClientConfiguration {
+  /**
+   * Provides the latest recommended configuration for a laptop development environment.  NOTE: this configuration may
+   * change in future releases to take advantage of improvements we identify for default configurations.
+   * @param {MomentoLoggerFactory} [loggerFactory=defaultLoggerFactory]
+   * @returns {CacheConfiguration}
+   */
+  static latest(
+    loggerFactory: MomentoLoggerFactory = defaultLoggerFactory
+  ): AuthClientConfiguration {
+    return new AuthClientConfiguration({
+      loggerFactory: loggerFactory,
+    });
+  }
+}

packages/client-sdk-nodejs/src/config/retry/eligibility-strategy.ts

@@ -1,9 +1,10 @@
-import {StatusObject} from '@grpc/grpc-js';
+import {Metadata, StatusObject} from '@grpc/grpc-js';
 import {ClientMethodDefinition} from '@grpc/grpc-js/build/src/make-client';
 
 export interface EligibleForRetryProps {
   grpcStatus: StatusObject;
   grpcRequest: ClientMethodDefinition<unknown, unknown>;
+  requestMetadata: Metadata;
 }
 
 export interface EligibilityStrategy {

packages/client-sdk-nodejs/src/config/retry/fixed-timeout-retry-strategy.ts

@@ -0,0 +1,57 @@
+import {
+  DeterminewhenToRetryRequestProps,
+  RetryStrategy,
+} from './retry-strategy';
+import {EligibilityStrategy} from './eligibility-strategy';
+import {MomentoLoggerFactory, MomentoLogger} from '../..';
+import {DefaultStorageEligibilityStrategy} from './storage-default-eligibility-strategy';
+
+export interface FixedTimeoutRetryStrategyProps {
+  loggerFactory: MomentoLoggerFactory;
+  eligibilityStrategy?: EligibilityStrategy;
+
+  // Retry request after a fixed time interval (defaults to 100ms)
+  retryDelayIntervalMillis?: number;
+
+  // Number of milliseconds the client is willing to wait for response data to be received before retrying (defaults to 1000ms). After the overarching GRPC config deadlineMillis has been reached, the client will terminate the RPC with a Cancelled error.
+  responseDataReceivedTimeoutMillis?: number;
+}
+
+export class FixedTimeoutRetryStrategy implements RetryStrategy {
+  private readonly logger: MomentoLogger;
+  private readonly eligibilityStrategy: EligibilityStrategy;
+  private readonly retryDelayIntervalMillis: number;
+  readonly responseDataReceivedTimeoutMillis: number;
+
+  constructor(props: FixedTimeoutRetryStrategyProps) {
+    this.logger = props.loggerFactory.getLogger(this);
+    this.eligibilityStrategy =
+      props.eligibilityStrategy ??
+      new DefaultStorageEligibilityStrategy(props.loggerFactory);
+    this.retryDelayIntervalMillis = props.retryDelayIntervalMillis ?? 100;
+    this.responseDataReceivedTimeoutMillis =
+      props.responseDataReceivedTimeoutMillis ?? 1000;
+  }
+
+  determineWhenToRetryRequest(
+    props: DeterminewhenToRetryRequestProps
+  ): number | null {
+    this.logger.debug(
+      `Determining whether request is eligible for retry; status code: ${props.grpcStatus.code}, request type: ${props.grpcRequest.path}, attemptNumber: ${props.attemptNumber}`
+    );
+    if (!this.eligibilityStrategy.isEligibleForRetry(props)) {
+      // null means do not retry
+      return null;
+    }
+
+    this.logger.debug(
+      `Request is eligible for retry (attempt ${props.attemptNumber}), retrying after ${this.retryDelayIntervalMillis} ms +/- jitter.`
+    );
+    // retry after a fixed time interval has passed (+/- some jitter)
+    return addJitter(this.retryDelayIntervalMillis);
+  }
+}
+
+function addJitter(whenToRetry: number): number {
+  return (0.2 * Math.random() + 0.9) * whenToRetry;
+}