support cacheSettings at a query level (#41)

awslabs · Jan 26, 2022 · 9ddaa58 · 9ddaa58
1 parent 76a9d63
commit 9ddaa58
Show file tree

Hide file tree

Showing 8 changed files with 103 additions and 5 deletions.
diff --git a/packages/core/src/data-module/IotAppKitDataModule.spec.ts b/packages/core/src/data-module/IotAppKitDataModule.spec.ts
@@ -808,6 +808,54 @@ describe('caching', () => {
   });
 });
 
+it('overrides module-level cache TTL if query-level cache TTL is provided', async () => {
+  const customCacheSettings = {
+    ttlDurationMapping: {
+      [MINUTE_IN_MS]: 0,
+      [5 * MINUTE_IN_MS]: 30 * SECOND_IN_MS,
+    },
+  };
+
+  const dataModule = new IotAppKitDataModule({ cacheSettings: customCacheSettings });
+  const dataSource = createMockSiteWiseDataSource([DATA_STREAM]);
+  dataModule.registerDataSource(dataSource);
+
+  const END = new Date();
+  const START = new Date(END.getTime() - HOUR_IN_MS);
+
+  const dataStreamCallback = jest.fn();
+  dataModule.subscribeToDataStreams(
+    {
+      queries: [
+        {
+          ...DATA_STREAM_QUERY,
+          cacheSettings: {
+            ttlDurationMapping: {
+              [MINUTE_IN_MS]: 0,
+              [10 * MINUTE_IN_MS]: 30 * SECOND_IN_MS,
+            },
+          },
+        },
+      ],
+      request: { viewport: { start: START, end: END }, settings: { refreshRate: MINUTE_IN_MS } },
+    },
+    dataStreamCallback
+  );
+
+  (dataSource.initiateRequest as Mock).mockClear();
+  jest.advanceTimersByTime(MINUTE_IN_MS);
+
+  expect(dataSource.initiateRequest).toBeCalledWith(expect.any(Object), [
+    {
+      id: DATA_STREAM_INFO.id,
+      resolution: DATA_STREAM_INFO.resolution,
+      // 1 minute time advancement invalidates 10 minutes of cache with query-level mapping, which is 9 minutes from END_1
+      start: new Date(END.getTime() - 9 * MINUTE_IN_MS),
+      end: END,
+    },
+  ]);
+});
+
 describe('request scheduler', () => {
   it('periodically requests duration based queries', async () => {
     const dataModule = new IotAppKitDataModule();

diff --git a/packages/core/src/data-module/IotAppKitDataModule.ts b/packages/core/src/data-module/IotAppKitDataModule.ts
@@ -100,14 +100,14 @@ export class IotAppKitDataModule implements DataModule {
     );
 
     const requests = requiredStreams
-      .map(({ resolution, id }) => {
+      .map(({ resolution, id, cacheSettings }) => {
         const dateRanges = getDateRangesToRequest({
           store: this.dataCache.getState(),
           start: adjustedStart,
           end: adjustedEnd,
           resolution,
           dataStreamId: id,
-          cacheSettings: this.cacheSettings,
+          cacheSettings: { ...this.cacheSettings, ...cacheSettings },
         });
 
         return {

diff --git a/packages/core/src/data-module/data-cache/caching/caching.spec.ts b/packages/core/src/data-module/data-cache/caching/caching.spec.ts
@@ -6,6 +6,7 @@ import {
   EMPTY_CACHE,
   getDateRangesToRequest,
   unexpiredCacheIntervals,
+  maxCacheDuration,
 } from './caching';
 import { DEFAULT_CACHE_SETTINGS } from '../../IotAppKitDataModule';
 import { HOUR_IN_MS, MINUTE_IN_MS, SECOND_IN_MS } from '../../../common/time';
@@ -1210,3 +1211,25 @@ describe('checkCacheForRecentPoint', () => {
     expect(presentInCache).toBeFalse();
   });
 });
+
+describe('maxCacheDuration', () => {
+  it('returns the maximum cache TTL duration', () => {
+    expect(
+      maxCacheDuration({
+        ttlDurationMapping: {
+          [1.2 * MINUTE_IN_MS]: 0,
+          [3 * MINUTE_IN_MS]: 30 * SECOND_IN_MS,
+          [20 * MINUTE_IN_MS]: 5 * MINUTE_IN_MS,
+        },
+      })
+    ).toBe(20 * MINUTE_IN_MS);
+  });
+
+  it('handles empty mappings', () => {
+    expect(
+      maxCacheDuration({
+        ttlDurationMapping: {},
+      })
+    ).toBe(0);
+  });
+});
diff --git a/packages/core/src/data-module/data-cache/caching/caching.ts b/packages/core/src/data-module/data-cache/caching/caching.ts
@@ -204,3 +204,14 @@ export const validateRequestConfig = (requestConfig: TimeSeriesDataRequestSettin
 
   return false;
 };
+
+// Returns the maximum duration for possible uncached data for given CacheSettings
+export const maxCacheDuration = (cacheSettings: CacheSettings) => {
+  const ttlDurations = Object.keys(cacheSettings.ttlDurationMapping).map((key) => Number(key));
+
+  if (ttlDurations.length === 0) {
+    return 0;
+  }
+
+  return Math.max(...ttlDurations);
+};
diff --git a/packages/core/src/data-module/data-source-store/dataSourceStore.ts b/packages/core/src/data-module/data-source-store/dataSourceStore.ts
@@ -43,7 +43,9 @@ export default class DataSourceStore {
     request: TimeSeriesDataRequest;
   }): RequestInformation[] => {
     const dataSource = this.getDataSource(query.source);
-    return dataSource.getRequestsFromQuery({ query, request });
+    return dataSource
+      .getRequestsFromQuery({ query, request })
+      .map((request) => ({ ...request, cacheSettings: query.cacheSettings }));
   };
 
   public initiateRequest = <Query extends DataStreamQuery>(

diff --git a/packages/core/src/data-module/subscription-store/subscriptionStore.ts b/packages/core/src/data-module/subscription-store/subscriptionStore.ts
@@ -4,6 +4,7 @@ import { CacheSettings } from '../data-cache/types';
 import DataSourceStore from '../data-source-store/dataSourceStore';
 import RequestScheduler from '../request-scheduler/requestScheduler';
 import { viewportEndDate } from '../../common/viewport';
+import { maxCacheDuration } from '../data-cache/caching/caching';
 
 /**
  * Subscription store
@@ -55,7 +56,11 @@ export default class SubscriptionStore {
             refreshRate: subscription.request.settings?.refreshRate,
             refreshExpiration:
               viewportEndDate(subscription.request.viewport).getTime() +
-              Math.max(...Object.keys(this.cacheSettings.ttlDurationMapping).map((key) => Number(key))),
+              Math.max(
+                ...subscription.queries.map((query) =>
+                  maxCacheDuration({ ...this.cacheSettings, ...query.cacheSettings })
+                )
+              ),
           });
         }
 

diff --git a/packages/core/src/data-module/types.d.ts b/packages/core/src/data-module/types.d.ts
@@ -13,8 +13,14 @@ import {
   ListAssociatedAssetsCommandOutput,
 } from '@aws-sdk/client-iotsitewise';
 import { RefId } from '../data-sources/site-wise/types';
+import { CacheSettings } from './data-cache/types';
 
-export type RequestInformation = { id: DataStreamId; resolution: Resolution; refId?: RefId };
+export type RequestInformation = {
+  id: DataStreamId;
+  resolution: Resolution;
+  refId?: RefId;
+  cacheSettings?: CacheSettings;
+};
 export type RequestInformationAndRange = RequestInformation & { start: Date; end: Date };
 
 export type DataSourceName = string;
@@ -55,6 +61,7 @@ export type DataModuleSubscription<Query extends DataStreamQuery> = {
 
 export type DataStreamQuery = {
   source: DataSourceName;
+  cacheSettings?: CacheSettings;
 };
 
 export type AnyDataStreamQuery = DataStreamQuery & any;

diff --git a/packages/core/src/data-sources/site-wise/types.d.ts b/packages/core/src/data-sources/site-wise/types.d.ts
@@ -1,3 +1,4 @@
+import { CacheSettings } from '../../data-module/data-cache/types';
 import { DataStreamQuery, SubscriptionUpdate } from '../../data-module/types.d';
 
 /**
@@ -17,6 +18,7 @@ export type PropertyQuery = {
   propertyId: string;
   refId?: RefId;
   resolution?: string;
+  cacheSettings?: CacheSettings;
 };
 
 export type AssetQuery = {