From e482a706a3c5ba9a9f9d7d9e15f1db72b0cb0471 Mon Sep 17 00:00:00 2001 From: RaenonX Date: Mon, 17 Jan 2022 23:08:40 -0600 Subject: [PATCH] Updated GA data to use the database as cache #37 Signed-off-by: RaenonX --- src/endpoints/info/homepage/handler.ts | 2 +- src/thirdparty/ga/controller.test.ts | 36 ++++++----------- src/thirdparty/ga/controller.ts | 39 +++++------------- src/thirdparty/ga/dbCache.ts | 56 ++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 54 deletions(-) create mode 100644 src/thirdparty/ga/dbCache.ts diff --git a/src/endpoints/info/homepage/handler.ts b/src/endpoints/info/homepage/handler.ts index 50c6791..7b81438 100644 --- a/src/endpoints/info/homepage/handler.ts +++ b/src/endpoints/info/homepage/handler.ts @@ -12,7 +12,7 @@ export const handleHomepageLanding = async ({ payload, mongoClient, }: HandlerParams): Promise => { - const gaData = await getGaData(); + const gaData = await getGaData(mongoClient); const data: HomepageData = { posts: { diff --git a/src/thirdparty/ga/controller.test.ts b/src/thirdparty/ga/controller.test.ts index 4e4310e..514279b 100644 --- a/src/thirdparty/ga/controller.test.ts +++ b/src/thirdparty/ga/controller.test.ts @@ -1,10 +1,10 @@ import {periodicActiveData, periodicCountryData, periodicLangData} from '../../../test/data/thirdparty/ga'; import {Application, createApp} from '../../app'; -import {CACHE_LIFE_SECS} from '../../utils/cache/const'; -import {getGaData, resetGaData} from './controller'; +import {getGaData} from './controller'; import * as periodicActive from './data/periodicActive'; import * as periodicCountry from './data/periodicCountry'; import * as periodicTotal from './data/periodicTotal'; +import {resetCache} from './dbCache'; describe('Google Analytics data cache', () => { @@ -18,6 +18,8 @@ describe('Google Analytics data cache', () => { }); beforeEach(async () => { + await app.reset(); + fnFetchTotal = jest.spyOn(periodicTotal, 'getPeriodicLanguageUser') .mockResolvedValue(periodicLangData); fnFetchCountry = jest.spyOn(periodicCountry, 'getPeriodicCountryUser') @@ -26,45 +28,37 @@ describe('Google Analytics data cache', () => { .mockResolvedValue(periodicActiveData); }); - afterEach(() => { - resetGaData(); - }); - afterAll(async () => { await app.close(); }); it('fetches data on initial request', async () => { - await getGaData(); + await getGaData(app.mongoClient); expect(fnFetchTotal).toHaveBeenCalledTimes(1); expect(fnFetchCountry).toHaveBeenCalledTimes(1); expect(fnFetchActive).toHaveBeenCalledTimes(1); }); it('does not re-fetch data twice', async () => { - await getGaData(); + await getGaData(app.mongoClient); fnFetchTotal.mockClear(); fnFetchCountry.mockClear(); fnFetchActive.mockClear(); - await getGaData(); + await getGaData(app.mongoClient); expect(fnFetchTotal).not.toHaveBeenCalled(); expect(fnFetchCountry).not.toHaveBeenCalled(); expect(fnFetchActive).not.toHaveBeenCalled(); }); it('re-fetches after the cache expires', async () => { - await getGaData(); + await getGaData(app.mongoClient); expect(fnFetchTotal).toHaveBeenCalledTimes(1); expect(fnFetchCountry).toHaveBeenCalledTimes(1); expect(fnFetchActive).toHaveBeenCalledTimes(1); - // Accelerate time - const now = Date.now(); - jest - .spyOn(Date, 'now') - .mockImplementation(() => now + CACHE_LIFE_SECS * 1000 + 100000); + await resetCache(app.mongoClient); - await getGaData(); + await getGaData(app.mongoClient); expect(fnFetchTotal).toHaveBeenCalledTimes(2); expect(fnFetchCountry).toHaveBeenCalledTimes(2); expect(fnFetchActive).toHaveBeenCalledTimes(2); @@ -73,18 +67,12 @@ describe('Google Analytics data cache', () => { }); it('does not re-fetch before the cache expires', async () => { - await getGaData(); + await getGaData(app.mongoClient); expect(fnFetchTotal).toHaveBeenCalledTimes(1); expect(fnFetchCountry).toHaveBeenCalledTimes(1); expect(fnFetchActive).toHaveBeenCalledTimes(1); - // Accelerate time - const now = Date.now(); - jest - .spyOn(Date, 'now') - .mockImplementation(() => now + CACHE_LIFE_SECS * 1000 / 2); - - await getGaData(); + await getGaData(app.mongoClient); expect(fnFetchTotal).toHaveBeenCalledTimes(1); expect(fnFetchCountry).toHaveBeenCalledTimes(1); expect(fnFetchActive).toHaveBeenCalledTimes(1); diff --git a/src/thirdparty/ga/controller.ts b/src/thirdparty/ga/controller.ts index c586f2d..2d65dd0 100644 --- a/src/thirdparty/ga/controller.ts +++ b/src/thirdparty/ga/controller.ts @@ -1,42 +1,21 @@ import env from 'env-var'; +import {MongoClient} from 'mongodb'; import {periodicActiveData, periodicCountryData, periodicLangData} from '../../../test/data/thirdparty/ga'; import {isCi} from '../../api-def/utils'; -import {isCacheExpired} from '../../utils/cache/func'; import {getPeriodicActiveUser} from './data/periodicActive'; import {getPeriodicCountryUser} from './data/periodicCountry'; import {getPeriodicLanguageUser} from './data/periodicTotal'; +import {getCache, setCache} from './dbCache'; import {GACache} from './type'; -const generateNewCache = (): GACache => ({ - data: { - perCountry: { - D1: {countries: [], total: 0}, - D7: {countries: [], total: 0}, - D30: {countries: [], total: 0}, - }, - perLang: { - data: [], - toppedLang: [], - }, - active: { - data: [], - }, - }, - lastFetchedEpoch: 0, -}); - -let cache: GACache = generateNewCache(); - -export const resetGaData = (): void => { - cache = generateNewCache(); -}; - -export const getGaData = async (): Promise => { +export const getGaData = async (mongoClient: MongoClient): Promise => { const currentEpoch = Math.round(Date.now() / 1000); - if (!isCacheExpired(cache, currentEpoch)) { + const cache = await getCache(mongoClient); + + if (cache) { return cache; } else if (!isCi() && env.get('GA_DEV').asBool()) { return { @@ -49,7 +28,7 @@ export const getGaData = async (): Promise => { }; } - cache = { + const newCache: GACache = { data: { perCountry: await getPeriodicCountryUser(6), perLang: await getPeriodicLanguageUser(30, 3), @@ -58,5 +37,7 @@ export const getGaData = async (): Promise => { lastFetchedEpoch: currentEpoch, }; - return cache; + await setCache(mongoClient, newCache); + + return newCache; }; diff --git a/src/thirdparty/ga/dbCache.ts b/src/thirdparty/ga/dbCache.ts new file mode 100644 index 0000000..13536ab --- /dev/null +++ b/src/thirdparty/ga/dbCache.ts @@ -0,0 +1,56 @@ +import {Collection, MongoClient} from 'mongodb'; + +import {CollectionInfo} from '../../base/controller/info'; +import {CACHE_LIFE_SECS} from '../../utils/cache/const'; +import {getCollection} from '../../utils/mongodb'; +import {GACache} from './type'; + + +const dbInfo: CollectionInfo = { + dbName: 'cache', + collectionName: 'ga', +}; + +export enum GACacheKey { + data = 'd', + generationTimestamp = 'g' +} + +export type GACacheDocument = { + [GACacheKey.data]: GACache, + [GACacheKey.generationTimestamp]: Date, +}; + +const getCacheCollection = (mongoClient: MongoClient): Collection => { + return getCollection(mongoClient, dbInfo, (collection) => { + // Enable data auto-expiration + collection.createIndex(GACacheKey.generationTimestamp, {expireAfterSeconds: CACHE_LIFE_SECS}); + }); +}; + +export const getCache = async (mongoClient: MongoClient): Promise => { + const collection = getCacheCollection(mongoClient); + + const cacheEntry = await collection.findOne(); + + if (!cacheEntry) { + return null; + } + + return cacheEntry[GACacheKey.data]; +}; + +export const setCache = async (mongoClient: MongoClient, data: GACache): Promise => { + const collection = getCacheCollection(mongoClient); + + await collection.insertOne({ + [GACacheKey.data]: data, + [GACacheKey.generationTimestamp]: new Date(), + }); +}; + +export const resetCache = async (mongoClient: MongoClient): Promise => { + const collection = getCacheCollection(mongoClient); + + await collection.deleteMany({}); +};