Sync tenants data stores on application bootstrap (#158)

apps/policy-engine/src/app/app.module.ts

@@ -1,18 +1,23 @@
 import { HttpModule } from '@nestjs/axios'
-import { Module, ValidationPipe } from '@nestjs/common'
+import { Module, OnApplicationBootstrap, ValidationPipe } from '@nestjs/common'
 import { ConfigModule } from '@nestjs/config'
 import { APP_PIPE } from '@nestjs/core'
 import { EncryptionModule } from '../encryption/encryption.module'
 import { load } from '../policy-engine.config'
 import { KeyValueModule } from '../shared/module/key-value/key-value.module'
 import { AppController } from './app.controller'
 import { AppService } from './app.service'
+import { DataStoreRepositoryFactory } from './core/factory/data-store-repository.factory'
+import { BootstrapService } from './core/service/bootstrap.service'
+import { DataStoreService } from './core/service/data-store.service'
 import { EngineService } from './core/service/engine.service'
 import { TenantService } from './core/service/tenant.service'
 import { TenantController } from './http/rest/controller/tenant.controller'
 import { OpaService } from './opa/opa.service'
 import { EngineRepository } from './persistence/repository/engine.repository'
 import { EntityRepository } from './persistence/repository/entity.repository'
+import { FileSystemDataStoreRepository } from './persistence/repository/file-system-data-store.repository'
+import { HttpDataStoreRepository } from './persistence/repository/http-data-store.repository'
 import { TenantRepository } from './persistence/repository/tenant.repository'
 
 @Module({
@@ -28,10 +33,15 @@ import { TenantRepository } from './persistence/repository/tenant.repository'
   controllers: [AppController, TenantController],
   providers: [
     AppService,
-    OpaService,
+    BootstrapService,
+    DataStoreRepositoryFactory,
+    DataStoreService,
     EngineRepository,
     EngineService,
     EntityRepository,
+    FileSystemDataStoreRepository,
+    HttpDataStoreRepository,
+    OpaService,
     TenantRepository,
     TenantService,
     {
@@ -40,4 +50,10 @@ import { TenantRepository } from './persistence/repository/tenant.repository'
     }
   ]
 })
-export class AppModule {}
+export class AppModule implements OnApplicationBootstrap {
+  constructor(private bootstrapService: BootstrapService) {}
+
+  async onApplicationBootstrap() {
+    await this.bootstrapService.boot()
+  }
+}

apps/policy-engine/src/app/core/service/__test__/unit/bootstrap.service.spec.ts

@@ -0,0 +1,64 @@
+import { Test } from '@nestjs/testing'
+import { MockProxy, mock } from 'jest-mock-extended'
+import { BootstrapService } from '../../bootstrap.service'
+import { TenantService } from '../../tenant.service'
+
+describe(BootstrapService.name, () => {
+  let bootstrapService: BootstrapService
+  let tenantServiceMock: MockProxy<TenantService>
+
+  const dataStore = {
+    entity: {
+      dataUrl: 'http://9.9.9.9:90',
+      signatureUrl: 'http://9.9.9.9:90',
+      keys: []
+    },
+    policy: {
+      dataUrl: 'http://9.9.9.9:90',
+      signatureUrl: 'http://9.9.9.9:90',
+      keys: []
+    }
+  }
+
+  const tenantOne = {
+    dataStore,
+    clientId: 'test-tenant-one-id',
+    clientSecret: 'unsafe-client-secret',
+    createdAt: new Date(),
+    updatedAt: new Date()
+  }
+
+  const tenantTwo = {
+    dataStore,
+    clientId: 'test-tenant-two-id',
+    clientSecret: 'unsafe-client-secret',
+    createdAt: new Date(),
+    updatedAt: new Date()
+  }
+
+  beforeEach(async () => {
+    tenantServiceMock = mock<TenantService>()
+    tenantServiceMock.findAll.mockResolvedValue([tenantOne, tenantTwo])
+
+    const module = await Test.createTestingModule({
+      providers: [
+        BootstrapService,
+        {
+          provide: TenantService,
+          useValue: tenantServiceMock
+        }
+      ]
+    }).compile()
+
+    bootstrapService = module.get<BootstrapService>(BootstrapService)
+  })
+
+  describe('boot', () => {
+    it('syncs tenants data stores', async () => {
+      await bootstrapService.boot()
+
+      expect(tenantServiceMock.syncDataStore).toHaveBeenNthCalledWith(1, tenantOne.clientId)
+      expect(tenantServiceMock.syncDataStore).toHaveBeenNthCalledWith(2, tenantTwo.clientId)
+    })
+  })
+})

apps/policy-engine/src/app/core/service/__test__/unit/tenant.service.spec.ts

@@ -0,0 +1,91 @@
+import { DataStoreConfiguration, FIXTURE } from '@narval/policy-engine-shared'
+import { Test } from '@nestjs/testing'
+import { MockProxy, mock } from 'jest-mock-extended'
+import { KeyValueService } from '../../../../../shared/module/key-value/core/service/key-value.service'
+import { InMemoryKeyValueRepository } from '../../../../../shared/module/key-value/persistence/repository/in-memory-key-value.repository'
+import { Tenant } from '../../../../../shared/type/domain.type'
+import { TenantRepository } from '../../../../persistence/repository/tenant.repository'
+import { DataStoreService } from '../../data-store.service'
+import { TenantService } from '../../tenant.service'
+
+describe(TenantService.name, () => {
+  let tenantService: TenantService
+  let tenantRepository: TenantRepository
+  let dataStoreServiceMock: MockProxy<DataStoreService>
+
+  const clientId = 'test-client-id'
+
+  const dataStoreConfiguration: DataStoreConfiguration = {
+    dataUrl: 'a-url-that-doesnt-need-to-exist-for-the-purpose-of-this-test',
+    signatureUrl: 'a-url-that-doesnt-need-to-exist-for-the-purpose-of-this-test',
+    keys: []
+  }
+
+  const tenant: Tenant = {
+    clientId,
+    clientSecret: 'test-client-secret',
+    dataStore: {
+      entity: dataStoreConfiguration,
+      policy: dataStoreConfiguration
+    },
+    createdAt: new Date(),
+    updatedAt: new Date()
+  }
+
+  const stores = {
+    entity: {
+      data: FIXTURE.ENTITIES,
+      signature: 'test-signature'
+    },
+    policy: {
+      data: FIXTURE.POLICIES,
+      signature: 'test-signature'
+    }
+  }
+
+  beforeEach(async () => {
+    dataStoreServiceMock = mock<DataStoreService>()
+    dataStoreServiceMock.fetch.mockResolvedValue(stores)
+
+    const module = await Test.createTestingModule({
+      providers: [
+        TenantService,
+        TenantRepository,
+        {
+          provide: DataStoreService,
+          useValue: dataStoreServiceMock
+        },
+        {
+          provide: KeyValueService,
+          useClass: InMemoryKeyValueRepository
+        }
+      ]
+    }).compile()
+
+    tenantService = module.get<TenantService>(TenantService)
+    tenantRepository = module.get<TenantRepository>(TenantRepository)
+  })
+
+  describe('syncDataStore', () => {
+    beforeEach(async () => {
+      await tenantRepository.save(tenant)
+    })
+
+    it('saves entity and policy stores', async () => {
+      expect(await tenantRepository.findEntityStore(clientId)).toEqual(null)
+      expect(await tenantRepository.findPolicyStore(clientId)).toEqual(null)
+
+      await tenantService.syncDataStore(clientId)
+
+      expect(await tenantRepository.findEntityStore(clientId)).toEqual(stores.entity)
+      expect(await tenantRepository.findPolicyStore(clientId)).toEqual(stores.policy)
+    })
+
+    it('fetches the data stores once', async () => {
+      await tenantService.syncDataStore(clientId)
+
+      expect(dataStoreServiceMock.fetch).toHaveBeenCalledTimes(1)
+      expect(dataStoreServiceMock.fetch).toHaveBeenCalledWith(tenant.dataStore)
+    })
+  })
+})

apps/policy-engine/src/app/core/service/bootstrap.service.ts

@@ -0,0 +1,51 @@
+import { Injectable, Logger } from '@nestjs/common'
+import { TenantService } from './tenant.service'
+
+@Injectable()
+export class BootstrapService {
+  private logger = new Logger(BootstrapService.name)
+
+  constructor(private tenantService: TenantService) {}
+
+  async boot(): Promise<void> {
+    this.logger.log('Start application bootstrap procedure')
+
+    await this.tenantService.onboard(
+      {
+        clientId: '012553b0-34e9-4b48-b217-ced3c906cd39',
+        clientSecret: 'unsafe-dev-secret',
+        dataStore: {
+          entity: {
+            dataUrl: 'http://127.0.0.1:4200/api/data-store',
+            signatureUrl: 'http://127.0.0.1:4200/api/data-store',
+            keys: []
+          },
+          policy: {
+            dataUrl: 'http://127.0.0.1:4200/api/data-store',
+            signatureUrl: 'http://127.0.0.1:4200/api/data-store',
+            keys: []
+          }
+        },
+        createdAt: new Date(),
+        updatedAt: new Date()
+      },
+      // Disable sync after the onboard because we'll sync it as part of the boot.
+      { syncAfter: false }
+    )
+
+    await this.syncTenants()
+  }
+
+  private async syncTenants(): Promise<void> {
+    const tenants = await this.tenantService.findAll()
+
+    this.logger.log('Start syncing tenants data stores', {
+      tenantsCount: tenants.length
+    })
+
+    // TODO: (@wcalderipe, 07/03/24) maybe change the execution to parallel?
+    for (const tenant of tenants) {
+      await this.tenantService.syncDataStore(tenant.clientId)
+    }
+  }
+}

apps/policy-engine/src/app/core/service/data-store.service.ts

@@ -1,7 +1,7 @@
 import {
   DataStoreConfiguration,
-  Entities,
-  Policy,
+  EntityStore,
+  PolicyStore,
   entityDataSchema,
   entitySignatureSchema,
   policyDataSchema,
@@ -17,14 +17,8 @@ export class DataStoreService {
   constructor(private dataStoreRepositoryFactory: DataStoreRepositoryFactory) {}
 
   async fetch(store: { entity: DataStoreConfiguration; policy: DataStoreConfiguration }): Promise<{
-    entity: {
-      data: Entities
-      signature: string
-    }
-    policy: {
-      data: Policy[]
-      signature: string
-    }
+    entity: EntityStore
+    policy: PolicyStore
   }> {
     const [entityData, entitySignature, policyData, policySignature] = await Promise.all([
       this.fetchByUrl(store.entity.dataUrl, entityDataSchema),

apps/policy-engine/src/app/core/service/tenant.service.ts

@@ -1,25 +1,101 @@
-import { HttpStatus, Injectable } from '@nestjs/common'
+import { EntityStore, PolicyStore } from '@narval/policy-engine-shared'
+import { HttpStatus, Injectable, Logger } from '@nestjs/common'
 import { ApplicationException } from '../../../shared/exception/application.exception'
 import { Tenant } from '../../../shared/type/domain.type'
 import { TenantRepository } from '../../persistence/repository/tenant.repository'
+import { DataStoreService } from './data-store.service'
 
 @Injectable()
 export class TenantService {
-  constructor(private tenantRepository: TenantRepository) {}
+  private logger = new Logger(TenantService.name)
+
+  constructor(
+    private tenantRepository: TenantRepository,
+    private dataStoreService: DataStoreService
+  ) {}
 
   async findByClientId(clientId: string): Promise<Tenant | null> {
     return this.tenantRepository.findByClientId(clientId)
   }
 
-  async create(tenant: Tenant): Promise<Tenant> {
-    if (await this.tenantRepository.findByClientId(tenant.clientId)) {
+  async onboard(tenant: Tenant, options?: { syncAfter?: boolean }): Promise<Tenant> {
+    const syncAfter = options?.syncAfter ?? true
+
+    const exists = await this.tenantRepository.findByClientId(tenant.clientId)
+
+    if (exists) {
       throw new ApplicationException({
         message: 'Tenant already exist',
         suggestedHttpStatusCode: HttpStatus.BAD_REQUEST,
         context: { clientId: tenant.clientId }
       })
     }
 
-    return this.tenantRepository.create(tenant)
+    try {
+      await this.tenantRepository.save(tenant)
+
+      if (syncAfter) {
+        const hasSynced = await this.syncDataStore(tenant.clientId)
+
+        if (!hasSynced) {
+          this.logger.warn('Failed to sync new tenant data store during the onboard')
+        }
+      }
+
+      return tenant
+    } catch (error) {
+      throw new ApplicationException({
+        message: 'Failed to onboard new tenant',
+        suggestedHttpStatusCode: HttpStatus.UNPROCESSABLE_ENTITY,
+        origin: error,
+        context: { tenant }
+      })
+    }
+  }
+
+  async syncDataStore(clientId: string): Promise<boolean> {
+    this.logger.log('Start syncing tenant data stores', { clientId })
+
+    try {
+      const tenant = await this.findByClientId(clientId)
+
+      if (tenant) {
+        this.logger.log('Sync tenant data stores', {
+          dataStore: tenant.dataStore
+        })
+
+        const stores = await this.dataStoreService.fetch(tenant.dataStore)
+
+        await Promise.all([
+          this.tenantRepository.saveEntityStore(clientId, stores.entity),
+          this.tenantRepository.savePolicyStore(clientId, stores.policy)
+        ])
+
+        this.logger.log('Tenant data stores synced', { clientId, stores })
+
+        return true
+      }
+
+      return false
+    } catch (error) {
+      this.logger.error('Failed to sync tenant data store', {
+        message: error.message,
+        stack: error.stack
+      })
+
+      return false
+    }
+  }
+
+  async findEntityStore(clientId: string): Promise<EntityStore | null> {
+    return this.tenantRepository.findEntityStore(clientId)
+  }
+
+  async findPolicyStore(clientId: string): Promise<PolicyStore | null> {
+    return this.tenantRepository.findPolicyStore(clientId)
+  }
+
+  async findAll(): Promise<Tenant[]> {
+    return this.tenantRepository.findAll()
   }
 }