Add github action for integration test

.github/workflows/integration-test.yml

@@ -0,0 +1,32 @@
+name: Integration Test
+
+on:
+  workflow_dispatch:
+    ## The tests take a long time to run and can be potentially quite costly, so set it to run manually
+    ## TODO Take push out
+  push:
+    branches:
+      - qt/integration-test
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/[email protected]
+
+      - uses: actions/[email protected]
+        with:
+          node-version: 18
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Trigger harvest and verify completion
+        run: npx mocha integration/test/harvestTest.js
+
+      - name: Verify computed definitions
+        run: npx mocha integration/test/serviceTest.js

integration/test/harvestTest.js

@@ -0,0 +1,36 @@
+const { components, devApiBaseUrl, harvest } = require('./testConfig')
+const Poller = require('../tools/poller')
+const Harvester = require('../tools/harvester')
+const assert = require('assert')
+
+describe('Tests for Harvester', function () {
+  it('should verify all harvests are complete', async function () {
+    this.timeout(harvest.timeout)
+    console.time('Harvest Test')
+    const status = await harvestTillCompletion(components)
+    for (const [coordinates, isHarvested] of status) {
+      assert.strictEqual(isHarvested, true, `Harvest for ${coordinates} is not complete`)
+    }
+    console.timeEnd('Harvest Test')
+  })
+})
+
+async function harvestTillCompletion(components) {
+  const { harvestToolVersions, poll } = harvest
+  const harvester = new Harvester(devApiBaseUrl, harvestToolVersions)
+  const poller = new Poller(poll.interval, poll.maxTime)
+
+  //make sure that we have one entire set of harvest results (old or new)
+  console.log('Ensure harvest results exit before starting tests')
+  const previousHarvests = await harvester.pollForCompletion(components, poller)
+  const previousHarvestsComplete = Array.from(previousHarvests.values()).every(v => v)
+  if (!previousHarvestsComplete) {
+    await harvester.harvest(components)
+    await harvester.pollForCompletion(components, poller)
+  }
+
+  //trigger a reharvest to overwrite the old result
+  console.log('Trigger reharvest to overwrite old results')
+  await harvester.harvest(components, true)
+  return harvester.pollForCompletion(components, poller, Date.now())
+}

integration/test/serviceTest.js

@@ -0,0 +1,61 @@
+const { omit, isEqual } = require('lodash')
+const expect = require('chai').expect
+const { callFetch } = require('../tools/fetch')
+const { devApiBaseUrl, prodApiBaseUrl, components, definition } = require('./testConfig')
+
+describe('Validate Definition between dev and prod', function () {
+  it('should get definition for a component and compare to production', async function () {
+    this.timeout(definition.timeout)
+    for (const coordinates of components) {
+      console.log(coordinates)
+      await compareDefintion(coordinates)
+    }
+  })
+})
+
+async function compareDefintion(coordinates) {
+  const recomputedDef = await callFetch(`${devApiBaseUrl}/definitions/${coordinates}?force=true`).then(r => r.json())
+  const expectedDef = await callFetch(`${prodApiBaseUrl}/definitions/${coordinates}`).then(r => r.json())
+  expect(recomputedDef.coordinates).to.be.deep.equals(expectedDef.coordinates)
+  compareLicensed(recomputedDef, expectedDef)
+  compareDescribed(recomputedDef, expectedDef)
+  compareFiles(recomputedDef, expectedDef)
+  expect(recomputedDef.score).to.be.deep.equal(expectedDef.score)
+}
+
+function compareLicensed(result, expectation) {
+  const actual = omit(result.licensed, ['facets'])
+  const expected = omit(expectation.licensed, ['facets'])
+  expect(actual).to.be.deep.equals(expected)
+}
+
+function compareDescribed(result, expectation) {
+  const actual = omit(result.described, ['tools'])
+  const expected = omit(expectation.described, ['tools'])
+  expect(actual).to.be.deep.equals(expected)
+}
+
+function compareFiles(result, expectation) {
+  const resultFiles = filesToMap(result)
+  const expectedFiles = filesToMap(expectation)
+  const extraInResult = result.files.filter(f => !expectedFiles.has(f.path))
+  const missingInResult = expectation.files.filter(f => !resultFiles.has(f.path))
+  const differentEntries = result.files.filter(f => expectedFiles.has(f.path) && !isEqual(expectedFiles.get(f.path), f))
+
+  const differences = [...extraInResult, ...missingInResult, ...differentEntries]
+  differences.forEach(f => logFiles(expectedFiles.get(f.path), resultFiles.get(f.path)))
+
+  expect(missingInResult.length).to.be.equal(0, 'Some files are missing in the result')
+  expect(extraInResult.length).to.be.equal(0, 'There are extra files in the result')
+  expect(differentEntries.length).to.be.equal(0, 'Some files are different between the result and the expectation')
+}
+
+function logFiles(expected, actual) {
+  console.log('-------------------')
+  console.log(`expected: ${JSON.stringify(expected || {})}`)
+  console.log(`actual:   ${JSON.stringify(actual || {})}`)
+}
+
+function filesToMap(result) {
+  return new Map(result.files.map(f => [f.path, f]))
+}

integration/test/testConfig.js

@@ -0,0 +1,43 @@
+const devApiBaseUrl = 'https://dev-api.clearlydefined.io'
+const prodApiBaseUrl = 'https://api.clearlydefined.io'
+
+const pollingInterval = 1000 * 60 * 5 // 5 minutes
+const pollingMaxTime = 1000 * 60 * 30 // 30 minutes
+
+const harvestToolVersions = [
+  ['licensee', '9.14.0'],
+  ['scancode', '30.3.0'],
+  ['reuse', '3.2.1']
+]
+
+const components = [
+  'maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.16',
+  'maven/mavengoogle/android.arch.lifecycle/common/1.0.1',
+  'maven/gradleplugin/io.github.lognet/grpc-spring-boot-starter-gradle-plugin/4.6.0',
+  'crate/cratesio/-/ratatui/0.26.0',
+  'npm/npmjs/-/redis/0.1.0',
+  'git/github/ratatui-org/ratatui/bcf43688ec4a13825307aef88f3cdcd007b32641',
+  'gem/rubygems/-/sorbet/0.5.11226',
+  'pypi/pypi/-/platformdirs/4.2.0',
+  'go/golang/rsc.io/quote/v1.3.0',
+  'nuget/nuget/-/HotChocolate/13.8.1'
+  // 'composer/packagist/symfony/polyfill-mbstring/1.11.0',
+  // 'pod/cocoapods/-/SoftButton/0.1.0',
+  // 'deb/debian/-/mini-httpd/1.30-0.2_arm64'
+  // 'debsrc/debian/-/mini-httpd/1.30-0.2_arm64',
+  // 'sourcearchive/mavencentral/org.apache.httpcomponents/httpcore/4.1'
+]
+
+module.exports = {
+  devApiBaseUrl,
+  prodApiBaseUrl,
+  components,
+  harvest: {
+    poll: { interval: pollingInterval, maxTime: pollingMaxTime },
+    harvestToolVersions,
+    timeout: 1000 * 60 * 60 * 2 // 2 hours
+  },
+  definition: {
+    timeout: 1000 * 20 // 20 seconds
+  }
+}

integration/test/toolTest.js

@@ -0,0 +1,113 @@
+const expect = require('chai').expect
+const { callFetch } = require('../tools/fetch')
+const Poller = require('../tools/poller')
+const Harvester = require('../tools/harvester')
+const { devApiBaseUrl } = require('./testConfig')
+const sinon = require('sinon')
+
+describe('Integration test against dev deployment', function () {
+  it('should get harvest for a component', async function () {
+    const coordinates = 'nuget/nuget/-/HotChocolate/13.8.1'
+    const result = await callFetch(`${devApiBaseUrl}/harvest/${coordinates}?form=list`).then(r => r.json())
+    expect(result.length).to.be.greaterThan(0)
+  })
+
+  it.skip('should harvest a component', async function () {
+    const coordinates = 'nuget/nuget/-/HotChocolate/13.8.1'
+    const harvester = new Harvester(devApiBaseUrl)
+    const result = await harvester.harvest([coordinates])
+    expect(result.status).to.be.equal(201)
+  })
+})
+
+describe('Tests for Harvester', function () {
+  const coordinates = 'nuget/nuget/-/HotChocolate/13.8.1'
+  let harvester
+  beforeEach(function () {
+    harvester = new Harvester(devApiBaseUrl)
+  })
+
+  it('should detect when a scan tool result for component is available', async function () {
+    sinon.stub(harvester, 'fetchHarvestResult').resolves(metadata())
+    const result = await harvester.isHarvestedbyTool(coordinates, 'licensee', '9.14.0')
+    expect(result).to.be.equal(true)
+  })
+
+  it('should detect when component is completely harvested', async function () {
+    sinon.stub(harvester, 'fetchHarvestResult').resolves(metadata())
+    const result = await harvester.isHarvestComplete(coordinates)
+    expect(result).to.be.equal(true)
+  })
+
+  it('should detect whether component is harvested after a timestamp', async function () {
+    const date = '2023-01-01T00:00:00.000Z'
+    sinon.stub(harvester, 'fetchHarvestResult').resolves(metadata(date))
+    const result = await harvester.isHarvestComplete(coordinates, Date.now())
+    expect(result).to.be.equal(false)
+  })
+})
+
+describe('Integration Tests for Harvester and Poller', function () {
+  const coordinates = 'nuget/nuget/-/HotChocolate/13.8.1'
+  const interval = 10 * 1
+  const maxTime = 10 * 2
+  let poller
+  let harvester
+
+  beforeEach(function () {
+    harvester = new Harvester(devApiBaseUrl)
+    poller = new Poller(interval, maxTime)
+  })
+
+  it('should poll until max time is reached', async function () {
+    sinon.stub(harvester, 'fetchHarvestResult').resolves({})
+    const result = await poller.poll(async () => await harvester.isHarvestComplete(coordinates, Date.now()))
+    expect(result).to.be.equal(false)
+  })
+
+  it('should poll for completion if results exist', async function () {
+    sinon.stub(harvester, 'fetchHarvestResult').resolves(metadata())
+    const status = await harvester.pollForCompletion([coordinates], poller)
+    expect(status.get(coordinates)).to.be.equal(true)
+  })
+
+  it('should poll for completion if results are stale', async function () {
+    const date = '2023-01-01T00:00:00.000Z'
+    sinon.stub(harvester, 'fetchHarvestResult').resolves(metadata(date))
+    const status = await harvester.pollForCompletion([coordinates], poller, Date.now())
+    expect(status.get(coordinates)).to.be.equal(false)
+  })
+})
+
+describe('Unit Tests for Poller', function () {
+  const interval = 10 * 1
+  const maxTime = 10 * 2
+  let poller
+
+  beforeEach(function () {
+    poller = new Poller(interval, maxTime)
+  })
+
+  it('should poll until max time reached', async function () {
+    const activity = sinon.stub().resolves(false)
+    const result = await poller.poll(activity)
+    expect(activity.callCount).to.be.equal(3)
+    expect(result).to.be.equal(false)
+  })
+
+  it('should handle when activity is done', async function () {
+    const activity = sinon.stub().resolves(true)
+    const result = await poller.poll(activity)
+    expect(activity.callCount).to.be.equal(1)
+    expect(result).to.be.equal(true)
+  })
+
+  it('should continue to poll until activity is done', async function () {
+    const activity = sinon.stub().resolves(false).onCall(1).resolves(true)
+    const result = await poller.poll(activity)
+    expect(activity.callCount).to.be.equal(2)
+    expect(result).to.be.equal(true)
+  })
+})
+
+const metadata = date => ({ _metadata: { fetchedAt: date || new Date().toISOString() } })

integration/tools/fetch.js

@@ -0,0 +1,20 @@
+const fetch = (...args) => import('node-fetch').then(({ default: fetch }) => fetch(...args))
+
+function buildPostOpts(json) {
+  return {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(json)
+  }
+}
+
+async function callFetch(url, fetchOpts) {
+  console.log(url, fetchOpts)
+  const response = await fetch(url, fetchOpts)
+  if (!response.ok) {
+    const { status, statusText } = response
+    throw new Error(`Error fetching ${url}: ${status}, ${statusText}`)
+  }
+  return response
+}
+module.exports = { callFetch, buildPostOpts }

integration/tools/harvester.js

@@ -0,0 +1,77 @@
+const { callFetch, buildPostOpts } = require('./fetch')
+
+const defaultToolChecks = [
+  ['licensee', '9.14.0'],
+  ['scancode', '30.3.0'],
+  ['reuse', '3.2.1']
+]
+
+class Harvester {
+  constructor(apiBaseUrl, harvestToolChecks) {
+    this.apiBaseUrl = apiBaseUrl
+    this.harvestToolChecks = harvestToolChecks || defaultToolChecks
+  }
+
+  async harvest(components, reharvest = false) {
+    return await callFetch(`${this.apiBaseUrl}/harvest`, buildPostOpts(this._buildPostJson(components, reharvest)))
+  }
+
+  _buildPostJson(components, reharvest = false) {
+    return components.map(coordinates => {
+      const result = { tool: 'component', coordinates }
+      if (reharvest) result.policy = 'always'
+      return result
+    })
+  }
+
+  async pollForCompletion(components, poller, startTime) {
+    const status = new Map()
+    for (const coordinates of components) {
+      const completed = await this._pollForOneCompletion(coordinates, poller, startTime)
+      status.set(coordinates, completed)
+    }
+
+    for (const coordinates of components) {
+      const completed = status.get(coordinates) || (await this.isHarvestComplete(coordinates, startTime))
+      status.set(coordinates, completed)
+    }
+    return status
+  }
+
+  async _pollForOneCompletion(coordinates, poller, startTime) {
+    try {
+      const completed = await poller.poll(async () => this.isHarvestComplete(coordinates, startTime))
+      console.log(`Completed ${coordinates}: ${completed}`)
+      return completed
+    } catch (e) {
+      console.error(`Failed to wait for harvest completion ${coordinates}: ${e.message}`)
+      return false
+    }
+  }
+
+  async isHarvestComplete(coordinates, startTime) {
+    const harvestChecks = this.harvestToolChecks.map(([tool, toolVersion]) =>
+      this.isHarvestedbyTool(coordinates, tool, toolVersion, startTime)
+    )
+
+    return Promise.all(harvestChecks)
+      .then(results => results.every(r => r))
+      .catch(() => false)
+  }
+
+  async isHarvestedbyTool(coordinates, tool, toolVersion, startTime = 0) {
+    const harvested = await this.fetchHarvestResult(coordinates, tool, toolVersion)
+    if (!harvested._metadata) return false
+    const fetchedAt = new Date(harvested._metadata.fetchedAt)
+    console.log(`${coordinates} ${tool}, ${toolVersion} fetched at ${fetchedAt}`)
+    return fetchedAt.getTime() > startTime
+  }
+
+  async fetchHarvestResult(coordinates, tool, toolVersion) {
+    return callFetch(`${this.apiBaseUrl}/harvest/${coordinates}/${tool}/${toolVersion}?form=raw`)
+      .then(r => r.json())
+      .catch(() => ({}))
+  }
+}
+
+module.exports = Harvester