test github action

.github/workflows/integration-test.yml

@@ -0,0 +1,29 @@
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - qt/test
+
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4 #TODO version
+
+      - uses: actions/setup-node@v4 #TODO version
+        with:
+          node-version: 18
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Trigger Harvest
+        run: node integration/tools/run-harvest.js
+
+      - name: Verify Definitions
+        run: npx mocha integration/test/sanityTest.js

integration/test/sanityTest.js

@@ -0,0 +1,59 @@
+const { omit, isEqual } = require('lodash')
+const expect = require('chai').expect
+const { callFetch } = require('../tools/fetch')
+const { devApiBaseUrl, prodApiBaseUrl, components } = require('../tools/testConfig')
+
+describe('Validate Definition between dev and prod', function () {
+  it('should be able to get definition for a component and compare to production', async function () {
+    this.timeout(300000)
+    for (const coordinates of components) {
+      console.log(coordinates)
+      await compareDefintion(coordinates)
+    }
+  })
+})
+
+async function compareDefintion(coordinates) {
+  const result = await callFetch(`${devApiBaseUrl}/definitions/${coordinates}?force=true`).then((r) => r.json())
+  const expected = await callFetch(`${prodApiBaseUrl}/definitions/${coordinates}`).then((r) => r.json())
+  expect(result.coordinates).to.be.deep.equals(expected.coordinates)
+  compareLicensed(result, expected)
+  compareDescribed(result, expected)
+  compareFiles(result, expected)
+  expect(result.score).to.be.deep.equal(expected.score)
+}
+
+function compareLicensed(result, expectation) {
+  const actual = omit(result.licensed, ['facets'])
+  const expected = omit(expectation.licensed, ['facets'])
+  expect(actual).to.be.deep.equals(expected)
+}
+
+function compareDescribed(result, expectation) {
+  const actual = omit(result.described, ['tools'])
+  const expected = omit(expectation.described, ['tools'])
+  expect(actual).to.be.deep.equals(expected)
+}
+
+function compareFiles(result, expectation) {
+  const rfiles = filesToMap(result)
+  const efiles = filesToMap(expectation)
+  const missingInResult = result.files.filter((f) => !efiles.has(f.path))
+  const extraInResult = expectation.files.filter((f) => !rfiles.has(f.path))
+  const differentEntries = result.files.filter((f) => {
+    const different = efiles.has(f.path) && !isEqual(efiles.get(f.path), f)
+    if (different) {
+      console.log('-------------------')
+      console.log(JSON.stringify(efiles.get(f.path)))
+      console.log(JSON.stringify(f))
+    }
+    return different
+  })
+  expect(missingInResult.length).to.be.equal(0)
+  expect(extraInResult.length).to.be.equal(0)
+  expect(differentEntries.length).to.be.equal(0)
+}
+
+function filesToMap(result) {
+  return new Map(result.files.map((f) => [f.path, f]))
+}

integration/test/toolTest.js

@@ -0,0 +1,60 @@
+const expect = require('chai').expect
+const { callFetch } = require('../tools/fetch')
+const Poller = require('../tools/poll')
+const Harvester = require('../tools/harvest')
+const { devApiBaseUrl } = require('../tools/testConfig')
+
+describe('Test against dev deployment', function () {
+  it('should be able to get harvest for a component', async function () {
+    const coordinates = 'nuget/nuget/-/HotChocolate/13.8.1'
+    const result = await callFetch(`${devApiBaseUrl}/harvest/${coordinates}?form=list`).then((r) => r.json())
+    expect(result.length).to.be.greaterThan(0)
+  })
+
+  it('should be able to harvest a component', async function () {
+    this.timeout(20000)
+    const coordinates = 'nuget/nuget/-/HotChocolate/13.8.1'
+    const harvester = new Harvester(devApiBaseUrl)
+    const result = await harvester.harvest([coordinates])
+    expect(result.status).to.be.equal(201)
+  })
+})
+
+describe('Test Harvester', function () {
+  let harvester
+  beforeEach(function () {
+    harvester = new Harvester(devApiBaseUrl)
+  })
+
+  it('should be able to detect when a tool for component is ran', async function () {
+    const coordinates = 'nuget/nuget/-/HotChocolate/13.8.1'
+    const result = await harvester.isHarvestCompletedRecently(coordinates, 'licensee', '9.14.0')
+    expect(result).to.be.equal(true)
+  })
+
+  it('should be able to detect when component is completely harvested', async function () {
+    const coordinates = 'nuget/nuget/-/HotChocolate/13.8.1'
+    const result = await harvester.isHarvestCompletedRecently(coordinates)
+    expect(result).to.be.equal(true)
+  })
+
+  it('should poll', async function () {
+    this.timeout(1000 * 60 * 5)
+    const coordinates = 'nuget/nuget/-/HotChocolate/13.8.1'
+    const interval = 1000 * 1
+    const maxTime = 1000 * 3
+    const result = await new Poller(interval, maxTime).poll(
+      async () => await harvester.isHarvestCompletedRecently(coordinates)
+    )
+    expect(result).to.be.equal(false)
+  })
+
+  it('should poll for complete', async function () {
+    this.timeout(1000 * 60 * 5)
+    const coordinates = 'nuget/nuget/-/HotChocolate/13.8.1'
+    const interval = 1000 * 1
+    const maxTime = 1000 * 3
+    const status = await harvester.waitForCompletion([coordinates], new Poller(interval, maxTime))
+    expect(status.get(coordinates)).to.be.equal(false)
+  })
+})

integration/tools/fetch.js

@@ -0,0 +1,22 @@
+const fetch = (...args) => import('node-fetch').then(({ default: fetch }) => fetch(...args))
+
+function buildPostOpts(json) {
+  return {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify(json)
+  }
+}
+
+async function callFetch(url, fetchOpts) {
+  console.log(url, fetchOpts)
+  const response = await fetch(url, fetchOpts)
+  if (!response.ok) {
+    const { status, statusText } = response
+    throw new Error(`Error fetching ${url}: ${status}, ${statusText}`)
+  }
+  return response
+}
+module.exports = { callFetch, buildPostOpts }

integration/tools/harvest.js

@@ -0,0 +1,60 @@
+const { callFetch, buildPostOpts } = require('./fetch')
+
+class Harvester {
+  constructor(apiBaseUrl) {
+    this.apiBaseUrl = apiBaseUrl
+  }
+
+  async harvest(components) {
+    return await callFetch(`${this.apiBaseUrl}/harvest`, buildPostOpts(this._buildPostJson(components)))
+  }
+
+  _buildPostJson(components) {
+    return components.map((coordinates) => {
+      const tool = 'component'
+      const policy = 'always'
+      return { tool, coordinates, policy }
+    })
+  }
+
+  async waitForCompletion(components, poller) {
+    const status = new Map()
+    for (const coordinates of components) {
+      try {
+        const completed = await poller.poll(() => this.isHarvestCompletedRecently(coordinates))
+        console.log(`Completed ${coordinates}: ${completed}`)
+        status.set(coordinates, completed)
+      } catch (e) {
+        console.error(`Failed to wait for harvest completion ${coordinates}: ${e.message}`)
+        status.set(coordinates, false)
+      }
+    }
+    return status
+  }
+
+  async isHarvestCompletedRecently(coordinates) {
+    const harvestChecks = [
+      ['licensee', '9.14.0'],
+      ['scancode', '30.3.0'],
+      ['reuse', '3.2.1']
+    ].map(([tool, toolVersion]) => this.isRecentlyHarvestedbyTool(coordinates, tool, toolVersion))
+
+    return Promise.all(harvestChecks).then((results) => results.every((r) => r))
+  }
+
+  async isRecentlyHarvestedbyTool(coordinates, tool, toolVersion) {
+    const harvested = await this.fetchHarvestResult(coordinates, tool, toolVersion)
+    if (!harvested._metadata) return false
+    const fetchedAt = new Date(harvested._metadata.fetchedAt)
+    console.log(`${coordinates} ${tool}, ${toolVersion} fetched at ${fetchedAt}`)
+    return Date.now() - fetchedAt.getTime() < 1000 * 60 * 60 // 1 hour
+  }
+
+  async fetchHarvestResult(coordinates, tool, toolVersion) {
+    return callFetch(`${this.apiBaseUrl}/harvest/${coordinates}/${tool}/${toolVersion}?form=raw`)
+      .then((r) => r.json())
+      .catch(() => ({}))
+  }
+}
+
+module.exports = Harvester

integration/tools/poll.js

@@ -0,0 +1,20 @@
+class Poller {
+  constructor(interval, maxTime) {
+    this.interval = interval
+    this.maxTime = maxTime
+  }
+
+  async poll(activity) {
+    let counter = 0
+    while (counter * this.interval <= this.maxTime) {
+      console.log(`Polling ${counter}`)
+      const result = await activity()
+      if (result) return true
+      await new Promise((resolve) => setTimeout(resolve, this.interval))
+      counter++
+    }
+    return false
+  }
+}
+
+module.exports = Poller

integration/tools/run-harvest.js

@@ -0,0 +1,11 @@
+const { components, devApiBaseUrl, interval, maxTime } = require('./testConfig')
+const Poller = require('./poll')
+const Harvester = require('./harvest')
+
+async function harvestTillCompletion(components) {
+  const harvester = new Harvester(devApiBaseUrl)
+  await harvester.harvest(components)
+  await harvester.waitForCompletion(components, new Poller(interval, maxTime))
+}
+
+harvestTillCompletion(components)

integration/tools/testConfig.js

@@ -0,0 +1,23 @@
+const devApiBaseUrl = 'https://dev-api.clearlydefined.io'
+const prodApiBaseUrl = 'https://api.clearlydefined.io'
+const interval = 1000 * 60 * 5 // 5 minutes
+const maxTime = 1000 * 60 * 30  // 30 minutes
+
+const components = [
+  'maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.16',
+  'maven/mavengoogle/android.arch.lifecycle/common/1.0.1',
+  'maven/gradleplugin/io.github.lognet/grpc-spring-boot-starter-gradle-plugin/4.6.0',
+  'crate/cratesio/-/ratatui/0.26.0',
+  'npm/npmjs/-/redis/0.1.0',
+  'git/github/ratatui-org/ratatui/bcf43688ec4a13825307aef88f3cdcd007b32641',
+  'gem/rubygems/-/sorbet/0.5.11226',
+  'pypi/pypi/-/platformdirs/4.2.0',
+  'go/golang/rsc.io/quote/v1.3.0',
+  'nuget/nuget/-/HotChocolate/13.8.1'
+  // 'composer/packagist/symfony/polyfill-mbstring/1.11.0',
+  // 'pod/cocoapods/-/SoftButton/0.1.0',
+  // 'deb/debian/-/mini-httpd/1.30-0.2_arm64'
+  // 'debsrc/debian/-/mini-httpd/1.30-0.2_arm64',
+  // 'sourcearchive/mavencentral/org.apache.httpcomponents/httpcore/4.1'
+]
+module.exports = { devApiBaseUrl, prodApiBaseUrl, components, interval, maxTime }