From 23426e946aed3ac80897153bb19d9322b29152b2 Mon Sep 17 00:00:00 2001 From: Jonathan Olson Date: Tue, 14 Apr 2020 18:52:12 -0600 Subject: [PATCH] Some CT server work for https://github.com/phetsims/aqua/issues/88 --- html/continuous-report.html | 1 + js/CTSnapshot.js | 143 +++++++++++++++ js/continuous-loop.js | 13 +- js/continuous-report.js | 16 +- js/local-server.js | 338 ++++++++++++++++++++++++++++++++++++ package.json | 4 +- 6 files changed, 504 insertions(+), 11 deletions(-) create mode 100644 js/CTSnapshot.js create mode 100644 js/local-server.js diff --git a/html/continuous-report.html b/html/continuous-report.html index 60af243..1121d59 100644 --- a/html/continuous-report.html +++ b/html/continuous-report.html @@ -94,6 +94,7 @@

Continuous Testing Results

+ diff --git a/js/CTSnapshot.js b/js/CTSnapshot.js new file mode 100644 index 0000000..96761d2 --- /dev/null +++ b/js/CTSnapshot.js @@ -0,0 +1,143 @@ +// Copyright 2020, University of Colorado Boulder + +/** + * Holds data related to a CT snapshot + * + * @author Jonathan Olson + */ + +'use strict'; + +const copyDirectory = require( '../../perennial/js/common/copyDirectory' ); +const createDirectory = require( '../../perennial/js/common/createDirectory' ); +const deleteDirectory = require( '../../perennial/js/common/deleteDirectory' ); +const execute = require( '../../perennial/js/common/execute' ); +const getRepoList = require( '../../perennial/js/common/getRepoList' ); +const gitRevParse = require( '../../perennial/js/common/gitRevParse' ); +const npmCommand = require( '../../perennial/js/common/npmCommand' ); +const fs = require( 'fs' ); +const _ = require( 'lodash' ); // eslint-disable-line + +// constants +const copyOptions = { filter: path => path.indexOf( 'node_modules' ) < 0 }; + +class CTSnapshot { + /** + * Creates this snapshot. + * @public + * + * @param {string} rootDir + * @param {function({string})} setSnapshotStatus + */ + async create( rootDir, setSnapshotStatus ) { + + // @private {string} + this.rootDir = rootDir; + + // @private {function} + this.setSnapshotStatus = setSnapshotStatus; + + const timestamp = Date.now(); + const snapshotDir = `${rootDir}/ct-snapshots`; + + this.setSnapshotStatus( `Initializing new snapshot: ${timestamp}` ); + + // @public {number} + this.timestamp = timestamp; + + // @public {string} + this.name = `snapshot-${timestamp}`; + + // @public {boolean} + this.exists = true; + + // @public {string} + this.phetDir = `${snapshotDir}/${timestamp}-phet`; + this.phetioDir = `${snapshotDir}/${timestamp}-phet-io`; + + if ( !fs.existsSync( snapshotDir ) ) { + await createDirectory( snapshotDir ); + } + await createDirectory( this.phetDir ); + await createDirectory( this.phetioDir ); + + this.setSnapshotStatus( 'Copying snapshot files' ); + + // @public {Array.} + this.repos = getRepoList( 'active-repos' ); + + // @public {Array.} + this.npmInstalledRepos = []; + + // @public {Object} - maps repo {string} => sha {string} + this.shas = {}; + for ( const repo of this.repos ) { + this.shas[ repo ] = await gitRevParse( repo, 'master' ); + } + + for ( const repo of this.repos ) { + await copyDirectory( `${rootDir}/${repo}`, `${this.phetDir}/${repo}`, copyOptions ); + await copyDirectory( `${rootDir}/${repo}`, `${this.phetioDir}/${repo}`, copyOptions ); + } + + // @public {Array.} + this.tests = JSON.parse( await execute( 'node', [ 'js/listContinuousTests.js' ], '../perennial' ) ).map( test => { + test.snapshot = this; + return test; + } ); + this.browserTests = this.tests.filter( test => [ 'sim-test', 'qunit-test', 'pageload-test' ].includes( test.type ) ).map( test => { + test.count = 0; + return test; + } ); + this.lintTests = this.tests.filter( test => test.type === 'lint' ).map( test => { + test.complete = false; + return test; + } ); + this.buildTests = this.tests.filter( test => test.type === 'build' ).map( test => { + test.complete = false; + test.success = false; + return test; + } ); + } + + async npmInstall() { + const npmRepos = this.repos.filter( repo => fs.existsSync( `../${repo}/package.json` ) ); + for ( const repo of npmRepos ) { + this.setSnapshotStatus( `Running npm update for ${repo}` ); + + await execute( npmCommand, [ 'update', `--cache=../npm-caches/${repo}`, '--tmp=../npm-tmp' ], `${this.phetDir}/${repo}` ); + await execute( npmCommand, [ 'update', `--cache=../npm-caches/${repo}`, '--tmp=../npm-tmp' ], `${this.phetioDir}/${repo}` ); + } + } + + /** + * Removes the snapshot's files. + * @public + */ + async remove() { + await deleteDirectory( this.phetDir ); + await deleteDirectory( this.phetioDir ); + + this.exists = false; + } + + getAvailableBrowserTests( es5Only ) { + return this.browserTests.filter( test => { + if ( es5Only && !test.es5 ) { + return false; + } + + if ( test.buildDependencies ) { + for ( const dependency of test.buildDependencies ) { + if ( !_.some( this.buildTests, buildTest => buildTest.repo === dependency && buildTest.brand === test.brand && buildTest.success ) ) { + return false; + } + } + } + + return true; + } ); + } +} + +module.exports = CTSnapshot; diff --git a/js/continuous-loop.js b/js/continuous-loop.js index 8345dd4..aca89c6 100644 --- a/js/continuous-loop.js +++ b/js/continuous-loop.js @@ -20,12 +20,15 @@ const options = QueryStringMachine.getAll( { }, old: { type: 'flag' + }, + server: { + type: 'string', + + // Ignore current port, keep protocol and host. + defaultValue: window.location.protocol + '//' + window.location.hostname } } ); -// Ignore current port, keep protocol and host. -const serverOrigin = window.location.protocol + '//' + window.location.hostname; - // iframe that will contain qunit-test.html/sim-test.html/etc. const iframe = document.createElement( 'iframe' ); iframe.setAttribute( 'frameborder', '0' ); @@ -85,7 +88,7 @@ function nextTest() { // On connection failure, just try again with a delay (don't hammer the server) setTimeout( nextTest, 60000 ); // 1min }; - req.open( 'get', serverOrigin + '/aquaserver/next-test?old=' + options.old, true ); + req.open( 'get', options.server + '/aquaserver/next-test?old=' + options.old, true ); req.send(); resetTimer(); } @@ -108,7 +111,7 @@ function sendTestResult( names, message, testInfo, passed ) { message: message, id: options.id }; - req.open( 'get', serverOrigin + '/aquaserver/test-result?result=' + encodeURIComponent( JSON.stringify( result ) ) ); + req.open( 'get', options.server + '/aquaserver/test-result?result=' + encodeURIComponent( JSON.stringify( result ) ) ); req.send(); resetTimer(); } diff --git a/js/continuous-report.js b/js/continuous-report.js index 92cbcef..2abfc60 100644 --- a/js/continuous-report.js +++ b/js/continuous-report.js @@ -10,8 +10,14 @@ 'use strict'; -// Origin for our server (ignoring current port), so that we don't require localhost -const serverOrigin = window.location.protocol + '//' + window.location.hostname; +const options = QueryStringMachine.getAll( { + server: { + type: 'string', + + // Origin for our server (ignoring current port), so that we don't require localhost + defaultValue: window.location.protocol + '//' + window.location.hostname + } +} ); /** * Returns a CSS class to use given the number of passing results and failing results. @@ -311,7 +317,7 @@ function recursiveResults( name, resultNode, snapshots, padding, path ) { setTimeout( mainLoop, 3000 ); console.log( 'XHR error?' ); }; - req.open( 'get', serverOrigin + '/aquaserver/results', true ); // enable CORS + req.open( 'get', options.server + '/aquaserver/results', true ); // enable CORS req.send(); })(); @@ -329,7 +335,7 @@ function recursiveResults( name, resultNode, snapshots, padding, path ) { element.innerHTML = 'Could not contact server'; console.log( 'XHR error?' ); }; - req.open( 'get', serverOrigin + '/aquaserver/snapshot-status', true ); // enable CORS + req.open( 'get', options.server + '/aquaserver/snapshot-status', true ); // enable CORS req.send(); })(); @@ -347,6 +353,6 @@ function recursiveResults( name, resultNode, snapshots, padding, path ) { element.innerHTML = 'Could not contact server'; console.log( 'XHR error?' ); }; - req.open( 'get', serverOrigin + '/aquaserver/test-status', true ); // enable CORS + req.open( 'get', options.server + '/aquaserver/test-status', true ); // enable CORS req.send(); })(); diff --git a/js/local-server.js b/js/local-server.js new file mode 100644 index 0000000..e819213 --- /dev/null +++ b/js/local-server.js @@ -0,0 +1,338 @@ +// Copyright 2020, University of Colorado Boulder + +/** + * @author Jonathan Olson + */ + +'use strict'; + +const asyncFilter = require( '../../perennial/js/common/asyncFilter' ); +const cloneMissingRepos = require( '../../perennial/js/common/cloneMissingRepos' ); +const getRepoList = require( '../../perennial/js/common/getRepoList' ); +const gitPull = require( '../../perennial/js/common/gitPull' ); +const isStale = require( '../../perennial/js/common/isStale' ); +const CTSnapshot = require( './CTSnapshot' ); +const http = require( 'http' ); +const _ = require( 'lodash' ); // eslint-disable-line +const path = require( 'path' ); +const url = require( 'url' ); +const winston = require( 'winston' ); + +const PORT = 45366; +const NUMBER_OF_DAYS_TO_KEEP_SNAPSHOTS = 2; // in days, any shapshots that are older will be removed from the continuous report + +const jsonHeaders = { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*' +}; + +// {Array.} All of our snapshots +const snapshots = []; + +// {Results} Main results, with the addition of the snapshots reference +const testResults = { + children: {}, + results: [], + snapshots: snapshots +}; + +// root of your GitHub working copy, relative to the name of the directory that the currently-executing script resides in +const rootDir = path.normalize( __dirname + '/../../' ); // eslint-disable-line no-undef + +// Gets update with the current status +let snapshotStatus = 'Starting up'; +const setSnapshotStatus = str => { + snapshotStatus = `[${new Date().toLocaleString().replace( /^.*, /g, '' ).replace( ' AM', 'am' ).replace( ' PM', 'pm' )}] ${str}`; + winston.info( `status: ${snapshotStatus}` ); +}; + +/** + * Adds a test result into our {Results} object. + * @private + * + * @param {boolean} passed + * @param {Snapshot} snapshot + * @param {Array.} test - The path + * @param {string} message + */ +const addResult = ( passed, snapshot, test, message ) => { + const localTest = test.slice(); + let container = testResults; + while ( localTest.length ) { + const testName = localTest.shift(); + if ( container.children[ testName ] ) { + container = container.children[ testName ]; + } + else { + const newContainer = { + children: {}, + results: [] + }; + container.children[ testName ] = newContainer; + container = newContainer; + } + } + + container.results.push( { + passed: passed, + snapshotName: snapshot.name, + snapshotTimestamp: snapshot.timestamp, + message: message + } ); + + // NOTE: we could remove stale tests here? +}; + +/** + * Records a test pass from any source. + * + * @param {Snapshot} snapshot + * @param {Array.} test - The path + * @param {string|undefined} message + */ +const testPass = ( snapshot, test, message ) => { + if ( snapshot === null ) { + throw new Error( 'Snapshot null: ' + JSON.stringify( test ) + ' + ' + JSON.stringify( message ) ); + } + winston.info( '[PASS] ' + snapshot.name + ' ' + test.join( ',' ) + ': ' + message ); + addResult( true, snapshot, test, message ); +}; + +/** + * Records a test failure from any source. + * + * @param {Snapshot} snapshot + * @param {Array.} test - The path + * @param {string|undefined} message + */ +const testFail = ( snapshot, test, message ) => { + winston.info( '[FAIL] ' + snapshot.name + ' ' + test.join( ',' ) + ': ' + message ); + addResult( false, snapshot, test, message ); +}; + +/** + * Respond to an HTTP request with a response with the given {Test}. + * @private + * + * @param {ServerResponse} res + * @param {Test|null} test + */ +const deliverTest = ( res, test ) => { + let url; + const base = test ? `../../${test.brand === 'phet-io' ? test.phetioDir : test.phetDir}` : ''; + + if ( test === null ) { + url = 'no-test.html'; + } + else if ( test.type === 'sim-test' ) { + url = 'sim-test.html?url=' + encodeURIComponent( `${base}/test.url` ) + '&simQueryParameters=' + encodeURIComponent( test.queryParameters ); + } + else if ( test.type === 'qunit-test' ) { + url = 'qunit-test.html?url=' + encodeURIComponent( `${base}/test.url` ); + } + else if ( test.type === 'pageload-test' ) { + url = 'pageload-test.html?url=' + encodeURIComponent( `${base}/test.url` ); + } + else { + url = 'no-test.html'; + } + + if ( test ) { + test.count++; + } + + const object = { + count: test ? test.count : 0, + snapshotName: test ? test.snapshot.name : null, + test: test ? test.test : null, + url: url + }; + + winston.info( 'Delivering test: ' + JSON.stringify( object, null, 2 ) ); + res.writeHead( 200, jsonHeaders ); + res.end( JSON.stringify( object ) ); +}; + +/** + * Respond to an HTTP request with an empty test (will trigger checking for a new test without testing anything). + * @private + * + * @param {ServerResponse} res + */ +const deliverEmptyTest = res => { + deliverTest( res, null ); +}; + +/** + * Sends a random browser test (from those with the lowest count) to the ServerResponse. + * @private + * + * @param {ServerResponse} res + * @param {boolean} es5Only + */ +const randomBrowserTest = ( res, es5Only ) => { + if ( snapshots.length === 0 ) { + deliverEmptyTest( res ); + return; + } + + // Pick from one of the first two snapshots + let queue = snapshots[ 0 ].getAvailableBrowserTests( es5Only ); + if ( snapshots.length > 1 ) { + queue = queue.concat( snapshots[ 1 ].getAvailableBrowserTests( es5Only ) ); + } + + let lowestCount = Infinity; + let lowestTests = []; + queue.forEach( test => { + if ( lowestCount > test.count ) { + lowestCount = test.count; + lowestTests = []; + } + if ( lowestCount === test.count ) { + lowestTests.push( test ); + } + } ); + + // Deliver a random available test currently + if ( lowestTests.length > 0 ) { + const test = lowestTests[ Math.floor( lowestTests.length * Math.random() ) ]; + deliverTest( res, test ); + } + else { + deliverEmptyTest( res ); + } +}; + +const startServer = () => { + // Main server creation + http.createServer( ( req, res ) => { + const requestInfo = url.parse( req.url, true ); + + if ( requestInfo.pathname === '/aquaserver/next-test' ) { + // ?old=true or ?old=false, determines whether ES6 or other newer features can be run directly in the browser + randomBrowserTest( res, requestInfo.query.old === 'true' ); + } + if ( requestInfo.pathname === '/aquaserver/test-result' ) { + const result = JSON.parse( requestInfo.query.result ); + + const snapshot = _.find( snapshots, snapshot => snapshot.name === result.snapshotName ); + if ( snapshot ) { + const test = result.test; + let message = result.message; + if ( !message || message.indexOf( 'errors.html#timeout' ) < 0 ) { + if ( !result.passed ) { + message = ( result.message ? ( result.message + '\n' ) : '' ) + 'id: ' + result.id; + } + if ( result.passed ) { + testPass( snapshot, test, message ); + } + else { + testFail( snapshot, test, message ); + } + } + } + else { + winston.info( `Could not find snapshot: ${snapshot}` ); + } + + res.writeHead( 200, jsonHeaders ); + res.end( JSON.stringify( { received: 'true' } ) ); + } + if ( requestInfo.pathname === '/aquaserver/results' ) { + res.writeHead( 200, jsonHeaders ); + res.end( JSON.stringify( testResults ) ); + } + if ( requestInfo.pathname === '/aquaserver/snapshot-status' ) { + res.writeHead( 200, jsonHeaders ); + res.end( JSON.stringify( { + status: snapshotStatus + } ) ); + } + if ( requestInfo.pathname === '/aquaserver/test-status' ) { + res.writeHead( 200, jsonHeaders ); + res.end( JSON.stringify( { + zeroCounts: snapshots[ 0 ] ? snapshots[ 0 ].browserTests.filter( test => test.count === 0 ).length : 0 + } ) ); + } + } ).listen( PORT ); + + winston.info( `running on port ${PORT}` ); +}; + +const removeResultsForSnapshot = ( container, snapshot ) => { + container.results = container.results.filter( testResult => testResult.snapshotName !== snapshot.name ); + + container.children && Object.keys( container.children ).forEach( childKey => { + removeResultsForSnapshot( container.children[ childKey ], snapshot ); + } ); +}; + +const cycleSnapshots = async () => { + // {boolean} Whether our last scan of SHAs found anything stale. + let wasStale = true; + + while ( true ) { // eslint-disable-line + try { + if ( wasStale ) { + setSnapshotStatus( 'Checking for commits (changes detected, waiting for stable SHAs)' ); + } + else { + setSnapshotStatus( 'Checking for commits (no changes since last snapshot)' ); + } + + const reposToCheck = getRepoList( 'active-repos' ).filter( repo => repo !== 'aqua' ); + + const staleRepos = await asyncFilter( reposToCheck, async repo => { + winston.info( `Checking stale: ${repo}` ); + return await isStale( repo ); + } ); + + if ( staleRepos.length ) { + wasStale = true; + + winston.info( `Stale repos: ${staleRepos.join( ', ' )}` ); + setSnapshotStatus( `Pulling repos: ${staleRepos.join( ', ' )}` ); + + for ( const repo of staleRepos ) { + await gitPull( repo ); + } + await cloneMissingRepos(); + } + else { + winston.info( 'No stale repos' ); + + if ( wasStale ) { + wasStale = false; + + winston.info( 'Stable point reached' ); + + const snapshot = new CTSnapshot(); + await snapshot.create( rootDir, setSnapshotStatus ); + + snapshots.unshift( snapshot ); + + await snapshot.npmInstall(); + + const cutoffTimestamp = Date.now() - 1000 * 60 * 60 * 24 * NUMBER_OF_DAYS_TO_KEEP_SNAPSHOTS; + while ( snapshots.length > 70 || snapshots[ snapshots.length - 1 ].timestamp < cutoffTimestamp && !snapshots[ snapshots.length - 1 ].exists ) { + removeResultsForSnapshot( testResults, snapshots.pop() ); + } + + setSnapshotStatus( 'Removing old snapshot files' ); + const numActiveSnapshots = 3; + if ( snapshots.length > numActiveSnapshots ) { + const lastSnapshot = snapshots[ numActiveSnapshots ]; + await lastSnapshot.remove(); + } + } + } + } + catch ( e ) { + winston.error( e ); + } + } +}; + +startServer(); +cycleSnapshots(); diff --git a/package.json b/package.json index 35aa4fa..519319b 100644 --- a/package.json +++ b/package.json @@ -8,9 +8,11 @@ }, "devDependencies": { "grunt": "~1.0.0", + "lodash": "^4.17.10", "ncp": "^2.0.0", "rimraf": "^2.5.4", - "puppeteer": "~1.11.0" + "puppeteer": "~1.11.0", + "winston": "^0.9.0" }, "eslintConfig": { "extends": "../chipper/eslint/node_eslintrc.js",