Add more structured output for tests

extensions/ql-vscode/CHANGELOG.md

@@ -4,6 +4,9 @@
 
 - Editors opened by navigating from the results view are no longer opened in _preview mode_. Now they are opened as a persistent editor. [#630](https://github.com/github/vscode-codeql/pull/630)
 - When comparing the results of a failed QL test run and the `.expected` file does not exist, an empty `.expected` file is created and compared against the `.actual` file. [#669](https://github.com/github/vscode-codeql/pull/669)
+- Alter structure of the _Test Explorer_ tree. It now follows the structure of the filesystem instead of the QL Packs. [#624](https://github.com/github/vscode-codeql/pull/624)
+- Alter structure of the _Test Explorer_ tree. It now follows the structure of the filesystem instead of the QL Packs. [#624](https://github.com/github/vscode-codeql/pull/624)
+- Add more structured output for tests. [#626](https://github.com/github/vscode-codeql/pull/626)
 
 ## 1.3.6 - 4 November 2020
 
@@ -33,6 +36,7 @@
 - Remove feature flag for the AST Viewer. For more information on how to use the AST Viewer, [see the documentation](https://help.semmle.com/codeql/codeql-for-vscode/procedures/exploring-the-structure-of-your-source-code.html).
 - The `codeQL.runningTests.numberOfThreads` setting is now used correctly when running tests.
 - Alter structure of the _Test Explorer_ tree. It now follows the structure of the filesystem instead of the qlpacks.
+- Ensure output of CodeQL test runs includes compilation error messages and test failure messages.
 
 ## 1.3.3 - 16 September 2020
 

extensions/ql-vscode/src/cli.ts

@@ -17,6 +17,7 @@ import { DistributionProvider, FindDistributionResultKind } from './distribution
 import { assertNever } from './pure/helpers-pure';
 import { QueryMetadata, SortDirection } from './pure/interface-types';
 import { Logger, ProgressReporter } from './logging';
+import { CompilationMessage } from './pure/messages';
 
 /**
  * The version of the SARIF format that we are using.
@@ -90,10 +91,11 @@ export interface TestRunOptions {
 export interface TestCompleted {
   test: string;
   pass: boolean;
-  messages: string[];
+  messages: CompilationMessage[];
   compilationMs: number;
   evaluationMs: number;
   expected: string;
+  diff: string[] | undefined;
 }
 
 /**

extensions/ql-vscode/src/test-adapter.ts

@@ -217,10 +217,26 @@ export class QLTestAdapter extends DisposableObject implements TestAdapter {
       cancellationToken: cancellationToken,
       logger: testLogger
     })) {
+      const state = event.pass
+        ? 'passed'
+        : event.messages?.length
+        ? 'errored'
+        : 'failed';
+      let message: string | undefined;
+      if (event.diff?.length) {
+        message = ['', `${state}: ${event.test}`, ...event.diff, ''].join('\n');
+        testLogger.log(message);
+      }
+      (event.diff || []).join('\n');
       this._testStates.fire({
         type: 'test',
-        state: event.pass ? 'passed' : 'failed',
-        test: event.test
+        state,
+        test: event.test,
+        message,
+        decorations: event.messages?.map(msg => ({
+          line: msg.position.line,
+          message: msg.message
+        }))
       });
     }
   }

extensions/ql-vscode/src/vscode-tests/no-workspace/databases.test.ts

@@ -24,8 +24,9 @@ describe('databases', () => {
     databaseManager = new DatabaseManager(
       {
         workspaceState: {
-          update: updateSpy
-        }
+          update: updateSpy,
+          get: sinon.stub()
+        },
       } as unknown as ExtensionContext,
       {} as QueryServerConfig,
       {} as Logger,

extensions/ql-vscode/src/vscode-tests/no-workspace/test-adapter.test.ts

@@ -0,0 +1,106 @@
+import 'vscode-test';
+import 'mocha';
+import * as sinon from 'sinon';
+import { Uri, WorkspaceFolder } from 'vscode';
+import { expect } from 'chai';
+
+import { QLTestAdapter } from '../../test-adapter';
+import { CodeQLCliServer } from '../../cli';
+
+describe('test-adapter', () => {
+  let adapter: QLTestAdapter;
+  let runTestsSpy: sinon.SinonStub;
+  let resolveTestsSpy: sinon.SinonStub;
+  let resolveQlpacksSpy: sinon.SinonStub;
+  let sandox: sinon.SinonSandbox;
+
+  beforeEach(() => {
+    sandox = sinon.createSandbox();
+    mockRunTests();
+    resolveQlpacksSpy = sandox.stub().resolves({});
+    resolveTestsSpy = sandox.stub().resolves([]);
+    adapter = new QLTestAdapter({
+      name: 'ABC',
+      uri: Uri.parse('file:/ab/c')
+    } as WorkspaceFolder, {
+      runTests: runTestsSpy,
+      resolveQlpacks: resolveQlpacksSpy,
+      resolveTests: resolveTestsSpy
+    } as unknown as CodeQLCliServer);
+  });
+
+  afterEach(() => {
+    sandox.restore();
+  });
+
+  it('should run some tests', async () => {
+
+    const listenerSpy = sandox.spy();
+    adapter.testStates(listenerSpy);
+    const testsPath = Uri.parse('file:/ab/c').fsPath;
+    const dPath = Uri.parse('file:/ab/c/d.ql').fsPath;
+    const gPath = Uri.parse('file:/ab/c/e/f/g.ql').fsPath;
+    const hPath = Uri.parse('file:/ab/c/e/f/h.ql').fsPath;
+
+    await adapter.run([testsPath]);
+
+    expect(listenerSpy.getCall(0).args).to.deep.eq([
+      { type: 'started', tests: [testsPath] }
+    ]);
+    expect(listenerSpy.getCall(1).args).to.deep.eq([{
+      type: 'test',
+      state: 'passed',
+      test: dPath,
+      message: undefined,
+      decorations: []
+    }]);
+    expect(listenerSpy.getCall(2).args).to.deep.eq([{
+      type: 'test',
+      state: 'errored',
+      test: gPath,
+      message: `\nerrored: ${gPath}\npqr\nxyz\n`,
+      decorations: [
+        { line: 1, message: 'abc' }
+      ]
+    }]);
+    expect(listenerSpy.getCall(3).args).to.deep.eq([{
+      type: 'test',
+      state: 'failed',
+      test: hPath,
+      message: `\nfailed: ${hPath}\njkh\ntuv\n`,
+      decorations: []
+    }]);
+    expect(listenerSpy.getCall(4).args).to.deep.eq([{ type: 'finished' }]);
+    expect(listenerSpy).to.have.callCount(5);
+  });
+
+  function mockRunTests() {
+    // runTests is an async generator function. This is not directly supported in sinon
+    // However, we can pretend the same thing by just returning an async array.
+    runTestsSpy = sandox.stub();
+    runTestsSpy.returns(
+      (async function*() {
+        yield Promise.resolve({
+          test: Uri.parse('file:/ab/c/d.ql').fsPath,
+          pass: true,
+          messages: []
+        });
+        yield Promise.resolve({
+          test: Uri.parse('file:/ab/c/e/f/g.ql').fsPath,
+          pass: false,
+          diff: ['pqr', 'xyz'],
+          // a compile error
+          messages: [
+            { position: { line: 1 }, message: 'abc' }
+          ]
+        });
+        yield Promise.resolve({
+          test: Uri.parse('file:/ab/c/e/f/h.ql').fsPath,
+          pass: false,
+          diff: ['jkh', 'tuv'],
+          messages: []
+        });
+      })()
+    );
+  }
+});