Forum sample rework (#2455)

doc/build_apps/js_app_tsoa.rst

@@ -41,10 +41,13 @@ The sample app has the following folder layout:
     │   │   └── site.ts
     │   ├── models
     │   │   └── poll.ts
+    │   ├── services
+    │   │   ├── csv.ts
+    │   │   └── poll.ts
     │   ├── authentication.ts
+    │   ├── constants.ts
     │   └── error_handler.ts
     ├── tsoa-support
-    │   ├── entry.ts
     │   ├── postprocess.js
     │   └── routes.ts.tmpl
     ├── app.tmpl.json
@@ -56,9 +59,11 @@ The sample app has the following folder layout:
 It contains these files:
 
 - ``src/controllers/*.ts``: :ref:`build_apps/js_app_tsoa:Controllers`.
-- ``src/models/*.ts``: Data models shared between endpoint handlers.
+- ``src/models/*.ts``: Data models shared between different app components.
+- ``src/services/*.ts``: Business logic, called by controllers.
 - ``src/authentication.ts``: `authentication module <https://tsoa-community.github.io/docs/authentication.html>`_. 
   See also :ref:`build_apps/auth/jwt_ms_example:JWT Authentication example using Microsoft Identity Platform`.
+- ``src/constants.ts``: app-wide constants.
 - ``src/error_handler.ts``: global error handler.
 - ``tsoa-support/*``: Supporting scripts used during :ref:`build_apps/js_app_tsoa:Conversion to an app bundle`.
 - ``app.tmpl.json``: :ref:`App metadata <build_apps/js_app_tsoa:Metadata>`.

samples/apps/forum/.gitignore

@@ -5,4 +5,5 @@ build/
 .venv_ccf_sandbox/
 .workspace_ccf/
 *_opinions.csv
-*.jwt
+*.jwt
+*.pem

samples/apps/forum/README.md

@@ -1,31 +1,68 @@
 # Confidential Forum sample app
 
-NOTE: This sample is a work-in-progress.
+See also the [TypeScript Application using tsoa](https://microsoft.github.io/CCF/main/build_apps/js_app_tsoa.html) documentation page for further details on how this sample is built using the tsoa framework.
 
-Install dependencies:
+## Getting started
+
+When running this sample against a CCF release, open `package.json` and replace the `file:` reference of `ccf-app` with a reference to a published version (adjust the version number accordingly):
+
+```
+"@microsoft/ccf-app": "~1.0.0",
+```
+
+Now you can continue with installing all dependencies:
 
 ```sh
 npm install
 ```
 
+To run the demo and end-to-end tests, define the following environment variable:
+
+```sh
+export CCF_BINARY_DIR=/opt/ccf-x.y.z/bin
+```
+
+If not defined, it assumes you built CCF from source and defaults to `CCF_BINARY_DIR=<repo_root>/build`.
+
+## Demo
+
 Start the sandbox:
 
 ```sh
 npm start
 ```
 
-Open your browser at https://127.0.0.1:8000/app/site
+(Use `VERBOSE=1 npm start` for verbose output)
 
-Generate opinions, user identities and submit:
+Open your browser at https://127.0.0.1:8000/app/site and create the sample polls and opinions.
+The statistics will still be empty since the opinion threshold has not been reached yet.
+
+Now, generate more opinions, user identities and submit:
 
 ```sh
-python3.8 test/demo/generate-opinions.py test/demo/polls.csv 9
-npm run ts test/demo/generate-jwts.ts . 9
-npm run ts test/demo/submit-opinions.ts .
+mkdir demo/data
+python3.8 demo/generate-opinions.py demo/data demo/polls.csv 9
+npm run ts demo/generate-jwts.ts demo/data 9
+npm run ts demo/submit-opinions.ts demo/data
 ```
 
+Return to the website and view the statistics which should be visible now.
+
+## Tests & debugging
+
 Run tests:
 
 ```sh
 npm test
+# or:
+npm run test:unit # unit tests
+npm run test:e2e # end-to-end tests
 ```
+
+Unit tests run outside CCF and end-to-end tests run against a single CCF sandbox node.
+
+The unit tests make use of the [`ccf-app` polyfill](https://microsoft.github.io/CCF/main/js/ccf-app/modules/polyfill.html) and can be easily debugged in VS Code.
+The simplest workflow is using the [JavaScript Debug Terminal](https://code.visualstudio.com/docs/nodejs/nodejs-debugging#_javascript-debug-terminal) in VS Code. Set a break point and run `npm run test:unit` inside the JavaScript Debug Terminal. The debugger will attach and stop automatically.
+
+Debugging of JavaScript code running in CCF is currently not possible.
+However, all uncaught exceptions and output from `console.log()` are dumped to the node's log files.

samples/apps/forum/app.tmpl.json

@@ -94,8 +94,8 @@
     },
     "/csv": {
       "get": {
-        "js_module": "CSVControllerProxy.js",
-        "js_function": "getOpinionsAsCSV",
+        "js_module": "CsvControllerProxy.js",
+        "js_function": "getOpinionsAsCsv",
         "forwarding_required": "always",
         "execute_outside_consensus": "never",
         "authn_policies": ["jwt"],
@@ -112,7 +112,7 @@
         }
       },
       "post": {
-        "js_module": "CSVControllerProxy.js",
+        "js_module": "CsvControllerProxy.js",
         "js_function": "submitOpinionsFromCSV",
         "forwarding_required": "always",
         "execute_outside_consensus": "never",

samples/apps/forum/test/demo/generate-jwts.ts → samples/apps/forum/demo/generate-jwts.ts

@@ -5,7 +5,7 @@ import * as fs from "fs";
 import * as path from "path";
 import jwt from "jsonwebtoken";
 
-const demoJwtKeyPath = "test/jwt_demo_key.pem";
+const demoJwtKeyPath = "demo/jwt_demo_key.pem";
 
 function main() {
   const args = process.argv.slice(2);

samples/apps/forum/test/demo/generate-opinions.py → samples/apps/forum/demo/generate-opinions.py

@@ -1,6 +1,7 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the Apache 2.0 License.
 
+import os
 import sys
 import csv
 import random
@@ -42,15 +43,15 @@ def spread(topic):
             assert False
 
 
-def main(polls_path, user_count):
+def main(out_dir, polls_path, user_count):
     entries = []
 
     with open(polls_path, "r") as pp:
         polls = csv.DictReader(pp)
         entries = [poll for poll in polls]
 
     for user in range(user_count):
-        with open(f"user{user}_opinions.csv", "w") as uf:
+        with open(os.path.join(out_dir, f"user{user}_opinions.csv"), "w") as uf:
             header = ["Topic", "Opinion"]
             writer = csv.DictWriter(uf, header)
             writer.writeheader()
@@ -68,4 +69,4 @@ def push(opinion):
 
 
 if __name__ == "__main__":
-    main(sys.argv[1], int(sys.argv[2]))
+    main(sys.argv[1], sys.argv[2], int(sys.argv[3]))

samples/apps/forum/test/start.ts → samples/apps/forum/demo/start.ts

@@ -4,14 +4,13 @@
 import { spawnSync } from "child_process";
 import * as fs from "fs";
 import * as crypto from "crypto";
-import selfsigned from "selfsigned";
 import tmp from "tmp";
 import bent from "bent";
 import forge from "node-forge";
-import * as util from "./util";
+import * as util from "../test/e2e/util";
 
-const demoJwtKeyPath = "test/jwt_demo_key.pem";
-const demoJwtCertPath = "test/jwt_demo_cert.pem";
+const demoJwtKeyPath = "demo/jwt_demo_key.pem";
+const demoJwtCertPath = "demo/jwt_demo_cert.pem";
 
 async function main() {
   tmp.setGracefulCleanup();
@@ -71,13 +70,20 @@ function generateKeyPair(keyPath: string, certPath: string) {
       format: "pem",
     },
   });
-  const certPem = selfsigned.generate(null, {
-    algorithm: "sha256",
-    keyPair: {
-      privateKey: keys.privateKey,
-      publicKey: keys.publicKey,
+  const cert = forge.pki.createCertificate();
+  const attrs = [
+    {
+      name: "commonName",
+      value: "Test",
     },
-  }).cert;
+  ];
+  cert.setIssuer(attrs);
+  cert.publicKey = forge.pki.publicKeyFromPem(keys.publicKey);
+  cert.sign(
+    forge.pki.privateKeyFromPem(keys.privateKey),
+    forge.md.sha256.create()
+  );
+  const certPem = forge.pki.certificateToPem(cert);
   fs.writeFileSync(keyPath, keys.privateKey);
   fs.writeFileSync(certPath, certPem);
 }

samples/apps/forum/test/demo/submit-opinions.ts → samples/apps/forum/demo/submit-opinions.ts

@@ -5,16 +5,16 @@ import * as fs from "fs";
 import * as path from "path";
 import glob from "glob";
 import bent from "bent";
-import { parse } from "papaparse";
-import { NODE_ADDR } from "../util";
-import { SubmitOpinionsRequest } from "../../src/controllers/poll";
+import papa from "papaparse";
+import { NODE_ADDR } from "../test/e2e/util";
+import { SubmitOpinionsRequest } from "../src/controllers/poll";
 
 const ENDPOINT_URL = `${NODE_ADDR}/app/polls`;
 
 function getAuth(jwt: string) {
   // See src/util.ts.
   return {
-    authorization: `Bearer ${jwt}'`,
+    authorization: `Bearer ${jwt}`,
   };
 }
 
@@ -36,7 +36,8 @@ async function main() {
     const jwtPath = path.join(folder, user + ".jwt");
     const jwt = fs.readFileSync(jwtPath, "utf8");
     const csv = fs.readFileSync(csvPath, "utf8");
-    const rows = parse(csv, { header: true }).data as CSVRow[];
+    const rows = papa.parse(csv, { header: true, skipEmptyLines: true })
+      .data as CSVRow[];
 
     const req: SubmitOpinionsRequest = { opinions: {} };
     for (const row of rows) {
@@ -45,6 +46,7 @@ async function main() {
       };
     }
     console.log("Submitting opinions for user " + user);
+    console.log(req);
     try {
       await bent("PUT", 204)(`${ENDPOINT_URL}`, req, getAuth(jwt));
     } catch (e) {

samples/apps/forum/package.json

@@ -2,24 +2,26 @@
   "private": true,
   "scripts": {
     "build": "del-cli -f dist/ build/ && tsoa spec-and-routes && node tsoa-support/postprocess.js && rollup --config && del-cli dist/src/build/endpoints.js",
-    "start": "npm run build && npm run ts test/start.ts",
-    "test": "npm run build && mocha -r esm -r ts-node/register test/**/*.test.ts",
-    "ts": "node --no-warnings --experimental-specifier-resolution=node --loader ts-node/esm"
+    "start": "npm run build && npm run ts demo/start.ts",
+    "test": "npm run test:unit && npm run test:e2e",
+    "test:unit": "mocha --loader=ts-node/esm --experimental-specifier-resolution=node --file=test/unit/setup.ts test/unit/*.test.ts",
+    "test:e2e": "npm run build && mocha --loader=ts-node/esm --experimental-specifier-resolution=node test/e2e/*.test.ts",
+    "ts": "node --no-warnings --loader=ts-node/esm --experimental-specifier-resolution=node"
   },
   "type": "module",
   "dependencies": {
     "@microsoft/ccf-app": "file:../../../js/ccf-app",
-    "@tsoa/runtime": "^3.3.0",
+    "@tsoa/runtime": "^3.6.1",
     "lodash-es": "^4.17.15",
-    "mathjs": "^7.5.1",
+    "mathjs": "9.3.0 || ^9.3.2",
     "papaparse": "^5.3.0"
   },
   "devDependencies": {
     "@apidevtools/swagger-parser": "^10.0.2",
     "@rollup/plugin-commonjs": "^14.0.0",
     "@rollup/plugin-node-resolve": "^8.4.0",
     "@rollup/plugin-typescript": "^5.0.2",
-    "@tsoa/cli": "^3.3.0",
+    "@tsoa/cli": "^3.6.1",
     "@types/bent": "^7.3.1",
     "@types/chai": "^4.2.13",
     "@types/jsonwebtoken": "^8.5.0",
@@ -33,18 +35,16 @@
     "bent": "^7.3.12",
     "chai": "^4.2.0",
     "del-cli": "^3.0.1",
-    "esm": "^3.2.25",
     "glob": "^7.1.6",
     "http-server": "^0.12.3",
     "json-merge-patch": "^1.0.1",
     "jsonwebtoken": "^8.5.1",
-    "mocha": "^8.1.3",
+    "mocha": "^8.3.2",
     "node-forge": "^0.10.0",
     "rollup": "^2.23.0",
-    "selfsigned": "^1.10.8",
     "tmp": "^0.2.1",
     "ts-node": "^9.1.0",
-    "tslib": "^2.0.1",
-    "typescript": "^4.0.2"
+    "tslib": "^2.2.0",
+    "typescript": "^4.2.4"
   }
 }

samples/apps/forum/src/constants.ts

@@ -0,0 +1 @@
+export const MINIMUM_OPINION_THRESHOLD = 10;