Merge pull request #765 from biothings/concurrency

Concurrency improvements

.prettierrc.yml → .prettierrc.yaml

@@ -3,5 +3,5 @@ tabWidth: 2
 semi: true
 singleQuote: false
 bracketSpacing: true
-printWidth: 120
+printWidth: 80
 arrowParens: avoid

package.json

@@ -18,7 +18,7 @@
     "lint:fix": "pnpm lint --fix",
     "test": "turbo run --parallel test",
     "test-cov": "turbo run --parallel test-cov",
-    "siege-local": "artillery run -e local --output report_local.json --config performance-test/config.yaml performance-test/scenarios/query.yaml",
+    "siege-local": "artillery run -e local --output report_local.json --config performance-test/config.yaml performance-test/scenarios/creative.yaml",
     "smartapi_sync": "node ./scripts/smartapi_sync.js",
     "get_rev": "./scripts/get_rev.sh",
     "basic-start": "node .",
@@ -28,6 +28,7 @@
     "debug": "DEBUG=biomedical-id-resolver,bte* pnpm run watch"
   },
   "engines": {
+    "node": ">=18.0.0",
     "pnpm": ">=8.9.2"
   },
   "repository": {
@@ -52,6 +53,8 @@
     "@typescript-eslint/eslint-plugin": "^6.8.0",
     "@typescript-eslint/parser": "^6.8.0",
     "artillery": "^1.7.9",
+    "async": "^3.2.4",
+    "commitlint": "^18.2.0",
     "cookie-parser": "~1.4.6",
     "coveralls": "^3.1.1",
     "eslint": "^7.32.0",

packages/packages.txt

@@ -6,3 +6,4 @@ biothings/bte_trapi_query_graph_handler.git  query_graph_handler
 biothings/node-expansion.git                 node-expansion
 biothings/biolink-model.js.git               biolink-model
 biothings/biomedical_id_resolver.js.git      biomedical_id_resolver
+biothings/bte-utils.git                      utils

performance-test/20-creative.js

@@ -0,0 +1,124 @@
+const debug = require("debug")("concurrency-test");
+const axios = require("axios");
+const async = require("async");
+
+const CONCURRENCY = 20;
+
+const TEST_URL = "https://api.bte.ncats.io";
+// const TEST_URL = "http://localhost:3000"
+const TEST_ENDPOINT = "/v1/asyncquery";
+
+const IDS = [
+  "MONDO:0016575",
+  "MONDO:0016575",
+  "MONDO:0005377",
+  "MONDO:0007035",
+  "MONDO:0001993",
+  "MONDO:0005015",
+  "MONDO:0019609",
+  "MONDO:0004975",
+  "HP:0002014",
+];
+
+const start = performance.now();
+
+let results = Array(CONCURRENCY).fill(0);
+
+console.log(`Beginning test of ${CONCURRENCY} simultanoues queries...`);
+
+results = async
+  .map(results, async () => {
+    return new Promise(async resolve => {
+      // jitter sendout time by a few ms
+      await new Promise((resolve) => setTimeout(() => resolve(), Math.floor(Math.random() * 10)));
+      const start = performance.now();
+      const id = IDS[Math.floor(Math.random() * IDS.length)]
+      const body = {
+        message: {
+          query_graph: {
+            edges: {
+              e0: {
+                subject: "n0",
+                predicates: ["biolink:treats"],
+                object: "n1",
+                knowledge_type: "inferred",
+              },
+            },
+            nodes: {
+              n0: {
+                categories: ["biolink:SmallMolecule"],
+              },
+              n1: {
+                ids: [id],
+                categories: ["biolink:Disease"],
+              },
+            },
+          },
+        },
+      };
+
+      setTimeout(() => resolve(`${id}: Timed out.`), 300000);
+      let isAccepted = false;
+
+      try {
+        isAccepted = await axios({
+          method: "post",
+          url: `${TEST_URL}${TEST_ENDPOINT}`,
+          headers: {
+            "Content-type": "Application/json",
+          },
+          data: JSON.stringify(body),
+          timeout: 300000,
+        });
+      } catch (error) {
+        resolve(`${id}: Query error.`);
+        return;
+      }
+
+      if (!isAccepted) {
+        resolve(`${id}: Query not accepted.`);
+        return;
+      }
+
+      const poll_url = isAccepted.data.job_url;
+
+      let response;
+
+      while (true) {
+        try {
+          response = await axios({
+            method: "get",
+            url: poll_url,
+            timeout: 300000,
+          });
+          if (response.data.status === "Completed") {
+            const end = performance.now();
+            resolve(`${id}: Finished in ${Math.ceil((end - start) / 1000)}s`);
+            return;
+            break;
+          } else if (response.data.status === "Failed") {
+            resolve(`${id}: Query failed.`);
+            return;
+            break;
+          }
+        } catch (error) {
+          console.log(error);
+        }
+        await new Promise(stopWaiting => setTimeout(() => stopWaiting(), 1000));
+      }
+    });
+  })
+  .then(value => {
+    let failed = 0;
+    value.forEach(result => {
+      console.log(result);
+      if (!result.includes("Finished") || result.includes("300s")) {
+        failed += 1;
+      }
+    });
+    end = performance.now();
+    console.log(`\nTest completed in ${Math.ceil((end - start) / 1000)}s`);
+    console.log(`Score: ${Math.round((CONCURRENCY - failed) / CONCURRENCY * 100)}%`)
+    console.log(`Test ${!failed ? "passed!" : "failed."}`);
+    process.exit();
+  });

performance-test/config.yaml

@@ -1,25 +1,25 @@
+---
 config:
-    environments:
-        production:
-            target: "https://api.bte.ncats.io"
-            phases:
-                - duration: 120
-                  arrivalRate: 0.5
-
-        local:
-            target: "http://localhost:3000"
-            phases:
-                - duration: 120
-                  arrivalRate: 0.5
-
-        dev:
-            target: "https://dev.api.bte.ncats.io"
-            phases:
-                - duration: 120
-                  arrivalRate: 0.5
-    payload:
-        path: "single_hop_examples.csv"
-        fields:
-            - "input_type"
-            - "input_id"
-            - "output_type"
+  environments:
+    local:
+      target: http://localhost:3000
+      http:
+        timeout: 300
+      phases:
+        - duration: 1
+          maxVusers: 20
+          arrivalCount: 20
+          name: Concurrency test
+    dev:
+      target: http://api.bte.ncats.io
+      http:
+        timeout: 300
+      phases:
+        - duration: 1
+          maxVusers: 20
+          arrivalCount: 20
+          name: Concurrency test
+  payload:
+    path: creative_examples.csv
+    fields: [input_id]
+  processor: functions.js

performance-test/scenarios/creative.yaml

@@ -0,0 +1,36 @@
+---
+scenarios:
+  - flow:
+      - post:
+          url: /v1/asyncquery
+          json:
+            message:
+              query_graph:
+                edges:
+                  e0:
+                    subject: n0
+                    predicates: [biolink:treats]
+                    object: n1
+                    knowledge_type: inferred
+                nodes:
+                  n0:
+                    categories: [biolink:SmallMolecule]
+                  n1:
+                    ids: ["{{ input_id }}"]
+                    categories: [biolink:Disease]
+          capture:
+            - json: $.status
+              as: status
+            - json: $.job_id
+              as: job_id
+          expect:
+            - statusCode: 200
+            - equals: [Accepted, "{{ status }}"]
+      - loop:
+          - think: 5
+          - get:
+              url: "/v1/asyncquery_status/{{ job_id }}"
+              capture:
+                - json: $.status
+                  as: status
+        whileTrue: statusComplete

performance-test/scenarios/creative_examples.csv

@@ -0,0 +1,10 @@
+MONDO:0016575
+MONDO:0016575
+MONDO:0005377
+MONDO:0007035
+MONDO:0001993
+MONDO:0005015
+MONDO:0019609
+MONDO:0004975
+HP:0002014
+

performance-test/scenarios/functions.js

@@ -0,0 +1,11 @@
+module.exports = {
+  statusComplete: statusComplete,
+};
+
+function statusComplete(context, next) {
+  const continueLooping = context.vars.status !== "Complete";
+  if (context.vars.status === "Error") {
+    return next(new Error("Query failed."))
+  }
+  return next(continueLooping);
+}

performance-test/scenarios/query.yaml

@@ -1,17 +1,18 @@
+---
 scenarios:
-    - flow:
-        - post:
-            url: "/v1/query"
-            json:
-                message:
-                    query_graph:
-                        edges:
-                          e0:
-                            subject: "n0"
-                            object: "n1"
-                        nodes:
-                          n0:
-                            id: "{{ input_id }}"
-                            category: "{{ input_type }}"
-                          n1:
-                            category: "{{ output_type }}"
+  - flow:
+      - post:
+          url: /v1/query
+          json:
+            message:
+              query_graph:
+                edges:
+                  e0:
+                    subject: n0
+                    object: n1
+                nodes:
+                  n0:
+                    ids: ["{{ input_id }}"]
+                    categories: ["{{ input_type }}"]
+                  n1:
+                    categoies: ["{{ output_type }}"]