Merge branch 'main' of https://github.com/biothings/bte-server into f…

…ix-857-bte

config/api_list.yaml

@@ -102,36 +102,9 @@ include:
     name: Text Mining Targeted Association API
   # TRAPI (Translator standard) APIs: Automat
   # not accessible by team or api-specific endpoints
-  # Notes: We don't ingest the following:
-  # - Automat-robokop: seems to repeat a lot of data that is in the other APIs
-  - id: f82c01b15c46e024212c1a3271aaef0b
-    name: Automat-ctd(Trapi v1.5.0)
-  - id: 673b9fc76973dfa5fe3ed151fdbfc807
-    name: Automat-drug-central(Trapi v1.5.0)
-  - id: eef72049e4e01c020b7799f711e0e65b
-    name: Automat-gtex(Trapi v1.5.0)
-  - id: 759df287a21c30cd514df323be02a84b
-    name: Automat-gtopdb(Trapi v1.5.0)
-  - id: 349fed5531c094c33f10c071efe9d0de
-    name: Automat-gwas-catalog(Trapi v1.5.0)
-  - id: a5fe24f987331b58191e67598118f369
-    name: Automat-hetionet(Trapi v1.5.0)
-  - id: 8671309d2b94e413a4c1f9a9f82e4660
-    name: Automat-hgnc(Trapi v1.5.0)
-  - id: 0a1c0f46f4950b82b1aa7dad27aad10a
-    name: Automat-hmdb(Trapi v1.5.0)
-  - id: cb7a43d444cb3dcbe8e3c78d314334cf
-    name: Automat-human-goa(Trapi v1.5.0)
-  - id: c64d583402f21cc85810d33befe49c86
-    name: Automat-icees-kg(Trapi v1.5.0)
-  - id: b4023595664163e0aec5e825da150e16
-    name: Automat-intact(Trapi v1.5.0)
-  - id: 3f78d3fb8a7a577fbc7cc0a913ac3fc5
-    name: Automat-panther(Trapi v1.5.0)
-  - id: 1f057c53d42694686369f0e542f965c6
-    name: Automat-pharos(Trapi v1.5.0)
-  - id: 2aca41fc6c3dc426ec6583d42603be02
-    name: Automat-viral-proteome(Trapi v1.5.0)
+  # using robokop to access all automat resources at once
+  - id: 4f9c8853b721ef1f14ecee6d92fc19b5
+    name: Automat-robokop(Trapi v1.5.0)
   # TRAPI (Translator standard) APIs: COHD
   # not accessible by team or api-specific endpoints
   # notes on COHD:
@@ -145,4 +118,22 @@ include:
   # not accessible by team or api-specific endpoints
   - id: 412af63e15b73e5a30778aac84ce313f
     name: Connections Hypothesis Provider API
+  # TRAPI (Translator standard) APIs: Molecular Data Provider
+  # not accessible by team or api-specific endpoints
+  - id: 1901bab8d33bb70b124f400ec1cfdba3
+    name: MolePro
+  # TRAPI (Translator standard) APIs: Multiomics (made with RTX team Plover)
+  # not accessible by team or api-specific endpoints
+  - id: e51073371d7049b9643e1edbdd61bcbd
+    name: Clinical Trials KP - TRAPI 1.5.0
+    includeFlipped: true
+  - id: edc04feaf16c12424737988ce2e90d60
+    name: Drug Approvals KP - TRAPI 1.5.0
+    includeFlipped: true
+  - id: a8be4ea3fe8fa80a952ead0b3c5e4bc1
+    name: Microbiome KP - TRAPI 1.5.0
+    includeFlipped: true
+  - id: 1b6de23ed3c4e0713b20794477ba1e39
+    name: Multiomics KP - TRAPI 1.5.0
+    includeFlipped: true
 exclude: [] # if adding exclude remove these square brackets

config/smartapi_overrides.yaml

@@ -1,4 +1,7 @@
 ---
 config:
   only_overrides: false
-apis: {}
+apis: {
+  "8f08d1446e0bb9c2b323713ce83e2bd3": "https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/refs/heads/remove-clinical-trials/mychem.info/openapi_full.yml",
+  "1138c3297e8e403b6ac10cff5609b319": "https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/refs/heads/remove-clinical-trials/repodb/smartapi.yaml"
+}

src/config/index.ts

@@ -74,23 +74,23 @@ export default class Config {
   setLimiter() {
     const slowLimiter = rateLimit({
       windowMs: 1 * 60 * 1000, //1min
-      max: parseInt(process.env.MAX_QUERIES_PER_MIN || "20"),
+      max: parseInt(process.env.MAX_QUERIES_PER_MIN || "50"),
     });
     const medLimiter = rateLimit({
       windowMs: 1 * 60 * 1000, //1min
-      max: parseInt(process.env.MAX_QUERIES_PER_MIN || "30"),
+      max: parseInt(process.env.MAX_QUERIES_PER_MIN || "100"),
     });
     const fastLimiter = rateLimit({
       windowMs: 1 * 60 * 1000, //1min
-      max: parseInt(process.env.MAX_QUERIES_PER_MIN || "6000"),
+      max: parseInt(process.env.MAX_QUERIES_PER_MIN || "10000"),
     });
     this.app.use("/", fastLimiter);
-    this.app.use("/v1/query", slowLimiter);
+    this.app.use("/v1/query", medLimiter);
     this.app.use("/v1/team/:team_name/query", medLimiter);
-    this.app.use("/v1/team/:smartapiID/query", medLimiter);
-    this.app.use("/v1/meta_knowledge_graph", medLimiter);
-    this.app.use("/v1/team/:teamName/meta_knowledge_graph", medLimiter);
-    this.app.use("/v1/smartapi/:smartapiID/meta_knowledge_graph", medLimiter);
+    this.app.use("/v1/team/:smartapiID/query", fastLimiter);
+    this.app.use("/v1/meta_knowledge_graph", fastLimiter);
+    this.app.use("/v1/team/:teamName/meta_knowledge_graph", fastLimiter);
+    this.app.use("/v1/smartapi/:smartapiID/meta_knowledge_graph", fastLimiter);
     this.app.use("/v1/asyncquery", fastLimiter);
     this.app.use("/v1/team/:teamName/asyncquery", fastLimiter);
     this.app.use("/v1/smartapi/:smartapiID/asyncquery", fastLimiter);

src/controllers/threading/threadHandler.ts

@@ -22,10 +22,10 @@ import { Queue } from "bull";
 const SYNC_MIN_CONCURRENCY = 2;
 const ASYNC_MIN_CONCURRENCY = 3;
 
-// On most instances, there are two nodes, one for Service Provider endpoints and one for everything else
-// On Dev and local instances, this isn't the case, so a lower concurrency is needed
-const CORE_CONCURRENCY_RATIO = parseInt(process.env.CORE_CONCURRENCY_RATIO) || 0.25;
-const MEM_CONCURRENCY_RATIO = parseFloat(process.env.MEM_CONCURRENCY_RATIO) || 0.6;
+// Ratio of Queryies per core
+const CORE_CONCURRENCY_RATIO = parseInt(process.env.CORE_CONCURRENCY_RATIO) || 1;
+// Ratio of Queries per GB of memory
+const MEM_CONCURRENCY_RATIO = parseFloat(process.env.MEM_CONCURRENCY_RATIO) || 2;
 
 const CORE_LIMIT = Math.ceil(os.cpus().length * CORE_CONCURRENCY_RATIO);
 

src/routes/index.ts

@@ -50,10 +50,11 @@ export const tasks: TaskByRoute = {
   asyncquery_v1: (taskInfo: TaskInfo) => V1AsyncQuery.task(taskInfo),
   asyncquery_v1_by_api: (taskInfo: TaskInfo) => V1AsyncQueryByAPI.task(taskInfo),
   asyncquery_v1_by_team: (taskInfo: TaskInfo) => V1AsyncQueryByTeam.task(taskInfo),
+  // load MetaKG from global
+  meta_knowledge_graph_v1: (taskInfo: TaskInfo) => V1MetaKG.task(taskInfo),
   // Not threaded due to being lightweight/speed being higher priority
   // performance: (taskInfo: TaskInfo) => Performance.task(taskInfo),
   // metakg: (taskInfo: TaskInfo) => MetaKG.task(taskInfo),
-  // meta_knowledge_graph_v1: (taskInfo: TaskInfo) => V1MetaKG.task(taskInfo),
   // meta_knowledge_graph_v1_by_api: (taskInfo: TaskInfo) => V1MetaKGByAPI.task(taskInfo),
   // meta_knowledge_graph_v1_by_team: (taskInfo: TaskInfo) => V1MetaKGByTeam.task(taskInfo),
 };

src/routes/v1/meta_knowledge_graph_v1.ts

@@ -1,5 +1,8 @@
+import path from "path";
+import { TaskInfo } from "@biothings-explorer/types";
 import handler from "../../controllers/meta_knowledge_graph";
 import * as utils from "../../utils/common";
+import { runTask, taskResponse, taskError } from "../../controllers/threading/threadHandler";
 import { Express, NextFunction, Request, Response, RequestHandler } from "express";
 
 class MetaKG {
@@ -8,16 +11,33 @@ class MetaKG {
       .route("/v1/meta_knowledge_graph")
       .get((async (_req: Request, res: Response, next: NextFunction) => {
         try {
-          const metaKGHandler = new handler(undefined);
-          const kg = await metaKGHandler.getKG();
+          const response = await runTask(_req, res, path.parse(__filename).name);
           res.setHeader("Content-Type", "application/json");
-          res.end(JSON.stringify(kg));
+          res.end(JSON.stringify(response));
         } catch (error) {
           next(error);
         }
       }) as RequestHandler)
       .all(utils.methodNotAllowed);
   }
+
+  async task(taskInfo: TaskInfo) {
+    try {
+      let kg = undefined;
+
+      // read metakg from files if not globally defined
+      if(!taskInfo.data.options.metakg) {
+        const metaKGHandler = new handler(undefined);
+        kg = await metaKGHandler.getKG(); 
+      } else {
+        kg = taskInfo.data.options.metakg;
+      }
+      // response.logs = utils.filterForLogLevel(response.logs, options.logLevel);
+      return taskResponse(kg);
+    } catch (error) {
+      taskError(error as Error);
+    }
+  }
 }
 
 export default new MetaKG();