✨ new admin scripts to add docs

admin/compare2Docs.mjs

@@ -0,0 +1,43 @@
+// example:  node admin/compare2Docs.mjs --a:9tCMkJIB0zu1NPYlGUy3 --b:_NCMkJIB0zu1NPYlGUy3 --entity:participant_centric
+import { Client } from '@elastic/elasticsearch';
+import { diffString } from 'json-diff';
+import assert from 'node:assert/strict';
+
+import { esHost } from '../dist/src/env.js';
+import { ENTITIES } from './releaseStatsUtils.mjs';
+
+const args = process.argv.slice(2);
+const aArgument = args.find(x => x.startsWith('--a:')) ?? '';
+const a = aArgument.split('--a:')[1];
+const bArgument = args.find(x => x.startsWith('--b:')) ?? '';
+const b = bArgument.split('--b:')[1];
+
+const entityArgument = args.find(x => x.startsWith('--entity:')) ?? '';
+const entity = entityArgument.split('--entity:')[1] || 'study_centric';
+assert(!!a && !!b, 'Missing docs values');
+assert(a !== b, 'a and b have the same value. Nothing to compare');
+assert(
+    Object.values(ENTITIES).some(x => x === entity),
+    'Entity invalid',
+);
+
+const client = new Client({ node: esHost });
+
+const resp = await client.search({
+    index: `${entity}*`,
+    body: {
+        query: {
+            ids: {
+                values: [a, b],
+            },
+        },
+    },
+});
+
+const left = resp.body.hits.hits[0];
+const right = resp.body.hits.hits[1];
+
+assert(left && right, 'Could not find at least one the 2 docs');
+assert(left._index.startsWith(right._index.split('_centric')[0]), 'Docs seem to be from different entities.');
+
+console.log(diffString(left, right, { sort: true }));

admin/compareStudiesDupesIfExist.mjs

@@ -0,0 +1,62 @@
+import { Client } from '@elastic/elasticsearch';
+import { diffString } from 'json-diff';
+import assert from 'node:assert/strict';
+
+import { esHost } from '../dist/src/env.js';
+import { ENTITIES } from './releaseStatsUtils.mjs';
+import { binomialCoefficient, pairIt } from './utils.mjs';
+
+const client = new Client({ node: esHost });
+
+const STUDY_SEARCH_SIZE = 50;
+const allStudiesSearchResponse = await client.search({
+    index: `${ENTITIES.study_centric}`,
+    track_total_hits: true,
+    body: {
+        size: STUDY_SEARCH_SIZE,
+    },
+});
+
+const total = allStudiesSearchResponse.body.hits.total.value;
+assert(total > 0, 'No study found');
+assert(total < STUDY_SEARCH_SIZE, 'Not all studies were fetched, increase size in the script if needed');
+
+const all = allStudiesSearchResponse.body.hits.hits;
+
+const dupes = all
+    .reduce((xs, x) => {
+        if (all.filter(s => x._index === s._index).length > 1) {
+            return [...xs, { ...x, studyCode: x._source.study_code }];
+        }
+        return xs;
+    }, [])
+    .reduce((xs, x) => {
+        const studyCode = x.studyCode;
+        return {
+            ...xs,
+            [studyCode]: xs[studyCode] ? [...xs[studyCode], x] : [x],
+        };
+    }, {});
+
+const numberComparisons = Object.entries(dupes).reduce((xs, x) => {
+    const n = x[1].length;
+    return xs + binomialCoefficient(n, 2);
+}, 0);
+
+const MAX_N_OF_COMPARISONS = 25; // Arbitrary value, need real-world testing.
+const willNotExplode = numberComparisons <= MAX_N_OF_COMPARISONS;
+
+assert(willNotExplode, `Avoiding to compare for there are ${numberComparisons} comparisons to compute`);
+
+const allPairs = Object.fromEntries(Object.entries(dupes).map(x => [x[0], pairIt(x[1])]));
+
+Object.entries(allPairs).forEach(([code, pairs]) => {
+    console.log(`----- Showing diff for duplicates of ${code} (total of ${pairs.length} pairs) -----`);
+    pairs.forEach((p, index) => {
+        const left = p[0];
+        const right = p[1];
+        console.log(`pair #${index + 1} : ${left._id} vs ${right._id}`);
+        const diff = diffString(left, right, { sort: true });
+        console.log(diff ? diff : `no diff found in this docs pair`);
+    });
+});

admin/findClinicalIndicesUsage.mjs

@@ -1,8 +1,9 @@
 /**
 (WIP) Helper script to find unused indices
  * */
-import assert from 'node:assert/strict';
 import { Client } from '@elastic/elasticsearch';
+import assert from 'node:assert/strict';
+
 import { esHost } from '../dist/src/env.js';
 import { cbKeepClinicalPlusTranscriptomicsIndicesOnly, isIndexNameFromTranscriptomics } from './utils.mjs';
 

admin/utils.mjs

@@ -6,4 +6,20 @@ export const cbKeepClinicalPlusTranscriptomicsIndicesOnly = x =>
         x.index.includes(stem),
     );
 
-export const isIndexNameFromTranscriptomics = index => index.includes('gene_exp');
+export const isIndexNameFromTranscriptomics = index => index.includes('gene_exp');
+
+//https://labex.io/tutorials/javascript-javascript-programming-fundamentals-28177
+export const binomialCoefficient = (n, k) => {
+    if (Number.isNaN(n) || Number.isNaN(k)) return NaN;
+    if (k < 0 || k > n) return 0;
+    if (k === 0 || k === n) return 1;
+    if (k === 1 || k === n - 1) return n;
+    if (n - k < k) k = n - k;
+
+    let res = n;
+    for (let i = 2; i <= k; i++) res *= (n - i + 1) / i;
+    return Math.round(res);
+};
+
+//https://stackoverflow.com/questions/22566379/how-to-get-all-pairs-of-array-javascript
+export const pairIt = l => l.map((v, i) => l.slice(i + 1).map(w => [v, w])).flat();

package.json

@@ -68,6 +68,7 @@
     "eslint-plugin-prettier": "^3.4.0",
     "eslint-plugin-simple-import-sort": "^7.0.0",
     "jest": "^27.0.6",
+    "json-diff": "^1.0.6",
     "jsonwebtoken": "^8.5.1",
     "prettier": "^1.19.1",
     "supertest": "^6.1.2",