diff --git a/src/batch_edge_query.js b/src/batch_edge_query.js index 30389df9..dcc79437 100644 --- a/src/batch_edge_query.js +++ b/src/batch_edge_query.js @@ -20,17 +20,17 @@ module.exports = class BatchEdgeQueryHandler { } /** - * @param {Array} qXEdges - an array of TRAPI Query Edges; + * @param {Array} qEdges - an array of TRAPI Query Edges; */ - setEdges(qXEdges) { - this.qXEdges = qXEdges; + setEdges(qEdges) { + this.qEdges = qEdges; } /** * */ getEdges() { - return this.qXEdges; + return this.qEdges; } /** @@ -64,23 +64,26 @@ module.exports = class BatchEdgeQueryHandler { * Remove curies which resolve to the same thing, keeping the first. * @private */ - async _rmEquivalentDuplicates(qXEdges) { - Object.values(qXEdges).forEach((qXEdge) => { + async _rmEquivalentDuplicates(qEdges) { + Object.values(qEdges).forEach((qEdge) => { const nodes = { - subject: qXEdge.subject, - object: qXEdge.object, + subject: qEdge.subject, + object: qEdge.object, }; const strippedCuries = []; Object.entries(nodes).forEach(([nodeType, node]) => { const reducedCuries = []; const nodeStrippedCuries = []; - if (!node.curie) { return; } + if (!node.curie) { + return; + } node.curie.forEach((curie) => { // if the curie is already present, or an equivalent is, remove it if (!reducedCuries.includes(curie)) { - const equivalentAlreadyIncluded = qXEdge.getInputNode().getEquivalentIDs()[curie][0].curies.some( - (equivalentCurie) => reducedCuries.includes(equivalentCurie), - ); + const equivalentAlreadyIncluded = qEdge + .getInputNode() + .getEquivalentIDs() + [curie][0].curies.some((equivalentCurie) => reducedCuries.includes(equivalentCurie)); if (!equivalentAlreadyIncluded) { reducedCuries.push(curie); } else { @@ -99,34 +102,34 @@ module.exports = class BatchEdgeQueryHandler { } }); strippedCuries.forEach((curie) => { - qXEdge.getInputNode().removeEquivalentID(curie); + qEdge.getInputNode().removeEquivalentID(curie); }); }); } - async query(qXEdges, unavailableAPIs = {}) { + async query(qEdges, unavailableAPIs = {}) { debug('Node Update Start'); //it's now a single edge but convert to arr to simplify refactoring - qXEdges = Array.isArray(qXEdges) ? qXEdges : [qXEdges]; - const nodeUpdate = new NodesUpdateHandler(qXEdges); + qEdges = Array.isArray(qEdges) ? qEdges : [qEdges]; + const nodeUpdate = new NodesUpdateHandler(qEdges); //difference is there is no previous edge info anymore - await nodeUpdate.setEquivalentIDs(qXEdges); - await this._rmEquivalentDuplicates(qXEdges); + await nodeUpdate.setEquivalentIDs(qEdges); + await this._rmEquivalentDuplicates(qEdges); debug('Node Update Success'); const cacheHandler = new CacheHandler(this.caching, this.metaKG, this.recordConfig); - const { cachedRecords, nonCachedQXEdges } = await cacheHandler.categorizeEdges(qXEdges); + const { cachedRecords, nonCachedQEdges } = await cacheHandler.categorizeEdges(qEdges); this.logs = [...this.logs, ...cacheHandler.logs]; let queryRecords; - if (nonCachedQXEdges.length === 0) { + if (nonCachedQEdges.length === 0) { queryRecords = []; if (parentPort) { parentPort.postMessage({ cacheDone: true }); } } else { - debug('Start to convert qXEdges into APIEdges....'); - const edgeConverter = new QEdge2APIEdgeHandler(nonCachedQXEdges, this.metaKG); - const APIEdges = await edgeConverter.convert(nonCachedQXEdges); + debug('Start to convert qEdges into APIEdges....'); + const edgeConverter = new QEdge2APIEdgeHandler(nonCachedQEdges, this.metaKG); + const APIEdges = await edgeConverter.convert(nonCachedQEdges); debug(`qEdges are successfully converted into ${APIEdges.length} APIEdges....`); this.logs = [...this.logs, ...edgeConverter.logs]; if (APIEdges.length === 0 && cachedRecords.length === 0) { @@ -141,7 +144,8 @@ module.exports = class BatchEdgeQueryHandler { debug(`Total number of records is (${queryRecords.length})`); if (!isMainThread) { cacheHandler.cacheEdges(queryRecords); - } else { // await caching if async so end of job doesn't cut it off + } else { + // await caching if async so end of job doesn't cut it off await cacheHandler.cacheEdges(queryRecords); } } diff --git a/src/cache_handler.js b/src/cache_handler.js index 69917b87..9b042077 100644 --- a/src/cache_handler.js +++ b/src/cache_handler.js @@ -98,20 +98,20 @@ module.exports = class { ); } - async categorizeEdges(qXEdges) { + async categorizeEdges(qEdges) { if (this.cacheEnabled === false || process.env.INTERNAL_DISABLE_REDIS) { return { cachedRecords: [], - nonCachedQXEdges: qXEdges, + nonCachedQEdges: qEdges, }; } - let nonCachedQXEdges = []; + let nonCachedQEdges = []; let cachedRecords = []; debug('Begin edge cache lookup...'); - await async.eachSeries(qXEdges, async (qXEdge) => { - const qXEdgeMetaKGHash = this._hashEdgeByMetaKG(qXEdge.getHashedEdgeRepresentation()); + await async.eachSeries(qEdges, async (qEdge) => { + const qEdgeMetaKGHash = this._hashEdgeByMetaKG(qEdge.getHashedEdgeRepresentation()); const unpackedRecords = await new Promise(async (resolve) => { - const redisID = 'bte:edgeCache:' + qXEdgeMetaKGHash; + const redisID = 'bte:edgeCache:' + qEdgeMetaKGHash; await redisClient.client.usingLock([`redisLock:${redisID}`], 600000, async (signal) => { try { const compressedRecordPack = await redisClient.client.hgetallTimeout(redisID); @@ -129,7 +129,7 @@ module.exports = class { recordStream .pipe(this.createDecodeStream()) .on('data', (obj) => recordPack.push(obj)) - .on('end', () => resolve(Record.unpackRecords(recordPack, qXEdge, this.recordConfig))); + .on('end', () => resolve(Record.unpackRecords(recordPack, qEdge, this.recordConfig))); } else { resolve(null); } @@ -145,47 +145,47 @@ module.exports = class { new LogEntry( 'DEBUG', null, - `BTE finds cached records for ${qXEdge.getID()}`, + `BTE finds cached records for ${qEdge.getID()}`, { type: 'cacheHit', - qEdgeID: qXEdge.getID(), + qEdgeID: qEdge.getID(), } ).getLog() ); cachedRecords = [...cachedRecords, ...unpackedRecords]; } else { - nonCachedQXEdges.push(qXEdge); + nonCachedQEdges.push(qEdge); } debug(`Found (${cachedRecords.length}) cached records.`); }); - return { cachedRecords, nonCachedQXEdges }; + return { cachedRecords, nonCachedQEdges }; } - _hashEdgeByMetaKG(qXEdgeHash) { + _hashEdgeByMetaKG(qEdgeHash) { if (!this.metaKG) { - return qXEdgeHash; + return qEdgeHash; } const len = String(this.metaKG.ops.length); const allIDs = Array.from(new Set(this.metaKG.ops.map((op) => op.association.smartapi.id))).join(''); - return helper._generateHash(qXEdgeHash + len + allIDs); + return helper._generateHash(qEdgeHash + len + allIDs); } - _groupQueryRecordsByQXEdgeHash(queryRecords) { + _groupQueryRecordsByQEdgeHash(queryRecords) { let groupedRecords = {}; queryRecords.map((record) => { try { - const qXEdgeMetaKGHash = this._hashEdgeByMetaKG(record.qXEdge.getHashedEdgeRepresentation()); - if (!(qXEdgeMetaKGHash in groupedRecords)) { - groupedRecords[qXEdgeMetaKGHash] = []; + const qEdgeMetaKGHash = this._hashEdgeByMetaKG(record.qEdge.getHashedEdgeRepresentation()); + if (!(qEdgeMetaKGHash in groupedRecords)) { + groupedRecords[qEdgeMetaKGHash] = []; } - groupedRecords[qXEdgeMetaKGHash].push(record); + groupedRecords[qEdgeMetaKGHash].push(record); } catch (e) { debug('skipping malformed record'); } }); - Object.entries(groupedRecords).forEach(([qXEdgeMetaKGHash, records]) => { - groupedRecords[qXEdgeMetaKGHash] = Record.packRecords(records); + Object.entries(groupedRecords).forEach(([qEdgeMetaKGHash, records]) => { + groupedRecords[qEdgeMetaKGHash] = Record.packRecords(records); }); return groupedRecords; } @@ -210,11 +210,11 @@ module.exports = class { } debug('Start to cache query records.'); try { - const groupedRecords = this._groupQueryRecordsByQXEdgeHash(queryRecords); - const qXEdgeHashes = Array.from(Object.keys(groupedRecords)); - debug(`Number of hashed edges: ${qXEdgeHashes.length}`); + const groupedRecords = this._groupQueryRecordsByQEdgeHash(queryRecords); + const qEdgeHashes = Array.from(Object.keys(groupedRecords)); + debug(`Number of hashed edges: ${qEdgeHashes.length}`); const failedHashes = []; - await async.eachSeries(qXEdgeHashes, async (hash) => { + await async.eachSeries(qEdgeHashes, async (hash) => { // lock to prevent caching to/reading from actively caching edge const redisID = 'bte:edgeCache:' + hash; if (parentPort) { @@ -236,7 +236,7 @@ module.exports = class { try { await redisClient.client.delTimeout(redisID); } catch (e) { - debug(`Unable to remove partial cache ${redisID} from redis during cache failure due to error ${error}. This may result in failed or improper cache retrieval of this qXEdge.`) + debug(`Unable to remove partial cache ${redisID} from redis during cache failure due to error ${error}. This may result in failed or improper cache retrieval of this qEdge.`) } } }) @@ -247,7 +247,7 @@ module.exports = class { await redisClient.client.expireTimeout(redisID, process.env.REDIS_KEY_EXPIRE_TIME || 1800); } catch (error) { failedHashes.push(hash); - debug(`Failed to cache qXEdge ${hash} records due to error ${error}. This does not stop other edges from caching nor terminate the query.`) + debug(`Failed to cache qEdge ${hash} records due to error ${error}. This does not stop other edges from caching nor terminate the query.`) } finally { if (parentPort) { parentPort.postMessage({ completeCacheKey: redisID }); @@ -261,7 +261,7 @@ module.exports = class { if (successCount) { debug(`Successfully cached (${successCount}) query records.`); } else { - debug(`qXEdge caching failed.`); + debug(`qEdge caching failed.`); } } catch (error) { debug(`Caching failed due to ${error}. This does not terminate the query.`); diff --git a/src/edge_manager.js b/src/edge_manager.js index 9429d831..c2931965 100644 --- a/src/edge_manager.js +++ b/src/edge_manager.js @@ -5,7 +5,7 @@ const debug = require('debug')('bte:biothings-explorer-trapi:edge-manager'); const config = require('./config'); -module.exports = class QueryExecutionEdgeManager { +module.exports = class QueryEdgeManager { constructor(edges) { // flatten list of all edges available this._qEdges = _.flatten(edges); @@ -42,11 +42,11 @@ module.exports = class QueryExecutionEdgeManager { //either object or subject OR no count last // available not yet executed let available_edges = this._qEdges - .filter(qXEdge => !qXEdge.executed); + .filter(qEdge => !qEdge.executed); //safeguard for making sure there's available //edges when calling getNext if (available_edges.length == 0) { - debug(`(5) Error: ${available_edges} available qXEdges found.`); + debug(`(5) Error: ${available_edges} available qEdges found.`); this.logs.push( new LogEntry( 'DEBUG', @@ -57,48 +57,48 @@ module.exports = class QueryExecutionEdgeManager { ); } //begin search - let nextQXEdge; + let nextQEdge; let lowest_entity_count; let current_obj_lowest = 0; let current_sub_lowest = 0; - available_edges.forEach((qXEdge) => { + available_edges.forEach((qEdge) => { if ( - qXEdge && - qXEdge.object.entity_count + qEdge && + qEdge.object.entity_count ) { - current_obj_lowest = qXEdge.object.entity_count; + current_obj_lowest = qEdge.object.entity_count; if (!lowest_entity_count) { //set current lowest if none lowest_entity_count = current_obj_lowest; } if (current_obj_lowest <= lowest_entity_count) { //lowest is now object count - nextQXEdge = qXEdge; + nextQEdge = qEdge; } } if ( - qXEdge && - qXEdge.subject.entity_count && - qXEdge.subject.entity_count > 0 + qEdge && + qEdge.subject.entity_count && + qEdge.subject.entity_count > 0 ) { - current_sub_lowest = qXEdge.subject.entity_count; + current_sub_lowest = qEdge.subject.entity_count; if (!lowest_entity_count) { //set current lowest if none lowest_entity_count = current_sub_lowest; } if (current_sub_lowest <= lowest_entity_count) { //lowest is now subject count - nextQXEdge = qXEdge; + nextQEdge = qEdge; } } }); - if (!nextQXEdge) { + if (!nextQEdge) { //if no edge with count found pick the first empty //edge available let all_empty = available_edges .filter((edge) => !edge.object.entity_count && !edge.subject.entity_count); if (all_empty.length == 0) { - debug(`(5) Error: No available qXEdges found.`); + debug(`(5) Error: No available qEdges found.`); this.logs.push( new LogEntry( 'DEBUG', @@ -110,27 +110,27 @@ module.exports = class QueryExecutionEdgeManager { debug(`(5) Sending next edge '${all_empty[0].getID()}' with NO entity count.`); return this.preSendOffCheck(all_empty[0]); } - debug(`(5) Sending next edge '${nextQXEdge.getID()}' ` + - `WITH entity count...(${nextQXEdge.subject.entity_count || nextQXEdge.object.entity_count})`); - return this.preSendOffCheck(nextQXEdge); + debug(`(5) Sending next edge '${nextQEdge.getID()}' ` + + `WITH entity count...(${nextQEdge.subject.entity_count || nextQEdge.object.entity_count})`); + return this.preSendOffCheck(nextQEdge); } logEntityCounts() { - this._qEdges.forEach((qXEdge) => { - debug(`'${qXEdge.getID()}'` + - ` : (${qXEdge.subject.entity_count || 0}) ` + - `${qXEdge.reverse ? '<--' : '-->'}` + - ` (${qXEdge.object.entity_count || 0})`); + this._qEdges.forEach((qEdge) => { + debug(`'${qEdge.getID()}'` + + ` : (${qEdge.subject.entity_count || 0}) ` + + `${qEdge.reverse ? '<--' : '-->'}` + + ` (${qEdge.object.entity_count || 0})`); }); } - checkEntityMax(nextQXedge) { + checkEntityMax(nextQEdge) { const max = config.ENTITY_MAX; //(MAX) --- (0) not allowed //(MAX) --- (MAX) not allowed //(MAX) --- (2) allowed, (2 will be used) - let sub_count = nextQXedge.object.getEntityCount(); - let obj_count = nextQXedge.subject.getEntityCount(); + let sub_count = nextQEdge.object.getEntityCount(); + let obj_count = nextQEdge.subject.getEntityCount(); debug(`Checking entity max : (${sub_count})--(${obj_count})`); if ( (obj_count == 0 && sub_count > max) || @@ -138,21 +138,21 @@ module.exports = class QueryExecutionEdgeManager { (obj_count > max && sub_count > max) ) { throw new BTEError( - `Max number of entities exceeded (${max}) in '${nextQXedge.getID()}'` + `Max number of entities exceeded (${max}) in '${nextQEdge.getID()}'` ); } } - preSendOffCheck(nextQXEdge) { - // next: qXEdge + preSendOffCheck(nextQEdge) { + // next: qEdge //check that edge entities are or have potential to stay //under max limit - this.checkEntityMax(nextQXEdge); - if (nextQXEdge.object.entity_count && nextQXEdge.subject.entity_count) { + this.checkEntityMax(nextQEdge); + if (nextQEdge.object.entity_count && nextQEdge.subject.entity_count) { //if at the time of being queried the edge has both //obj and sub entity counts //chose obj/suj lower entity count for query - nextQXEdge.chooseLowerEntityValue(); + nextQEdge.chooseLowerEntityValue(); this.logs.push( new LogEntry('DEBUG', null, @@ -160,21 +160,21 @@ module.exports = class QueryExecutionEdgeManager { ); } else if ( - (nextQXEdge.object.entity_count && !nextQXEdge.subject.entity_count) || - (!nextQXEdge.object.entity_count && !nextQXEdge.subject.entity_count) + (nextQEdge.object.entity_count && !nextQEdge.subject.entity_count) || + (!nextQEdge.object.entity_count && !nextQEdge.subject.entity_count) ) { debug(`(5) Checking direction of edge with one set of entities...`); //check direction is correct if edge only has one set of entities //before sending off - nextQXEdge.reverse = nextQXEdge.subject.entity_count ? false : true; + nextQEdge.reverse = nextQEdge.subject.entity_count ? false : true; } this.logs.push( new LogEntry('DEBUG', null, - `Edge manager is sending next qEdge '${nextQXEdge.getID()}' for execution.`).getLog(), + `Edge manager is sending next qEdge '${nextQEdge.getID()}' for execution.`).getLog(), ); this.logEntityCounts(); - return nextQXEdge; + return nextQEdge; } getEdgesNotExecuted() { @@ -185,17 +185,17 @@ module.exports = class QueryExecutionEdgeManager { return not_executed; } - _filterEdgeRecords(qXEdge) { + _filterEdgeRecords(qEdge) { let keep = []; - let records = qXEdge.records; - let sub_count = qXEdge.subject.curie; - let obj_count = qXEdge.object.curie; - debug(`'${qXEdge.getID()}' Reversed[${qXEdge.reverse}] (${JSON.stringify(sub_count.length || 0)})` + + let records = qEdge.records; + let sub_count = qEdge.subject.curie; + let obj_count = qEdge.object.curie; + debug(`'${qEdge.getID()}' Reversed[${qEdge.reverse}] (${JSON.stringify(sub_count.length || 0)})` + `--(${JSON.stringify(obj_count.length || 0)}) entities / (${records.length}) records.`); // debug(`IDS SUB ${JSON.stringify(sub_count)}`) // debug(`IDS OBJ ${JSON.stringify(obj_count)}`) - let object_node_ids = qXEdge.reverse ? sub_count : obj_count; - let subject_node_ids = qXEdge.reverse ? obj_count : sub_count; + let object_node_ids = qEdge.reverse ? sub_count : obj_count; + let subject_node_ids = qEdge.reverse ? obj_count : sub_count; records.forEach((record) => { //check sub curies against $input ids @@ -288,12 +288,12 @@ module.exports = class QueryExecutionEdgeManager { keep.push(record); } }); - debug(`'${qXEdge.getID()}' dropped (${records.length - keep.length}) records.`); + debug(`'${qEdge.getID()}' dropped (${records.length - keep.length}) records.`); this.logs.push( new LogEntry( 'DEBUG', null, - `'${qXEdge.getID()}' kept (${keep.length}) / dropped (${records.length - keep.length}) records.` + `'${qEdge.getID()}' kept (${keep.length}) / dropped (${records.length - keep.length}) records.` ).getLog(), ); return keep; @@ -308,9 +308,9 @@ module.exports = class QueryExecutionEdgeManager { let brokenEdges = []; debug(`(11) Collecting records...`); //First: go through edges and filter that each edge is holding - this._qEdges.forEach((qXEdge) => { - let qEdgeID = qXEdge.getID(); - let filteredRecords = qXEdge.records.map(record => record.queryDirection()); + this._qEdges.forEach((qEdge) => { + let qEdgeID = qEdge.getID(); + let filteredRecords = qEdge.records.map(record => record.queryDirection()); if (filteredRecords.length == 0) { this.logs.push( new LogEntry( @@ -322,10 +322,10 @@ module.exports = class QueryExecutionEdgeManager { brokenChain = true; brokenEdges.push(qEdgeID); } - this.logs = [...this.logs, ...qXEdge.logs]; + this.logs = [...this.logs, ...qEdge.logs]; //collect records combinedRecords = combinedRecords.concat(filteredRecords); - let connections = qXEdge.subject.getConnections().concat(qXEdge.object.getConnections()); + let connections = qEdge.subject.getConnections().concat(qEdge.object.getConnections()); connections = connections.filter(id => id !== qEdgeID); connections = new Set(connections); recordsByQEdgeID[qEdgeID] = { @@ -352,7 +352,7 @@ module.exports = class QueryExecutionEdgeManager { `resulted in (0) records. No complete paths can be formed.` ).getLog(), ); - debug(`(12) qXEdges ${JSON.stringify(brokenEdges)} ` + + debug(`(12) qEdges ${JSON.stringify(brokenEdges)} ` + `resulted in (0) records. No complete paths can be formed.`); } //Organized by edge: update query records @@ -379,22 +379,22 @@ module.exports = class QueryExecutionEdgeManager { ); } - updateEdgeRecords(currentQXEdge) { + updateEdgeRecords(currentQEdge) { //1. filter edge records based on current status - let filteredRecords = this._filterEdgeRecords(currentQXEdge); + let filteredRecords = this._filterEdgeRecords(currentQEdge); //2.trigger node update / entity update based on new status - currentQXEdge.storeRecords(filteredRecords); + currentQEdge.storeRecords(filteredRecords); } - updateNeighborsEdgeRecords(currentQXEdge) { + updateNeighborsEdgeRecords(currentQEdge) { //update and filter only immediate neighbors debug(`Updating neighbors...`); - let currentQEdgeID = currentQXEdge.getID(); + let currentQEdgeID = currentQEdge.getID(); //get neighbors of this edges subject that are not this edge - let left_connections = currentQXEdge.subject.getConnections(); + let left_connections = currentQEdge.subject.getConnections(); left_connections = left_connections.filter((qEdgeID) => qEdgeID !== currentQEdgeID); //get neighbors of this edges object that are not this edge - let right_connections = currentQXEdge.object.getConnections(); + let right_connections = currentQEdge.object.getConnections(); right_connections = right_connections.filter((qEdgeID) => qEdgeID !== currentQEdgeID); debug(`(${left_connections})<--edge neighbors-->(${right_connections})`); if (left_connections.length) { @@ -422,15 +422,15 @@ module.exports = class QueryExecutionEdgeManager { } } - updateAllOtherEdges(currentQXEdge) { + updateAllOtherEdges(currentQEdge) { //update and filter all other edges debug(`Updating all other edges...`); - let currentQEdgeID = currentQXEdge.getID(); - this._qEdges.forEach((qXEdge) => { - if (qXEdge.getID() !== currentQEdgeID && qXEdge.records.length) { - debug(`Updating "${qXEdge.getID()}"...`); - this.updateEdgeRecords(qXEdge); - this.updateEdgeRecords(currentQXEdge); + let currentQEdgeID = currentQEdge.getID(); + this._qEdges.forEach((qEdge) => { + if (qEdge.getID() !== currentQEdgeID && qEdge.records.length) { + debug(`Updating "${qEdge.getID()}"...`); + this.updateEdgeRecords(qEdge); + this.updateEdgeRecords(currentQEdge); } }); } diff --git a/src/index.js b/src/index.js index d41993c6..f957f99e 100644 --- a/src/index.js +++ b/src/index.js @@ -97,9 +97,9 @@ exports.TRAPIQueryHandler = class TRAPIQueryHandler { async _processQueryGraph(queryGraph) { try { let queryGraphHandler = new QueryGraph(queryGraph); - let queryExecutionEdges = await queryGraphHandler.calculateEdges(); + let queryEdges = await queryGraphHandler.calculateEdges(); this.logs = [...this.logs, ...queryGraphHandler.logs]; - return queryExecutionEdges; + return queryEdges; } catch (err) { if (err instanceof InvalidQueryGraphError || err instanceof id_resolver.SRIResolverFailiure) { throw err; @@ -110,17 +110,17 @@ exports.TRAPIQueryHandler = class TRAPIQueryHandler { } } - _createBatchEdgeQueryHandlersForCurrent(currentQXEdge, metaKG) { + _createBatchEdgeQueryHandlersForCurrent(currentQEdge, metaKG) { let handler = new BatchEdgeQueryHandler(metaKG, this.resolveOutputIDs, { caching: this.options.caching, submitter: this.options.submitter, recordHashEdgeAttributes: config.EDGE_ATTRIBUTES_USED_IN_RECORD_HASH, }); - handler.setEdges(currentQXEdge); + handler.setEdges(currentQEdge); return handler; } - async _edgesSupported(qXEdges, metaKG) { + async _edgesSupported(qEdges, metaKG) { if (this.options.dryrun) { let log_msg = 'Running dryrun of query, no API calls will be performed. Actual query execution order may vary based on API responses received.'; @@ -128,26 +128,26 @@ exports.TRAPIQueryHandler = class TRAPIQueryHandler { } // _.cloneDeep() is resource-intensive but only runs once per query - qXEdges = _.cloneDeep(qXEdges); - const manager = new EdgeManager(qXEdges); + qEdges = _.cloneDeep(qEdges); + const manager = new EdgeManager(qEdges); const qEdgesMissingOps = {}; while (manager.getEdgesNotExecuted()) { - let currentQXEdge = manager.getNext(); - const edgeConverter = new QEdge2APIEdgeHandler([currentQXEdge], metaKG); - const metaXEdges = edgeConverter.getMetaXEdges(currentQXEdge); + let currentQEdge = manager.getNext(); + const edgeConverter = new QEdge2APIEdgeHandler([currentQEdge], metaKG); + const metaXEdges = edgeConverter.getMetaXEdges(currentQEdge); if (this.options.dryrun) { let apiNames = [...new Set(metaXEdges.map((metaXEdge) => metaXEdge.association.api_name))]; let log_msg; - if (currentQXEdge.reverse) { - log_msg = `qEdge ${currentQXEdge.id} (reversed): ${currentQXEdge.object.category} > ${ - currentQXEdge.predicate ? `${currentQXEdge.predicate} > ` : '' - }${currentQXEdge.subject.category}`; + if (currentQEdge.reverse) { + log_msg = `qEdge ${currentQEdge.id} (reversed): ${currentQEdge.object.category} > ${ + currentQEdge.predicate ? `${currentQEdge.predicate} > ` : '' + }${currentQEdge.subject.category}`; } else { - log_msg = `qEdge ${currentQXEdge.id}: ${currentQXEdge.subject.category} > ${ - currentQXEdge.predicate ? `${currentQXEdge.predicate} > ` : '' - }${currentQXEdge.object.category}`; + log_msg = `qEdge ${currentQEdge.id}: ${currentQEdge.subject.category} > ${ + currentQEdge.predicate ? `${currentQEdge.predicate} > ` : '' + }${currentQEdge.object.category}`; } this.logs.push(new LogEntry('INFO', null, log_msg).getLog()); @@ -163,21 +163,21 @@ exports.TRAPIQueryHandler = class TRAPIQueryHandler { } if (!metaXEdges.length) { - qEdgesMissingOps[currentQXEdge.id] = currentQXEdge.reverse; + qEdgesMissingOps[currentQEdge.id] = currentQEdge.reverse; } // assume results so next edge may be reversed or not - currentQXEdge.executed = true; + currentQEdge.executed = true; //use # of APIs as estimate of # of records if (metaXEdges.length) { - if (currentQXEdge.reverse) { - currentQXEdge.subject.entity_count = currentQXEdge.object.entity_count * metaXEdges.length; + if (currentQEdge.reverse) { + currentQEdge.subject.entity_count = currentQEdge.object.entity_count * metaXEdges.length; } else { - currentQXEdge.object.entity_count = currentQXEdge.subject.entity_count * metaXEdges.length; + currentQEdge.object.entity_count = currentQEdge.subject.entity_count * metaXEdges.length; } } else { - currentQXEdge.object.entity_count = 1; - currentQXEdge.subject.entity_count = 1; + currentQEdge.object.entity_count = 1; + currentQEdge.subject.entity_count = 1; } } @@ -357,7 +357,7 @@ exports.TRAPIQueryHandler = class TRAPIQueryHandler { ).getLog(), ); } - let queryExecutionEdges = await this._processQueryGraph(this.queryGraph); + let queryEdges = await this._processQueryGraph(this.queryGraph); // TODO remove this when constraints implemented if (await this._checkContraints()) { return; @@ -368,7 +368,7 @@ exports.TRAPIQueryHandler = class TRAPIQueryHandler { debug(message); return; } - debug(`(3) All edges created ${JSON.stringify(queryExecutionEdges)}`); + debug(`(3) All edges created ${JSON.stringify(queryEdges)}`); if (this._queryUsesInferredMode() && this._queryIsOneHop()) { await this._handleInferredEdges(); return; @@ -378,38 +378,38 @@ exports.TRAPIQueryHandler = class TRAPIQueryHandler { this.logs.push(new LogEntry('WARNING', null, message).getLog()); return; } - if (!(await this._edgesSupported(queryExecutionEdges, metaKG))) { + if (!(await this._edgesSupported(queryEdges, metaKG))) { return; } - const manager = new EdgeManager(queryExecutionEdges); + const manager = new EdgeManager(queryEdges); const unavailableAPIs = {}; while (manager.getEdgesNotExecuted()) { //next available/most efficient edge - let currentQXEdge = manager.getNext(); + let currentQEdge = manager.getNext(); //crate queries from edge - let handler = this._createBatchEdgeQueryHandlersForCurrent(currentQXEdge, metaKG); + let handler = this._createBatchEdgeQueryHandlersForCurrent(currentQEdge, metaKG); this.logs.push( new LogEntry( 'INFO', null, - `Executing ${currentQXEdge.getID()}${currentQXEdge.isReversed() ? ' (reversed)' : ''}: ${ - currentQXEdge.subject.id - } ${currentQXEdge.isReversed() ? '<--' : '-->'} ${currentQXEdge.object.id}`, + `Executing ${currentQEdge.getID()}${currentQEdge.isReversed() ? ' (reversed)' : ''}: ${ + currentQEdge.subject.id + } ${currentQEdge.isReversed() ? '<--' : '-->'} ${currentQEdge.object.id}`, ).getLog(), ); - debug(`(5) Executing current edge >> "${currentQXEdge.getID()}"`); + debug(`(5) Executing current edge >> "${currentQEdge.getID()}"`); //execute current edge query - let queryRecords = await handler.query(handler.qXEdges, unavailableAPIs); + let queryRecords = await handler.query(handler.qEdges, unavailableAPIs); this.logs = [...this.logs, ...handler.logs]; // create an edge execution summary let success = 0, fail = 0, total = 0; let cached = this.logs.filter( - ({ data }) => data?.qEdgeID === currentQXEdge.id && data?.type === 'cacheHit', + ({ data }) => data?.qEdgeID === currentQEdge.id && data?.type === 'cacheHit', ).length; this.logs - .filter(({ data }) => data?.qEdgeID === currentQXEdge.id && data?.type === 'query') + .filter(({ data }) => data?.qEdgeID === currentQEdge.id && data?.type === 'query') .forEach(({ data }) => { !data.error ? success++ : fail++; total++; @@ -418,43 +418,43 @@ exports.TRAPIQueryHandler = class TRAPIQueryHandler { new LogEntry( 'INFO', null, - `${currentQXEdge.id} execution: ${total} queries (${success} success/${fail} fail) and (${cached}) cached qEdges return (${queryRecords.length}) records`, + `${currentQEdge.id} execution: ${total} queries (${success} success/${fail} fail) and (${cached}) cached qEdges return (${queryRecords.length}) records`, {}, ).getLog(), ); if (queryRecords.length === 0) { this._logSkippedQueries(unavailableAPIs); - debug(`(X) Terminating..."${currentQXEdge.getID()}" got 0 records.`); + debug(`(X) Terminating..."${currentQEdge.getID()}" got 0 records.`); this.logs.push( new LogEntry( 'WARNING', null, - `qEdge (${currentQXEdge.getID()}) got 0 records. Your query terminates.`, + `qEdge (${currentQEdge.getID()}) got 0 records. Your query terminates.`, ).getLog(), ); return; } //storing records will trigger a node entity count update - currentQXEdge.storeRecords(queryRecords); + currentQEdge.storeRecords(queryRecords); //filter records - manager.updateEdgeRecords(currentQXEdge); + manager.updateEdgeRecords(currentQEdge); //update and filter neighbors - manager.updateAllOtherEdges(currentQXEdge); + manager.updateAllOtherEdges(currentQEdge); // check that any records are kept - if (!currentQXEdge.records.length) { + if (!currentQEdge.records.length) { this._logSkippedQueries(unavailableAPIs); - debug(`(X) Terminating..."${currentQXEdge.getID()}" kept 0 records.`); + debug(`(X) Terminating..."${currentQEdge.getID()}" kept 0 records.`); this.logs.push( new LogEntry( 'WARNING', null, - `qEdge (${currentQXEdge.getID()}) kept 0 records. Your query terminates.`, + `qEdge (${currentQEdge.getID()}) kept 0 records. Your query terminates.`, ).getLog(), ); return; } // edge all done - currentQXEdge.executed = true; + currentQEdge.executed = true; debug(`(10) Edge successfully queried.`); } this._logSkippedQueries(unavailableAPIs); diff --git a/src/qedge2apiedge.js b/src/qedge2apiedge.js index 4135b568..9ac3d5ed 100644 --- a/src/qedge2apiedge.js +++ b/src/qedge2apiedge.js @@ -29,34 +29,34 @@ module.exports = class QEdge2APIEdgeHandler { * Get SmartAPI Edges based on TRAPI Query Edge. * @private * @param {object} metaKG - SmartAPI Knowledge Graph Object - * @param {object} qXEdge - TRAPI Query Edge Object + * @param {object} qEdge - TRAPI Query Edge Object */ - getMetaXEdges(qXEdge, metaKG = this.metaKG) { - debug(`Input node is ${qXEdge.getInputNode().id}`); - debug(`Output node is ${qXEdge.getOutputNode().id}`); + getMetaXEdges(qEdge, metaKG = this.metaKG) { + debug(`Input node is ${qEdge.getInputNode().id}`); + debug(`Output node is ${qEdge.getOutputNode().id}`); this.logs.push( new LogEntry( 'DEBUG', null, - `BTE is trying to find metaKG edges (smartAPI registry, x-bte annotation) connecting from ${qXEdge.getInputNode().getCategories()} to ${qXEdge + `BTE is trying to find metaKG edges (smartAPI registry, x-bte annotation) connecting from ${qEdge.getInputNode().getCategories()} to ${qEdge .getOutputNode() - .getCategories()} with predicate ${qXEdge.getPredicate()}`, + .getCategories()} with predicate ${qEdge.getPredicate()}`, ).getLog(), ); let filterCriteria = { - input_type: qXEdge.getInputNode().getCategories(), - output_type: qXEdge.getOutputNode().getCategories(), - predicate: qXEdge.getPredicate(), + input_type: qEdge.getInputNode().getCategories(), + output_type: qEdge.getOutputNode().getCategories(), + predicate: qEdge.getPredicate(), }; debug(`KG Filters: ${JSON.stringify(filterCriteria, null, 2)}`); let metaXEdges = metaKG.filter(filterCriteria).map((metaEdge) => { - metaEdge.reasoner_edge = qXEdge; + metaEdge.reasoner_edge = qEdge; return metaEdge; }); if (metaXEdges.length === 0) { - debug(`No smartapi edge found for ${qXEdge.getID()}`); + debug(`No smartapi edge found for ${qEdge.getID()}`); this.logs.push( - new LogEntry('WARNING', null, `BTE didn't find any metaKG edges corresponding to ${qXEdge.getID()}`).getLog(), + new LogEntry('WARNING', null, `BTE didn't find any metaKG edges corresponding to ${qEdge.getID()}`).getLog(), ); } else { this.logs.push( @@ -65,7 +65,7 @@ module.exports = class QEdge2APIEdgeHandler { null, `BTE found ${ metaXEdges.length - } metaKG edges corresponding to ${qXEdge.getID()}. These metaKG edges comes from ${ + } metaKG edges corresponding to ${qEdge.getID()}. These metaKG edges comes from ${ new Set(this._findAPIsFromMetaEdges(metaXEdges)).size } unique APIs. They are ${Array.from(new Set(this._findAPIsFromMetaEdges(metaXEdges))).join(',')}`, ).getLog(), @@ -320,10 +320,10 @@ module.exports = class QEdge2APIEdgeHandler { return APIEdges; } - async convert(qXEdges) { + async convert(qEdges) { let APIEdges = []; - await Promise.all(qXEdges.map(async (qXEdge) => { - const metaXedges = await this.getMetaXEdges(qXEdge); + await Promise.all(qEdges.map(async (qEdge) => { + const metaXedges = await this.getMetaXEdges(qEdge); const apis = _.uniq(metaXedges.map(api => api.association.api_name)); debug(`${apis.length} APIs being used:`, JSON.stringify(apis)); debug(`${metaXedges.length} SmartAPI edges are retrieved....`); @@ -331,7 +331,7 @@ module.exports = class QEdge2APIEdgeHandler { let newEdges = await this._createAPIEdges(metaXEdge); debug(`${newEdges.length} metaKG are created....`); newEdges = newEdges.map((e) => { - e.filter = qXEdge.filter; + e.filter = qEdge.filter; return e; }); APIEdges = [...APIEdges, ...newEdges]; diff --git a/src/query_edge.js b/src/query_edge.js index 934c4fb0..15ed58b7 100644 --- a/src/query_edge.js +++ b/src/query_edge.js @@ -1,5 +1,5 @@ const helper = require('./helper'); -const debug = require('debug')('bte:biothings-explorer-trapi:MegaQEdge'); +const debug = require('debug')('bte:biothings-explorer-trapi:QEdge'); const utils = require('./utils'); const biolink = require('./biolink'); const { Record } = require('@biothings-explorer/api-response-transform'); @@ -20,25 +20,21 @@ module.exports = class QEdge { this.reverse = this.subject?.getCurie?.() === undefined && this.object?.getCurie?.() !== undefined; - this.reverse = info.reverse !== undefined - ? info.reverse - : this.reverse; + this.reverse = info.reverse !== undefined ? info.reverse : this.reverse; this.reverse = reverse !== undefined ? reverse : this.reverse; this.init(); - //edge has been fully executed this.executed = info.executed === undefined ? false : info.executed; //run initial checks this.logs = info.logs === undefined ? [] : info.logs; //this edges query response records - if (info.records && info.frozen === true) this.records = info.records.map(recordJSON => new Record(recordJSON)); + if (info.records && info.frozen === true) this.records = info.records.map((recordJSON) => new Record(recordJSON)); else this.records = []; - debug(`(2) Created Edge` + - ` ${JSON.stringify(this.getID())} Reverse = ${this.reverse}`) + debug(`(2) Created Edge` + ` ${JSON.stringify(this.getID())} Reverse = ${this.reverse}`); } freeze() { @@ -52,8 +48,8 @@ module.exports = class QEdge { subject: this.subject.freeze(), object: this.object.freeze(), predicate: this.predicate, - records: this.records.map(record => record.freeze()), - frozen: true + records: this.records.map((record) => record.freeze()), + frozen: true, }; } @@ -67,7 +63,10 @@ module.exports = class QEdge { getHashedEdgeRepresentation() { const toBeHashed = - this.getInputNode().getCategories() + this.getPredicate() + this.getOutputNode().getCategories() + this.getInputCurie(); + this.getInputNode().getCategories() + + this.getPredicate() + + this.getOutputNode().getCategories() + + this.getInputCurie(); return helper._generateHash(toBeHashed); } @@ -100,8 +99,7 @@ module.exports = class QEdge { this.reverse = false; this.object.holdCurie(); debug(`(8) Sub - Obj were same but chose subject (${this.subject.entity_count})`); - } - else if (this.object.entity_count > this.subject.entity_count) { + } else if (this.object.entity_count > this.subject.entity_count) { //(#) ---> () this.reverse = false; //tell node to hold curie in a temp field @@ -123,19 +121,18 @@ module.exports = class QEdge { //will give you all curies found by semantic type, each type will have //a main ID and all of it's aliases debug(`(7) Updating Entities in "${this.getID()}"`); - let typesToInclude = isReversed ? - this.subject.getCategories() : - this.object.getCategories(); + let typesToInclude = isReversed ? this.subject.getCategories() : this.object.getCategories(); debug(`(7) Collecting Types: "${JSON.stringify(typesToInclude)}"`); let all = {}; records.forEach((record) => { - record.subject.normalizedInfo.forEach((o) => { //create semantic type if not included let type = o._leafSemanticType; - if (typesToInclude.includes(type) || + if ( + typesToInclude.includes(type) || typesToInclude.includes('NamedThing') || - typesToInclude.toString().includes(type)) { + typesToInclude.toString().includes(type) + ) { if (!Object.hasOwnProperty.call(all, type)) { all[type] = {}; } @@ -182,7 +179,7 @@ module.exports = class QEdge { } //else #2 check curie else if (Object.hasOwnProperty.call(o, 'curie')) { - if (Array.isArray( o.curie)) { + if (Array.isArray(o.curie)) { all[type][original] = o.curie; } else { all[type][original] = [o.curie]; @@ -198,9 +195,11 @@ module.exports = class QEdge { record.object.normalizedInfo.forEach((o) => { //create semantic type if not included let type = o._leafSemanticType; - if (typesToInclude.includes(type) || + if ( + typesToInclude.includes(type) || typesToInclude.includes('NamedThing') || - typesToInclude.toString().includes(type)) { + typesToInclude.toString().includes(type) + ) { if (!Object.hasOwnProperty.call(all, type)) { all[type] = {}; } @@ -248,7 +247,7 @@ module.exports = class QEdge { } //else #2 check curie else if (Object.hasOwnProperty.call(o, 'curie')) { - if (Array.isArray( o.curie)) { + if (Array.isArray(o.curie)) { all[type][original] = o.curie; } else { all[type][original] = [o.curie]; @@ -260,7 +259,6 @@ module.exports = class QEdge { } } }); - }); // {Gene:{'id': ['alias']}} debug(`Collected entity ids in records: ${JSON.stringify(Object.keys(all))}`); @@ -271,7 +269,7 @@ module.exports = class QEdge { //combine all curies in case there are //multiple categories in this node since //they are separated by type - let combined = {}; + let combined = {}; for (const type in curies) { for (const original in curies[type]) { combined[original] = curies[type][original]; @@ -284,15 +282,11 @@ module.exports = class QEdge { //update node queried (1) ---> (update) let curies_by_semantic_type = this.extractCuriesFromRecords(records, this.reverse); let combined_curies = this._combineCuries(curies_by_semantic_type); - this.reverse ? - this.subject.updateCuries(combined_curies) : - this.object.updateCuries(combined_curies); + this.reverse ? this.subject.updateCuries(combined_curies) : this.object.updateCuries(combined_curies); //update node used as input (1 [update]) ---> () let curies_by_semantic_type_2 = this.extractCuriesFromRecords(records, !this.reverse); let combined_curies_2 = this._combineCuries(curies_by_semantic_type_2); - !this.reverse ? - this.subject.updateCuries(combined_curies_2) : - this.object.updateCuries(combined_curies_2); + !this.reverse ? this.subject.updateCuries(combined_curies_2) : this.object.updateCuries(combined_curies_2); } applyNodeConstraints() { @@ -310,7 +304,7 @@ module.exports = class QEdge { //apply constraints for (let x = 0; x < sub_constraints.length; x++) { const constraint = sub_constraints[x]; - keep = this.meetsConstraint(constraint, res, from) + keep = this.meetsConstraint(constraint, res, from); } //pass or not if (keep) { @@ -330,7 +324,7 @@ module.exports = class QEdge { //apply constraints for (let x = 0; x < obj_constraints.length; x++) { const constraint = obj_constraints[x]; - keep = this.meetsConstraint(constraint, res, from) + keep = this.meetsConstraint(constraint, res, from); } //pass or not if (keep) { @@ -340,7 +334,7 @@ module.exports = class QEdge { } if (save_kept) { //only override recordss if there was any filtering done. - this.records = kept; + this.records = kept; debug(`(6) Reduced to (${this.records.length}) records.`); } else { debug(`(6) No constraints. Skipping...`); @@ -351,7 +345,7 @@ module.exports = class QEdge { //list of attribute ids in node let available_attributes = new Set(); for (const key in record[from].attributes) { - available_attributes.add(key) + available_attributes.add(key); } available_attributes = [...available_attributes]; // debug(`ATTRS ${JSON.stringify(record[from].normalizedInfo[0]._leafSemanticType)}` + @@ -368,53 +362,59 @@ module.exports = class QEdge { node_attributes[filter] = record[from].attributes[filter]; }); switch (constraint.operator) { - case "==": - for (const key in node_attributes) { - if (!isNaN(constraint.value)) { - if (Array.isArray(node_attributes[key])) { - if (node_attributes[key].includes(constraint.value) || - node_attributes[key].includes(constraint.value.toString())) { - return true; - } - } else { - if (node_attributes[key] == constraint.value || - node_attributes[key] == constraint.value.toString() || - node_attributes[key] == parseInt(constraint.value)) { - return true; - } + case '==': + for (const key in node_attributes) { + if (!isNaN(constraint.value)) { + if (Array.isArray(node_attributes[key])) { + if ( + node_attributes[key].includes(constraint.value) || + node_attributes[key].includes(constraint.value.toString()) + ) { + return true; } } else { - if (Array.isArray(node_attributes[key])) { - if (node_attributes[key].includes(constraint.value)) { - return true; - } - } else { - if (node_attributes[key] == constraint.value || - node_attributes[key] == constraint.value.toString() || - node_attributes[key] == parseInt(constraint.value)) { - return true; - } + if ( + node_attributes[key] == constraint.value || + node_attributes[key] == constraint.value.toString() || + node_attributes[key] == parseInt(constraint.value) + ) { + return true; } } - } - return false; - case ">": - for (const key in node_attributes) { + } else { if (Array.isArray(node_attributes[key])) { - for (let index = 0; index < node_attributes[key].length; index++) { - const element = node_attributes[key][index]; - if (parseInt(element) > parseInt(constraint.value)) { - return true; - } + if (node_attributes[key].includes(constraint.value)) { + return true; } } else { - if (parseInt(node_attributes[key]) > parseInt(constraint.value)) { + if ( + node_attributes[key] == constraint.value || + node_attributes[key] == constraint.value.toString() || + node_attributes[key] == parseInt(constraint.value) + ) { return true; } } } - return false; - case ">=": + } + return false; + case '>': + for (const key in node_attributes) { + if (Array.isArray(node_attributes[key])) { + for (let index = 0; index < node_attributes[key].length; index++) { + const element = node_attributes[key][index]; + if (parseInt(element) > parseInt(constraint.value)) { + return true; + } + } + } else { + if (parseInt(node_attributes[key]) > parseInt(constraint.value)) { + return true; + } + } + } + return false; + case '>=': for (const key in node_attributes) { if (Array.isArray(node_attributes[key])) { for (let index = 0; index < node_attributes[key].length; index++) { @@ -430,7 +430,7 @@ module.exports = class QEdge { } } return false; - case "<": + case '<': for (const key in node_attributes) { if (Array.isArray(node_attributes[key])) { for (let index = 0; index < node_attributes[key].length; index++) { @@ -446,7 +446,7 @@ module.exports = class QEdge { } } return false; - case "<=": + case '<=': for (const key in node_attributes) { if (Array.isArray(node_attributes[key])) { for (let index = 0; index < node_attributes[key].length; index++) { @@ -465,7 +465,7 @@ module.exports = class QEdge { default: debug(`Node operator not handled ${constraint.operator}`); return false; - }; + } } } @@ -481,7 +481,10 @@ module.exports = class QEdge { getHashedEdgeRepresentation() { const toBeHashed = - this.getInputNode().getCategories() + this.getPredicate() + this.getOutputNode().getCategories() + this.getInputCurie(); + this.getInputNode().getCategories() + + this.getPredicate() + + this.getOutputNode().getCategories() + + this.getInputCurie(); return helper._generateHash(toBeHashed); } diff --git a/src/query_node.js b/src/query_node.js index f897b343..33c07f62 100644 --- a/src/query_node.js +++ b/src/query_node.js @@ -1,240 +1,239 @@ const _ = require('lodash'); const utils = require('./utils'); const biolink = require('./biolink'); -const debug = require('debug')('bte:biothings-explorer-trapi:NewQNode'); +const debug = require('debug')('bte:biothings-explorer-trapi:QNode'); const InvalidQueryGraphError = require('./exceptions/invalid_query_graph_error'); module.exports = class QNode { - /** - * - * @param {object} info - Qnode info, e.g. ID, curie, category - */ - constructor(info) { - this.id = info.id; - this.category = info.categories || 'NamedThing'; - // mainIDs - this.curie = info.ids; - //is_set - this.is_set = info.is_set; - //mainID : its equivalent ids - this.expanded_curie = info.expanded_curie !== undefined ? info.expanded_curie : {}; - this.entity_count = info.ids ? info.ids.length : 0; - debug(`(1) Node "${this.id}" has (${this.entity_count}) entities at start.`); - //when choosing a lower entity count a node with higher count - // might be told to store its curies temporarily - this.held_curie = info.held_curie !== undefined ? info.held_curie : []; - this.held_expanded = info.held_expanded !== undefined ? info.held_expanded : {}; - //node constraints - this.constraints = info.constraints; - //list of edge ids that are connected to this node - this.connected_to = info.connected_to !== undefined ? new Set(info.connected_to) : new Set(); - //object-ify array of initial curies - if (info.expanded_curie === undefined) this.expandCurie(); - this.validateConstraints(); - } - - freeze() { - return { - category: this.category, - connected_to: Array.from(this.connected_to), - constraints: this.constraints, - curie: this.curie, - entity_count: this.entity_count, - equivalentIDs: this.equivalentIDs, - expanded_curie: this.expanded_curie, - held_curie: this.held_curie, - held_expanded: this.held_expanded, - id: this.id, - is_set: this.is_set - } - } - - isSet() { - //query node specified as set - return this.is_set ? true : false; - } - - validateConstraints() { - const required = ['id', 'operator', 'value']; - if (this.constraints && this.constraints.length) { - this.constraints.forEach((constraint) => { - let constraint_keys = Object.keys(constraint); - if (_.intersection(constraint_keys, required).length < 3) { - throw new InvalidQueryGraphError( - `Invalid constraint specification must include (${required})`); - } - }); - } - } - - expandCurie() { - if (this.curie && this.curie.length) { - this.curie.forEach((id) => { - if (!Object.hasOwnProperty.call(id, this.expanded_curie)) { - this.expanded_curie[id] = [id]; - } - }); - debug(`(1) Node "${this.id}" expanded initial curie. ${JSON.stringify(this.expanded_curie)}`); - } - } - - updateConnection(qEdgeID) { - this.connected_to.add(qEdgeID); - debug(`"${this.id}" connected to "${[...this.connected_to]}"`); - } - - getConnections() { - return [...this.connected_to]; - } - - holdCurie() { - //hold curie aside temp - debug(`(8) Node "${this.id}" holding ${JSON.stringify(this.curie)} aside.`); - this.held_curie = this.curie; - this.held_expanded = this.expanded_curie; - this.curie = undefined; - this.expanded_curie = {}; - } - - updateCuries(curies) { - // {originalID : [aliases]} - if (!this.curie) { - this.curie = []; - } - //bring back held curie - if (this.held_curie.length) { - debug(`(8) Node "${this.id}" restored curie.`); - //restore - this.curie = this.held_curie; - this.expanded_curie = this.held_expanded; - //reset holds - this.held_curie = []; - this.held_expanded = {}; - } - if (!this.curie.length) { - debug(`Node "${this.id}" saving (${Object.keys(curies).length}) curies...`); - this.curie = Object.keys(curies); - this.expanded_curie = curies; - } else { - debug(`Node "${this.id}" intersecting (${this.curie.length})/(${Object.keys(curies).length}) curies...`); - // let intersection = this.intersectCuries(this.curie, curies); - // this.curie = intersection; - // debug(`Node "${this.id}" kept (${intersection.length}) curies...`); - this.intersectWithExpandedCuries(curies); - } - this.entity_count = this.curie.length; - } - - _combineCuriesIntoList(curies) { - // curies {originalID : ['aliasID']} - //combine all curies into single list for easy intersection - let combined = new Set(); - for (const original in curies) { - !Array.isArray(curies[original]) ? - combined.add(curies[original]) : - curies[original].forEach((curie) => { - combined.add(curie); - }); - } - return [...combined]; - } - - intersectWithExpandedCuries(newCuries) { - let keep = {}; - for (const mainID in newCuries) { - let current_list_of_aliases = newCuries[mainID]; - for (const existingMainID in this.expanded_curie) { - let existing_list_of_aliases = this.expanded_curie[existingMainID]; - let idsMatchFound = _.intersection(current_list_of_aliases, existing_list_of_aliases); - if (idsMatchFound.length) { - if (!Object.hasOwnProperty.call(keep, mainID)) { - keep[mainID] = current_list_of_aliases; - } - } - } + /** + * + * @param {object} info - Qnode info, e.g. ID, curie, category + */ + constructor(info) { + this.id = info.id; + this.category = info.categories || 'NamedThing'; + // mainIDs + this.curie = info.ids; + //is_set + this.is_set = info.is_set; + //mainID : its equivalent ids + this.expanded_curie = info.expanded_curie !== undefined ? info.expanded_curie : {}; + this.entity_count = info.ids ? info.ids.length : 0; + debug(`(1) Node "${this.id}" has (${this.entity_count}) entities at start.`); + //when choosing a lower entity count a node with higher count + // might be told to store its curies temporarily + this.held_curie = info.held_curie !== undefined ? info.held_curie : []; + this.held_expanded = info.held_expanded !== undefined ? info.held_expanded : {}; + //node constraints + this.constraints = info.constraints; + //list of edge ids that are connected to this node + this.connected_to = info.connected_to !== undefined ? new Set(info.connected_to) : new Set(); + //object-ify array of initial curies + if (info.expanded_curie === undefined) this.expandCurie(); + this.validateConstraints(); + } + + freeze() { + return { + category: this.category, + connected_to: Array.from(this.connected_to), + constraints: this.constraints, + curie: this.curie, + entity_count: this.entity_count, + equivalentIDs: this.equivalentIDs, + expanded_curie: this.expanded_curie, + held_curie: this.held_curie, + held_expanded: this.held_expanded, + id: this.id, + is_set: this.is_set, + }; + } + + isSet() { + //query node specified as set + return this.is_set ? true : false; + } + + validateConstraints() { + const required = ['id', 'operator', 'value']; + if (this.constraints && this.constraints.length) { + this.constraints.forEach((constraint) => { + let constraint_keys = Object.keys(constraint); + if (_.intersection(constraint_keys, required).length < 3) { + throw new InvalidQueryGraphError(`Invalid constraint specification must include (${required})`); } - //save expanded curies (main + aliases) - this.expanded_curie = keep; - //save curies (main ids) - this.curie = Object.keys(keep); - debug(`Node "${this.id}" kept (${Object.keys(keep).length}) curies...`); - } - - intersectCuries(curies, newCuries) { - //curies is a list ['ID'] - // new curies {originalID : ['aliasID']} - let all_new_curies = this._combineCuriesIntoList(newCuries); - return _.intersection(curies, all_new_curies ); - } - - getID() { - return this.id; - } - - getCurie() { - return this.curie; + }); } + } - getEquivalentIDs() { - return this.equivalentIDs ?? {}; - } - - removeEquivalentID(id) { - delete this.equivalentIDs[id]; - } - - getCategories() { - if (this.hasEquivalentIDs() === false) { - const categories = utils.toArray(this.category); - let expanded_categories = []; - categories.map((category) => { - expanded_categories = [ - ...expanded_categories, - ...biolink.getDescendantClasses(utils.removeBioLinkPrefix(category)), - ]; - }); - return utils.getUnique(expanded_categories); + expandCurie() { + if (this.curie && this.curie.length) { + this.curie.forEach((id) => { + if (!Object.hasOwnProperty.call(id, this.expanded_curie)) { + this.expanded_curie[id] = [id]; } - let categories = []; - Object.values(this.equivalentIDs).map((entities) => { - entities.map((entity) => { - categories = [...categories, ...entity.semanticTypes.map(semantic => utils.removeBioLinkPrefix(semantic))]; - }); - }); - return utils.getUnique(categories); - } - - getEntities() { - return Object.values(this.equivalentIDs).reduce((res, entities) => { - return [...res, ...entities]; - }, []); - } - - getPrimaryIDs() { - return this.getEntities().map((entity) => entity.primaryID); - } - - setEquivalentIDs(equivalentIDs) { - this.equivalentIDs = equivalentIDs; - } - - updateEquivalentIDs(equivalentIDs) { - if (this.equivalentIDs === undefined) { - this.equivalentIDs = equivalentIDs; - } else { - this.equivalentIDs = { ...this.equivalentIDs, ...equivalentIDs }; + }); + debug(`(1) Node "${this.id}" expanded initial curie. ${JSON.stringify(this.expanded_curie)}`); + } + } + + updateConnection(qEdgeID) { + this.connected_to.add(qEdgeID); + debug(`"${this.id}" connected to "${[...this.connected_to]}"`); + } + + getConnections() { + return [...this.connected_to]; + } + + holdCurie() { + //hold curie aside temp + debug(`(8) Node "${this.id}" holding ${JSON.stringify(this.curie)} aside.`); + this.held_curie = this.curie; + this.held_expanded = this.expanded_curie; + this.curie = undefined; + this.expanded_curie = {}; + } + + updateCuries(curies) { + // {originalID : [aliases]} + if (!this.curie) { + this.curie = []; + } + //bring back held curie + if (this.held_curie.length) { + debug(`(8) Node "${this.id}" restored curie.`); + //restore + this.curie = this.held_curie; + this.expanded_curie = this.held_expanded; + //reset holds + this.held_curie = []; + this.held_expanded = {}; + } + if (!this.curie.length) { + debug(`Node "${this.id}" saving (${Object.keys(curies).length}) curies...`); + this.curie = Object.keys(curies); + this.expanded_curie = curies; + } else { + debug(`Node "${this.id}" intersecting (${this.curie.length})/(${Object.keys(curies).length}) curies...`); + // let intersection = this.intersectCuries(this.curie, curies); + // this.curie = intersection; + // debug(`Node "${this.id}" kept (${intersection.length}) curies...`); + this.intersectWithExpandedCuries(curies); + } + this.entity_count = this.curie.length; + } + + _combineCuriesIntoList(curies) { + // curies {originalID : ['aliasID']} + //combine all curies into single list for easy intersection + let combined = new Set(); + for (const original in curies) { + !Array.isArray(curies[original]) + ? combined.add(curies[original]) + : curies[original].forEach((curie) => { + combined.add(curie); + }); + } + return [...combined]; + } + + intersectWithExpandedCuries(newCuries) { + let keep = {}; + for (const mainID in newCuries) { + let current_list_of_aliases = newCuries[mainID]; + for (const existingMainID in this.expanded_curie) { + let existing_list_of_aliases = this.expanded_curie[existingMainID]; + let idsMatchFound = _.intersection(current_list_of_aliases, existing_list_of_aliases); + if (idsMatchFound.length) { + if (!Object.hasOwnProperty.call(keep, mainID)) { + keep[mainID] = current_list_of_aliases; + } } + } } - - hasInput() { - return !(this.curie === undefined || this.curie === null); - } - - hasEquivalentIDs() { - return !(typeof this.equivalentIDs === 'undefined' || this.equivalentIDs === {}); - } - - getEntityCount() { - return this.curie ? this.curie.length : 0; - } + //save expanded curies (main + aliases) + this.expanded_curie = keep; + //save curies (main ids) + this.curie = Object.keys(keep); + debug(`Node "${this.id}" kept (${Object.keys(keep).length}) curies...`); + } + + intersectCuries(curies, newCuries) { + //curies is a list ['ID'] + // new curies {originalID : ['aliasID']} + let all_new_curies = this._combineCuriesIntoList(newCuries); + return _.intersection(curies, all_new_curies); + } + + getID() { + return this.id; + } + + getCurie() { + return this.curie; + } + + getEquivalentIDs() { + return this.equivalentIDs ?? {}; + } + + removeEquivalentID(id) { + delete this.equivalentIDs[id]; + } + + getCategories() { + if (this.hasEquivalentIDs() === false) { + const categories = utils.toArray(this.category); + let expanded_categories = []; + categories.map((category) => { + expanded_categories = [ + ...expanded_categories, + ...biolink.getDescendantClasses(utils.removeBioLinkPrefix(category)), + ]; + }); + return utils.getUnique(expanded_categories); + } + let categories = []; + Object.values(this.equivalentIDs).map((entities) => { + entities.map((entity) => { + categories = [...categories, ...entity.semanticTypes.map((semantic) => utils.removeBioLinkPrefix(semantic))]; + }); + }); + return utils.getUnique(categories); + } + + getEntities() { + return Object.values(this.equivalentIDs).reduce((res, entities) => { + return [...res, ...entities]; + }, []); + } + + getPrimaryIDs() { + return this.getEntities().map((entity) => entity.primaryID); + } + + setEquivalentIDs(equivalentIDs) { + this.equivalentIDs = equivalentIDs; + } + + updateEquivalentIDs(equivalentIDs) { + if (this.equivalentIDs === undefined) { + this.equivalentIDs = equivalentIDs; + } else { + this.equivalentIDs = { ...this.equivalentIDs, ...equivalentIDs }; + } + } + + hasInput() { + return !(this.curie === undefined || this.curie === null); + } + + hasEquivalentIDs() { + return !(typeof this.equivalentIDs === 'undefined' || this.equivalentIDs === {}); + } + + getEntityCount() { + return this.curie ? this.curie.length : 0; + } }; diff --git a/src/update_nodes.js b/src/update_nodes.js index c530c459..019a1968 100644 --- a/src/update_nodes.js +++ b/src/update_nodes.js @@ -2,27 +2,24 @@ const id_resolver = require('biomedical_id_resolver'); const debug = require('debug')('bte:biothings-explorer-trapi:nodeUpdateHandler'); module.exports = class NodesUpdateHandler { - constructor(qXEdges) { - this.qXEdges = qXEdges; + constructor(qEdges) { + this.qEdges = qEdges; } /** * @private * s */ - _getCuries(qXEdges) { + _getCuries(qEdges) { let curies = {}; - qXEdges.map((qXEdge) => { - if (qXEdge.hasInputResolved()) { - return; - } - if (qXEdge.hasInput()) { - const inputCategories = qXEdge.getInputNode().getCategories(); + qEdges.map((qEdge) => { + if (qEdge.hasInput()) { + const inputCategories = qEdge.getInputNode().getCategories(); inputCategories.map((category) => { if (!(category in curies)) { curies[category] = []; } - curies[category] = [...curies[category], ...qXEdge.getInputCurie()]; + curies[category] = [...curies[category], ...qEdge.getInputCurie()]; }); } }); @@ -40,20 +37,20 @@ module.exports = class NodesUpdateHandler { return equivalentIDs; } - async setEquivalentIDs(qXEdges) { + async setEquivalentIDs(qEdges) { debug(`Getting equivalent IDs...`); - const curies = this._getCuries(this.qXEdges); + const curies = this._getCuries(this.qEdges); debug(`curies: ${JSON.stringify(curies)}`); const equivalentIDs = await this._getEquivalentIDs(curies); - qXEdges.map((qXEdge) => { + qEdges.map((qEdge) => { const edgeEquivalentIDs = Object.keys(equivalentIDs) - .filter((key) => qXEdge.getInputCurie().includes(key)) + .filter((key) => qEdge.getInputCurie().includes(key)) .reduce((res, key) => { return { ...res, [key]: equivalentIDs[key] }; }, {}); - debug(`Got Edge Equivalent IDs successfully.`); + debug(`Got Edge Equivalent IDs successfully.`); if (Object.keys(edgeEquivalentIDs).length > 0) { - qXEdge.getInputNode().setEquivalentIDs(edgeEquivalentIDs); + qEdge.getInputNode().setEquivalentIDs(edgeEquivalentIDs); } }); return; @@ -80,11 +77,8 @@ module.exports = class NodesUpdateHandler { // ); // }) queryRecords.map((record) => { - if ( - record && - !(record.object.curie in record.qXEdge.getOutputNode().getEquivalentIDs()) - ) { - record.qXEdge.getOutputNode().updateEquivalentIDs({[record.object.curie]: record.object.normalizedInfo}); + if (record && !(record.object.curie in record.qEdge.getOutputNode().getEquivalentIDs())) { + record.qEdge.getOutputNode().updateEquivalentIDs({ [record.object.curie]: record.object.normalizedInfo }); } }); }