Updated the genes route to support multiple data sources and multiple…

… gene names per source via URL matrix parameters. This revealed many bugs that were subsequently fixed: Get effects now use unique action IDs to determine if a service call has actually been made for a requested entity. The Levenshtein distance metric now uses memoization to prevent deep redundant recursion. The gene famiy character set that's used to construct the MSA HMM is now being filtered by how many times families occur in the set of tracks. All families excluded from the character set are now added to the omit set used during model training. The MSA now correctly handles reverse oriented alignments internally. MSA alignments that contain inversions are no longer throwing an error when they are converted to MSA column coordinates.

client/src/app/core/utils/counter.util.ts

@@ -0,0 +1,12 @@
+// keeps an internal count that is incremented at each get call
+class Counter {
+
+  private _count = 0;
+
+  getCount() {
+    return this._count++;
+  }
+}
+
+// singleton
+export const counter = new Counter();

client/src/app/core/utils/index.ts

@@ -1,5 +1,6 @@
 import { arrayFlatten } from './array-flatten.util';
 import { arrayIsEqual } from './array-is-equal.util';
+import { counter } from './counter.util';
 import { elementIsVisible } from './element-is-visible.util';
 import { memoizeArray } from './memoize-array.util';
 import { saveFile } from './save-file.util';
@@ -8,6 +9,7 @@ import { saveFile } from './save-file.util';
 export const utils: any[] = [
   arrayFlatten,
   arrayIsEqual,
+  counter,
   elementIsVisible,
   memoizeArray,
   saveFile,
@@ -16,6 +18,7 @@ export const utils: any[] = [
 
 export * from './array-flatten.util';
 export * from './array-is-equal.util';
+export * from './counter.util';
 export * from './element-is-visible.util';
 export * from './memoize-array.util';
 export * from './save-file.util';

client/src/app/gene/components/viewers/micro/micro.shim.ts

@@ -84,7 +84,7 @@ function _tracksToData(tracks, genes) {
 export function microShim(clusterID, queryTracks, tracks, genes) {
   // create data
   const filteredTracks = tracks.filter((t) => t.cluster == clusterID);
-  const data = _tracksToData(tracks, genes);
+  const data = _tracksToData(filteredTracks, genes);
   // identify bold tracks
   const bold = [];
   filteredTracks.forEach((t, i) => {

client/src/app/gene/gene-routing.module.ts

@@ -7,9 +7,13 @@ import { GeneComponent, HeaderLeftComponent, HeaderRightComponent }
 
 export const routes: Routes = [
   {
-    path: ':source/:gene',
+    path: '',
     component: GeneComponent,
   },
+  //{
+  //  path: ':source/:gene',
+  //  component: GeneComponent,
+  //},
   {
     path: '',
     component: HeaderLeftComponent,

client/src/app/gene/models/params/clustering.model.ts

@@ -11,7 +11,7 @@ export class ClusteringParams implements Params {
 
   constructor(
     public linkage: string = 'average',  // TODO: remove magic string
-    public threshold: number = 10) {
+    public threshold: number = 20) {
   }
 
   formControls(): any {

client/src/app/gene/models/shims/family-count-map.ts

@@ -0,0 +1,14 @@
+export type FamilyCountMap = {[key: string]: number};
+
+
+export function familyCountMap(families: string[]): FamilyCountMap {
+  const reducer = (accumulator, family) => {
+      if (!(family in accumulator)) {
+        accumulator[family] = 0;
+      }
+      accumulator[family] += 1;
+      return accumulator;
+    };
+  const map = families.reduce(reducer, {});
+  return map;
+}

client/src/app/gene/models/shims/index.ts

@@ -1,5 +1,6 @@
 export * from './block-index-map';
 export * from './endpoint-genes';
+export * from './family-count-map';
 export * from './gene-map';
 export * from './macro-blocks';
 export * from './name-source-id';

client/src/app/gene/store/actions/chromosome.actions.ts

@@ -1,4 +1,5 @@
 import { Action } from '@ngrx/store';
+import { counter } from '@gcv/core/utils';
 import { Track } from '@gcv/gene/models';
 
 export const GET = '[CHROMOSOME] GET';
@@ -7,6 +8,7 @@ export const GET_FAILURE = '[CHROMOSOME] GET_FAILURE';
 
 export class Get implements Action {
   readonly type = GET;
+  readonly id = counter.getCount();
   constructor(public payload: {name: string, source: string}) { }
 }
 

client/src/app/gene/store/actions/gene.actions.ts

@@ -1,12 +1,15 @@
 import { Action } from '@ngrx/store';
+import { counter } from '@gcv/core/utils';
 import { Gene, Track } from '@gcv/gene/models';
 
 export const GET = '[GENE] GET';
 export const GET_SUCCESS = '[GENE] GET_SUCCESS';
 export const GET_FAILURE = '[GENE] GET_FAILURE';
 
+
 export class Get implements Action {
   readonly type = GET;
+  readonly id = counter.getCount();
   constructor(public payload: {names: string[], source: string}) { }
 }
 

client/src/app/gene/store/effects/chromosome.effects.ts

@@ -3,11 +3,13 @@ import { Injectable } from '@angular/core';
 // store
 import { Effect, Actions, ofType } from '@ngrx/effects';
 import { of } from 'rxjs';
-import { catchError, filter, map, mergeMap, switchMap } from 'rxjs/operators';
+import { catchError, filter, map, mergeMap, switchMap, withLatestFrom }
+  from 'rxjs/operators';
 import { Store } from '@ngrx/store';
 import * as chromosomeActions from '@gcv/gene/store/actions/chromosome.actions';
 import * as fromRoot from '@gcv/gene/store/reducers';
 import * as fromChromosome from '@gcv/gene/store/selectors/chromosome/';
+import { TrackID, trackID } from '@gcv/gene/store/utils';
 // app
 import { ChromosomeService } from '@gcv/gene/services';
 
@@ -24,15 +26,28 @@ export class ChromosomeEffects {
   getSelected$ = this.store
   .select(fromChromosome.getUnloadedSelectedChromosomeIDs).pipe(
     filter((ids) => ids.length > 0),
-    switchMap((ids) =>  ids.map((id) => new chromosomeActions.Get(id))),
+    mergeMap((ids) => ids.map((id) => new chromosomeActions.Get(id))),
   );
 
   // get chromosome via the chromosome service
   @Effect()
   getChromosome$ = this.actions$.pipe(
     ofType(chromosomeActions.GET),
-    map((action: chromosomeActions.Get) => action.payload),
-    mergeMap(({name, source}) => {
+    map((action: chromosomeActions.Get) => {
+      return {action: action.id, ...action.payload};
+    }),
+    withLatestFrom(this.store.select(fromChromosome.getLoading)),
+    mergeMap(([{action, name, source}, loading]) => {
+      // get loaded/loading genes
+      const actionTrackID =
+        ({name, source, action}) => `${trackID(name, source)}:${action}`;
+      const loadingIDs = new Set(loading.map(actionTrackID));
+      // only load chromosome if action is loading
+      const id = actionTrackID({action, name, source});
+      const chromosomeID = trackID(name, source);
+      if (!loadingIDs.has(id)) {
+        return [];
+      }
       return this.chromosomeService.getChromosome(name, source).pipe(
         map((chromosome) => {
           return new chromosomeActions.GetSuccess({chromosome});

client/src/app/gene/store/effects/gene.effects.ts

@@ -29,34 +29,45 @@ export class GeneEffects {
   getSelected$ = this.store.select(fromGene.getUnloadedSelectedGeneIDs).pipe(
     filter((ids) => ids.length > 0),
     switchMap((ids) => {
-      const id2action = ({name, source}) => {
-          return new geneActions.Get({names: [name], source});
+      // bin ids by source
+      const reducer = (accumulator, {name, source}) => {
+          if (!(source in accumulator)) {
+            accumulator[source] = [];
+          }
+          accumulator[source].push(name);
+          return accumulator;
         };
-      const actions = ids.map(id2action);
+      const geneBins = ids.reduce(reducer, {});
+      // make a gene request action for each source bin
+      const actions = Object.entries(geneBins)
+        .map(([source, names]: [string, string[]]) => {
+          return new geneActions.Get({names, source});
+        });
       return actions;
     }),
   );
 
-  // TODO: when loading multi route, take loading and failed genes into
-  // consideration
-
   // get genes via the gene service
   @Effect()
   getGenes$ = this.actions$.pipe(
     ofType(geneActions.GET),
-    map((action: geneActions.Get) => action.payload),
-    withLatestFrom(this.store.select(fromGene.getGeneState)),
-    concatMap(([{names, source}, state]) => {
+    map((action: geneActions.Get) => ({action: action.id, ...action.payload})),
+    withLatestFrom(this.store.select(fromGene.getLoading)),
+    concatMap(([{action, names, source}, loading]) => {
       // get loaded/loading genes
-      const loadingIDs = new Set(state.loading.map(geneID));
-      const loadedIDs = new Set(state.loaded.map(geneID));
-      // filter out genes that are already loaded or loading
-      const filteredNamess = names.filter((n) => {
-          const gID = geneID(n, source);
-          return !loadingIDs.has(gID) && !loadedIDs.has(gID);
+      const actionGeneID =
+        ({name, source, action}) => `${geneID(name, source)}:${action}`;
+      const loadingIDs = new Set(loading.map(actionGeneID));
+      // only keep genes whose action is loaded
+      const filteredNames = names.filter((name) => {
+          const id = actionGeneID({name, source, action});
+          return loadingIDs.has(id);
         });
       // get the genes
-      return this.geneService.getGenes(names, source).pipe(
+      if (filteredNames.length == 0) {
+        return [];
+      }
+      return this.geneService.getGenes(filteredNames, source).pipe(
         map((genes: Gene[]) =>  new geneActions.GetSuccess({genes})),
         catchError((error) => of(new geneActions.GetFailure({names, source})))
       );
@@ -66,9 +77,7 @@ export class GeneEffects {
   // get all genes for the selected micro-tracks
   @Effect()
   getMicroTrackGenes$ = this.store.select(fromMicroTracks.getAllMicroTracks).pipe(
-    switchMap((tracks) => {
-      return geneActions.tracksToGetGeneActions(tracks);
-    }),
+    switchMap((tracks) => geneActions.tracksToGetGeneActions(tracks)),
   );
 
 }

client/src/app/gene/store/reducers/chromosome.reducer.ts

@@ -1,17 +1,16 @@
 // A chromosome is an instance of Track that represents an entire chromosome
 // as an ordered list of genes and a corresponding list of gene families. This
-// file provides an NgRx reducer and selectors for storing and accessing
-// chromosome data. Specifically, a chromosome is loaded as a Track for each
-// gene provided by the user. These Tracks are stored by the chromosome reducer
-// and made available via selectors. This includes a selector that provides the
-// neighborhood each user provided gene occurs is as a slice of the gene's
-// chromosome.
+// file provides an NgRx reducer for storing chromosome data. Specifically, a
+// chromosome is loaded as a Track for each gene provided by the user. These
+// Tracks are stored by the chromosome reducer and made available via selectors.
+// This includes a selector that provides the neighborhood each user provided
+// gene occurs is as a slice of the gene's chromosome - a micro-track.
 
 // NgRx
 import { createEntityAdapter, EntityState } from '@ngrx/entity';
 // store
 import * as chromosomeActions from '@gcv/gene/store/actions/chromosome.actions';
-import { TrackID, trackID } from '@gcv/gene/store/utils';
+import { ActionID, TrackID, trackID } from '@gcv/gene/store/utils';
 // app
 import { Track } from '@gcv/gene/models';
 
@@ -23,17 +22,14 @@ const adapter = createEntityAdapter<Track>({
   selectId: (e) => trackID(e.name, e.source)
 });
 
-// TODO: is loaded even necessary or can it be derived from entity ids and
 // selectedChromosomeIDs selector?
 export interface State extends EntityState<Track> {
   failed: TrackID[];
-  loaded: TrackID[];
-  loading: TrackID[];
+  loading: (TrackID & ActionID)[];
 }
 
 const initialState: State = adapter.getInitialState({
   failed: [],
-  loaded: [],
   loading: [],
 });
 
@@ -43,9 +39,20 @@ export function reducer(
 ): State {
   switch (action.type) {
     case chromosomeActions.GET:
+      const loadingIDs = new Set(state.loading.map(trackID));
+      const id = trackID(action.payload);
+      const loading = [];
+      if (!loadingIDs.has(id) && !(id in state.entities)) {
+        loading.push({action: action.id, ...action.payload});
+      }
+      const failed = state.failed.filter(({name, source}) => {
+          const fID = trackID(name, source);
+          return fID === id;
+        });
       return {
         ...state,
-        loading: state.loading.concat([action.payload]),
+        loading: state.loading.concat(loading),
+        failed,
       };
     case chromosomeActions.GET_SUCCESS:
     {
@@ -55,7 +62,6 @@ export function reducer(
         chromosome,
         {
           ...state,
-          loaded: state.loaded.concat(id),
           loading: state.loading.filter(({name, source}) => {
             return !(name === id.name && source === id.source);
           }),

client/src/app/gene/store/reducers/gene.reducer.ts

@@ -1,15 +1,16 @@
 // A Gene is a second class citizen in the GCV, that is, other than dictating
 // what chromosomes are loaded, all visualizations, algorithms, and auxiliary
 // models are derived from the gene families of the Track model. As such, genes
-// are loaded on an as needed basis. This file contains an NgRx reducer and
-// selectors for storing and accessing Genes.
+// are loaded on an as needed basis. This file contains an NgRx reducer for
+// storing Genes.
 
 // NgRx
 import { createEntityAdapter, EntityState } from '@ngrx/entity';
 // store
 import * as geneActions from '@gcv/gene/store/actions/gene.actions';
 // app
 import { Gene } from '@gcv/gene/models';
+import { ActionID } from '@gcv/gene/store/utils';
 
 declare var Object: any;  // because TypeScript doesn't support Object.values
 
@@ -32,17 +33,14 @@ const adapter = createEntityAdapter<Gene>({
   selectId: (e) => geneID(e.name, e.source)
 });
 
-// TODO: is loaded even necessary or can it be derived from entity ids and
 // selectedGeneIDs selector?
 export interface State extends EntityState<Gene> {
   failed: GeneID[];
-  loaded: GeneID[];
-  loading: GeneID[];
+  loading: (GeneID & ActionID)[];
 }
 
 const initialState: State = adapter.getInitialState({
   failed: [],
-  loaded: [],
   loading: [],
 });
 
@@ -53,10 +51,24 @@ export function reducer(
   switch (action.type) {
     case geneActions.GET:
       const source = action.payload.source;
-      const loading = action.payload.names.map((name) => ({name, source}));
+      const loadingIDs = new Set(state.loading.map(geneID));
+      const loading = action.payload.names
+        .filter((name) => {
+          const id = geneID(name, source);
+          return !loadingIDs.has(id) && !(id in state.entities);
+        })
+        .map((name) => {
+          return {name, source, action: action.id};
+        });
+      const newLoadingIDs = new Set(loading.map(geneID));
+      const failed = state.failed.filter(({name, source}) => {
+          const id = geneID(name, source);
+          return !newLoadingIDs.has(id);
+        });
       return {
         ...state,
         loading: state.loading.concat(loading),
+        failed,
       };
     case geneActions.GET_SUCCESS:
     {
@@ -72,7 +84,6 @@ export function reducer(
         genes,
         {
           ...state,
-          loaded: state.loaded.concat(loaded),
           loading,
         },
       );

client/src/app/gene/store/selectors/chromosome/chromosome-state.selector.ts

@@ -10,3 +10,9 @@ export const getChromosomeState = createSelector(
   fromModule.getGeneModuleState,
   state => state[chromosomeFeatureKey]
 );
+
+
+export const getLoading = createSelector(
+  getChromosomeState,
+  state => state.loading,
+);