Offline methods (#1019)

* feat(core): add a compression service for blob * feat(geo): adding service to store/retrieve data within indexed-db * wip compression handling for non blob object * wip * feat(vector): method to retrieve data from online or offline source (indexeddb) * wip - adjust vector layer custom loader * wip * chore(*): ngx-indexed-db version update Co-authored-by: Pierre-Étienne Lord <[email protected]>
infra-geo-ouverte · Mar 21, 2023 · 7d405b7 · 7d405b7
1 parent c43ae61
commit 7d405b7
Show file tree

Hide file tree

Showing 17 changed files with 508 additions and 5 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -113,6 +113,7 @@
     "jwt-decode": "^2.2.0",
     "moment": "^2.29.2",
     "ngx-color-picker": "^10.1.0",
+    "ngx-indexed-db": "^11.0.2",
     "ngx-toastr": "^14.1.3",
     "nosleep.js": "^0.12.0",
     "ol": "^6.8.1",

diff --git a/packages/core/src/lib/compression/compressedData.interface.ts b/packages/core/src/lib/compression/compressedData.interface.ts
@@ -0,0 +1,5 @@
+export interface CompressedData {
+    length: number;
+    type: string;
+    object: any;
+}
diff --git a/packages/core/src/lib/compression/compression.service.ts b/packages/core/src/lib/compression/compression.service.ts
@@ -0,0 +1,143 @@
+import { Injectable } from '@angular/core';
+import { Observable, Observer } from 'rxjs';
+import { CompressedData } from './compressedData.interface';
+
+function getNumber(v: number, endposition: number, length: number) {
+  /* tslint:disable:no-bitwise*/
+  const mask = ((1 << length) - 1);
+  return (v >> endposition) & mask;
+  /* tslint:enable:no-bitwise*/
+}
+
+@Injectable({
+  providedIn: 'root'
+})
+export class CompressionService {
+  private base64Index = new Map<string, number>();
+  private indexBase64 = new Map<number, string>();
+
+  constructor() {
+    this.generateBase64Index();
+  }
+
+  private generateBase64Index() {
+    // https://fr.wikipedia.org/wiki/Base64
+    // A-Z => [0, 25]
+    for (let i = 0; i < 26; i++) {
+      this.base64Index.set(String.fromCharCode('A'.charCodeAt(0) + i), i);
+      this.indexBase64.set(i, String.fromCharCode('A'.charCodeAt(0) + i));
+    }
+    // a-z => [26, 51]
+    for (let i = 0; i < 26; i++) {
+      this.base64Index.set(String.fromCharCode('a'.charCodeAt(0) + i), i + 26);
+      this.indexBase64.set(i + 26, String.fromCharCode('a'.charCodeAt(0) + i));
+    }
+    // 0-9 => [52, 61]
+    for (let i = 0; i < 10; i++) {
+      this.base64Index.set(String.fromCharCode('0'.charCodeAt(0) + i), i + 52);
+      this.indexBase64.set(i + 52, String.fromCharCode('0'.charCodeAt(0) + i));
+    }
+    // + / => [62, 63]
+    this.base64Index.set('+', 62);
+    this.base64Index.set('/', 63);
+    this.indexBase64.set(62, '+');
+    this.indexBase64.set(63, '/');
+  }
+
+  compressBlob(blob: Blob): Observable<CompressedData> {
+    if (!blob) {
+      return;
+    }
+
+    const observable = new Observable((observer: Observer<CompressedData>) => {
+      const reader = new FileReader();
+      reader.readAsDataURL(blob);
+      reader.onload = () => {
+        const base64 = reader.result.valueOf() as string;
+        const text64 = base64.substr(base64.indexOf(',') + 1);
+        const compressed = this.compressStringBase64(text64);
+        const compressedData: CompressedData = { length: text64.length,
+                                                 type: blob.type,
+                                                 object: compressed };
+        observer.next(compressedData);
+      };
+    });
+    return observable;
+  }
+
+  decompressBlob(compressedData: CompressedData): Blob {
+    /* tslint:disable:no-bitwise*/
+    const object = compressedData.object;
+    const length = compressedData.length;
+    const decompressed: string = this.decompressStringBase64(object, length);
+    const byteCharacters = atob(decompressed);
+    const byteNumbers = new Array(byteCharacters.length);
+    for (let i = 0; i < byteCharacters.length; i++) {
+        byteNumbers[i] = byteCharacters.charCodeAt(i);
+    }
+    const byteArray = new Uint8Array(byteNumbers);
+    const blob = new Blob([byteArray], {type: compressedData.type});
+    /* tslint:enable:no-bitwise*/
+    return blob;
+  }
+
+  private compressStringBase64(s: string): string {
+    /* tslint:disable:no-bitwise*/
+    let out = '';
+    let bits = 16;
+    let chr = 0;
+    let rem = 0;
+    for (const c of s) {
+      const value = this.base64Index.get(c);
+      if (bits > 6) {
+        bits -= 6;
+        chr += value << bits;
+      } else {
+        rem = 6 - bits;
+        chr += value >> rem;
+        out += String.fromCharCode(chr);
+        chr = (value) << (16 - rem);
+        bits = 16 - rem;
+      }
+    }
+    if (s.length % 8 !== 0) {
+      out += String.fromCharCode(chr);
+    }
+    /* tslint:enable:no-bitwise*/
+    return String.fromCharCode(9731) + out;
+  }
+
+  private decompressStringBase64(c: string, length: number): string {
+     /* tslint:disable:no-bitwise*/
+    if (!c) {
+      return;
+    }
+
+    if (c.charCodeAt(0) !== 9731) {
+      return c;
+    }
+
+    let chr = 0;
+    let rem = 0;
+    let bits = 16;
+    let out = '';
+    let j = 1;
+    let value = c.charCodeAt(j);
+    for (let i = 0; i < length; i++) {
+      if (bits > 6) {
+        bits -= 6;
+        chr = getNumber(value, bits, 6);
+        out += this.indexBase64.get(chr);
+      } else {
+        rem = 6 - bits;
+        chr = getNumber(value, 0, bits) << rem;
+        value = c.charCodeAt(++j);
+        chr += getNumber(value, 16 - rem, rem);
+        out += this.indexBase64.get(chr);
+        bits = 16 - rem;
+      }
+    }
+    return out;
+    /* tslint:enable:no-bitwise*/
+  }
+}
diff --git a/packages/core/src/lib/compression/index.ts b/packages/core/src/lib/compression/index.ts
@@ -0,0 +1,2 @@
+export * from './compressedData.interface';
+export * from './compression.service';
diff --git a/packages/core/src/lib/core.module.ts b/packages/core/src/lib/core.module.ts
@@ -9,7 +9,19 @@ import { IgoConfigModule } from './config/config.module';
 import { IgoLanguageModule } from './language/language.module';
 import { IgoMessageModule } from './message/message.module';
 import { IgoErrorModule } from './request/error.module';
+import { DBConfig, NgxIndexedDBModule } from 'ngx-indexed-db';
 
+const dbConfig: DBConfig = {
+  name: 'igo2DB',
+  version: 1,
+  objectStoresMeta: [{
+    store: 'geoData',
+    storeConfig: { keyPath: 'url', autoIncrement: false },
+    storeSchema: [
+      { name: 'regionID', keypath: 'regionID', options: { unique: false }}
+    ]
+  }]
+};
 @NgModule({
   imports: [
     CommonModule,
@@ -18,7 +30,8 @@ import { IgoErrorModule } from './request/error.module';
     IgoConfigModule.forRoot(),
     IgoErrorModule.forRoot(),
     IgoLanguageModule.forRoot(),
-    IgoMessageModule.forRoot()
+    IgoMessageModule.forRoot(),
+    NgxIndexedDBModule.forRoot(dbConfig)
   ],
   declarations: [],
   exports: [

diff --git a/packages/core/src/public_api.ts b/packages/core/src/public_api.ts
@@ -25,4 +25,5 @@ export * from './lib/media';
 export * from './lib/message';
 export * from './lib/request';
 export * from './lib/storage';
+export * from './lib/compression';
 export * from './lib/network';
diff --git a/packages/geo/src/lib/layer/shared/layer.service.ts b/packages/geo/src/lib/layer/shared/layer.service.ts
@@ -43,6 +43,7 @@ import {
 import { computeMVTOptionsOnHover } from '../utils/layer.utils';
 import { StyleService } from './style.service';
 import { LanguageService, MessageService } from '@igo2/core';
+import { GeoNetworkService } from '../../offline/shared/geo-network.service';
 import { StyleLike as OlStyleLike } from 'ol/style/Style';
 
 @Injectable({
@@ -53,6 +54,7 @@ export class LayerService {
     private http: HttpClient,
     private styleService: StyleService,
     private dataSourceService: DataSourceService,
+    private geoNetwork: GeoNetworkService,
     private messageService: MessageService,
     private languageService: LanguageService,
     @Optional() private authInterceptor: AuthInterceptor
@@ -152,7 +154,7 @@ export class LayerService {
           resolution
         );
       };
-      igoLayer = new VectorLayer(layerOptions, this.messageService, this.authInterceptor);
+      igoLayer = new VectorLayer(layerOptions, this.messageService, this.authInterceptor, this.geoNetwork);
     }
 
     if (layerOptions.source instanceof ClusterDataSource) {
@@ -166,15 +168,15 @@ export class LayerService {
           baseStyle
         );
       };
-      igoLayer = new VectorLayer(layerOptions, this.messageService, this.authInterceptor);
+      igoLayer = new VectorLayer(layerOptions, this.messageService, this.authInterceptor, this.geoNetwork);
     }
 
     const layerOptionsOl = Object.assign({}, layerOptions, {
       style
     });
 
     if (!igoLayer) {
-      igoLayer = new VectorLayer(layerOptionsOl, this.messageService, this.authInterceptor);
+      igoLayer = new VectorLayer(layerOptionsOl, this.messageService, this.authInterceptor, this.geoNetwork);
     }
 
     this.applyMapboxStyle(igoLayer, layerOptionsOl as any);

diff --git a/packages/geo/src/lib/layer/shared/layers/vector-layer.ts b/packages/geo/src/lib/layer/shared/layers/vector-layer.ts
@@ -25,6 +25,10 @@ import { MessageService } from '@igo2/core';
 import { WFSDataSourceOptions } from '../../../datasource/shared/datasources/wfs-datasource.interface';
 import { buildUrl, defaultMaxFeatures } from '../../../datasource/shared/datasources/wms-wfs.utils';
 import { OgcFilterableDataSourceOptions } from '../../../filter/shared/ogc-filter.interface';
+import { GeoNetworkService, SimpleGetOptions } from '../../../offline/shared/geo-network.service';
+import { catchError, concatMap, first } from 'rxjs/operators';
+import { GeoDBService } from '../../../offline/geoDB/geoDB.service';
+import { of } from 'rxjs';
 export class VectorLayer extends Layer {
   public dataSource:
     | FeatureDataSource
@@ -48,7 +52,9 @@ export class VectorLayer extends Layer {
   constructor(
     options: VectorLayerOptions,
     public messageService?: MessageService,
-    public authInterceptor?: AuthInterceptor
+    public authInterceptor?: AuthInterceptor,
+    private geoNetworkService?: GeoNetworkService,
+    private geoDBService?: GeoDBService
   ) {
     super(options, messageService, authInterceptor);
     this.watcher = new VectorWatcher(this);
@@ -392,6 +398,57 @@ export class VectorLayer extends Layer {
     } else {
       modifiedUrl = url(extent, resolution, projection);
     }
+
+    if (this.geoNetworkService && typeof url !== 'function') {
+      const format = vectorSource.getFormat();
+      const type = format.getType();
+
+      let responseType = type;
+      const onError = () => {
+        vectorSource.removeLoadedExtent(extent);
+        failure();
+      };
+
+      const options: SimpleGetOptions = { responseType };
+      this.geoNetworkService.geoDBService.get(url).pipe(concatMap(r =>
+        r ? of(r) : this.geoNetworkService.get(modifiedUrl, options)
+          .pipe(
+            first(),
+            catchError((res) => {
+              onError();
+              throw res;
+            })
+          )
+      ))
+      .subscribe((content) => {
+          if (content) {
+            const format = vectorSource.getFormat();
+            const type = format.getType();
+            let source;
+            if (type === FormatType.JSON || type === FormatType.TEXT) {
+              source = content;
+            } else if (type === FormatType.XML) {
+              source = content;
+              if (!source) {
+                source = new DOMParser().parseFromString(
+                  content,
+                  'application/xml'
+                );
+              }
+            } else if (type === FormatType.ARRAY_BUFFER) {
+              source = content;
+            }
+            if (source) {
+              const features = format.readFeatures(source, { extent, featureProjection: projection });
+              vectorSource.addFeatures(features, format.readProjection(source));
+              success(features);
+            } else {
+              onError();
+            }
+          }
+        });
+
+    } else {
     xhr.open( 'GET', modifiedUrl);
     const format = vectorSource.getFormat();
     if (format.getType() === FormatType.ARRAY_BUFFER) {
@@ -437,4 +494,5 @@ export class VectorLayer extends Layer {
     };
     xhr.send();
   }
+  }
 }
diff --git a/packages/geo/src/lib/offline/geoDB/geoDB.enums.ts b/packages/geo/src/lib/offline/geoDB/geoDB.enums.ts
@@ -0,0 +1,4 @@
+export enum InsertSourceInsertDBEnum {
+    System = 'system',
+    User = 'user'
+  }
diff --git a/packages/geo/src/lib/offline/geoDB/geoDB.interface.ts b/packages/geo/src/lib/offline/geoDB/geoDB.interface.ts
@@ -0,0 +1,10 @@
+import { InsertSourceInsertDBEnum } from "./geoDB.enums";
+
+export interface GeoDBData {
+    url: string;
+    regionID: number;
+    object: any;
+    compressed: boolean;
+    insertSource: InsertSourceInsertDBEnum;
+    insertEvent: string;
+}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		export * from './compressedData.interface';
		export * from './compression.service';