diff --git a/package-lock.json b/package-lock.json index 4dc72d7161..7d8d52bef2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14313,6 +14313,14 @@ "tslib": "^2.0.0" } }, + "ngx-indexed-db": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/ngx-indexed-db/-/ngx-indexed-db-11.0.2.tgz", + "integrity": "sha512-CC0E8PyENobmLxw1XGgQdhFQMfqOU0u4UtzD3NlSMNBw/5gxc15xWt4XGjfx92OtZtRRfrMIO54lq1pJ+AwyYg==", + "requires": { + "tslib": "^2.0.0" + } + }, "ngx-toastr": { "version": "14.1.3", "resolved": "https://registry.npmjs.org/ngx-toastr/-/ngx-toastr-14.1.3.tgz", diff --git a/package.json b/package.json index c8a4c8a856..ae11c3b0c3 100644 --- a/package.json +++ b/package.json @@ -113,6 +113,7 @@ "jwt-decode": "^2.2.0", "moment": "^2.29.2", "ngx-color-picker": "^10.1.0", + "ngx-indexed-db": "^11.0.2", "ngx-toastr": "^14.1.3", "nosleep.js": "^0.12.0", "ol": "^6.8.1", diff --git a/packages/core/src/lib/compression/compressedData.interface.ts b/packages/core/src/lib/compression/compressedData.interface.ts new file mode 100644 index 0000000000..7b20955913 --- /dev/null +++ b/packages/core/src/lib/compression/compressedData.interface.ts @@ -0,0 +1,5 @@ +export interface CompressedData { + length: number; + type: string; + object: any; +} diff --git a/packages/core/src/lib/compression/compression.service.ts b/packages/core/src/lib/compression/compression.service.ts new file mode 100644 index 0000000000..4134721662 --- /dev/null +++ b/packages/core/src/lib/compression/compression.service.ts @@ -0,0 +1,143 @@ +import { Injectable } from '@angular/core'; +import { Observable, Observer } from 'rxjs'; +import { CompressedData } from './compressedData.interface'; + +function getNumber(v: number, endposition: number, length: number) { + /* tslint:disable:no-bitwise*/ + const mask = ((1 << length) - 1); + return (v >> endposition) & mask; + /* tslint:enable:no-bitwise*/ +} + +@Injectable({ + providedIn: 'root' +}) +export class CompressionService { + private base64Index = new Map(); + private indexBase64 = new Map(); + + constructor() { + this.generateBase64Index(); + } + + private generateBase64Index() { + // https://fr.wikipedia.org/wiki/Base64 + // A-Z => [0, 25] + for (let i = 0; i < 26; i++) { + this.base64Index.set(String.fromCharCode('A'.charCodeAt(0) + i), i); + this.indexBase64.set(i, String.fromCharCode('A'.charCodeAt(0) + i)); + } + // a-z => [26, 51] + for (let i = 0; i < 26; i++) { + this.base64Index.set(String.fromCharCode('a'.charCodeAt(0) + i), i + 26); + this.indexBase64.set(i + 26, String.fromCharCode('a'.charCodeAt(0) + i)); + } + // 0-9 => [52, 61] + for (let i = 0; i < 10; i++) { + this.base64Index.set(String.fromCharCode('0'.charCodeAt(0) + i), i + 52); + this.indexBase64.set(i + 52, String.fromCharCode('0'.charCodeAt(0) + i)); + } + // + / => [62, 63] + this.base64Index.set('+', 62); + this.base64Index.set('/', 63); + this.indexBase64.set(62, '+'); + this.indexBase64.set(63, '/'); + } + + compressBlob(blob: Blob): Observable { + if (!blob) { + return; + } + + const observable = new Observable((observer: Observer) => { + const reader = new FileReader(); + reader.readAsDataURL(blob); + reader.onload = () => { + const base64 = reader.result.valueOf() as string; + const text64 = base64.substr(base64.indexOf(',') + 1); + const compressed = this.compressStringBase64(text64); + const compressedData: CompressedData = { length: text64.length, + type: blob.type, + object: compressed }; + observer.next(compressedData); + }; + }); + return observable; + } + + decompressBlob(compressedData: CompressedData): Blob { + /* tslint:disable:no-bitwise*/ + const object = compressedData.object; + const length = compressedData.length; + const decompressed: string = this.decompressStringBase64(object, length); + const byteCharacters = atob(decompressed); + const byteNumbers = new Array(byteCharacters.length); + for (let i = 0; i < byteCharacters.length; i++) { + byteNumbers[i] = byteCharacters.charCodeAt(i); + } + const byteArray = new Uint8Array(byteNumbers); + const blob = new Blob([byteArray], {type: compressedData.type}); + /* tslint:enable:no-bitwise*/ + return blob; + } + + private compressStringBase64(s: string): string { + /* tslint:disable:no-bitwise*/ + let out = ''; + let bits = 16; + let chr = 0; + let rem = 0; + for (const c of s) { + const value = this.base64Index.get(c); + if (bits > 6) { + bits -= 6; + chr += value << bits; + } else { + rem = 6 - bits; + chr += value >> rem; + out += String.fromCharCode(chr); + chr = (value) << (16 - rem); + bits = 16 - rem; + } + } + if (s.length % 8 !== 0) { + out += String.fromCharCode(chr); + } + /* tslint:enable:no-bitwise*/ + return String.fromCharCode(9731) + out; + } + + private decompressStringBase64(c: string, length: number): string { + /* tslint:disable:no-bitwise*/ + if (!c) { + return; + } + + if (c.charCodeAt(0) !== 9731) { + return c; + } + + let chr = 0; + let rem = 0; + let bits = 16; + let out = ''; + let j = 1; + let value = c.charCodeAt(j); + for (let i = 0; i < length; i++) { + if (bits > 6) { + bits -= 6; + chr = getNumber(value, bits, 6); + out += this.indexBase64.get(chr); + } else { + rem = 6 - bits; + chr = getNumber(value, 0, bits) << rem; + value = c.charCodeAt(++j); + chr += getNumber(value, 16 - rem, rem); + out += this.indexBase64.get(chr); + bits = 16 - rem; + } + } + return out; + /* tslint:enable:no-bitwise*/ + } +} diff --git a/packages/core/src/lib/compression/index.ts b/packages/core/src/lib/compression/index.ts new file mode 100644 index 0000000000..931ffbc7cc --- /dev/null +++ b/packages/core/src/lib/compression/index.ts @@ -0,0 +1,2 @@ +export * from './compressedData.interface'; +export * from './compression.service'; diff --git a/packages/core/src/lib/core.module.ts b/packages/core/src/lib/core.module.ts index 1b5bdc2987..e3d30f2ff3 100644 --- a/packages/core/src/lib/core.module.ts +++ b/packages/core/src/lib/core.module.ts @@ -9,7 +9,19 @@ import { IgoConfigModule } from './config/config.module'; import { IgoLanguageModule } from './language/language.module'; import { IgoMessageModule } from './message/message.module'; import { IgoErrorModule } from './request/error.module'; +import { DBConfig, NgxIndexedDBModule } from 'ngx-indexed-db'; +const dbConfig: DBConfig = { + name: 'igo2DB', + version: 1, + objectStoresMeta: [{ + store: 'geoData', + storeConfig: { keyPath: 'url', autoIncrement: false }, + storeSchema: [ + { name: 'regionID', keypath: 'regionID', options: { unique: false }} + ] + }] +}; @NgModule({ imports: [ CommonModule, @@ -18,7 +30,8 @@ import { IgoErrorModule } from './request/error.module'; IgoConfigModule.forRoot(), IgoErrorModule.forRoot(), IgoLanguageModule.forRoot(), - IgoMessageModule.forRoot() + IgoMessageModule.forRoot(), + NgxIndexedDBModule.forRoot(dbConfig) ], declarations: [], exports: [ diff --git a/packages/core/src/public_api.ts b/packages/core/src/public_api.ts index 769578c8ed..3833a08c74 100644 --- a/packages/core/src/public_api.ts +++ b/packages/core/src/public_api.ts @@ -25,4 +25,5 @@ export * from './lib/media'; export * from './lib/message'; export * from './lib/request'; export * from './lib/storage'; +export * from './lib/compression'; export * from './lib/network'; diff --git a/packages/geo/src/lib/layer/shared/layer.service.ts b/packages/geo/src/lib/layer/shared/layer.service.ts index e45d2b7417..c4e6e55914 100644 --- a/packages/geo/src/lib/layer/shared/layer.service.ts +++ b/packages/geo/src/lib/layer/shared/layer.service.ts @@ -43,6 +43,7 @@ import { import { computeMVTOptionsOnHover } from '../utils/layer.utils'; import { StyleService } from './style.service'; import { LanguageService, MessageService } from '@igo2/core'; +import { GeoNetworkService } from '../../offline/shared/geo-network.service'; import { StyleLike as OlStyleLike } from 'ol/style/Style'; @Injectable({ @@ -53,6 +54,7 @@ export class LayerService { private http: HttpClient, private styleService: StyleService, private dataSourceService: DataSourceService, + private geoNetwork: GeoNetworkService, private messageService: MessageService, private languageService: LanguageService, @Optional() private authInterceptor: AuthInterceptor @@ -152,7 +154,7 @@ export class LayerService { resolution ); }; - igoLayer = new VectorLayer(layerOptions, this.messageService, this.authInterceptor); + igoLayer = new VectorLayer(layerOptions, this.messageService, this.authInterceptor, this.geoNetwork); } if (layerOptions.source instanceof ClusterDataSource) { @@ -166,7 +168,7 @@ export class LayerService { baseStyle ); }; - igoLayer = new VectorLayer(layerOptions, this.messageService, this.authInterceptor); + igoLayer = new VectorLayer(layerOptions, this.messageService, this.authInterceptor, this.geoNetwork); } const layerOptionsOl = Object.assign({}, layerOptions, { @@ -174,7 +176,7 @@ export class LayerService { }); if (!igoLayer) { - igoLayer = new VectorLayer(layerOptionsOl, this.messageService, this.authInterceptor); + igoLayer = new VectorLayer(layerOptionsOl, this.messageService, this.authInterceptor, this.geoNetwork); } this.applyMapboxStyle(igoLayer, layerOptionsOl as any); diff --git a/packages/geo/src/lib/layer/shared/layers/vector-layer.ts b/packages/geo/src/lib/layer/shared/layers/vector-layer.ts index e7e42defc7..1872bf721d 100644 --- a/packages/geo/src/lib/layer/shared/layers/vector-layer.ts +++ b/packages/geo/src/lib/layer/shared/layers/vector-layer.ts @@ -25,6 +25,10 @@ import { MessageService } from '@igo2/core'; import { WFSDataSourceOptions } from '../../../datasource/shared/datasources/wfs-datasource.interface'; import { buildUrl, defaultMaxFeatures } from '../../../datasource/shared/datasources/wms-wfs.utils'; import { OgcFilterableDataSourceOptions } from '../../../filter/shared/ogc-filter.interface'; +import { GeoNetworkService, SimpleGetOptions } from '../../../offline/shared/geo-network.service'; +import { catchError, concatMap, first } from 'rxjs/operators'; +import { GeoDBService } from '../../../offline/geoDB/geoDB.service'; +import { of } from 'rxjs'; export class VectorLayer extends Layer { public dataSource: | FeatureDataSource @@ -48,7 +52,9 @@ export class VectorLayer extends Layer { constructor( options: VectorLayerOptions, public messageService?: MessageService, - public authInterceptor?: AuthInterceptor + public authInterceptor?: AuthInterceptor, + private geoNetworkService?: GeoNetworkService, + private geoDBService?: GeoDBService ) { super(options, messageService, authInterceptor); this.watcher = new VectorWatcher(this); @@ -392,6 +398,57 @@ export class VectorLayer extends Layer { } else { modifiedUrl = url(extent, resolution, projection); } + + if (this.geoNetworkService && typeof url !== 'function') { + const format = vectorSource.getFormat(); + const type = format.getType(); + + let responseType = type; + const onError = () => { + vectorSource.removeLoadedExtent(extent); + failure(); + }; + + const options: SimpleGetOptions = { responseType }; + this.geoNetworkService.geoDBService.get(url).pipe(concatMap(r => + r ? of(r) : this.geoNetworkService.get(modifiedUrl, options) + .pipe( + first(), + catchError((res) => { + onError(); + throw res; + }) + ) + )) + .subscribe((content) => { + if (content) { + const format = vectorSource.getFormat(); + const type = format.getType(); + let source; + if (type === FormatType.JSON || type === FormatType.TEXT) { + source = content; + } else if (type === FormatType.XML) { + source = content; + if (!source) { + source = new DOMParser().parseFromString( + content, + 'application/xml' + ); + } + } else if (type === FormatType.ARRAY_BUFFER) { + source = content; + } + if (source) { + const features = format.readFeatures(source, { extent, featureProjection: projection }); + vectorSource.addFeatures(features, format.readProjection(source)); + success(features); + } else { + onError(); + } + } + }); + + } else { xhr.open( 'GET', modifiedUrl); const format = vectorSource.getFormat(); if (format.getType() === FormatType.ARRAY_BUFFER) { @@ -437,4 +494,5 @@ export class VectorLayer extends Layer { }; xhr.send(); } + } } diff --git a/packages/geo/src/lib/offline/geoDB/geoDB.enums.ts b/packages/geo/src/lib/offline/geoDB/geoDB.enums.ts new file mode 100644 index 0000000000..eece95bbaa --- /dev/null +++ b/packages/geo/src/lib/offline/geoDB/geoDB.enums.ts @@ -0,0 +1,4 @@ +export enum InsertSourceInsertDBEnum { + System = 'system', + User = 'user' + } diff --git a/packages/geo/src/lib/offline/geoDB/geoDB.interface.ts b/packages/geo/src/lib/offline/geoDB/geoDB.interface.ts new file mode 100644 index 0000000000..2b16432ad0 --- /dev/null +++ b/packages/geo/src/lib/offline/geoDB/geoDB.interface.ts @@ -0,0 +1,10 @@ +import { InsertSourceInsertDBEnum } from "./geoDB.enums"; + +export interface GeoDBData { + url: string; + regionID: number; + object: any; + compressed: boolean; + insertSource: InsertSourceInsertDBEnum; + insertEvent: string; +} diff --git a/packages/geo/src/lib/offline/geoDB/geoDB.service.ts b/packages/geo/src/lib/offline/geoDB/geoDB.service.ts new file mode 100644 index 0000000000..4060a2babe --- /dev/null +++ b/packages/geo/src/lib/offline/geoDB/geoDB.service.ts @@ -0,0 +1,180 @@ +import { Injectable } from '@angular/core'; +import { DBMode, NgxIndexedDBService } from 'ngx-indexed-db'; +import { Observable, of, Subject } from 'rxjs'; +import { concatMap, first, map, take } from 'rxjs/operators'; +import { CompressionService } from '@igo2/core'; +import { GeoDBData } from './geoDB.interface'; +import { InsertSourceInsertDBEnum } from './geoDB.enums'; + +@Injectable({ + providedIn: 'root' +}) +export class GeoDBService { + readonly dbName: string = 'geoData'; + public collisionsMap: Map = new Map(); + public _newTiles: number = 0; + + constructor( + private ngxIndexedDBService: NgxIndexedDBService, + private compression: CompressionService + ) { } + + /** + * Only blob can be will be compressed + * @param url + * @param regionID + * @param object object to handle + * @param insertSource type of event user or system + * @param insertEvent Name of the event where the insert has been triggered + * @returns + */ + update(url: string, regionID: number, object: any, insertSource: InsertSourceInsertDBEnum, insertEvent: string): Observable { + if (!object) { + return; + } + let compress = false; + + const subject: Subject = new Subject(); + let object$: Observable = of(object); + if (object instanceof Blob) { + object$ = this.compression.compressBlob(object); + compress = true; + } + let geoDBData: GeoDBData; + object$.pipe( + first(), + concatMap(object => { + geoDBData = { + url, + regionID, + object: object, + compressed: compress, + insertSource, + insertEvent + }; + return this.ngxIndexedDBService.getByID(this.dbName, url); + }), + concatMap((dbObject: GeoDBData) => { + if (!dbObject) { + this._newTiles++; + return this.ngxIndexedDBService.add(this.dbName, geoDBData); + } else { + const currentRegionID = dbObject.regionID; + if (currentRegionID !== regionID) { + const collisions = this.collisionsMap.get(currentRegionID); + if (collisions !== undefined) { + collisions.push(dbObject.url); + this.collisionsMap.set(currentRegionID, collisions); + } else { + this.collisionsMap.set(currentRegionID, [dbObject.url]); + } + } + return this.customUpdate(geoDBData); + } + }) + ).subscribe((response) => { + subject.next(response); + subject.complete(); + }); + return subject; + } + + private customUpdate(geoDBData: GeoDBData): Observable { + const subject: Subject = new Subject(); + const deleteRequest = this.ngxIndexedDBService.deleteByKey(this.dbName, geoDBData.url); + deleteRequest.pipe( + concatMap(isDeleted => isDeleted ? this.ngxIndexedDBService.add(this.dbName, geoDBData) : of(undefined)) + ) + .subscribe(object => { + if (object) { + subject.next(object); + } + subject.complete(); + }); + return subject; + } + + get(url: string): Observable { + return this.ngxIndexedDBService.getByID(this.dbName, url).pipe( + map((data: GeoDBData) => { + if (data) { + const object = data.object; + if (!data.compressed) { + return object; + } + return this.compression.decompressBlob(object); + } + }) + ); + } + + getRegionTileCountByID(id: number): Observable { + const subject: Subject = new Subject(); + const dbRequest = this.getRegionByID(id) + .subscribe((tiles) => { + subject.next(tiles.length); + subject.complete(); + }); + return subject; + } + + getRegionByID(id: number): Observable { + if (!id) { + return; + } + + const IDBKey: IDBKeyRange = IDBKeyRange.only(id); + const dbRequest = this.ngxIndexedDBService.getAllByIndex(this.dbName, 'regionID', IDBKey); + return dbRequest; + } + + deleteByRegionID(id: number): Observable { + if (!id) { + return; + } + + const IDBKey: IDBKeyRange = IDBKeyRange.only(id); + const dbRequest = this.ngxIndexedDBService.getAllByIndex(this.dbName, 'regionID', IDBKey); + dbRequest.subscribe((tiles: GeoDBData[]) => { + tiles.forEach((tile) => { + this.ngxIndexedDBService.deleteByKey(this.dbName, tile.url); + }); + }); + return dbRequest; + } + + openCursor( + keyRange: IDBKeyRange = IDBKeyRange.lowerBound(0), + mode: DBMode = DBMode.readonly + ) { + const request = this.ngxIndexedDBService.openCursorByIndex(this.dbName, 'regionID', keyRange, mode); + return request; + } + + resetCounters() { + this.resetCollisionsMap(); + this._newTiles = 0; + } + + resetCollisionsMap() { + this.collisionsMap = new Map(); + } + + revertCollisions() { + for (const [regionID, collisions] of this.collisionsMap) { + for (const url of collisions) { + this.ngxIndexedDBService.getByKey(this.dbName, url) + .pipe(take(1)) + .subscribe((dbObject: GeoDBData) => { + const updatedObject = dbObject; + updatedObject.regionID = regionID; + this.customUpdate(dbObject); + }); + } + } + } + + get newTiles(): number { + return this._newTiles; + } +} diff --git a/packages/geo/src/lib/offline/geoDB/index.ts b/packages/geo/src/lib/offline/geoDB/index.ts new file mode 100644 index 0000000000..872b724d1f --- /dev/null +++ b/packages/geo/src/lib/offline/geoDB/index.ts @@ -0,0 +1,3 @@ +export * from './geoDB.enums'; +export * from './geoDB.interface'; +export * from './geoDB.service'; diff --git a/packages/geo/src/lib/offline/index.ts b/packages/geo/src/lib/offline/index.ts new file mode 100644 index 0000000000..7a36a3783b --- /dev/null +++ b/packages/geo/src/lib/offline/index.ts @@ -0,0 +1,2 @@ +export * from './geoDB'; +export * from './shared'; diff --git a/packages/geo/src/lib/offline/shared/geo-network.service.ts b/packages/geo/src/lib/offline/shared/geo-network.service.ts new file mode 100644 index 0000000000..274c4e450f --- /dev/null +++ b/packages/geo/src/lib/offline/shared/geo-network.service.ts @@ -0,0 +1,69 @@ +import { HttpClient } from '@angular/common/http'; +import { Injectable } from '@angular/core'; +import { ConnectionState, NetworkService } from '@igo2/core'; +import { Observable } from 'rxjs'; +import { GeoDBService } from '../geoDB/geoDB.service'; + +export enum ResponseType { + Arraybuffer= 'arraybuffer', + Blob= 'blob', + Text= 'text', + Json= 'json', +} +export interface SimpleGetOptions { + responseType: ResponseType; + withCredentials?: boolean; +} +@Injectable({ + providedIn: 'root' +}) +export class GeoNetworkService { + private networkOnline: boolean = true; + constructor( + private http: HttpClient, + public geoDBService: GeoDBService, + private networkService: NetworkService, + ) { + this.networkService.currentState().subscribe((state: ConnectionState) => { + this.networkOnline = state.connection; + }); + } + + get(url: string, simpleGetOptions: SimpleGetOptions): Observable { + if (window.navigator.onLine && this.networkOnline) { + return this.getOnline(url, simpleGetOptions); + } + return this.getOffline(url); + } + + private getOnline(url: string, simpleGetOptions: SimpleGetOptions): Observable { + let request; + switch (simpleGetOptions.responseType) { + // TODO Ajuster pour autre formats + case 'arraybuffer': + request = this.http.get(url, { responseType: 'arraybuffer', withCredentials: simpleGetOptions.withCredentials }); + break; + case 'blob': + request = this.http.get(url, { responseType: 'blob', withCredentials: simpleGetOptions.withCredentials }); + break; + case 'text': + request = this.http.get(url, { responseType: 'text', withCredentials: simpleGetOptions.withCredentials }); + break; + case 'json': + request = this.http.get(url, { responseType: 'json', withCredentials: simpleGetOptions.withCredentials }); + break; + default: + request = this.http.get(url, { responseType: 'blob', withCredentials: simpleGetOptions.withCredentials }); + break; + } + return request; + } + + private getOffline(url: string): Observable { + return this.geoDBService.get(url); + } + + public isOnline() { + return this.networkOnline && window.navigator.onLine; + } +} diff --git a/packages/geo/src/lib/offline/shared/index.ts b/packages/geo/src/lib/offline/shared/index.ts new file mode 100644 index 0000000000..0adafae430 --- /dev/null +++ b/packages/geo/src/lib/offline/shared/index.ts @@ -0,0 +1 @@ +export * from './geo-network.service'; diff --git a/packages/geo/src/public_api.ts b/packages/geo/src/public_api.ts index 35e8230d21..638ae4a62c 100644 --- a/packages/geo/src/public_api.ts +++ b/packages/geo/src/public_api.ts @@ -54,3 +54,4 @@ export * from './lib/toast'; export * from './lib/utils'; export * from './lib/wkt'; export * from './lib/workspace'; +export * from './lib/offline';