Merge pull request #358 from swisstopo/develop

INT Release

.editorconfig

@@ -13,3 +13,6 @@ max_line_length = 120
 [*.md]
 max_line_length = off
 trim_trailing_whitespace = false
+
+[*.py]
+indent_size = 4

.github/scripts/find-version.js

@@ -10,6 +10,7 @@ const findNextVersion = (tags, branch) => {
     if (version.preRelease == null || version.patch !== 0) {
       version.minor += 1;
       version.patch = 0;
+      version.preRelease = null;
     }
   } else {
     // It's a patch.

.github/workflows/publish-edge.yml

@@ -19,6 +19,9 @@ on:
         description: "Tag the commit and published image with `edge`."
         default: true
 
+concurrency:
+  group: "publish-edge"
+
 permissions: write-all
 
 env:

.github/workflows/publish-rc.yml

@@ -15,6 +15,9 @@ on:
         required: false
         default: "edge"
 
+concurrency:
+  group: "publish-rc"
+
 permissions: write-all
 
 env:

.github/workflows/release.yml

@@ -21,6 +21,9 @@ on:
           Assign the `release-candidate` tag to this release.
         default: true
 
+concurrency:
+  group: "release"
+
 permissions: write-all
 
 env:

README.md

@@ -89,7 +89,7 @@ Be aware that you need to manually insert the `{DB_*}` values beforehand.
 
 ```bash
 cd development
-docker compose exec db sh -c 'pg_dump --dbname=postgresql://{DB_USERNAME}:{DB_PASSWORD}@{DB_HOST}:5432/{DB_DATABASE} --data-only --exclude-table asset_user --exclude-table workgroups_on_users --exclude-table _prisma_migrations -n public > /dump.sql'
+docker compose exec db sh -c 'pg_dump --dbname=postgresql://{DB_USERNAME}:{DB_PASSWORD}@{DB_HOST}:5432/{DB_DATABASE} --data-only --exclude-table asset_user --exclude-table workgroups_on_users --exclude-table _prisma_migrations --exclude-table asset_test --exclude-table asset_user_bak --exclude-table favorite -n public > /dump.sql'
 ```
 
 > The export will output warnings related to circular foreign-key constraints.

apps/client-asset-sg/project.json

@@ -19,7 +19,7 @@
         "assets": ["apps/client-asset-sg/src/favicon.ico", "apps/client-asset-sg/src/assets"],
         "styles": ["apps/client-asset-sg/src/styles.scss"],
         "scripts": [],
-        "allowedCommonJsDependencies": ["tsafe", "xml-utils", "pbf", "rbush", "earcut", "@prisma/client"]
+        "allowedCommonJsDependencies": ["tsafe", "xml-utils", "pbf", "rbush", "earcut", "@prisma/client", "validator"]
       },
       "configurations": {
         "production": {

apps/server-asset-sg/src/features/asset-edit/asset-edit.repo.ts

@@ -284,18 +284,6 @@ export class AssetEditRepo implements Repo<AssetEditDetail, number, AssetEditDat
             Asset: { none: {} },
           },
         });
-
-        await this.prismaService.assetFormatItem.deleteMany({
-          where: {
-            assets: { none: {} },
-          },
-        });
-
-        await this.prismaService.assetKindItem.deleteMany({
-          where: {
-            assets: { none: {} },
-          },
-        });
       });
       return true;
     } catch (e) {

libs/asset-viewer/src/lib/components/map/map.component.ts

@@ -21,7 +21,8 @@ import {
   selectAssetSearchPolygon,
   selectAssetSearchResultData,
   selectCurrentAssetDetail,
-  selectHasDefaultFilters,
+  selectHasNoActiveFilters,
+  selectIsSearchQueryEmpty,
   selectStudies,
 } from '../../state/asset-search/asset-search.selector';
 import { AppStateWithMapControl } from '../../state/map-control/map-control.reducer';
@@ -129,8 +130,8 @@ export class MapComponent implements AfterViewInit, OnChanges, OnDestroy {
     this.controller.addControl(this.controls.draw);
 
     this.subscription.add(
-      this.store.select(selectHasDefaultFilters).subscribe((hasDefaultFilters) => {
-        if (hasDefaultFilters) {
+      this.store.select(selectIsSearchQueryEmpty).subscribe((isSearchQueryEmpty) => {
+        if (isSearchQueryEmpty) {
           this.controls.zoom.resetZoom();
         }
       })
@@ -155,8 +156,8 @@ export class MapComponent implements AfterViewInit, OnChanges, OnDestroy {
 
   private initializeStoreBindings() {
     this.subscription.add(
-      this.store.select(selectHasDefaultFilters).subscribe((showStudies) => {
-        this.controller.setShowHeatmap(showStudies);
+      this.store.select(selectHasNoActiveFilters).subscribe((hasNoActiveFilters) => {
+        this.controller.setShowHeatmap(hasNoActiveFilters);
       })
     );
     this.subscription.add(

libs/asset-viewer/src/lib/state/asset-search/asset-search.effects.ts

@@ -18,7 +18,7 @@ import {
   selectAssetSearchIsInitialized,
   selectAssetSearchQuery,
   selectCurrentAssetDetail,
-  selectHasDefaultFilters,
+  selectIsSearchQueryEmpty,
   selectSearchLoadingState,
   selectStudies,
 } from './asset-search.selector';
@@ -79,15 +79,16 @@ export class AssetSearchEffects {
         return !deepEqual(params.query, storeQuery) || params.assetId != storeDetail?.assetId;
       }),
       map(([params, storeQuery, storeDetail, isMostRecentPage]) => {
-        const hasNoQueryParams = Object.values(params.query).every((v) => v == null || v == false);
+        const { favoritesOnly, ...query } = params.query;
+        const hasNoQueryParams = Object.values(query).every((v) => v == null);
         const hasQueryOrAssetIdInStore =
           !Object.values(storeQuery).every((v) => v == null || v == false) || storeDetail;
         // We only use the values from the store if all of the below are true:
         // - There are no Url query params
         // - There are query or assetId values in the store
         // - The current page is the most recent page
         if (hasNoQueryParams && hasQueryOrAssetIdInStore && isMostRecentPage) {
-          return actions.runCombinedSearch({ query: storeQuery, assetId: storeDetail?.assetId });
+          return actions.runCombinedSearch({ query: { ...storeQuery, favoritesOnly }, assetId: storeDetail?.assetId });
         }
         return actions.runCombinedSearch({ query: params.query, assetId: params.assetId });
       })
@@ -220,9 +221,9 @@ export class AssetSearchEffects {
     this.actions$.pipe(
       ofType(actions.updateResults),
       map(({ results }) => results.page.total !== 0),
-      withLatestFrom(this.store.select(selectHasDefaultFilters)),
-      map(([hasResults, hasNoFilters]) =>
-        !hasResults || hasNoFilters ? actions.closeResults() : actions.openResults()
+      withLatestFrom(this.store.select(selectIsSearchQueryEmpty)),
+      map(([hasResults, isSearchQueryEmpty]) =>
+        !hasResults || isSearchQueryEmpty ? actions.closeResults() : actions.openResults()
       )
     )
   );

libs/asset-viewer/src/lib/state/asset-search/asset-search.selector.ts

@@ -60,11 +60,16 @@ export const selectAssetSearchTotalResults = createSelector(assetSearchFeature,
 export const selectCurrentAssetDetail = createSelector(assetSearchFeature, (state) => state.currentAsset);
 export const selectStudies = createSelector(assetSearchFeature, (state) => state.studies);
 
-export const selectHasDefaultFilters = createSelector(
+export const selectHasNoActiveFilters = createSelector(assetSearchFeature, ({ query }) => hasNoActiveFilters(query));
+
+export const selectIsSearchQueryEmpty = createSelector(
   assetSearchFeature,
-  ({ query, currentAsset }) => currentAsset == null && Object.values(query).every((value) => value === undefined)
+  ({ query, currentAsset }) => currentAsset == null && hasNoActiveFilters(query)
 );
 
+const hasNoActiveFilters = (query: AssetSearchQuery): boolean =>
+  Object.values(query).every((value) => value === undefined || value == false);
+
 export const selectCurrentAssetDetailVM = createSelector(
   fromAppShared.selectRDReferenceData,
   selectCurrentAssetDetail,

libs/client-shared/src/lib/components/disclaimer-dialog/disclaimer-dialog.component.ts

@@ -3,6 +3,7 @@ import { Component, inject } from '@angular/core';
 import { MatDialogActions, MatDialogContent, MatDialogRef } from '@angular/material/dialog';
 import { MatDivider } from '@angular/material/divider';
 import { TranslateModule, TranslateService } from '@ngx-translate/core';
+import { LetModule } from '@rx-angular/template/let';
 import { map } from 'rxjs';
 import { ButtonComponent } from '../../components/button';
 import { LanguageSelectorComponent } from '../../components/language-selector';
@@ -23,6 +24,7 @@ const LEGAL_BASE_URL = 'https://www.swissgeol.ch/datenschutz';
     ButtonComponent,
     AsyncPipe,
     TranslateModule,
+    LetModule,
   ],
 })
 export class DisclaimerDialogComponent {

scripts/2024-10-01_import-new-geometries/main.py

@@ -5,11 +5,16 @@
 from geopandas import GeoDataFrame
 import os
 
+from shapely import Polygon
+
+verbose = False
+
 # The following parameters need to be adjusted to the correct values:
 filedir = ''
 filename = 'geometries.gpkg'
 output_filename = 'import_geometries.sql'
 output_sgsids_filename = 'sgsids.txt'
+output_sgsids_simplified_filename = 'sgsids_simplified.txt'
 
 # The following sgsids are not in the database, they were manually translated to the probably correct sgsid:
 sgsid_translation_dict = {}
@@ -18,7 +23,7 @@
 allowed_bemerkung = ['erraten', 'erraten_B', 'neu', 'neu (?)', 'unsich kein B']
 
 def read_gdf(file: str, layer: str, allowed_bemerkung: list, translation_dict: dict):
-    '''Read a geopandas dataframe from a file and layer, filter out entries with Bemerkung not in allowed_bemerkung and translate the sgsid according to translationDict.'''
+    """Read a geopandas dataframe from a file and layer, filter out entries with Bemerkung not in allowed_bemerkung and translate the sgsid according to translationDict."""
     gdf = gpd.read_file(file, layer=layer)
     print(gdf['Bemerkung'].value_counts())
     gdf = gdf[(gdf['Bemerkung'].isin(allowed_bemerkung))]
@@ -30,13 +35,67 @@ def write_insert_statements(f: TextIO, gdf: GeoDataFrame, geom_type: str):
         f.write(
             f"INSERT INTO study_{geom_type} (asset_id, geom_quality_item_code, geom) VALUES ((SELECT asset_id FROM asset WHERE sgs_id = {row['IDSGS_neu']}), 'revised', ST_GeomFromText('{row['geometry'].wkt}', 2056));\n")
 
+def remove_interior_polygons(gdf: GeoDataFrame):
+    '''Remove interior polygons from a GeoDataFrame of polygons.'''
+    gdf['geometry'] = gdf['geometry'].apply(lambda x: Polygon(x.exterior.coords))
+    return gdf
+
+simplified_sgsids = []
+simplified_rows = [0]
+
+def simplify_geometry(gdf: GeoDataFrame):
+    """ Simplify the geometries of large Polygons.
+    Use different tolerances depending on the number of points in the geometry.
+    1000 - 10000 points: 100
+    10000 - 50000 points: 1000
+    > 50000 points: 2000
+    """
+    def simplify_by(row, tolerance):
+        num_points = count_points(row['geometry'])
+        simplified = row['geometry'].simplify(tolerance, preserve_topology=True)
+        simplified_sgsids.append(row['IDSGS_neu'])
+        simplified_rows[0] += num_points - count_points(simplified)
+        if verbose:
+            print(
+                f" ||| SGSID Simplified {num_points} - {len(simplified.exterior.coords)} = {num_points - len(simplified.exterior.coords)}")
+        return simplified
+
+    def count_points(geom):
+        if geom.geom_type == 'Polygon':
+            return len(geom.exterior.coords)
+        elif geom.geom_type == 'MultiPolygon':
+            return sum([count_points(p) for p in geom.geoms])
+        else:
+            return 0
+
+    def simplify(row):
+        if row['geometry'].geom_type in ('Polygon', 'MultiPolygon'):
+            num_points = count_points(row['geometry'])
+        else:
+            return row['geometry']
+
+        if num_points < 1000:
+            return row['geometry']
+        elif num_points < 10000:
+            return simplify_by(row, 100)
+        elif num_points < 50000:
+            return simplify_by(row, 1000)
+        else:
+            return simplify_by(row, 2000)
+
+    gdf['geometry'] = gdf.apply(simplify, axis=1)
+    print(f"Simplified {len(simplified_sgsids)} geometries, removed {simplified_rows[0]} points.")
+    return gdf
+
 def main():
     path = os.path.join(filedir, filename)
     gdf_points = read_gdf(file=path, layer='backup_20240829__points', allowed_bemerkung=allowed_bemerkung, translation_dict=sgsid_translation_dict)
     gdf_lines = read_gdf(file=path, layer='backup_20240829__lines', allowed_bemerkung=allowed_bemerkung, translation_dict=sgsid_translation_dict)
     gdf_polygons = read_gdf(file=path, layer='backup_20240829__polygons', allowed_bemerkung=allowed_bemerkung, translation_dict=sgsid_translation_dict)
     gdf_lines = gdf_lines.explode() # explode multi-linestrings to linestrings
-    gdf_polygons = gdf_polygons.explode() # explode multi-polygons to polygons
+    gdf_polygons = simplify_geometry(gdf_polygons)
+    gdf_polygons = gdf_polygons.explode()  # explode multi-polygons to polygons
+    gdf_polygons = remove_interior_polygons(gdf_polygons) # remove holes from polygons
     gdf_points = gdf_points[~gdf_points['geometry'].is_empty] # remove empty geometries
 
     sgs_ids = np.unique(np.concatenate(
@@ -45,6 +104,7 @@ def main():
          gdf_polygons['IDSGS_neu'].unique()]))
 
     np.savetxt(os.path.join(filedir, output_sgsids_filename), sgs_ids.astype(int), fmt='%d', delimiter=',')
+    np.savetxt(os.path.join(filedir, output_sgsids_simplified_filename), np.unique(simplified_sgsids).astype(int), fmt='%d', delimiter=',')
 
     # for each sgsid write a sql delete statement to delete all entries with this sgsid. Commit the commands every 1000 rows.
     with open(os.path.join(filedir, output_filename), 'w') as f: