866: add example Akzeptanzpartner-daten.csv, improve local testing an…

…d creation process of sozialpass csv output, add documentation for imports
digitalfabrik · May 9, 2023 · e5bdbf7 · e5bdbf7
1 parent 2c4f841
commit e5bdbf7
Show file tree

Hide file tree

Showing 4 changed files with 195 additions and 18 deletions.
diff --git a/...src/main/kotlin/app/ehrenamtskarte/backend/stores/importer/bayern/EhrenamtskarteBayern.kt b/...src/main/kotlin/app/ehrenamtskarte/backend/stores/importer/bayern/EhrenamtskarteBayern.kt
@@ -18,19 +18,22 @@ object EhrenamtskarteBayern : Pipeline {
     private val httpClient = HttpClient()
 
     override fun import(config: ImportConfig, logger: Logger) {
-        Unit.addStep(DownloadLbe(config, logger, httpClient), logger) { logger.info("== Download lbe data ==") }
-            .addStep(FilterLbe(config, logger), logger) { logger.info("== Filter lbe data ==") }
-            .addStep(MapFromLbe(config, logger), logger) { logger.info("== Map lbe to internal data ==") }
-            .addStep(SanitizeAddress(config, logger), logger) { logger.info("== Sanitize address ==") }
-            .addStep(
-                SanitizeGeocode(config, logger, httpClient),
-                logger
-            ) { logger.info("== Sanitize data with geocoding ==") }
-            .addStep(
-                PostSanitizeFilter(config, logger, httpClient),
-                logger
-            ) { logger.info("== Filter sanitized data ==") }
-            .addStep(FilterDuplicates(config, logger), logger) { logger.info("== Filter duplicated data ==") }
-            .addStep(Store(config, logger), logger) { logger.info("== Store remaining data to db ==") }
+        // to speedup testing and creating csv import files for nuernberg, the bavaria import will only be executed if csvWriter is not enabled.
+        if (!config.backendConfig.csvWriter.enabled) {
+            Unit.addStep(DownloadLbe(config, logger, httpClient), logger) { logger.info("== Download lbe data ==") }
+                .addStep(FilterLbe(config, logger), logger) { logger.info("== Filter lbe data ==") }
+                .addStep(MapFromLbe(config, logger), logger) { logger.info("== Map lbe to internal data ==") }
+                .addStep(SanitizeAddress(config, logger), logger) { logger.info("== Sanitize address ==") }
+                .addStep(
+                    SanitizeGeocode(config, logger, httpClient),
+                    logger
+                ) { logger.info("== Sanitize data with geocoding ==") }
+                .addStep(
+                    PostSanitizeFilter(config, logger, httpClient),
+                    logger
+                ) { logger.info("== Filter sanitized data ==") }
+                .addStep(FilterDuplicates(config, logger), logger) { logger.info("== Filter duplicated data ==") }
+                .addStep(Store(config, logger), logger) { logger.info("== Store remaining data to db ==") }
+        }
     }
 }
diff --git a/...src/main/kotlin/app/ehrenamtskarte/backend/stores/importer/nuernberg/steps/DownloadCsv.kt b/...src/main/kotlin/app/ehrenamtskarte/backend/stores/importer/nuernberg/steps/DownloadCsv.kt
@@ -6,9 +6,9 @@ import app.ehrenamtskarte.backend.stores.importer.nuernberg.types.CSVAcceptingSt
 import org.apache.commons.csv.CSVFormat
 import org.apache.commons.csv.CSVRecord
 import org.slf4j.Logger
+import java.io.InputStream
 import java.io.InputStreamReader
 import java.net.URL
-import java.net.URLConnection
 
 /**
  * Downloads the CSV File.
@@ -20,10 +20,8 @@ class DownloadCsv(config: ImportConfig, private val logger: Logger) :
 
     override fun execute(input: Unit): List<CSVAcceptingStore> {
         try {
-            val url = URL(config.findProject().importUrl)
-            val urlConn: URLConnection = url.openConnection()
             val inputCSV = InputStreamReader(
-                urlConn.getInputStream(),
+                getCSVInputStream(),
                 Charsets.UTF_8
             )
             val records: Iterable<CSVRecord> = CSVFormat.RFC4180.parse(inputCSV)
@@ -62,4 +60,13 @@ class DownloadCsv(config: ImportConfig, private val logger: Logger) :
         }
         return stores
     }
+
+// csvWriter is used to create a local csv output file with coordinates of the stores which will be uploaded to the server
+    private fun getCSVInputStream(): InputStream {
+        if (config.backendConfig.csvWriter.enabled) {
+            return ClassLoader.getSystemResourceAsStream("import/Akzeptanzpartner-daten.csv")!!
+        }
+        val url = URL(config.findProject().importUrl)
+        return url.openConnection().getInputStream()
+    }
 }