From 9f236a9b90582ee17af4382c00c4b134a3c87c19 Mon Sep 17 00:00:00 2001 From: Kamil Czaja Date: Thu, 24 Oct 2024 16:42:38 +0200 Subject: [PATCH] feat: import crawler from edc-ce and cleanup --- .../actions/build-connector-image/action.yml | 75 +++ .github/workflows/ci.yml | 14 +- CHANGELOG.md | 11 +- .../sovity/authorityportal/api/UiResource.kt | 2 +- .../web/thirdparty/daps/DapsClient.kt | 4 + .../src/main/resources/application.properties | 2 + authority-portal-backend/build.gradle.kts | 4 + .../catalog-crawler}/Dockerfile | 15 +- .../catalog-crawler/README.md | 41 ++ .../build.gradle.kts | 9 +- .../build.gradle.kts | 2 +- .../catalog-crawler/build.gradle.kts | 25 +- .../catalog/crawler/CrawlerConfigProps.java | 549 ++++++++++++++++++ .../ext/catalog/crawler/CrawlerExtension.java | 85 +++ .../crawler/CrawlerExtensionContext.java | 0 .../CrawlerExtensionContextBuilder.java | 29 +- .../catalog/crawler/CrawlerInitializer.java | 0 .../sovity/edc/ext/catalog/crawler/Main.java | 22 + .../crawler/crawling/ConnectorCrawler.java | 4 +- .../crawling/OfflineConnectorCleaner.java | 0 .../fetching/FetchedCatalogBuilder.java | 0 .../fetching/FetchedCatalogMappingUtils.java | 0 .../fetching/FetchedCatalogService.java | 0 .../fetching/model/FetchedCatalog.java | 0 .../fetching/model/FetchedContractOffer.java | 0 .../fetching/model/FetchedDataOffer.java | 0 .../logging/ConnectorChangeTracker.java | 0 .../logging/CrawlerEventErrorMessage.java | 2 +- .../crawling/logging/CrawlerEventLogger.java | 8 +- .../logging/CrawlerExecutionTimeLogger.java | 6 +- .../crawling/writing/CatalogPatchBuilder.java | 4 +- .../writing/ConnectorUpdateCatalogWriter.java | 0 .../writing/ConnectorUpdateFailureWriter.java | 4 +- .../writing/ConnectorUpdateSuccessWriter.java | 4 +- .../writing/DataOfferLimitsEnforcer.java | 6 +- .../crawling/writing/utils/ChangeTracker.java | 0 .../crawling/writing/utils/DiffUtils.java | 0 .../catalog/crawler/dao/CatalogCleaner.java | 2 +- .../ext/catalog/crawler/dao/CatalogPatch.java | 4 +- .../crawler/dao/CatalogPatchApplier.java | 0 .../crawler/dao/config/DataSourceFactory.java | 20 +- .../crawler/dao/config/DslContextFactory.java | 2 +- .../dao/connectors/ConnectorQueries.java | 12 +- .../crawler/dao/connectors/ConnectorRef.java | 0 .../connectors/ConnectorStatusUpdater.java | 4 +- .../contract_offers/ContractOfferQueries.java | 4 +- .../ContractOfferRecordUpdater.java | 4 +- .../dao/data_offers/DataOfferQueries.java | 4 +- .../data_offers/DataOfferRecordUpdater.java | 5 +- .../catalog/crawler/dao/utils/JsonbUtils.java | 0 .../crawler/dao/utils/PostgresqlUtils.java | 0 .../crawler/dao/utils/RecordPatch.java | 0 .../orchestration/config/CrawlerConfig.java | 0 .../config/CrawlerConfigFactory.java | 24 +- .../config/EdcConfigPropertyUtils.java | 0 .../orchestration/queue/ConnectorQueue.java | 0 .../queue/ConnectorQueueFiller.java | 2 +- .../queue/ConnectorRefreshPriority.java | 0 .../orchestration/queue/ThreadPool.java | 0 .../orchestration/queue/ThreadPoolTask.java | 0 .../queue/ThreadPoolTaskQueue.java | 0 .../schedules/DeadConnectorRefreshJob.java | 2 +- .../schedules/OfflineConnectorCleanerJob.java | 0 .../schedules/OfflineConnectorRefreshJob.java | 2 +- .../schedules/OnlineConnectorRefreshJob.java | 2 +- .../schedules/QuartzScheduleInitializer.java | 0 .../schedules/utils/CronJobRef.java | 0 .../schedules/utils/JobFactoryImpl.java | 0 .../crawler/utils/CollectionUtils2.java | 0 .../ext/catalog/crawler/utils/JsonUtils2.java | 0 .../ext/catalog/crawler/utils/MapUtils.java | 0 .../catalog/crawler/utils/StringUtils2.java | 0 ...rg.eclipse.edc.spi.system.ServiceExtension | 0 .../ext/catalog/crawler/AssertionUtils.java | 0 .../ext/catalog/crawler/CrawlerTestDb.java | 14 +- .../ext/catalog/crawler/JsonTestUtils.java | 0 .../edc/ext/catalog/crawler/TestData.java | 10 +- .../logging/CrawlerEventLoggerTest.java | 2 +- .../writing/ConnectorSuccessWriterTest.java | 8 +- .../ConnectorUpdateCatalogWriterTest.java | 2 +- .../writing/DataOfferLimitsEnforcerTest.java | 6 +- .../DataOfferWriterTestDataHelper.java | 4 +- .../DataOfferWriterTestDataModels.java | 0 .../writing/DataOfferWriterTestDydi.java | 0 .../DataOfferWriterTestResultHelper.java | 6 +- .../crawling/writing/DiffUtilsTest.java | 0 .../dao/connectors/ConnectorRefTest.java | 0 .../queue/ThreadPoolQueueTest.java | 0 .../OfflineConnectorRemovalJobTest.java | 4 +- .../crawler/utils/CollectionUtils2Test.java | 0 .../catalog/crawler/utils/JsonUtils2Test.java | 0 .../crawler/utils/StringUtils2Test.java | 0 .../src/test/resources/logging.properties | 0 .../V9999__Make_Columns_Nullable.sql | 25 + .../catalog-crawler}/docker-entrypoint.sh | 6 - .../catalog-crawler}/logging.dev.properties | 0 .../catalog-crawler}/logging.properties | 0 .../gradle/libs.versions.toml | 86 ++- authority-portal-backend/settings.gradle.kts | 4 + .../goals/production}/.env.catalog-crawler | 0 .../goals/production/README.md | 63 +- extensions/catalog-crawler/README.md | 44 -- .../ext/catalog/crawler/CrawlerExtension.java | 149 ----- .../crawler/dao/config/FlywayService.java | 54 -- launchers/README.md | 186 ------ .../catalog-crawler-dev/build.gradle.kts | 23 - 106 files changed, 1094 insertions(+), 622 deletions(-) create mode 100644 .github/actions/build-connector-image/action.yml rename {launchers => authority-portal-backend/catalog-crawler}/Dockerfile (71%) create mode 100644 authority-portal-backend/catalog-crawler/README.md rename {launchers/connectors/catalog-crawler-ce => authority-portal-backend/catalog-crawler/catalog-crawler-ce-launcher}/build.gradle.kts (55%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts (86%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/build.gradle.kts (61%) create mode 100644 authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerConfigProps.java create mode 100644 authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContext.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java (95%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerInitializer.java (100%) create mode 100644 authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/Main.java rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/ConnectorCrawler.java (97%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/OfflineConnectorCleaner.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogBuilder.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogMappingUtils.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogService.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedCatalog.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedContractOffer.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedDataOffer.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/ConnectorChangeTracker.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventErrorMessage.java (95%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLogger.java (94%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerExecutionTimeLogger.java (86%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/CatalogPatchBuilder.java (96%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriter.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateFailureWriter.java (92%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateSuccessWriter.java (94%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcer.java (94%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/ChangeTracker.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/DiffUtils.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogCleaner.java (95%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogPatch.java (86%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogPatchApplier.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DataSourceFactory.java (62%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DslContextFactory.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorQueries.java (86%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorRef.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorStatusUpdater.java (88%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferQueries.java (84%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferRecordUpdater.java (94%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferQueries.java (85%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferRecordUpdater.java (96%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/JsonbUtils.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/PostgresqlUtils.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/RecordPatch.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfig.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfigFactory.java (53%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/EdcConfigPropertyUtils.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorQueue.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorQueueFiller.java (92%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorRefreshPriority.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPool.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolTask.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolTaskQueue.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/DeadConnectorRefreshJob.java (94%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorCleanerJob.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRefreshJob.java (94%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OnlineConnectorRefreshJob.java (94%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/QuartzScheduleInitializer.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/utils/CronJobRef.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/utils/JobFactoryImpl.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/CollectionUtils2.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/JsonUtils2.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/MapUtils.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/StringUtils2.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/AssertionUtils.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerTestDb.java (79%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/JsonTestUtils.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/TestData.java (84%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLoggerTest.java (97%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorSuccessWriterTest.java (95%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriterTest.java (98%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcerTest.java (97%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDataHelper.java (97%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDataModels.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDydi.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestResultHelper.java (88%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DiffUtilsTest.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorRefTest.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolQueueTest.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRemovalJobTest.java (95%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/CollectionUtils2Test.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/JsonUtils2Test.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/StringUtils2Test.java (100%) rename {extensions => authority-portal-backend}/catalog-crawler/catalog-crawler/src/test/resources/logging.properties (100%) create mode 100644 authority-portal-backend/catalog-crawler/catalog-crawler/src/test/resources/migration-test-utils/V9999__Make_Columns_Nullable.sql rename {launchers => authority-portal-backend/catalog-crawler}/docker-entrypoint.sh (94%) rename {launchers => authority-portal-backend/catalog-crawler}/logging.dev.properties (100%) rename {launchers => authority-portal-backend/catalog-crawler}/logging.properties (100%) rename {launchers => docs/deployment-guide/goals/production}/.env.catalog-crawler (100%) delete mode 100644 extensions/catalog-crawler/README.md delete mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java delete mode 100644 extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/FlywayService.java delete mode 100644 launchers/README.md delete mode 100644 launchers/connectors/catalog-crawler-dev/build.gradle.kts diff --git a/.github/actions/build-connector-image/action.yml b/.github/actions/build-connector-image/action.yml new file mode 100644 index 000000000..41638a2a7 --- /dev/null +++ b/.github/actions/build-connector-image/action.yml @@ -0,0 +1,75 @@ +name: "Build EDC Connector Image" +description: "Builds and deploys the React frontend to AWS S3" +inputs: + registry-url: + required: true + description: "Docker Registry" + registry-user: + required: true + description: "Docker Registry Login Username" + registry-password: + required: true + description: "Docker Registry Login Password" + image-base-name: + required: true + description: "Docker Image Base Name (Company)" + image-name: + required: true + description: "Docker Image Name (Artifact Name)" + connector-name: + required: true + description: "EDC Connector Name in launchers/connectors/{connector-name}" + title: + required: true + description: "Docker Image Title" + description: + required: true + description: "Docker Image Description" +runs: + using: "composite" + steps: + - name: "Docker: Log in to the Container registry" + uses: docker/login-action@v2 + with: + registry: ${{ inputs.registry-url }} + username: ${{ inputs.registry-user }} + password: ${{ inputs.registry-password }} + - name: "Docker: Store last commit info and build date" + id: last-commit-information + shell: bash + run: | + echo "LAST_COMMIT_INFO<> $GITHUB_ENV + export LAST_COMMIT_INFO=$(git log -1) + echo "$LAST_COMMIT_INFO" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + echo "BUILD_DATE=$(date --utc +%FT%TZ)" >> $GITHUB_ENV + - name: "Docker: Extract metadata (tags, labels)" + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ inputs.registry-url }}/${{ inputs.image-base-name }}/${{ inputs.image-name }} + labels: | + org.opencontainers.image.title=${{ inputs.title }} + org.opencontainers.image.description=${{ inputs.description }} + tags: | + type=schedule + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=ref,event=branch + type=ref,event=pr + type=sha + type=raw,value=latest,enable={{is_default_branch}} + type=raw,value=release,enable=${{ startsWith(github.ref, 'refs/tags/') }} + - name: "Docker: Build and Push" + uses: docker/build-push-action@v5 + with: + file: authority-portal-backend/catalog-crawler/Dockerfile + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + CONNECTOR_NAME=${{ inputs.connector-name }} + "EDC_LAST_COMMIT_INFO_ARG=${{ env.LAST_COMMIT_INFO }}" + EDC_BUILD_DATE_ARG=${{ env.BUILD_DATE }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3995e7a47..e42af9d7c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,7 +92,7 @@ jobs: type=sha type=raw,value=latest,enable={{is_default_branch}} type=raw,value=release,enable=${{ startsWith(github.ref, 'refs/tags/') }} - - name: "Docker: Build and Push Image" + - name: "Docker: Build and Push Image (authority-portal-backend)" uses: docker/build-push-action@v4 with: file: authority-portal-backend/authority-portal-quarkus/src/main/docker/Dockerfile.jvm @@ -101,7 +101,17 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} network: host - + - name: "Docker: Build and Push Image (authority-portal-crawler)" + uses: ./.github/actions/build-connector-image + with: + registry-url: ${{ env.REGISTRY }} + registry-user: ${{ github.actor }} + registry-password: ${{ secrets.GITHUB_TOKEN }} + image-base-name: ${{ env.IMAGE_NAME_BASE }} + image-name: "authority-portal-crawler" + connector-name: "catalog-crawler-ce" + title: "Catalog Crawler (Community Edition, DAPS)" + description: "sovity CE Catalog crawler for the sovity CE Authority Portal. Requires DAPS dataspace credentials to join an existing dataspace." frontend: name: Frontend diff --git a/CHANGELOG.md b/CHANGELOG.md index 201433d0f..4d4f4a65b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ please see [changelog_updates.md](docs/dev/changelog_updates.md). #### Major +- The Catalog Crawler has been moved to the AP repository. + - To prevent versioning conflicts with the image from EDC CE up to version 10.4.1, the image is now named differently. See [compatible versions](#compatible-versions) below. + #### Minor #### Patch @@ -24,11 +27,15 @@ please see [changelog_updates.md](docs/dev/changelog_updates.md). ### Deployment Migration Notes +- Please change the image used for the Catalog Crawler. The old image is no longer updated and will not work with future versions of the Portal. + - Previously: `ghcr.io/sovity/catalog-crawler-ce` + - Now: `ghcr.io/sovity/authority-portal-crawler` + #### Compatible Versions - Authority Portal Backend Docker Image: `ghcr.io/sovity/authority-portal-backend:{{ version }}` - Authority Portal Frontend Docker Image: `ghcr.io/sovity/authority-portal-frontend:{{ version }}` -- Catalog Crawler CE: `ghcr.io/sovity/catalog-crawler-ce:{{ CE VERSION }}` +- Catalog Crawler CE: `ghcr.io/sovity/authority-portal-crawler:{{ version }}` - Sovity EDC CE: {{ CE Release Link }} ## [v4.1.2] - 2024-09-26 @@ -255,7 +262,7 @@ MDS 2.2 intermediate release - All brokers can be undeployed including their databases. - Keycloak - - Keycloak IAM must be updated to version `24.0.4`. Follow the [Keycloak upgrade guide](https://www.keycloak.org/docs/24.0.0/upgrading/) for more information. + - Keycloak IAM must be updated to version `24.0.4`. Follow the [Keycloak upgrade guide](https://www.keycloak.org/docs) for more information. - Portal Backend - Following environment variables have been added and **must be configured** for each environment diff --git a/authority-portal-backend/authority-portal-api/src/main/java/de/sovity/authorityportal/api/UiResource.kt b/authority-portal-backend/authority-portal-api/src/main/java/de/sovity/authorityportal/api/UiResource.kt index b3e7ee32d..0745d5fee 100644 --- a/authority-portal-backend/authority-portal-api/src/main/java/de/sovity/authorityportal/api/UiResource.kt +++ b/authority-portal-backend/authority-portal-api/src/main/java/de/sovity/authorityportal/api/UiResource.kt @@ -17,11 +17,11 @@ import de.sovity.authorityportal.api.model.CentralComponentCreateRequest import de.sovity.authorityportal.api.model.CentralComponentDto import de.sovity.authorityportal.api.model.ComponentStatusOverview import de.sovity.authorityportal.api.model.ConfigureProvidedConnectorWithCertificateRequest +import de.sovity.authorityportal.api.model.ConfigureProvidedConnectorWithJwksRequest import de.sovity.authorityportal.api.model.ConnectorDetailsDto import de.sovity.authorityportal.api.model.ConnectorOverviewResult import de.sovity.authorityportal.api.model.CreateCaasRequest import de.sovity.authorityportal.api.model.CreateConnectorRequest -import de.sovity.authorityportal.api.model.ConfigureProvidedConnectorWithJwksRequest import de.sovity.authorityportal.api.model.CreateConnectorResponse import de.sovity.authorityportal.api.model.DeploymentEnvironmentDto import de.sovity.authorityportal.api.model.IdResponse diff --git a/authority-portal-backend/authority-portal-quarkus/src/main/kotlin/de/sovity/authorityportal/web/thirdparty/daps/DapsClient.kt b/authority-portal-backend/authority-portal-quarkus/src/main/kotlin/de/sovity/authorityportal/web/thirdparty/daps/DapsClient.kt index c26b2e6f3..4e32c6ac0 100644 --- a/authority-portal-backend/authority-portal-quarkus/src/main/kotlin/de/sovity/authorityportal/web/thirdparty/daps/DapsClient.kt +++ b/authority-portal-backend/authority-portal-quarkus/src/main/kotlin/de/sovity/authorityportal/web/thirdparty/daps/DapsClient.kt @@ -16,6 +16,7 @@ package de.sovity.authorityportal.web.thirdparty.daps import de.sovity.authorityportal.web.environment.DeploymentEnvironmentConfiguration.DeploymentEnvironment.DapsConfig import de.sovity.authorityportal.web.thirdparty.daps.ext.CustomKeycloakResource import de.sovity.authorityportal.web.thirdparty.daps.ext.instantiateResource +import io.quarkus.logging.Log import org.keycloak.admin.client.KeycloakBuilder import org.keycloak.representations.idm.ClientRepresentation import org.keycloak.representations.idm.ProtocolMapperRepresentation @@ -43,7 +44,9 @@ class DapsClient(dapsConfig: DapsConfig): AutoCloseable { } fun createClient(clientId: String) { + Log.info("Creating client $clientId in realm $realmName") keycloak.realm(realmName).clients().create(buildClientRepresentation(clientId)) + Log.info("Client $clientId created in realm $realmName") } fun deleteClient(clientId: String) { @@ -66,6 +69,7 @@ class DapsClient(dapsConfig: DapsConfig): AutoCloseable { } fun addJwksUrl(clientId: String, jwksUrl: String) { + Log.info("Getting client $clientId in realm $realmName") val client = getClientById(clientId) ?: error("Client not found") client.attributes["jwks.url"] = jwksUrl diff --git a/authority-portal-backend/authority-portal-quarkus/src/main/resources/application.properties b/authority-portal-backend/authority-portal-quarkus/src/main/resources/application.properties index 470688e82..96ad48184 100644 --- a/authority-portal-backend/authority-portal-quarkus/src/main/resources/application.properties +++ b/authority-portal-backend/authority-portal-quarkus/src/main/resources/application.properties @@ -78,6 +78,8 @@ quarkus.arc.exclude-types=io.swagger.v3.jaxrs2.** %test.quarkus.log.console.json=false %test.quarkus.otel.traces.exporter=none +quarkus.otel.sdk.disabled=true + # Rest Client quarkus.rest-client.http2=true diff --git a/authority-portal-backend/build.gradle.kts b/authority-portal-backend/build.gradle.kts index 8b5b8158f..91a03e847 100644 --- a/authority-portal-backend/build.gradle.kts +++ b/authority-portal-backend/build.gradle.kts @@ -23,6 +23,10 @@ subprojects { password = project.findProperty("gpr.key") as String? ?: System.getenv("GPR_KEY") } } + maven { + url = uri("https://pkgs.dev.azure.com/sovity/41799556-91c8-4df6-8ddb-4471d6f15953/_packaging/core-edc/maven/v1") + name = "AzureRepo" + } } configurations.all { diff --git a/launchers/Dockerfile b/authority-portal-backend/catalog-crawler/Dockerfile similarity index 71% rename from launchers/Dockerfile rename to authority-portal-backend/catalog-crawler/Dockerfile index 06d01939c..f03775cad 100644 --- a/launchers/Dockerfile +++ b/authority-portal-backend/catalog-crawler/Dockerfile @@ -9,29 +9,24 @@ RUN adduser -D -H -s /sbin/nologin edc USER edc:edc # Which app.jar to include -ARG CONNECTOR_NAME="sovity-ce" -ARG CONNECTOR_TYPE="connector" +ARG CONNECTOR_NAME="catalog-crawler-ce" # For last-commit-info extension ARG EDC_LAST_COMMIT_INFO_ARG="The docker container was built outside of github actions and you didn't provide the build arg EDC_LAST_COMMIT_INFO_ARG, so there's no last commit info." ARG EDC_BUILD_DATE_ARG="The docker container was built outside of github actions and you didn't provide the build arg EDC_BUILD_DATE_ARG, so there's no build date." WORKDIR /app -COPY ./launchers/connectors/$CONNECTOR_NAME/build/libs/app.jar /app -COPY ./launchers/logging.properties /app -COPY ./launchers/logging.dev.properties /app -COPY ./launchers/.env.$CONNECTOR_TYPE /app/.env +COPY ./authority-portal-backend/catalog-crawler/${CONNECTOR_NAME}-launcher/build/libs/app.jar /app +COPY ./authority-portal-backend/catalog-crawler/logging.properties /app +COPY ./authority-portal-backend/catalog-crawler/logging.dev.properties /app RUN touch /app/empty-properties-file.properties -# Replaces var statements so when they are sourced as bash they don't overwrite existing env vars -RUN sed -ri 's/^\s*(\S+)=(.*)$/\1=${\1:-"\2"}/' .env - ENV EDC_LAST_COMMIT_INFO=$EDC_LAST_COMMIT_INFO_ARG ENV EDC_BUILD_DATE=$EDC_BUILD_DATE_ARG ENV JVM_ARGS="" -COPY ./launchers/docker-entrypoint.sh /app/entrypoint.sh +COPY ./authority-portal-backend/catalog-crawler/docker-entrypoint.sh /app/entrypoint.sh ENTRYPOINT ["/app/entrypoint.sh"] CMD ["start"] diff --git a/authority-portal-backend/catalog-crawler/README.md b/authority-portal-backend/catalog-crawler/README.md new file mode 100644 index 000000000..05633cd3f --- /dev/null +++ b/authority-portal-backend/catalog-crawler/README.md @@ -0,0 +1,41 @@ + +
+
+ + Logo + + +

EDC-Connector Extension:
Catalog Crawler

+ +

+ Report Bug + · + Request Feature +

+
+ +## About this Extension + +The Catalog Crawler is an additional deployment unit needed to determine the online status of registered connectors and populate the Data Catalog: + +- It is a modified EDC connector with the task to crawl the other connectors' public data offers. +- It periodically checks the Authority Portal's connector list for its environment. +- It crawls the given connectors in regular intervals. +- It writes the data offers and connector statuses into the Authority Portal DB. +- Each environment configured in the Authority Portal requires its own Catalog Crawler with credentials for that environment's DAPS. + +## Why does this component exist? + +The Authority Portal uses a non-EDC stack and thus it cannot read the catalogs of participating connectors directly. + +## Deployment + +Please see the [Productive Deployment Guide](../../docs/deployment-guide/goals/production/README.md) for more information. + +## License + +Apache License 2.0 - see [LICENSE](../../LICENSE) + +## Contact + +sovity GmbH - contact@sovity.de diff --git a/launchers/connectors/catalog-crawler-ce/build.gradle.kts b/authority-portal-backend/catalog-crawler/catalog-crawler-ce-launcher/build.gradle.kts similarity index 55% rename from launchers/connectors/catalog-crawler-ce/build.gradle.kts rename to authority-portal-backend/catalog-crawler/catalog-crawler-ce-launcher/build.gradle.kts index e72aa1f52..4b535bded 100644 --- a/launchers/connectors/catalog-crawler-ce/build.gradle.kts +++ b/authority-portal-backend/catalog-crawler/catalog-crawler-ce-launcher/build.gradle.kts @@ -5,19 +5,22 @@ plugins { } dependencies { - implementation(project(":extensions:catalog-crawler:catalog-crawler-launcher-base")) + implementation(project(":catalog-crawler:catalog-crawler-launcher-base")) api(libs.edc.monitorJdkLogger) api(libs.edc.apiObservability) - implementation(project(":launchers:common:auth-daps")) + implementation(libs.edc.oauth2Core) + implementation(libs.edc.vaultFilesystem) } application { - mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") + mainClass.set("de.sovity.edc.ext.catalog.crawler.Main") } tasks.withType { mergeServiceFiles() archiveFileName.set("app.jar") } + +group = libs.versions.sovityEdcGroup.get() diff --git a/extensions/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts b/authority-portal-backend/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts similarity index 86% rename from extensions/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts rename to authority-portal-backend/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts index 2c376a085..3b30edd30 100644 --- a/extensions/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts +++ b/authority-portal-backend/catalog-crawler/catalog-crawler-launcher-base/build.gradle.kts @@ -13,7 +13,7 @@ dependencies { api(libs.edc.jsonLd) // Data Catalog Crawler - api(project(":extensions:catalog-crawler:catalog-crawler")) + api(project(":catalog-crawler:catalog-crawler")) } group = libs.versions.sovityEdcGroup.get() diff --git a/extensions/catalog-crawler/catalog-crawler/build.gradle.kts b/authority-portal-backend/catalog-crawler/catalog-crawler/build.gradle.kts similarity index 61% rename from extensions/catalog-crawler/catalog-crawler/build.gradle.kts rename to authority-portal-backend/catalog-crawler/catalog-crawler/build.gradle.kts index 7c2b46f7e..e4e30b170 100644 --- a/extensions/catalog-crawler/catalog-crawler/build.gradle.kts +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/build.gradle.kts @@ -10,18 +10,19 @@ dependencies { implementation(libs.edc.managementApiConfiguration) implementation(libs.quartz.quartz) - implementation(libs.apache.commonsLang) - implementation(project(":utils:versions")) + implementation(libs.commons.lang3) + implementation(libs.quarkus.jooq) - api(project(":utils:catalog-parser")) - api(project(":utils:json-and-jsonld-utils")) - api(project(":extensions:wrapper:wrapper-common-mappers")) - api(project(":extensions:catalog-crawler:catalog-crawler-db")) - api(project(":extensions:postgres-flyway-core")) + api(libs.sovity.edc.catalogParser) + api(libs.sovity.edc.jsonAndJsonLdUtils) + api(libs.sovity.edc.wrapperCommonMappers) + api(libs.sovity.edc.ext.postgresFlywayCore) + api(libs.sovity.edc.config) + api(project(":authority-portal-db")) testAnnotationProcessor(libs.lombok) testCompileOnly(libs.lombok) - testImplementation(project(":utils:test-utils")) + testImplementation(libs.sovity.edc.ext.testUtils) testImplementation(libs.assertj.core) testImplementation(libs.mockito.core) testImplementation(libs.restAssured.restAssured) @@ -38,11 +39,3 @@ tasks.getByName("test") { useJUnitPlatform() maxParallelForks = 1 } - -publishing { - publications { - create(project.name) { - from(components["java"]) - } - } -} diff --git a/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerConfigProps.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerConfigProps.java new file mode 100644 index 000000000..b62f3e3b1 --- /dev/null +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerConfigProps.java @@ -0,0 +1,549 @@ +/* + * Copyright (c) 2024 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial implementation + */ + +package de.sovity.edc.ext.catalog.crawler; + +import de.sovity.edc.utils.config.ConfigUtils; +import de.sovity.edc.utils.config.model.ConfigProp; +import de.sovity.edc.utils.config.utils.UrlPathUtils; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; +import lombok.experimental.UtilityClass; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Supplier; + +@UtilityClass +public class CrawlerConfigProps { + public static final List ALL_CRAWLER_PROPS = new ArrayList<>(); + + /* Crawler-specific Configuration */ + public static final ConfigProp CRAWLER_ENVIRONMENT_ID = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.BASIC) + .property("crawler.environment.id") + .description("Environment ID") + .required(true) + ); + + public static final ConfigProp CRAWLER_DB_JDBC_URL = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.BASIC) + .property("crawler.db.jdbc.url") + .description("PostgreSQL DB Connection: JDBC URL") + .required(true) + ); + + public static final ConfigProp CRAWLER_DB_JDBC_USER = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.BASIC) + .property("crawler.db.jdbc.user") + .description("PostgreSQL DB Connection: Username") + .required(true) + ); + + public static final ConfigProp CRAWLER_DB_JDBC_PASSWORD = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.BASIC) + .property("crawler.db.jdbc.password") + .description("PostgreSQL DB Connection: Password") + .required(true) + ); + + public static final ConfigProp CRAWLER_DB_CONNECTION_POOL_SIZE = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.ADVANCED) + .property("crawler.db.connection.pool.size") + .description("Size of the Hikari Connection Pool") + .defaultValue("30") + ); + + public static final ConfigProp CRAWLER_DB_CONNECTION_TIMEOUT_IN_MS = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.ADVANCED) + .property("crawler.db.connection.timeout.in.ms") + .description("Sets the connection timeout for the datasource in milliseconds.") + .defaultValue("30000") + ); + + public static final ConfigProp CRAWLER_CRON_ONLINE_CONNECTOR_REFRESH = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.ADVANCED) + .property("crawler.cron.online.connector.refresh") + .description("Cron expression for crawling ONLINE connectors") + .defaultValue("*/20 * * ? * *") + ); + + public static final ConfigProp CRAWLER_CRON_OFFLINE_CONNECTOR_REFRESH = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.ADVANCED) + .property("crawler.cron.offline.connector.refresh") + .description("Cron expression for crawling OFFLINE connectors") + .defaultValue("0 */5 * ? * *") + ); + + public static final ConfigProp CRAWLER_CRON_DEAD_CONNECTOR_REFRESH = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.ADVANCED) + .property("crawler.cron.dead.connector.refresh") + .description("Cron expression for crawling DEAD connectors") + .defaultValue("0 0 * ? * *") + ); + + public static final ConfigProp CRAWLER_SCHEDULED_KILL_OFFLINE_CONNECTORS = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.ADVANCED) + .property("crawler.scheduled.kill.offline.connectors") + .description("Scheduled task for marking connectors as DEAD") + .defaultValue("0 0 2 ? * *") + ); + + public static final ConfigProp CRAWLER_KILL_OFFLINE_CONNECTORS_AFTER = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.ADVANCED) + .property("crawler.kill.offline.connectors.after") + .description("Time in Java Duration Format after which an OFFLINE connector is marked as DEAD") + .defaultValue("P5D") + ); + + public static final ConfigProp CRAWLER_HIDE_OFFLINE_DATA_OFFERS_AFTER = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.ADVANCED) + .property("crawler.hide.offline.data.offers.after") + .description("Time in Java Duration Format after which an OFFLINE data offer is hidden") + .defaultValue("P1D") + ); + + public static final ConfigProp CRAWLER_NUM_THREADS = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.ADVANCED) + .property("crawler.num.threads") + .description("Number of threads for crawling") + .defaultValue("32") + ); + + public static final ConfigProp CRAWLER_MAX_DATA_OFFERS_PER_CONNECTOR = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.ADVANCED) + .property("crawler.max.data.offers.per.connector") + .description("Maximum number of data offers per connector") + .defaultValue("50") + ); + + public static final ConfigProp CRAWLER_MAX_CONTRACT_OFFERS_PER_DATA_OFFER = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.ADVANCED) + .property("crawler.max.contract.offers.per.data.offer") + .description("Maximum number of contract offers per data offer") + .defaultValue("10") + ); + + public static final ConfigProp MY_EDC_NETWORK_TYPE = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.ADVANCED) + .property("my.edc.network.type") + .description("Configuring EDCs for different environments. Available values are: %s".formatted( + String.join(", ", CrawlerConfigProps.NetworkType.ALL_NETWORK_TYPES))) + .warnIfOverridden(true) + .defaultValue(CrawlerConfigProps.NetworkType.PRODUCTION) + ); + + /* Basic Configuration */ + + public static final ConfigProp MY_EDC_PARTICIPANT_ID = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.BASIC) + .property("my.edc.participant.id") + .description("Participant ID / Connector ID") + .defaultValue("broker") + .warnIfOverridden(true) + .required(true) + ); + + public static final ConfigProp MY_EDC_FQDN = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.BASIC) + .property("my.edc.fqdn") + .description("Fully Qualified Domain Name of where the Connector is hosted, e.g. my-connector.myorg.com") + .requiredIf(props -> CrawlerConfigProps.NetworkType.isProduction(props) || CrawlerConfigProps.NetworkType.isLocalDemoDockerCompose(props)) + .defaultValueFn(props -> new CrawlerConfigProps.NetworkTypeMatcher(props).unitTest(() -> "localhost").orElseThrow()) + ); + + /* Auth */ + + public static final ConfigProp MY_EDC_C2C_IAM_TYPE = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.C2C_IAM) + .property("my.edc.c2c.iam.type") + .description("Type of Connector-to-Connector IAM / Authentication Mechanism used. " + + "Available values are: 'daps-sovity', 'daps-omejdn', 'mock-iam'. Default: 'daps-sovity'") + .warnIfOverridden(true) + .defaultValue("daps-sovity") + ); + + public static final ConfigProp EDC_OAUTH_TOKEN_URL = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.C2C_IAM) + .property("edc.oauth.token.url") + .description("OAuth2 / DAPS: Token URL") + .relevantIf(props -> MY_EDC_C2C_IAM_TYPE.getRaw(props).startsWith("daps")) + .required(true) + ); + + public static final ConfigProp EDC_OAUTH_PROVIDER_JWKS_URL = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.C2C_IAM) + .property("edc.oauth.provider.jwks.url") + .description("OAuth2 / DAPS: JWKS URL") + .relevantIf(props -> MY_EDC_C2C_IAM_TYPE.getRaw(props).startsWith("daps")) + .required(true) + ); + + public static final ConfigProp EDC_OAUTH_CLIENT_ID = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.C2C_IAM) + .property("edc.oauth.client.id") + .description("OAuth2 / DAPS: Client ID. Defaults to Participant ID") + .relevantIf(props -> MY_EDC_C2C_IAM_TYPE.getRaw(props).startsWith("daps")) + .defaultValueFn(MY_EDC_PARTICIPANT_ID::getRaw) + ); + + public static final ConfigProp EDC_KEYSTORE = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.C2C_IAM) + .property("edc.keystore") + .description("File-Based Vault: Keystore file (.jks)") + .relevantIf(props -> MY_EDC_C2C_IAM_TYPE.getRaw(props).startsWith("daps")) + .required(true) + ); + + public static final ConfigProp EDC_KEYSTORE_PASSWORD = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.C2C_IAM) + .property("edc.keystore.password") + .description("File-Based Vault: Keystore password") + .relevantIf(props -> MY_EDC_C2C_IAM_TYPE.getRaw(props).startsWith("daps")) + .required(true) + ); + + public static final ConfigProp EDC_OAUTH_CERTIFICATE_ALIAS = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.C2C_IAM) + .property("edc.oauth.certificate.alias") + .description("OAuth2 / DAPS: Certificate Vault Entry for the Public Key. Default: '1'") + .relevantIf(props -> MY_EDC_C2C_IAM_TYPE.getRaw(props).startsWith("daps")) + .defaultValue("1") + ); + + public static final ConfigProp EDC_OAUTH_PRIVATE_KEY_ALIAS = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.C2C_IAM) + .property("edc.oauth.private.key.alias") + .description("OAuth2 / DAPS: Certificate Vault Entry for the Private Key. Default: '1'") + .relevantIf(props -> MY_EDC_C2C_IAM_TYPE.getRaw(props).startsWith("daps")) + .defaultValue("1") + ); + + public static final ConfigProp EDC_OAUTH_PROVIDER_AUDIENCE = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.C2C_IAM) + .property("edc.oauth.provider.audience") + .description("OAuth2 / DAPS: Provider Audience") + .relevantIf(props -> MY_EDC_C2C_IAM_TYPE.getRaw(props).startsWith("daps")) + .warnIfOverridden(true) + .defaultValueFn(props -> { + if ("daps-omejdn".equals(MY_EDC_C2C_IAM_TYPE.getRaw(props))) { + return "idsc:IDS_CONNECTORS_ALL"; + } + + // daps-sovity + return EDC_OAUTH_TOKEN_URL.getRaw(props); + }) + ); + + public static final ConfigProp EDC_OAUTH_ENDPOINT_AUDIENCE = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.C2C_IAM) + .property("edc.oauth.endpoint.audience") + .description("OAuth2 / DAPS: Endpoint Audience") + .relevantIf(props -> MY_EDC_C2C_IAM_TYPE.getRaw(props).startsWith("daps")) + .warnIfOverridden(true) + .defaultValue("idsc:IDS_CONNECTORS_ALL") + ); + + public static final ConfigProp EDC_AGENT_IDENTITY_KEY = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.C2C_IAM) + .property("edc.agent.identity.key") + .description("OAuth2 / DAPS: Agent Identity Key") + .relevantIf(props -> MY_EDC_C2C_IAM_TYPE.getRaw(props).startsWith("daps")) + .warnIfOverridden(true) + .defaultValueFn(props -> { + if ("daps-omejdn".equals(MY_EDC_C2C_IAM_TYPE.getRaw(props))) { + return "client_id"; + } + + // daps-sovity + return "referringConnector"; + }) + ); + + /* Advanced */ + + public static final ConfigProp MY_EDC_FIRST_PORT = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.ADVANCED) + .property("my.edc.first.port") + .description("The first port of several ports to be used for the several API endpoints. " + + "Useful when starting two EDCs on the host machine network / during tests") + .warnIfOverridden(true) + .defaultValue("11000") + ); + + public static final ConfigProp EDC_WEB_REST_CORS_ENABLED = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.ADVANCED) + .property("edc.web.rest.cors.enabled") + .description("Enable CORS") + .warnIfOverridden(true) + .relevantIf(props -> !CrawlerConfigProps.NetworkType.isProduction(props)) + .defaultValue("true") + ); + + public static final ConfigProp EDC_WEB_REST_CORS_HEADERS = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.ADVANCED) + .property("edc.web.rest.cors.headers") + .description("CORS: Allowed Headers") + .warnIfOverridden(true) + .relevantIf(props -> !CrawlerConfigProps.NetworkType.isProduction(props)) + .defaultValue("origin,content-type,accept,authorization,X-Api-Key") + ); + + public static final ConfigProp EDC_WEB_REST_CORS_ORIGINS = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.ADVANCED) + .property("edc.web.rest.cors.origins") + .description("CORS: Allowed Origins") + .warnIfOverridden(true) + .relevantIf(props -> !CrawlerConfigProps.NetworkType.isProduction(props)) + .defaultValue("*") + ); + + /* Defaults of EDC Configuration */ + + public static final ConfigProp MY_EDC_PROTOCOL = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.RAW_EDC_CONFIG_DEFAULTS) + .property("my.edc.protocol") + .description("HTTP Protocol for when the EDC exposes its own URL for callbacks") + .warnIfOverridden(true) + .defaultValueFn(props -> CrawlerConfigProps.NetworkType.isProduction(props) ? "https://" : "http://") + ); + + public static final ConfigProp MY_EDC_BASE_PATH = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.RAW_EDC_CONFIG_DEFAULTS) + .property("my.edc.base.path") + .description("Optional prefix to be added before all API paths") + .warnIfOverridden(true) + .defaultValue("/") + ); + + public static final ConfigProp WEB_HTTP_PATH = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.RAW_EDC_CONFIG_DEFAULTS) + .property("web.http.path") + .description("API Group 'Web' contains misc API endpoints, usually not meant to be public, this is the base path.") + .warnIfOverridden(true) + .defaultValueFn(props -> UrlPathUtils.urlPathJoin(MY_EDC_BASE_PATH.getRaw(props), "api")) + ); + + public static final ConfigProp WEB_HTTP_PORT = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.RAW_EDC_CONFIG_DEFAULTS) + .property("web.http.port") + .description("API Group 'Web' contains misc API endpoints, usually not meant to be public, this is the port.") + .warnIfOverridden(true) + .defaultValueFn(props -> plus(props, MY_EDC_FIRST_PORT, 1)) + ); + + public static final ConfigProp WEB_HTTP_MANAGEMENT_PATH = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.RAW_EDC_CONFIG_DEFAULTS) + .property("web.http.management.path") + .description("API Group 'Management' contains API endpoints for EDC interaction and " + + "should be protected from unauthorized access. This is the base path.") + .warnIfOverridden(true) + .defaultValueFn(props -> UrlPathUtils.urlPathJoin(MY_EDC_BASE_PATH.getRaw(props), "api/management")) + ); + + public static final ConfigProp WEB_HTTP_MANAGEMENT_PORT = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.RAW_EDC_CONFIG_DEFAULTS) + .property("web.http.management.port") + .description( + "API Group 'Management' contains API endpoints for EDC interaction and " + + "should be protected from unauthorized access. This is the port.") + .warnIfOverridden(true) + .defaultValueFn(props -> plus(props, MY_EDC_FIRST_PORT, 2)) + ); + + public static final ConfigProp WEB_HTTP_PROTOCOL_PATH = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.RAW_EDC_CONFIG_DEFAULTS) + .property("web.http.protocol.path") + .description("API Group 'Protocol' must be public as it is used for connector to connector communication, this is the base path.") + .warnIfOverridden(true) + .defaultValueFn(props -> UrlPathUtils.urlPathJoin(MY_EDC_BASE_PATH.getRaw(props), "api/dsp")) + ); + + public static final ConfigProp WEB_HTTP_PROTOCOL_PORT = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.RAW_EDC_CONFIG_DEFAULTS) + .property("web.http.protocol.port") + .description("API Group 'Protocol' must be public as it is used for connector to connector communication, this is the port.") + .warnIfOverridden(true) + .defaultValueFn(props -> plus(props, MY_EDC_FIRST_PORT, 3)) + ); + + public static final ConfigProp WEB_HTTP_CONTROL_PATH = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.RAW_EDC_CONFIG_DEFAULTS) + .property("web.http.control.path") + .description("API Group 'Control' contains API endpoints for control plane/data plane interaction and " + + "should be non-public, this is the base path.") + .warnIfOverridden(true) + .defaultValueFn(props -> UrlPathUtils.urlPathJoin(MY_EDC_BASE_PATH.getRaw(props), "api/control")) + ); + + public static final ConfigProp WEB_HTTP_CONTROL_PORT = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.RAW_EDC_CONFIG_DEFAULTS) + .property("web.http.control.port") + .description("API Group 'Control' contains API endpoints for control plane/data plane interaction and " + + "should be non-public, this is the port.") + .warnIfOverridden(true) + .defaultValueFn(props -> plus(props, MY_EDC_FIRST_PORT, 4)) + ); + + public static final ConfigProp WEB_HTTP_PUBLIC_PATH = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.RAW_EDC_CONFIG_DEFAULTS) + .property("web.http.public.path") + .description("API Group 'Public' contains public data plane API endpoints. This is the base path.") + .warnIfOverridden(true) + .defaultValueFn(props -> UrlPathUtils.urlPathJoin(MY_EDC_BASE_PATH.getRaw(props), "api/public")) + ); + + public static final ConfigProp WEB_HTTP_PUBLIC_PORT = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.RAW_EDC_CONFIG_DEFAULTS) + .property("web.http.public.port") + .description("API Group 'Public' contains public data plane API endpoints. This is the port.") + .warnIfOverridden(true) + .defaultValueFn(props -> plus(props, MY_EDC_FIRST_PORT, 5)) + ); + + public static final ConfigProp EDC_JSONLD_HTTPS_ENABLED = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.RAW_EDC_CONFIG_DEFAULTS) + .property("edc.jsonld.https.enabled") + .description("Required to be set since Eclipse EDC 0.2.1") + .warnIfOverridden(true) + .defaultValue("true") + ); + + public static final ConfigProp EDC_CONNECTOR_NAME = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.RAW_EDC_CONFIG_DEFAULTS) + .property("edc.connector.name") + .description("Connector Name") + .warnIfOverridden(true) + .defaultValueFn(MY_EDC_PARTICIPANT_ID::getRaw) + ); + + public static final ConfigProp EDC_PARTICIPANT_ID = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.RAW_EDC_CONFIG_DEFAULTS) + .property("edc.participant.id") + .description("Participant ID / Connector ID") + .warnIfOverridden(true) + .defaultValueFn(MY_EDC_PARTICIPANT_ID::getRaw) + ); + + public static final ConfigProp EDC_HOSTNAME = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.RAW_EDC_CONFIG_DEFAULTS) + .property("edc.hostname") + .description("Same as %s".formatted(MY_EDC_FQDN.getProperty())) + .warnIfOverridden(true) + .defaultValueFn(MY_EDC_FQDN::getRaw) + ); + + public static final ConfigProp EDC_DSP_CALLBACK_ADDRESS = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.RAW_EDC_CONFIG_DEFAULTS) + .property("edc.dsp.callback.address") + .description("Full URL for the DSP callback address") + .warnIfOverridden(true) + .defaultValueFn(ConfigUtils::getProtocolApiUrl) + ); + + public static final ConfigProp EDC_VAULT = addCeProp(builder -> builder + .category(CrawlerConfigProps.Category.RAW_EDC_CONFIG_DEFAULTS) + .property("edc.vault") + .description("This file could contain an entry replacing the EDC_KEYSTORE ENV var, " + + "but for some reason it is required, and EDC won't start up if it isn't configured." + + "It is created in the Dockerfile") + .relevantIf(CrawlerConfigProps.NetworkType::isProduction) + .defaultValue("/app/empty-properties-file.properties") + ); + + /* Helpers */ + + private static ConfigProp addCeProp(Consumer builderFn) { + var builder = ConfigProp.builder(); + builderFn.accept(builder); + var built = builder.build(); + + // Register the property in the list of all available CE properties + // Order matters here, as the property defaults are calculated in order + built.also(ALL_CRAWLER_PROPS::add); + return built; + } + + private static String plus(Map props, ConfigProp prop, int add) { + var raw = prop.getRaw(props); + var result = Integer.parseInt(raw == null ? "0" : raw) + add; + return String.valueOf(result); + } + + @UtilityClass + public static class NetworkType { + public static final String PRODUCTION = "production"; + public static final String LOCAL_DEMO_DOCKER_COMPOSE = "local-demo-docker-compose"; + public static final String UNIT_TEST = "unit-test"; + public static final List ALL_NETWORK_TYPES = List.of(PRODUCTION, LOCAL_DEMO_DOCKER_COMPOSE, UNIT_TEST); + + public static boolean isProduction(Map props) { + return CrawlerConfigProps.NetworkType.PRODUCTION.equals(MY_EDC_NETWORK_TYPE.getRaw(props)); + } + + public static boolean isLocalDemoDockerCompose(Map props) { + return CrawlerConfigProps.NetworkType.LOCAL_DEMO_DOCKER_COMPOSE.equals(MY_EDC_NETWORK_TYPE.getRaw(props)); + } + + public static boolean isUnitTest(Map props) { + return CrawlerConfigProps.NetworkType.UNIT_TEST.equals(MY_EDC_NETWORK_TYPE.getRaw(props)); + } + } + + @Setter + @Accessors(fluent = true, chain = true) + @RequiredArgsConstructor + public static class NetworkTypeMatcher { + private final Map props; + private Supplier production; + private Supplier localDemoDockerCompose; + private Supplier unitTest; + + public T orElse(Supplier elseFn) { + if (production != null && CrawlerConfigProps.NetworkType.isProduction(props)) { + return production.get(); + } + + if (localDemoDockerCompose != null && CrawlerConfigProps.NetworkType.isLocalDemoDockerCompose(props)) { + return localDemoDockerCompose.get(); + } + + if (unitTest != null && CrawlerConfigProps.NetworkType.isUnitTest(props)) { + return unitTest.get(); + } + + return elseFn.get(); + } + + public T orElseThrow() { + return orElse(() -> { + var msg = "Unhandled %s: %s".formatted( + MY_EDC_NETWORK_TYPE.getProperty(), + MY_EDC_NETWORK_TYPE.getRaw(props) + ); + throw new IllegalArgumentException(msg); + }); + } + } + + @UtilityClass + private static class Category { + public static final String BASIC = "Basic Configuration"; + public static final String ADVANCED = "Advanced configuration"; + public static final String C2C_IAM = "Connector-to-Connector IAM"; + public static final String RAW_EDC_CONFIG_DEFAULTS = "EDC Config Defaults / Overrides"; + } +} diff --git a/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java new file mode 100644 index 000000000..00bda31d6 --- /dev/null +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2023 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial API and implementation + * + */ + +package de.sovity.edc.ext.catalog.crawler; + +import de.sovity.edc.ext.wrapper.api.common.mappers.PlaceholderEndpointService; +import org.eclipse.edc.connector.api.management.configuration.transform.ManagementApiTypeTransformerRegistry; +import org.eclipse.edc.connector.spi.catalog.CatalogService; +import org.eclipse.edc.jsonld.spi.JsonLd; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Provides; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.spi.types.TypeManager; + +@Provides({CrawlerExtensionContext.class}) +public class CrawlerExtension implements ServiceExtension { + + public static final String EXTENSION_NAME = "Authority Portal Data Catalog Crawler"; + + @Inject + private TypeManager typeManager; + + @Inject + private ManagementApiTypeTransformerRegistry typeTransformerRegistry; + + @Inject + private JsonLd jsonLd; + + @Inject + private CatalogService catalogService; + + /** + * Manual Dependency Injection Result + */ + private CrawlerExtensionContext services; + + @Override + public String name() { + return EXTENSION_NAME; + } + + @Override + public void initialize(ServiceExtensionContext context) { + services = CrawlerExtensionContextBuilder.buildContext( + context.getConfig(), + context.getMonitor(), + typeManager, + typeTransformerRegistry, + jsonLd, + catalogService, + new PlaceholderEndpointService("http://0.0.0.0/") + ); + + // Provide access for the tests + context.registerService(CrawlerExtensionContext.class, services); + } + + @Override + public void start() { + if (services == null) { + return; + } + services.crawlerInitializer().onStartup(); + } + + @Override + public void shutdown() { + if (services == null) { + return; + } + services.dataSource().close(); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContext.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContext.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContext.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContext.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java similarity index 95% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java index 89aa9e491..6aaf52998 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java @@ -33,7 +33,6 @@ import de.sovity.edc.ext.catalog.crawler.dao.CatalogPatchApplier; import de.sovity.edc.ext.catalog.crawler.dao.config.DataSourceFactory; import de.sovity.edc.ext.catalog.crawler.dao.config.DslContextFactory; -import de.sovity.edc.ext.catalog.crawler.dao.config.FlywayService; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorQueries; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorStatusUpdater; import de.sovity.edc.ext.catalog.crawler.dao.contract_offers.ContractOfferQueries; @@ -112,8 +111,6 @@ public static CrawlerExtensionContext buildContext( // DB var dataSourceFactory = new DataSourceFactory(config); var dataSource = dataSourceFactory.newDataSource(); - var flywayService = new FlywayService(config, monitor, dataSource); - flywayService.validateOrMigrateInTests(); // Dao var dataOfferQueries = new DataOfferQueries(); @@ -183,10 +180,10 @@ public static CrawlerExtensionContext buildContext( // Schedules List> jobs = List.of( - getOnlineConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller), - getOfflineConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller), - getDeadConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller), - getOfflineConnectorCleanerCronJob(dslContextFactory, offlineConnectorCleaner) + getOnlineConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller, config), + getOfflineConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller, config), + getDeadConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller, config), + getOfflineConnectorCleanerCronJob(dslContextFactory, offlineConnectorCleaner, config) ); // Startup @@ -264,9 +261,9 @@ private static AssetMapper newAssetMapper( @NotNull private static CronJobRef getOfflineConnectorCleanerCronJob(DslContextFactory dslContextFactory, - OfflineConnectorCleaner offlineConnectorCleaner) { + OfflineConnectorCleaner offlineConnectorCleaner, Config config) { return new CronJobRef<>( - CrawlerExtension.SCHEDULED_KILL_OFFLINE_CONNECTORS, + CrawlerConfigProps.CRAWLER_SCHEDULED_KILL_OFFLINE_CONNECTORS.getProperty(), OfflineConnectorCleanerJob.class, () -> new OfflineConnectorCleanerJob(dslContextFactory, offlineConnectorCleaner) ); @@ -275,10 +272,11 @@ private static CronJobRef getOfflineConnectorCleaner @NotNull private static CronJobRef getOnlineConnectorRefreshCronJob( DslContextFactory dslContextFactory, - ConnectorQueueFiller connectorQueueFiller + ConnectorQueueFiller connectorQueueFiller, + Config config ) { return new CronJobRef<>( - CrawlerExtension.CRON_ONLINE_CONNECTOR_REFRESH, + CrawlerConfigProps.CRAWLER_CRON_ONLINE_CONNECTOR_REFRESH.getProperty(), OnlineConnectorRefreshJob.class, () -> new OnlineConnectorRefreshJob(dslContextFactory, connectorQueueFiller) ); @@ -287,10 +285,11 @@ private static CronJobRef getOnlineConnectorRefreshCr @NotNull private static CronJobRef getOfflineConnectorRefreshCronJob( DslContextFactory dslContextFactory, - ConnectorQueueFiller connectorQueueFiller + ConnectorQueueFiller connectorQueueFiller, + Config config ) { return new CronJobRef<>( - CrawlerExtension.CRON_OFFLINE_CONNECTOR_REFRESH, + CrawlerConfigProps.CRAWLER_CRON_OFFLINE_CONNECTOR_REFRESH.getProperty(), OfflineConnectorRefreshJob.class, () -> new OfflineConnectorRefreshJob(dslContextFactory, connectorQueueFiller) ); @@ -298,9 +297,9 @@ private static CronJobRef getOfflineConnectorRefresh @NotNull private static CronJobRef getDeadConnectorRefreshCronJob(DslContextFactory dslContextFactory, - ConnectorQueueFiller connectorQueueFiller) { + ConnectorQueueFiller connectorQueueFiller, Config config) { return new CronJobRef<>( - CrawlerExtension.CRON_DEAD_CONNECTOR_REFRESH, + CrawlerConfigProps.CRAWLER_CRON_DEAD_CONNECTOR_REFRESH.getProperty(), DeadConnectorRefreshJob.class, () -> new DeadConnectorRefreshJob(dslContextFactory, connectorQueueFiller) ); diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerInitializer.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerInitializer.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerInitializer.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerInitializer.java diff --git a/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/Main.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/Main.java new file mode 100644 index 000000000..457d78e24 --- /dev/null +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/Main.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2024 sovity GmbH + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * sovity GmbH - initial implementation + */ + +package de.sovity.edc.ext.catalog.crawler; + +import de.sovity.edc.utils.config.SovityEdcRuntime; + +public class Main { + public static void main(String[] args) { + SovityEdcRuntime.boot(CrawlerConfigProps.ALL_CRAWLER_PROPS); + } +} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/ConnectorCrawler.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/ConnectorCrawler.java similarity index 97% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/ConnectorCrawler.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/ConnectorCrawler.java index dca422308..3611d24e6 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/ConnectorCrawler.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/ConnectorCrawler.java @@ -14,6 +14,7 @@ package de.sovity.edc.ext.catalog.crawler.crawling; +import de.sovity.authorityportal.db.jooq.enums.MeasurementErrorStatus; import de.sovity.edc.ext.catalog.crawler.crawling.fetching.FetchedCatalogService; import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerExecutionTimeLogger; import de.sovity.edc.ext.catalog.crawler.crawling.writing.ConnectorUpdateFailureWriter; @@ -21,7 +22,6 @@ import de.sovity.edc.ext.catalog.crawler.dao.config.DslContextFactory; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorQueries; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.MeasurementErrorStatus; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.time.StopWatch; import org.eclipse.edc.spi.monitor.Monitor; @@ -52,7 +52,7 @@ public void crawlConnector(ConnectorRef connectorRef) { try { monitor.info("Updating connector: " + connectorRef); - + var catalog = fetchedCatalogService.fetchCatalog(connectorRef); // Update connector in a single transaction diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/OfflineConnectorCleaner.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/OfflineConnectorCleaner.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/OfflineConnectorCleaner.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/OfflineConnectorCleaner.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogBuilder.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogBuilder.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogBuilder.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogBuilder.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogMappingUtils.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogMappingUtils.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogMappingUtils.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogMappingUtils.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogService.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogService.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogService.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/FetchedCatalogService.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedCatalog.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedCatalog.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedCatalog.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedCatalog.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedContractOffer.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedContractOffer.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedContractOffer.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedContractOffer.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedDataOffer.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedDataOffer.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedDataOffer.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/fetching/model/FetchedDataOffer.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/ConnectorChangeTracker.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/ConnectorChangeTracker.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/ConnectorChangeTracker.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/ConnectorChangeTracker.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventErrorMessage.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventErrorMessage.java similarity index 95% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventErrorMessage.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventErrorMessage.java index 9d7ca039a..92bc595ed 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventErrorMessage.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventErrorMessage.java @@ -20,7 +20,7 @@ /** * Helper Dto that contains User Message + Error Stack Trace to be written into - * {@link de.sovity.edc.ext.catalog.crawler.db.jooq.tables.CrawlerEventLog}. + * {@link de.sovity.authorityportal.db.jooq.tables.CrawlerEventLog}. *
* This class exists so that logging exceptions has a consistent format. * diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLogger.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLogger.java similarity index 94% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLogger.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLogger.java index 396e2095e..80f32f864 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLogger.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLogger.java @@ -14,11 +14,11 @@ package de.sovity.edc.ext.catalog.crawler.crawling.logging; +import de.sovity.authorityportal.db.jooq.Tables; +import de.sovity.authorityportal.db.jooq.enums.CrawlerEventStatus; +import de.sovity.authorityportal.db.jooq.enums.CrawlerEventType; +import de.sovity.authorityportal.db.jooq.tables.records.CrawlerEventLogRecord; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; -import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.CrawlerEventStatus; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.CrawlerEventType; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.CrawlerEventLogRecord; import lombok.RequiredArgsConstructor; import org.jooq.DSLContext; diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerExecutionTimeLogger.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerExecutionTimeLogger.java similarity index 86% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerExecutionTimeLogger.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerExecutionTimeLogger.java index b04e757f4..cacf23daf 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerExecutionTimeLogger.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerExecutionTimeLogger.java @@ -14,10 +14,10 @@ package de.sovity.edc.ext.catalog.crawler.crawling.logging; +import de.sovity.authorityportal.db.jooq.Tables; +import de.sovity.authorityportal.db.jooq.enums.MeasurementErrorStatus; +import de.sovity.authorityportal.db.jooq.enums.MeasurementType; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; -import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.MeasurementErrorStatus; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.MeasurementType; import lombok.RequiredArgsConstructor; import org.jooq.DSLContext; diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/CatalogPatchBuilder.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/CatalogPatchBuilder.java similarity index 96% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/CatalogPatchBuilder.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/CatalogPatchBuilder.java index 309e8d8fc..e5b726c27 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/CatalogPatchBuilder.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/CatalogPatchBuilder.java @@ -14,6 +14,8 @@ package de.sovity.edc.ext.catalog.crawler.crawling.writing; +import de.sovity.authorityportal.db.jooq.tables.records.ContractOfferRecord; +import de.sovity.authorityportal.db.jooq.tables.records.DataOfferRecord; import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedContractOffer; import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedDataOffer; import de.sovity.edc.ext.catalog.crawler.crawling.writing.utils.DiffUtils; @@ -23,8 +25,6 @@ import de.sovity.edc.ext.catalog.crawler.dao.contract_offers.ContractOfferRecordUpdater; import de.sovity.edc.ext.catalog.crawler.dao.data_offers.DataOfferQueries; import de.sovity.edc.ext.catalog.crawler.dao.data_offers.DataOfferRecordUpdater; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ContractOfferRecord; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.DataOfferRecord; import lombok.RequiredArgsConstructor; import org.jooq.DSLContext; diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriter.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriter.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriter.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriter.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateFailureWriter.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateFailureWriter.java similarity index 92% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateFailureWriter.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateFailureWriter.java index 05b7f1bb1..efa4614af 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateFailureWriter.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateFailureWriter.java @@ -14,11 +14,11 @@ package de.sovity.edc.ext.catalog.crawler.crawling.writing; +import de.sovity.authorityportal.db.jooq.enums.ConnectorOnlineStatus; +import de.sovity.authorityportal.db.jooq.tables.records.ConnectorRecord; import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerEventErrorMessage; import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerEventLogger; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ConnectorRecord; import lombok.RequiredArgsConstructor; import org.eclipse.edc.spi.monitor.Monitor; import org.jooq.DSLContext; diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateSuccessWriter.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateSuccessWriter.java similarity index 94% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateSuccessWriter.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateSuccessWriter.java index 32b5ebe5e..ce2ba322b 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateSuccessWriter.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateSuccessWriter.java @@ -14,12 +14,12 @@ package de.sovity.edc.ext.catalog.crawler.crawling.writing; +import de.sovity.authorityportal.db.jooq.enums.ConnectorOnlineStatus; +import de.sovity.authorityportal.db.jooq.tables.records.ConnectorRecord; import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedCatalog; import de.sovity.edc.ext.catalog.crawler.crawling.logging.ConnectorChangeTracker; import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerEventLogger; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ConnectorRecord; import lombok.RequiredArgsConstructor; import org.jooq.DSLContext; diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcer.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcer.java similarity index 94% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcer.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcer.java index 0583b93b3..1997b0b79 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcer.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcer.java @@ -14,12 +14,12 @@ package de.sovity.edc.ext.catalog.crawler.crawling.writing; +import de.sovity.authorityportal.db.jooq.enums.ConnectorContractOffersExceeded; +import de.sovity.authorityportal.db.jooq.enums.ConnectorDataOffersExceeded; +import de.sovity.authorityportal.db.jooq.tables.records.ConnectorRecord; import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedDataOffer; import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerEventLogger; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorContractOffersExceeded; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorDataOffersExceeded; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ConnectorRecord; import de.sovity.edc.ext.catalog.crawler.orchestration.config.CrawlerConfig; import lombok.RequiredArgsConstructor; import org.jooq.DSLContext; diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/ChangeTracker.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/ChangeTracker.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/ChangeTracker.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/ChangeTracker.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/DiffUtils.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/DiffUtils.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/DiffUtils.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/utils/DiffUtils.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogCleaner.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogCleaner.java similarity index 95% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogCleaner.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogCleaner.java index 87b17d337..3be0e8809 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogCleaner.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogCleaner.java @@ -14,9 +14,9 @@ package de.sovity.edc.ext.catalog.crawler.dao; +import de.sovity.authorityportal.db.jooq.Tables; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; import de.sovity.edc.ext.catalog.crawler.dao.utils.PostgresqlUtils; -import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; import lombok.RequiredArgsConstructor; import org.jooq.DSLContext; diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogPatch.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogPatch.java similarity index 86% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogPatch.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogPatch.java index 8ad4ebc1e..2a3df4ab4 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogPatch.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogPatch.java @@ -14,9 +14,9 @@ package de.sovity.edc.ext.catalog.crawler.dao; +import de.sovity.authorityportal.db.jooq.tables.records.ContractOfferRecord; +import de.sovity.authorityportal.db.jooq.tables.records.DataOfferRecord; import de.sovity.edc.ext.catalog.crawler.dao.utils.RecordPatch; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ContractOfferRecord; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.DataOfferRecord; import lombok.AccessLevel; import lombok.Getter; import lombok.Setter; diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogPatchApplier.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogPatchApplier.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogPatchApplier.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/CatalogPatchApplier.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DataSourceFactory.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DataSourceFactory.java similarity index 62% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DataSourceFactory.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DataSourceFactory.java index b66997470..4322be3ba 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DataSourceFactory.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DataSourceFactory.java @@ -15,11 +15,10 @@ package de.sovity.edc.ext.catalog.crawler.dao.config; import com.zaxxer.hikari.HikariDataSource; -import de.sovity.edc.ext.catalog.crawler.CrawlerExtension; +import de.sovity.edc.ext.catalog.crawler.CrawlerConfigProps; import de.sovity.edc.extension.postgresql.HikariDataSourceFactory; import de.sovity.edc.extension.postgresql.JdbcCredentials; import lombok.RequiredArgsConstructor; -import org.apache.commons.lang3.Validate; import org.eclipse.edc.spi.system.configuration.Config; import javax.sql.DataSource; @@ -36,8 +35,8 @@ public class DataSourceFactory { */ public HikariDataSource newDataSource() { var jdbcCredentials = getJdbcCredentials(); - int maxPoolSize = config.getInteger(CrawlerExtension.DB_CONNECTION_POOL_SIZE); - int connectionTimeoutInMs = config.getInteger(CrawlerExtension.DB_CONNECTION_TIMEOUT_IN_MS); + int maxPoolSize = CrawlerConfigProps.CRAWLER_DB_CONNECTION_POOL_SIZE.getInt(config); + int connectionTimeoutInMs = CrawlerConfigProps.CRAWLER_DB_CONNECTION_TIMEOUT_IN_MS.getInt(config); return HikariDataSourceFactory.newDataSource( jdbcCredentials, maxPoolSize, @@ -48,16 +47,9 @@ public HikariDataSource newDataSource() { public JdbcCredentials getJdbcCredentials() { return new JdbcCredentials( - getRequiredStringProperty(config, CrawlerExtension.JDBC_URL), - getRequiredStringProperty(config, CrawlerExtension.JDBC_USER), - getRequiredStringProperty(config, CrawlerExtension.JDBC_PASSWORD) + CrawlerConfigProps.CRAWLER_DB_JDBC_URL.getStringOrThrow(config), + CrawlerConfigProps.CRAWLER_DB_JDBC_USER.getStringOrThrow(config), + CrawlerConfigProps.CRAWLER_DB_JDBC_PASSWORD.getStringOrThrow(config) ); } - - private String getRequiredStringProperty(Config config, String name) { - String value = config.getString(name, ""); - Validate.notBlank(value, "EDC Property '%s' is required".formatted(name)); - return value; - } - } diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DslContextFactory.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DslContextFactory.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DslContextFactory.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DslContextFactory.java index 6f97871bb..3d4f10bfd 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DslContextFactory.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DslContextFactory.java @@ -5,9 +5,9 @@ import org.jooq.SQLDialect; import org.jooq.impl.DSL; +import javax.sql.DataSource; import java.util.function.Consumer; import java.util.function.Function; -import javax.sql.DataSource; /** * Quickly launch {@link org.jooq.DSLContext}s from EDC configuration. diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorQueries.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorQueries.java similarity index 86% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorQueries.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorQueries.java index b77a1b708..75ff26274 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorQueries.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorQueries.java @@ -14,11 +14,11 @@ package de.sovity.edc.ext.catalog.crawler.dao.connectors; -import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.Connector; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.Organization; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ConnectorRecord; +import de.sovity.authorityportal.db.jooq.Tables; +import de.sovity.authorityportal.db.jooq.enums.ConnectorOnlineStatus; +import de.sovity.authorityportal.db.jooq.tables.Connector; +import de.sovity.authorityportal.db.jooq.tables.Organization; +import de.sovity.authorityportal.db.jooq.tables.records.ConnectorRecord; import de.sovity.edc.ext.catalog.crawler.orchestration.config.CrawlerConfig; import lombok.RequiredArgsConstructor; import org.jetbrains.annotations.NotNull; @@ -65,7 +65,7 @@ private Set queryConnectorRefs( ) .from(c) .leftJoin(o).on(c.ORGANIZATION_ID.eq(o.ID)) - .where(condition.apply(c, o), c.ENVIRONMENT.eq(crawlerConfig.getEnvironmentId())) + .where(condition.apply(c, o), c.ENVIRONMENT.eq(crawlerConfig.getEnvironmentId()), c.ENDPOINT_URL.isNotNull()) .fetchInto(ConnectorRef.class); return new HashSet<>(query); diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorRef.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorRef.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorRef.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorRef.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorStatusUpdater.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorStatusUpdater.java similarity index 88% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorStatusUpdater.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorStatusUpdater.java index 6dfad8125..f806c2f03 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorStatusUpdater.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorStatusUpdater.java @@ -14,9 +14,9 @@ package de.sovity.edc.ext.catalog.crawler.dao.connectors; +import de.sovity.authorityportal.db.jooq.Tables; +import de.sovity.authorityportal.db.jooq.enums.ConnectorOnlineStatus; import de.sovity.edc.ext.catalog.crawler.dao.utils.PostgresqlUtils; -import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; import org.jooq.DSLContext; import java.util.Collection; diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferQueries.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferQueries.java similarity index 84% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferQueries.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferQueries.java index 0d350e429..8b35d791f 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferQueries.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferQueries.java @@ -14,8 +14,8 @@ package de.sovity.edc.ext.catalog.crawler.dao.contract_offers; -import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ContractOfferRecord; +import de.sovity.authorityportal.db.jooq.Tables; +import de.sovity.authorityportal.db.jooq.tables.records.ContractOfferRecord; import org.jooq.DSLContext; import java.util.List; diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferRecordUpdater.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferRecordUpdater.java similarity index 94% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferRecordUpdater.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferRecordUpdater.java index ed23d2f07..b0f5da153 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferRecordUpdater.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/contract_offers/ContractOfferRecordUpdater.java @@ -14,11 +14,11 @@ package de.sovity.edc.ext.catalog.crawler.dao.contract_offers; +import de.sovity.authorityportal.db.jooq.tables.records.ContractOfferRecord; +import de.sovity.authorityportal.db.jooq.tables.records.DataOfferRecord; import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedContractOffer; import de.sovity.edc.ext.catalog.crawler.crawling.writing.utils.ChangeTracker; import de.sovity.edc.ext.catalog.crawler.dao.utils.JsonbUtils; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ContractOfferRecord; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.DataOfferRecord; import de.sovity.edc.ext.catalog.crawler.utils.JsonUtils2; import lombok.RequiredArgsConstructor; import org.jooq.JSONB; diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferQueries.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferQueries.java similarity index 85% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferQueries.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferQueries.java index f4a82b14c..e9b9f5ee9 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferQueries.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferQueries.java @@ -14,8 +14,8 @@ package de.sovity.edc.ext.catalog.crawler.dao.data_offers; -import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.DataOfferRecord; +import de.sovity.authorityportal.db.jooq.Tables; +import de.sovity.authorityportal.db.jooq.tables.records.DataOfferRecord; import lombok.RequiredArgsConstructor; import org.jooq.DSLContext; diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferRecordUpdater.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferRecordUpdater.java similarity index 96% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferRecordUpdater.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferRecordUpdater.java index c09a3d0b8..9f1c3e676 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferRecordUpdater.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/data_offers/DataOfferRecordUpdater.java @@ -14,17 +14,18 @@ package de.sovity.edc.ext.catalog.crawler.dao.data_offers; +import de.sovity.authorityportal.db.jooq.tables.records.DataOfferRecord; import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedDataOffer; import de.sovity.edc.ext.catalog.crawler.crawling.writing.utils.ChangeTracker; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; import de.sovity.edc.ext.catalog.crawler.dao.utils.JsonbUtils; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.DataOfferRecord; import de.sovity.edc.ext.catalog.crawler.utils.JsonUtils2; import de.sovity.edc.ext.wrapper.api.common.mappers.asset.utils.ShortDescriptionBuilder; import lombok.RequiredArgsConstructor; import org.jooq.JSONB; import java.time.OffsetDateTime; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -128,7 +129,7 @@ public boolean updateDataOffer( emptyIfNull(record.getKeywords()), emptyIfNull(asset.getKeywords()), it -> { - record.setKeywords(it.toArray(new String[0])); + record.setKeywords(new ArrayList<>(it)); record.setKeywordsCommaJoined(String.join(", ", it)); } ); diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/JsonbUtils.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/JsonbUtils.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/JsonbUtils.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/JsonbUtils.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/PostgresqlUtils.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/PostgresqlUtils.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/PostgresqlUtils.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/PostgresqlUtils.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/RecordPatch.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/RecordPatch.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/RecordPatch.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/utils/RecordPatch.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfig.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfig.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfig.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfig.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfigFactory.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfigFactory.java similarity index 53% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfigFactory.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfigFactory.java index f40e6b6af..c208c6052 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfigFactory.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfigFactory.java @@ -14,10 +14,8 @@ package de.sovity.edc.ext.catalog.crawler.orchestration.config; -import de.sovity.edc.ext.catalog.crawler.CrawlerExtension; -import lombok.NonNull; +import de.sovity.edc.ext.catalog.crawler.CrawlerConfigProps; import lombok.RequiredArgsConstructor; -import org.apache.commons.lang3.StringUtils; import org.eclipse.edc.spi.system.configuration.Config; import java.time.Duration; @@ -27,11 +25,11 @@ public class CrawlerConfigFactory { private final Config config; public CrawlerConfig buildCrawlerConfig() { - var environmentId = config.getString(CrawlerExtension.ENVIRONMENT_ID); - var numThreads = config.getInteger(CrawlerExtension.NUM_THREADS, 1); - var killOfflineConnectorsAfter = getDuration(CrawlerExtension.KILL_OFFLINE_CONNECTORS_AFTER, Duration.ofDays(5)); - var maxDataOffers = config.getInteger(CrawlerExtension.MAX_DATA_OFFERS_PER_CONNECTOR, -1); - var maxContractOffers = config.getInteger(CrawlerExtension.MAX_CONTRACT_OFFERS_PER_DATA_OFFER, -1); + var environmentId = CrawlerConfigProps.CRAWLER_ENVIRONMENT_ID.getStringOrThrow(config); + var numThreads = CrawlerConfigProps.CRAWLER_NUM_THREADS.getInt(config); + var killOfflineConnectorsAfter = Duration.parse(CrawlerConfigProps.CRAWLER_KILL_OFFLINE_CONNECTORS_AFTER.getStringOrThrow(config)); + var maxDataOffers = CrawlerConfigProps.CRAWLER_MAX_DATA_OFFERS_PER_CONNECTOR.getInt(config); + var maxContractOffers = CrawlerConfigProps.CRAWLER_MAX_CONTRACT_OFFERS_PER_DATA_OFFER.getInt(config); return CrawlerConfig.builder() .environmentId(environmentId) @@ -41,14 +39,4 @@ public CrawlerConfig buildCrawlerConfig() { .maxContractOffersPerDataOffer(maxContractOffers) .build(); } - - private Duration getDuration(@NonNull String configProperty, Duration defaultValue) { - var value = config.getString(configProperty, ""); - - if (StringUtils.isBlank(value)) { - return defaultValue; - } - - return Duration.parse(value); - } } diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/EdcConfigPropertyUtils.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/EdcConfigPropertyUtils.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/EdcConfigPropertyUtils.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/EdcConfigPropertyUtils.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorQueue.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorQueue.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorQueue.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorQueue.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorQueueFiller.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorQueueFiller.java similarity index 92% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorQueueFiller.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorQueueFiller.java index aaf2d7da9..350b61545 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorQueueFiller.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorQueueFiller.java @@ -14,8 +14,8 @@ package de.sovity.edc.ext.catalog.crawler.orchestration.queue; +import de.sovity.authorityportal.db.jooq.enums.ConnectorOnlineStatus; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorQueries; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; import lombok.RequiredArgsConstructor; import org.jooq.DSLContext; diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorRefreshPriority.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorRefreshPriority.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorRefreshPriority.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ConnectorRefreshPriority.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPool.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPool.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPool.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPool.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolTask.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolTask.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolTask.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolTask.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolTaskQueue.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolTaskQueue.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolTaskQueue.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolTaskQueue.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/DeadConnectorRefreshJob.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/DeadConnectorRefreshJob.java similarity index 94% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/DeadConnectorRefreshJob.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/DeadConnectorRefreshJob.java index 4f229b7fc..df86a21b1 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/DeadConnectorRefreshJob.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/DeadConnectorRefreshJob.java @@ -14,8 +14,8 @@ package de.sovity.edc.ext.catalog.crawler.orchestration.schedules; +import de.sovity.authorityportal.db.jooq.enums.ConnectorOnlineStatus; import de.sovity.edc.ext.catalog.crawler.dao.config.DslContextFactory; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; import de.sovity.edc.ext.catalog.crawler.orchestration.queue.ConnectorQueueFiller; import de.sovity.edc.ext.catalog.crawler.orchestration.queue.ConnectorRefreshPriority; import lombok.RequiredArgsConstructor; diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorCleanerJob.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorCleanerJob.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorCleanerJob.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorCleanerJob.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRefreshJob.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRefreshJob.java similarity index 94% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRefreshJob.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRefreshJob.java index 18965edf6..e9fc1a5f8 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRefreshJob.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRefreshJob.java @@ -14,8 +14,8 @@ package de.sovity.edc.ext.catalog.crawler.orchestration.schedules; +import de.sovity.authorityportal.db.jooq.enums.ConnectorOnlineStatus; import de.sovity.edc.ext.catalog.crawler.dao.config.DslContextFactory; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; import de.sovity.edc.ext.catalog.crawler.orchestration.queue.ConnectorQueueFiller; import de.sovity.edc.ext.catalog.crawler.orchestration.queue.ConnectorRefreshPriority; import lombok.RequiredArgsConstructor; diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OnlineConnectorRefreshJob.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OnlineConnectorRefreshJob.java similarity index 94% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OnlineConnectorRefreshJob.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OnlineConnectorRefreshJob.java index 6cad1899f..0b39d1bd9 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OnlineConnectorRefreshJob.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OnlineConnectorRefreshJob.java @@ -14,8 +14,8 @@ package de.sovity.edc.ext.catalog.crawler.orchestration.schedules; +import de.sovity.authorityportal.db.jooq.enums.ConnectorOnlineStatus; import de.sovity.edc.ext.catalog.crawler.dao.config.DslContextFactory; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; import de.sovity.edc.ext.catalog.crawler.orchestration.queue.ConnectorQueueFiller; import de.sovity.edc.ext.catalog.crawler.orchestration.queue.ConnectorRefreshPriority; import lombok.RequiredArgsConstructor; diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/QuartzScheduleInitializer.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/QuartzScheduleInitializer.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/QuartzScheduleInitializer.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/QuartzScheduleInitializer.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/utils/CronJobRef.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/utils/CronJobRef.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/utils/CronJobRef.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/utils/CronJobRef.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/utils/JobFactoryImpl.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/utils/JobFactoryImpl.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/utils/JobFactoryImpl.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/utils/JobFactoryImpl.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/CollectionUtils2.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/CollectionUtils2.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/CollectionUtils2.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/CollectionUtils2.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/JsonUtils2.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/JsonUtils2.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/JsonUtils2.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/JsonUtils2.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/MapUtils.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/MapUtils.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/MapUtils.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/MapUtils.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/StringUtils2.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/StringUtils2.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/StringUtils2.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/utils/StringUtils2.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/AssertionUtils.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/AssertionUtils.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/AssertionUtils.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/AssertionUtils.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerTestDb.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerTestDb.java similarity index 79% rename from extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerTestDb.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerTestDb.java index a4057c57b..801f3d311 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerTestDb.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerTestDb.java @@ -2,8 +2,8 @@ import com.zaxxer.hikari.HikariDataSource; import de.sovity.edc.ext.catalog.crawler.dao.config.DslContextFactory; -import de.sovity.edc.ext.catalog.crawler.dao.config.FlywayService; import de.sovity.edc.extension.e2e.db.TestDatabaseViaTestcontainers; +import de.sovity.edc.extension.postgresql.FlywayExecutionParams; import de.sovity.edc.extension.postgresql.FlywayUtils; import de.sovity.edc.extension.postgresql.HikariDataSourceFactory; import de.sovity.edc.extension.postgresql.JdbcCredentials; @@ -39,7 +39,7 @@ public void beforeAll(ExtensionContext extensionContext) throws Exception { dslContextFactory = new DslContextFactory(dataSource); // Migrate DB - var params = FlywayService.baseConfig("classpath:db-crawler/migration-test-utils") + var params = baseConfig("classpath:/migration-test-utils") .migrate(true) .build(); try { @@ -59,4 +59,14 @@ public void afterAll(ExtensionContext extensionContext) throws Exception { // Close DB db.afterAll(extensionContext); } + + public static FlywayExecutionParams.FlywayExecutionParamsBuilder baseConfig(String additionalMigrationLocations) { + var migrationLocations = FlywayUtils.parseFlywayLocations( + "classpath:/db/migration,%s".formatted(additionalMigrationLocations) + ); + + return FlywayExecutionParams.builder() + .migrationLocations(migrationLocations) + .table("flyway_schema_history"); + } } diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/JsonTestUtils.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/JsonTestUtils.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/JsonTestUtils.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/JsonTestUtils.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/TestData.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/TestData.java similarity index 84% rename from extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/TestData.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/TestData.java index 102beb54c..fdab3b1b2 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/TestData.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/TestData.java @@ -14,12 +14,12 @@ package de.sovity.edc.ext.catalog.crawler; +import de.sovity.authorityportal.db.jooq.Tables; +import de.sovity.authorityportal.db.jooq.enums.ConnectorContractOffersExceeded; +import de.sovity.authorityportal.db.jooq.enums.ConnectorDataOffersExceeded; +import de.sovity.authorityportal.db.jooq.enums.ConnectorOnlineStatus; +import de.sovity.authorityportal.db.jooq.tables.records.ConnectorRecord; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; -import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorContractOffersExceeded; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorDataOffersExceeded; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ConnectorRecord; import lombok.experimental.UtilityClass; import org.jooq.DSLContext; diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLoggerTest.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLoggerTest.java similarity index 97% rename from extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLoggerTest.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLoggerTest.java index f48ba78c8..f1fed1954 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLoggerTest.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/logging/CrawlerEventLoggerTest.java @@ -14,9 +14,9 @@ package de.sovity.edc.ext.catalog.crawler.crawling.logging; +import de.sovity.authorityportal.db.jooq.Tables; import de.sovity.edc.ext.catalog.crawler.CrawlerTestDb; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; -import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; import org.jooq.DSLContext; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorSuccessWriterTest.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorSuccessWriterTest.java similarity index 95% rename from extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorSuccessWriterTest.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorSuccessWriterTest.java index 4e7d6a603..e7e2c3952 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorSuccessWriterTest.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorSuccessWriterTest.java @@ -14,15 +14,15 @@ package de.sovity.edc.ext.catalog.crawler.crawling.writing; +import de.sovity.authorityportal.db.jooq.Tables; +import de.sovity.authorityportal.db.jooq.enums.ConnectorContractOffersExceeded; +import de.sovity.authorityportal.db.jooq.enums.ConnectorDataOffersExceeded; +import de.sovity.authorityportal.db.jooq.enums.ConnectorOnlineStatus; import de.sovity.edc.ext.catalog.crawler.CrawlerTestDb; import de.sovity.edc.ext.catalog.crawler.TestData; import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedCatalog; import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedContractOffer; import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedDataOffer; -import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorContractOffersExceeded; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorDataOffersExceeded; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; import de.sovity.edc.ext.wrapper.api.common.model.UiAsset; import org.assertj.core.data.TemporalUnitLessThanOffset; import org.jooq.impl.DSL; diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriterTest.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriterTest.java similarity index 98% rename from extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriterTest.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriterTest.java index 249d36764..d5ac0b883 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriterTest.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/ConnectorUpdateCatalogWriterTest.java @@ -14,10 +14,10 @@ package de.sovity.edc.ext.catalog.crawler.crawling.writing; +import de.sovity.authorityportal.db.jooq.tables.records.DataOfferRecord; import de.sovity.edc.ext.catalog.crawler.AssertionUtils; import de.sovity.edc.ext.catalog.crawler.CrawlerTestDb; import de.sovity.edc.ext.catalog.crawler.crawling.logging.ConnectorChangeTracker; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.DataOfferRecord; import org.assertj.core.data.TemporalUnitLessThanOffset; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcerTest.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcerTest.java similarity index 97% rename from extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcerTest.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcerTest.java index cb0bafdb5..9d486bffc 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcerTest.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferLimitsEnforcerTest.java @@ -14,13 +14,13 @@ package de.sovity.edc.ext.catalog.crawler.crawling.writing; +import de.sovity.authorityportal.db.jooq.enums.ConnectorContractOffersExceeded; +import de.sovity.authorityportal.db.jooq.enums.ConnectorDataOffersExceeded; +import de.sovity.authorityportal.db.jooq.tables.records.ConnectorRecord; import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedContractOffer; import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedDataOffer; import de.sovity.edc.ext.catalog.crawler.crawling.logging.CrawlerEventLogger; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorContractOffersExceeded; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorDataOffersExceeded; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ConnectorRecord; import de.sovity.edc.ext.catalog.crawler.orchestration.config.CrawlerConfig; import org.assertj.core.api.Assertions; import org.jooq.DSLContext; diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDataHelper.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDataHelper.java similarity index 97% rename from extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDataHelper.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDataHelper.java index 997141ab5..fd5501ab2 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDataHelper.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDataHelper.java @@ -14,12 +14,12 @@ package de.sovity.edc.ext.catalog.crawler.crawling.writing; +import de.sovity.authorityportal.db.jooq.tables.records.ContractOfferRecord; +import de.sovity.authorityportal.db.jooq.tables.records.DataOfferRecord; import de.sovity.edc.ext.catalog.crawler.TestData; import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedContractOffer; import de.sovity.edc.ext.catalog.crawler.crawling.fetching.model.FetchedDataOffer; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ContractOfferRecord; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.DataOfferRecord; import de.sovity.edc.ext.wrapper.api.common.model.UiAsset; import de.sovity.edc.utils.JsonUtils; import jakarta.json.Json; diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDataModels.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDataModels.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDataModels.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDataModels.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDydi.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDydi.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDydi.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestDydi.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestResultHelper.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestResultHelper.java similarity index 88% rename from extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestResultHelper.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestResultHelper.java index e5dfa39bb..c982b2b55 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestResultHelper.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DataOfferWriterTestResultHelper.java @@ -14,9 +14,9 @@ package de.sovity.edc.ext.catalog.crawler.crawling.writing; -import de.sovity.edc.ext.catalog.crawler.db.jooq.Tables; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.ContractOfferRecord; -import de.sovity.edc.ext.catalog.crawler.db.jooq.tables.records.DataOfferRecord; +import de.sovity.authorityportal.db.jooq.Tables; +import de.sovity.authorityportal.db.jooq.tables.records.ContractOfferRecord; +import de.sovity.authorityportal.db.jooq.tables.records.DataOfferRecord; import org.jetbrains.annotations.NotNull; import org.jooq.DSLContext; diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DiffUtilsTest.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DiffUtilsTest.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DiffUtilsTest.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/crawling/writing/DiffUtilsTest.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorRefTest.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorRefTest.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorRefTest.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/dao/connectors/ConnectorRefTest.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolQueueTest.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolQueueTest.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolQueueTest.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/orchestration/queue/ThreadPoolQueueTest.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRemovalJobTest.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRemovalJobTest.java similarity index 95% rename from extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRemovalJobTest.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRemovalJobTest.java index f054ff5b8..c8282c369 100644 --- a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRemovalJobTest.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/orchestration/schedules/OfflineConnectorRemovalJobTest.java @@ -14,6 +14,7 @@ package de.sovity.edc.ext.catalog.crawler.orchestration.schedules; +import de.sovity.authorityportal.db.jooq.enums.ConnectorOnlineStatus; import de.sovity.edc.ext.catalog.crawler.CrawlerTestDb; import de.sovity.edc.ext.catalog.crawler.TestData; import de.sovity.edc.ext.catalog.crawler.crawling.OfflineConnectorCleaner; @@ -22,7 +23,6 @@ import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorQueries; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorRef; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorStatusUpdater; -import de.sovity.edc.ext.catalog.crawler.db.jooq.enums.ConnectorOnlineStatus; import de.sovity.edc.ext.catalog.crawler.orchestration.config.CrawlerConfig; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -31,7 +31,7 @@ import java.time.Duration; import java.time.OffsetDateTime; -import static de.sovity.edc.ext.catalog.crawler.db.jooq.tables.Connector.CONNECTOR; +import static de.sovity.authorityportal.db.jooq.tables.Connector.CONNECTOR; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/CollectionUtils2Test.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/CollectionUtils2Test.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/CollectionUtils2Test.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/CollectionUtils2Test.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/JsonUtils2Test.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/JsonUtils2Test.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/JsonUtils2Test.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/JsonUtils2Test.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/StringUtils2Test.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/StringUtils2Test.java similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/StringUtils2Test.java rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/utils/StringUtils2Test.java diff --git a/extensions/catalog-crawler/catalog-crawler/src/test/resources/logging.properties b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/resources/logging.properties similarity index 100% rename from extensions/catalog-crawler/catalog-crawler/src/test/resources/logging.properties rename to authority-portal-backend/catalog-crawler/catalog-crawler/src/test/resources/logging.properties diff --git a/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/resources/migration-test-utils/V9999__Make_Columns_Nullable.sql b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/resources/migration-test-utils/V9999__Make_Columns_Nullable.sql new file mode 100644 index 000000000..bc4f9fd8c --- /dev/null +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/resources/migration-test-utils/V9999__Make_Columns_Nullable.sql @@ -0,0 +1,25 @@ +do +$$ + declare + r record; + begin + for r in (select 'alter table "' || c.table_schema || '"."' || c.table_name || '" alter column "' || c.column_name || + '" drop not null;' as command + from information_schema.columns c + where c.table_schema not in ('pg_catalog', 'information_schema') -- exclude system schemas + and c.table_name in ('connector', 'organization', 'user') -- only selected AP tables + and c.is_nullable = 'NO' + and not exists (SELECT tc.constraint_type + FROM information_schema.table_constraints AS tc + JOIN information_schema.key_column_usage AS kcu + ON tc.constraint_name = kcu.constraint_name + AND tc.table_schema = kcu.table_schema + WHERE tc.table_schema = c.table_schema + and tc.table_name = c.table_name + AND kcu.column_name = c.column_name + AND tc.constraint_type = 'PRIMARY KEY')) -- exclude primary keys + loop + execute r.command; + end loop; + end +$$; diff --git a/launchers/docker-entrypoint.sh b/authority-portal-backend/catalog-crawler/docker-entrypoint.sh similarity index 94% rename from launchers/docker-entrypoint.sh rename to authority-portal-backend/catalog-crawler/docker-entrypoint.sh index 1f463be66..5daf083f5 100755 --- a/launchers/docker-entrypoint.sh +++ b/authority-portal-backend/catalog-crawler/docker-entrypoint.sh @@ -7,12 +7,6 @@ # Do not set -u to permit unset variables in .env set -eo pipefail -# Apply ENV Vars on JAR startup -set -a -source /app/.env -set +a - - if [[ "x${1:-}" == "xstart" ]]; then cmd=(java ${JAVA_ARGS:-}) diff --git a/launchers/logging.dev.properties b/authority-portal-backend/catalog-crawler/logging.dev.properties similarity index 100% rename from launchers/logging.dev.properties rename to authority-portal-backend/catalog-crawler/logging.dev.properties diff --git a/launchers/logging.properties b/authority-portal-backend/catalog-crawler/logging.properties similarity index 100% rename from launchers/logging.properties rename to authority-portal-backend/catalog-crawler/logging.properties diff --git a/authority-portal-backend/gradle/libs.versions.toml b/authority-portal-backend/gradle/libs.versions.toml index fe22adff4..c6fc1553f 100644 --- a/authority-portal-backend/gradle/libs.versions.toml +++ b/authority-portal-backend/gradle/libs.versions.toml @@ -3,7 +3,18 @@ java = "17" kotlin = "1.9.22" npmVersion = "8.15.0" -sovity-edcCe = "10.4.1" +edcGroup = "org.eclipse.edc" +sovityCatalogCrawlerGroup = "de.sovity.edc.catalog.crawler" +sovityEdcExtensionGroup = "de.sovity.edc.ext" +sovityEdcGroup = "de.sovity.edc" + +sovity-edcCe = "0.0.1-SNAPSHOT" +edc = "0.2.1.4" +quartz = "2.3.2" +shadow = "7.1.2" +junit = "5.10.0" +flyway = "9.0.1" +jsonAssert = "1.5.1" quarkus = "3.9.2" quarkus-keycloakAdminClientReactive = "3.6.6" @@ -26,9 +37,16 @@ mockitoKotlin = "5.1.0" awaitility = "4.2.1" commons-lang = "3.14.0" +restAssured = "5.4.0" [libraries] sovity-edc-wrapperCommonApi = { module = "de.sovity.edc:wrapper-common-api", version.ref = "sovity-edcCe" } +sovity-edc-wrapperCommonMappers = { module = "de.sovity.edc:wrapper-common-mappers", version.ref = "sovity-edcCe" } +sovity-edc-jsonAndJsonLdUtils = { module = "de.sovity.edc:json-and-jsonld-utils", version.ref = "sovity-edcCe" } +sovity-edc-catalogParser = { module = "de.sovity.edc:catalog-parser", version.ref = "sovity-edcCe" } +sovity-edc-config = { module = "de.sovity.edc:config", version.ref = "sovity-edcCe" } +sovity-edc-ext-postgresFlywayCore = { module = "de.sovity.edc.ext:postgres-flyway-core", version.ref = "sovity-edcCe" } +sovity-edc-ext-testUtils = { module = "de.sovity.edc.ext:test-utils", version.ref = "sovity-edcCe" } quarkus-universeBom = { module = "io.quarkus.platform:quarkus-bom", version.ref = "quarkus" } quarkus-keycloakAdminClientReactive = { module = "io.quarkus:quarkus-keycloak-admin-client-reactive", version.ref = "quarkus-keycloakAdminClientReactive" } @@ -44,6 +62,8 @@ postgresql = { module = "org.postgresql:postgresql", version.ref = "postgresql" swaggerCore-annotations = { module = "io.swagger.core.v3:swagger-annotations-jakarta", version.ref = "swaggerCore" } swaggerCore-jaxrs2 = { module = "io.swagger.core.v3:swagger-jaxrs2-jakarta", version.ref = "swaggerCore" } testcontainers-postgresql = { module = "org.testcontainers:postgresql", version.ref = "testcontainers" } +testcontainers-testcontainers = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" } +testcontainers-junitJupiter = { module = "org.testcontainers:junit-jupiter", version.ref = "testcontainers" } assertj-core = { module = "org.assertj:assertj-core", version.ref = "assertj" } mockito-core = { module = "org.mockito:mockito-core", version.ref = "mockito" } @@ -54,6 +74,68 @@ awaitility = { module = "org.awaitility:awaitility", version.ref = "awaitility" commons-lang3 = { module = "org.apache.commons:commons-lang3", version.ref = "commons-lang" } +edc-apiCore = { module = "org.eclipse.edc:api-core", version.ref = "edc" } +edc-apiObservability = { module = "org.eclipse.edc:api-observability", version.ref = "edc" } +edc-authSpi = { module = "org.eclipse.edc:auth-spi", version.ref = "edc" } +edc-authTokenbased = { module = "org.eclipse.edc:auth-tokenbased", version.ref = "edc" } +edc-boot = { module = "org.eclipse.edc:boot", version.ref = "edc" } +edc-configurationFilesystem = { module = "org.eclipse.edc:configuration-filesystem", version.ref = "edc" } +edc-connectorCore = { module = "org.eclipse.edc:connector-core", version.ref = "edc" } +edc-contractDefinitionApi = { module = "org.eclipse.edc:contract-definition-api", version.ref = "edc" } +edc-contractNegotiationStoreSql = { module = "org.eclipse.edc:contract-negotiation-store-sql", version.ref = "edc" } +edc-contractSpi = { module = "org.eclipse.edc:contract-spi", version.ref = "edc" } +edc-controlPlaneAggregateServices = { module = "org.eclipse.edc:control-plane-aggregate-services", version.ref = "edc" } +edc-controlPlaneCore = { module = "org.eclipse.edc:control-plane-core", version.ref = "edc" } +edc-controlPlaneSpi = { module = "org.eclipse.edc:control-plane-spi", version.ref = "edc" } +edc-controlPlaneSql = { module = "org.eclipse.edc:control-plane-sql", version.ref = "edc" } +edc-coreSpi = { module = "org.eclipse.edc:core-spi", version.ref = "edc" } +edc-dataPlaneCore = { module = "org.eclipse.edc:data-plane-core", version.ref = "edc" } +edc-dataPlaneFramework = { module = "org.eclipse.edc:data-plane-framework", version.ref = "edc" } +edc-dataPlaneHttp = { module = "org.eclipse.edc:data-plane-http", version.ref = "edc" } +edc-dataPlaneSelectorClient = { module = "org.eclipse.edc:data-plane-selector-client", version.ref = "edc" } +edc-dataPlaneSelectorCore = { module = "org.eclipse.edc:data-plane-selector-core", version.ref = "edc" } +edc-dataPlaneUtil = { module = "org.eclipse.edc:data-plane-util", version.ref = "edc" } +edc-dsp = { module = "org.eclipse.edc:dsp", version.ref = "edc" } +edc-dspApiConfiguration = { module = "org.eclipse.edc:dsp-api-configuration", version.ref = "edc" } +edc-dspNegotiationTransform = { module = "org.eclipse.edc:dsp-negotiation-transform", version.ref = "edc" } +edc-dspHttpSpi = { module = "org.eclipse.edc:dsp-http-spi", version.ref = "edc" } +edc-dspHttpCore = { module = "org.eclipse.edc:dsp-http-core", version.ref = "edc" } +edc-http = { module = "org.eclipse.edc:http", version.ref = "edc" } +edc-httpSpi = { module = "org.eclipse.edc:http-spi", version.ref = "edc" } +edc-iamMock = { module = "org.eclipse.edc:iam-mock", version.ref = "edc" } +edc-jsonLd = { module = "org.eclipse.edc:json-ld", version.ref = "edc" } +edc-jsonLdSpi = { module = "org.eclipse.edc:json-ld-spi", version.ref = "edc" } +edc-junit = { module = "org.eclipse.edc:junit", version.ref = "edc" } +edc-managementApi = { module = "org.eclipse.edc:management-api", version.ref = "edc" } +edc-managementApiConfiguration = { module = "org.eclipse.edc:management-api-configuration", version.ref = "edc" } +edc-monitorJdkLogger = { module = "org.eclipse.edc:monitor-jdk-logger", version.ref = "edc" } +edc-oauth2Core = { module = "org.eclipse.edc:oauth2-core", version.ref = "edc" } +edc-policyDefinitionApi = { module = "org.eclipse.edc:policy-definition-api", version.ref = "edc" } +edc-policyEngineSpi = { module = "org.eclipse.edc:policy-engine-spi", version.ref = "edc" } +edc-policyModel = { module = "org.eclipse.edc:policy-model", version.ref = "edc" } +edc-policySpi = { module = "org.eclipse.edc:policy-spi", version.ref = "edc" } +edc-runtimeMetamodel = { module = "org.eclipse.edc:runtime-metamodel", version = "0.2.1" } +edc-sqlCore = { module = "org.eclipse.edc:sql-core", version.ref = "edc" } +edc-transactionLocal = { module = "org.eclipse.edc:transaction-local", version.ref = "edc" } +edc-transferDataPlane = { module = "org.eclipse.edc:transfer-data-plane", version.ref = "edc" } +edc-transferProcessApi = { module = "org.eclipse.edc:transfer-process-api", version.ref = "edc" } +edc-transferProcessStoreSql = { module = "org.eclipse.edc:transfer-process-store-sql", version.ref = "edc" } +edc-transferSpi = { module = "org.eclipse.edc:transfer-spi", version.ref = "edc" } +edc-transformCore = { module = "org.eclipse.edc:transform-core", version.ref = "edc" } +edc-transformSpi = { module = "org.eclipse.edc:transform-spi", version.ref = "edc" } +edc-vaultFilesystem = { module = "org.eclipse.edc:vault-filesystem", version.ref = "edc" } +quartz-quartz = { module = "org.quartz-scheduler:quartz", version.ref = "quartz" } + +junit-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "junit" } +junit-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "junit" } +junit-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "junit" } + +jsonAssert = { module = "org.skyscreamer:jsonassert", version.ref = "jsonAssert" } + +flyway-core = { module = "org.flywaydb:flyway-core", version.ref = "flyway" } + +restAssured-restAssured = { module = "io.rest-assured:rest-assured", version.ref = "restAssured" } + [bundles] mockito = ["mockito-core", "mockito-kotlin", "mockito-junitJupiter"] assertj = ["assertj-core"] @@ -63,5 +145,7 @@ quarkus = { id = "io.quarkus", version.ref = "quarkus" } flyway = { id = "org.flywaydb.flyway", version = "9.20.1" } jooq = { id = "nu.studer.jooq", version = "7.1.1" } +shadow = { id = "com.github.johnrengelman.shadow", version.ref = "shadow" } + openapiYamlGen = { id = "io.swagger.core.v3.swagger-gradle-plugin", version = "2.2.21" } openapiCodegen = { id = "org.openapi.generator", version = "6.6.0" } diff --git a/authority-portal-backend/settings.gradle.kts b/authority-portal-backend/settings.gradle.kts index d613ba1f8..a8c09d6e2 100644 --- a/authority-portal-backend/settings.gradle.kts +++ b/authority-portal-backend/settings.gradle.kts @@ -11,3 +11,7 @@ pluginManagement { include(":authority-portal-api") include(":authority-portal-db") include(":authority-portal-quarkus") +include(":catalog-crawler") +include(":catalog-crawler:catalog-crawler") +include(":catalog-crawler:catalog-crawler-launcher-base") +include(":catalog-crawler:catalog-crawler-ce-launcher") diff --git a/launchers/.env.catalog-crawler b/docs/deployment-guide/goals/production/.env.catalog-crawler similarity index 100% rename from launchers/.env.catalog-crawler rename to docs/deployment-guide/goals/production/.env.catalog-crawler diff --git a/docs/deployment-guide/goals/production/README.md b/docs/deployment-guide/goals/production/README.md index 4b93a2e80..9ca2d6379 100644 --- a/docs/deployment-guide/goals/production/README.md +++ b/docs/deployment-guide/goals/production/README.md @@ -42,16 +42,16 @@ The respective compatible versions can be found in the [CHANGELOG.md](../../../. ### Deployment Units -| Deployment Unit | Version / Details | -|---------------------------|----------------------------------------------------------------------------------------------------------| -| Reverse Proxy / Ingress | _Infrastructure dependent_ | -| Keycloak Deployment | Version 24.0.4 or compatible version | -| OAuth2 Proxy | quay.io/oauth2-proxy/oauth2-proxy:7.5.0 | -| Caddy behind OAuth2 Proxy | caddy:2.7 | -| Authority Portal Backend | authority-portal-backend, see [CHANGELOG.md](../../../../CHANGELOG.md) for compatible versions. | -| Authority Portal Frontend | authority-portal-frontend, see [CHANGELOG.md](../../../../CHANGELOG.md) for compatible versions. | -| Catalog Crawler | ghcr.io/sovity/catalog-crawler-ce, see [CHANGELOG.md](../../../../CHANGELOG.md) for compatible versions. | -| Postgresql | Version 16 or compatible version | +| Deployment Unit | Version / Details | +|---------------------------|---------------------------------------------------------------------------------------------------| +| Reverse Proxy / Ingress | _Infrastructure dependent_ | +| Keycloak Deployment | Version 24.0.4 or compatible version | +| OAuth2 Proxy | quay.io/oauth2-proxy/oauth2-proxy:7.5.0 | +| Caddy behind OAuth2 Proxy | caddy:2.7 | +| Authority Portal Backend | authority-portal-backend, see [CHANGELOG.md](../../../../CHANGELOG.md) for compatible versions. | +| Authority Portal Frontend | authority-portal-frontend, see [CHANGELOG.md](../../../../CHANGELOG.md) for compatible versions. | +| Catalog Crawler | authority-portal-crawler, see [CHANGELOG.md](../../../../CHANGELOG.md) for compatible versions. | +| Postgresql | Version 16 or compatible version | ### Configuration @@ -324,7 +324,48 @@ AUTHORITY_PORTAL_FRONTEND_PORTAL_DISPLAY_NAME: "Authority Portal" # Portal name - Each deployment environment requires a Data Catalog Crawler. - A Data Catalog Crawler is based on the EDC Connector and crawls the catalogs of all connectors in the dataspace. - You will need an SKI/AKI client ID to register the crawler. Please refer to the [EDC documentation](https://github.com/sovity/edc-ce/tree/main/docs/getting-started#faq) on how to generate one. -- See the [Catalog Crawler Productive Deployment Guide](https://github.com/sovity/edc-ce/blob/v10.4.1/docs/deployment-guide/goals/catalog-crawler-production/README.md) + +#### Reverse Proxy Configuration + +- The catalog crawler is meant to be served via TLS/HTTPS. +- The catalog crawler is meant to be deployed with a reverse proxy terminating TLS / providing HTTPS. +- All requests are meant to be redirected to the deployment's `11003` port. + +#### Catalog Crawler Configuration + +A productive configuration will require you to join a DAPS. + +For that you will need a SKI/AKI client ID. Please refer +to [edc-extension's Getting Started Guide](https://github.com/sovity/edc-ce/tree/main/docs/getting-started#faq) +on how to generate one. + +The DAPS needs to contain the claim `referringConnector=broker` for the broker. +Although it is discouraged to do so, the expected value `broker` could be overridden by specifying a different value for `MY_EDC_PARTICIPANT_ID`. + +```yaml +# Required: Fully Qualified Domain Name +MY_EDC_FQDN: "crawler.test.example.com" + +# Required: Authority Portal Environment ID +CRAWLER_ENVIRONMENT_ID: test + +# Required: Authority Portal Postgresql DB Access +CRAWLER_DB_JDBC_URL: jdbc:postgresql://authority-portal:5432/portal +CRAWLER_DB_JDBC_USER: portal +CRAWLER_DB_JDBC_PASSWORD: portal + +# Required: DAPS credentials +EDC_OAUTH_TOKEN_URL: 'https://daps.test.mobility-dataspace.eu/token' +EDC_OAUTH_PROVIDER_JWKS_URL: 'https://daps.test.mobility-dataspace.eu/jwks' +EDC_OAUTH_CLIENT_ID: '_your SKI/AKI_' +EDC_KEYSTORE: '_your keystore file_' # Needs to be available as file in the running container +EDC_KEYSTORE_PASSWORD: '_your keystore password_' +EDC_OAUTH_CERTIFICATE_ALIAS: 1 +EDC_OAUTH_PRIVATE_KEY_ALIAS: 1 +``` + +Additional available configuration options can be found +in [.env.catalog-crawler](.env.catalog-crawler). ## Initial Setup diff --git a/extensions/catalog-crawler/README.md b/extensions/catalog-crawler/README.md deleted file mode 100644 index 881dbbf40..000000000 --- a/extensions/catalog-crawler/README.md +++ /dev/null @@ -1,44 +0,0 @@ - -
-
- - Logo - - -

EDC-Connector Extension:
Catalog Crawler

- -

- Report Bug - · - Request Feature -

-
- -## About this Extension - -The catalog crawler is a deployment unit depending on an existing Authority Portal's database: - -- It is a modified EDC connector with the task to crawl the other connector's public data offers. -- It periodically checks the Authority Portal's connector list for its environment. -- It crawls the given connectors in regular intervals. -- It writes the data offers and connector statuses back into the Authority Portal DB. -- Each Environment configured in the Authority Portal requires its own Catalog Crawler with credentials for that environment's DAPS. - -## Why does this component exist? - -The Authority Portal uses a non-EDC stack, and the EDC stack cannot handle multiple sources of authority at once. - -With the `DB -> UI` part of the broker having been moved to the Authority Portal, only the `Catalog -> DB` part remains as the Catalog Crawler, -as it requires Connector-to-Connector IAM within the given Dataspace. - -## Deployment - -Please see the [Catalog Crawler Productive Deployment Guide](../../docs/deployment-guide/goals/catalog-crawler-production/README.md) for more information. - -## License - -Apache License 2.0 - see [LICENSE](../../LICENSE) - -## Contact - -sovity GmbH - contact@sovity.de diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java deleted file mode 100644 index ae1f0eca4..000000000 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtension.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (c) 2023 sovity GmbH - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * sovity GmbH - initial API and implementation - * - */ - -package de.sovity.edc.ext.catalog.crawler; - -import de.sovity.edc.ext.wrapper.api.common.mappers.PlaceholderEndpointService; -import org.eclipse.edc.connector.api.management.configuration.transform.ManagementApiTypeTransformerRegistry; -import org.eclipse.edc.connector.spi.catalog.CatalogService; -import org.eclipse.edc.jsonld.spi.JsonLd; -import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.runtime.metamodel.annotation.Provides; -import org.eclipse.edc.runtime.metamodel.annotation.Setting; -import org.eclipse.edc.spi.system.ServiceExtension; -import org.eclipse.edc.spi.system.ServiceExtensionContext; -import org.eclipse.edc.spi.types.TypeManager; - -import static de.sovity.edc.ext.catalog.crawler.orchestration.config.EdcConfigPropertyUtils.toEdcProp; - -@Provides({CrawlerExtensionContext.class}) -public class CrawlerExtension implements ServiceExtension { - - public static final String EXTENSION_NAME = "Authority Portal Data Catalog Crawler"; - - @Setting(required = true) - public static final String EXTENSION_ENABLED = toEdcProp("CRAWLER_EXTENSION_ENABLED"); - - @Setting(required = true) - public static final String ENVIRONMENT_ID = toEdcProp("CRAWLER_ENVIRONMENT_ID"); - - @Setting(required = true) - public static final String JDBC_URL = toEdcProp("CRAWLER_DB_JDBC_URL"); - - @Setting(required = true) - public static final String JDBC_USER = toEdcProp("CRAWLER_DB_JDBC_USER"); - - @Setting(required = true) - public static final String JDBC_PASSWORD = toEdcProp("CRAWLER_DB_JDBC_PASSWORD"); - - @Setting - public static final String DB_CONNECTION_POOL_SIZE = toEdcProp("CRAWLER_DB_CONNECTION_POOL_SIZE"); - - @Setting - public static final String DB_CONNECTION_TIMEOUT_IN_MS = toEdcProp("CRAWLER_DB_CONNECTION_TIMEOUT_IN_MS"); - - @Setting - public static final String DB_MIGRATE = toEdcProp("CRAWLER_DB_MIGRATE"); - - @Setting - public static final String DB_CLEAN = toEdcProp("CRAWLER_DB_CLEAN"); - - @Setting - public static final String DB_CLEAN_ENABLED = toEdcProp("CRAWLER_DB_CLEAN_ENABLED"); - - @Setting - public static final String DB_ADDITIONAL_FLYWAY_MIGRATION_LOCATIONS = toEdcProp("CRAWLER_DB_ADDITIONAL_FLYWAY_LOCATIONS"); - - @Setting - public static final String NUM_THREADS = toEdcProp("CRAWLER_NUM_THREADS"); - - @Setting - public static final String MAX_DATA_OFFERS_PER_CONNECTOR = toEdcProp("CRAWLER_MAX_DATA_OFFERS_PER_CONNECTOR"); - - @Setting - public static final String MAX_CONTRACT_OFFERS_PER_DATA_OFFER = toEdcProp("CRAWLER_MAX_CONTRACT_OFFERS_PER_DATA_OFFER"); - - @Setting - public static final String CRON_ONLINE_CONNECTOR_REFRESH = toEdcProp("CRAWLER_CRON_ONLINE_CONNECTOR_REFRESH"); - - @Setting - public static final String CRON_OFFLINE_CONNECTOR_REFRESH = toEdcProp("CRAWLER_CRON_OFFLINE_CONNECTOR_REFRESH"); - - @Setting - public static final String CRON_DEAD_CONNECTOR_REFRESH = toEdcProp("CRAWLER_CRON_DEAD_CONNECTOR_REFRESH"); - - @Setting - public static final String SCHEDULED_KILL_OFFLINE_CONNECTORS = toEdcProp("CRAWLER_SCHEDULED_KILL_OFFLINE_CONNECTORS"); - @Setting - public static final String KILL_OFFLINE_CONNECTORS_AFTER = toEdcProp("CRAWLER_KILL_OFFLINE_CONNECTORS_AFTER"); - - @Inject - private TypeManager typeManager; - - @Inject - private ManagementApiTypeTransformerRegistry typeTransformerRegistry; - - @Inject - private JsonLd jsonLd; - - @Inject - private CatalogService catalogService; - - /** - * Manual Dependency Injection Result - */ - private CrawlerExtensionContext services; - - @Override - public String name() { - return EXTENSION_NAME; - } - - @Override - public void initialize(ServiceExtensionContext context) { - if (!Boolean.TRUE.equals(context.getConfig().getBoolean(EXTENSION_ENABLED, false))) { - context.getMonitor().info("Crawler extension is disabled."); - return; - } - - services = CrawlerExtensionContextBuilder.buildContext( - context.getConfig(), - context.getMonitor(), - typeManager, - typeTransformerRegistry, - jsonLd, - catalogService, - new PlaceholderEndpointService("http://0.0.0.0/") - ); - - // Provide access for the tests - context.registerService(CrawlerExtensionContext.class, services); - } - - @Override - public void start() { - if (services == null) { - return; - } - services.crawlerInitializer().onStartup(); - } - - @Override - public void shutdown() { - if (services == null) { - return; - } - services.dataSource().close(); - } -} diff --git a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/FlywayService.java b/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/FlywayService.java deleted file mode 100644 index dee88e32b..000000000 --- a/extensions/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/FlywayService.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2023 sovity GmbH - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * sovity GmbH - initial implementation - * - */ - -package de.sovity.edc.ext.catalog.crawler.dao.config; - -import de.sovity.edc.ext.catalog.crawler.CrawlerExtension; -import de.sovity.edc.extension.postgresql.FlywayExecutionParams; -import de.sovity.edc.extension.postgresql.FlywayUtils; -import lombok.RequiredArgsConstructor; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.system.configuration.Config; - -import javax.sql.DataSource; - -@RequiredArgsConstructor -public class FlywayService { - private final Config config; - private final Monitor monitor; - private final DataSource dataSource; - - public void validateOrMigrateInTests() { - var additionalLocations = config.getString(CrawlerExtension.DB_ADDITIONAL_FLYWAY_MIGRATION_LOCATIONS, ""); - - var params = baseConfig(additionalLocations) - .clean(config.getBoolean(CrawlerExtension.DB_CLEAN, false)) - .cleanEnabled(config.getBoolean(CrawlerExtension.DB_CLEAN_ENABLED, false)) - .migrate(config.getBoolean(CrawlerExtension.DB_MIGRATE, false)) - .infoLogger(monitor::info) - .build(); - - FlywayUtils.cleanAndMigrate(params, dataSource); - } - - public static FlywayExecutionParams.FlywayExecutionParamsBuilder baseConfig(String additionalMigrationLocations) { - var migrationLocations = FlywayUtils.parseFlywayLocations( - "classpath:db-crawler/migration,%s".formatted(additionalMigrationLocations) - ); - - return FlywayExecutionParams.builder() - .migrationLocations(migrationLocations) - .table("flyway_schema_history"); - } -} diff --git a/launchers/README.md b/launchers/README.md deleted file mode 100644 index dd9d43c90..000000000 --- a/launchers/README.md +++ /dev/null @@ -1,186 +0,0 @@ - -
-
- - Logo - - -

sovity Community Edition EDC:
Docker Images

- -

- Report Bug - · - Request Feature -

-
- -## sovity Community Edition EDC Docker Images - -[Eclipse Dataspace Components](https://github.com/eclipse-edc) (EDC) is a framework -for building dataspaces, exchanging data securely with ensured data -sovereignty. - -[sovity](https://sovity.de/) extends the EDC Connector's functionality with extensions to offer -enterprise-ready managed services like "Connector-as-a-Service", out-of-the-box fully configured DAPS -and integrations to existing other dataspace technologies. - -We believe in open source and actively contribute to open source community. Our sovity Community Edition EDC packages -open source EDC Extensions and combines them with [our own open source EDC Extensions](../extensions) to build -ready-to-use EDC Docker Images. - -Together with our [EDC UI](https://github.com/sovity/EDC-UI) Docker Images, it offers several of our extended EDC -functionalities for self-hosting purposes. - -## Different Image Types - -Our sovity Community Edition EDC is built as several docker image variants in different configurations. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Docker ImageTypePurposeFeatures
- edc-dev - Development -
    -
  • Local manual testing
    • -
  • Local demos
    • -
- 		-
    -
  • Control- and Data-Plane
    • -
  • sovity Community Edition EDC Extensions
    • -
  • Management API Auth via API Keys
    • -
  • PostgreSQL Persistence & Flyway
    • -
  • Mock IAM
    • -
-
- edc-ce - sovity Community Edition -
    -
  • Self-Deploy a productive sovity EDC
    • -
- 		-
    -
  • Control- and Data-Plane
    • -
  • sovity Community Edition EDC Extensions
    • -
  • Management API Auth via API Keys
    • -
  • PostgreSQL Persistence & Flyway
    • -
  • DAPS Authentication
    • -
-
- edc-ce-mds - MDS Community Edition -
    -
  • Self-Deploy a productive MDS EDC
    • -
- 		-
    -
  • Control- and Data-Plane
    • -
  • sovity Community Edition EDC Extensions
    • -
  • Management API Auth via API Keys
    • -
  • PostgreSQL Persistence & Flyway
    • -
  • DAPS Authentication
    • -
  • Clearing House Extension
    • -
-
edc-eeCommercial -
    -
  • Productive use
    • -
  • Professional users
    • -
  • Our Connector-as-a-Service (CaaS) customers
    • -
  • Request Demo -
- 		-
    -
  • Managed Control- and Data Planes, individually scalable
    • -
  • Hosted on highly performant infrastructure
    • -
  • Management API Auth via Service Accounts
    • -
  • Managed User Auth via standalone IAM (SSO)
    • -
  • Automatic Dataspace Roll-In, for example to Data Spaces like Catena-X or Mobility Data Space
    • -
  • Managed DAPS Authentication
    • -
  • Support & Tutorials
    • -
  • Automatic updates to newest version and new features
    • -
  • Off-the-shelf extensions for use cases available
    • -
  • EDC available within minutes
    • -
  • Can be combined with Data Space as a Service (DSaaS)
    • -
-
- catalog-crawler-dev - Development -
    -
  • Local Demo
    • -
  • E2E Testing
    • -
- 		-
    -
  • Catalog Crawler for one environment
    • -
  • Mock IAM
    • -
-
catalog-crawler-ceCommunity Edition -
    -
  • Productive Deployment
    • -
- 		-
    -
  • Catalog Crawler for one environment
    • -
  • DAPS Authentication
    • -
-
- -## Image Tags - -| Tag | Description | -|---------|----------------------------------------------------| -| latest | latest version of our main branch | -| release | latest release of our sovity Community Edition EDC | - -## Configuration - -For available configurations please refer to our [Getting Started Guide](../docs/getting-started/README.md). - -## License - -Apache License 2.0 - see [LICENSE](../LICENSE) - -## Contact - -sovity GmbH - contact@sovity.de diff --git a/launchers/connectors/catalog-crawler-dev/build.gradle.kts b/launchers/connectors/catalog-crawler-dev/build.gradle.kts deleted file mode 100644 index c68026e6c..000000000 --- a/launchers/connectors/catalog-crawler-dev/build.gradle.kts +++ /dev/null @@ -1,23 +0,0 @@ -plugins { - `java-library` - id("application") - alias(libs.plugins.shadow) -} - -dependencies { - implementation(project(":extensions:catalog-crawler:catalog-crawler-launcher-base")) - - api(libs.edc.monitorJdkLogger) - api(libs.edc.apiObservability) - - implementation(project(":launchers:common:auth-mock")) -} - -application { - mainClass.set("org.eclipse.edc.boot.system.runtime.BaseRuntime") -} - -tasks.withType { - mergeServiceFiles() - archiveFileName.set("app.jar") -}