From 9643664abba89152a5163a129ca8c5e486db4261 Mon Sep 17 00:00:00 2001 From: ndc-dxc <162444006+ndc-dxc@users.noreply.github.com> Date: Mon, 26 Aug 2024 10:27:25 +0200 Subject: [PATCH] Atomic Writer (#136) * upgrades gradle upgrades spring security upgrades SecurityConfig fixes docker image in compose to run on latest mysql * upgrade Spring * fixes integration tests * optimize imports * fixes LOB mapping to tinytext * Atomic Writer * fixes integration tests * fixes owasp * Update README.md --- build.gradle | 6 +- .../ndc/integration/BaseIntegrationTest.java | 4 +- .../integration/InMemoryInstanceManager.java | 69 ++++++++++++++++ .../integration/RestApiIntegrationTests.java | 17 ++++ ...ssetMetadataRepositoryIntegrationTest.java | 8 +- .../TripleStoreRepositoryTest.java | 5 +- .../SimpleHarvestRepositoryProcessor.java | 17 +++- .../ndc/eventhandler/event/ConfigService.java | 2 +- .../event/HarvesterStartedEvent.java | 6 +- .../handler/HarvesterRunUpdatingHandler.java | 1 + .../ndc/harvester/HarvesterService.java | 81 ++++++++++++------- .../ndc/harvester/SemanticAssetHarvester.java | 3 +- .../context/HarvestExecutionContext.java | 3 + .../BaseSemanticAssetHarvester.java | 3 +- .../ControlledVocabularyHarvester.java | 5 +- .../model/BaseSemanticAssetModel.java | 8 +- .../model/ControlledVocabularyModel.java | 12 +-- .../ndc/harvester/model/Instance.java | 14 ++++ .../ndc/harvester/model/OntologyModel.java | 12 +-- .../ndc/harvester/model/SchemaModel.java | 13 +-- .../model/SemanticAssetModelFactory.java | 10 ++- .../model/index/SemanticAssetMetadata.java | 3 +- .../ControlledVocabularyPathProcessor.java | 5 +- .../service/ActualConfigService.java | 13 ++- .../service/HarvesterRunService.java | 78 +++++++++--------- .../ndc/model/harvester/HarvesterRun.java | 1 + .../SemanticAssetMetadataRepository.java | 61 +++++++++++--- .../ndc/repository/TripleStoreRepository.java | 66 ++++++++++++--- .../ndc/service/DefaultInstanceManager.java | 56 +++++++++++++ .../ndc/service/InstanceManager.java | 27 +++++++ .../ControlledVocabularyValidator.java | 3 +- .../ndc/validator/OntologyValidator.java | 3 +- .../ndc/validator/SchemaValidator.java | 3 +- .../db/migration/V7__instance_manager.sql | 3 + .../ndc/harvester/HarvesterServiceTest.java | 34 +------- .../ControlledVocabularyHarvesterTest.java | 5 +- .../model/BaseSemanticAssetModelTest.java | 2 +- .../model/ControlledVocabularyModelTest.java | 39 ++++----- .../harvester/model/OntologyModelTest.java | 21 ++--- .../ndc/harvester/model/SchemaModelTest.java | 57 ++++++------- .../model/SemanticAssetModelFactoryTest.java | 12 ++- ...ControlledVocabularyPathProcessorTest.java | 11 +-- .../service/ActualConfigServiceTest.java | 4 +- .../SemanticAssetMetadataRepositoryTest.java | 12 ++- .../repository/TripleStoreRepositoryTest.java | 9 ++- .../ndc/validator/ValidatorTest.java | 6 +- 46 files changed, 592 insertions(+), 241 deletions(-) create mode 100644 src/integration/java/it/gov/innovazione/ndc/integration/InMemoryInstanceManager.java create mode 100644 src/main/java/it/gov/innovazione/ndc/harvester/model/Instance.java create mode 100644 src/main/java/it/gov/innovazione/ndc/service/DefaultInstanceManager.java create mode 100644 src/main/java/it/gov/innovazione/ndc/service/InstanceManager.java create mode 100644 src/main/resources/db/migration/V7__instance_manager.sql diff --git a/build.gradle b/build.gradle index 47b99359..4765a6f6 100644 --- a/build.gradle +++ b/build.gradle @@ -160,13 +160,13 @@ jacocoTestCoverageVerification { limit { counter = 'LINE' value = 'COVEREDRATIO' - minimum = 0.9 + minimum = 0.8 } excludes = [ 'it.gov.innovazione.ndc.config.*', 'it.gov.innovazione.ndc.Application', 'it.gov.innovazione.ndc.harvester.util.*', - 'it.gov.innovazione.ndc.model.profiles.*', + 'it.gov.innovazione.ndc.model.*', 'it.gov.innovazione.ndc.gen.*', 'it.gov.innovazione.ndc.validator.BaseSemanticAssetValidator', 'it.gov.innovazione.ndc.controller.*', @@ -189,6 +189,8 @@ jacocoTestCoverageVerification { 'it.gov.innovazione.ndc.harvester.harvesters.utils.PathUtils', 'it.gov.innovazione.ndc.harvester.service.OnceLogger', 'it.gov.innovazione.ndc.harvester.service.ConfigReaderService', + 'it.gov.innovazione.ndc.service.DefaultInstanceManager', + 'it.gov.innovazione.ndc.repository.TripleStoreRepository', 'it.gov.innovazione.ndc.service.EventCleaner', 'it.gov.innovazione.ndc.service.TemplateService', 'it.gov.innovazione.ndc.alerter.*' diff --git a/src/integration/java/it/gov/innovazione/ndc/integration/BaseIntegrationTest.java b/src/integration/java/it/gov/innovazione/ndc/integration/BaseIntegrationTest.java index 349ce6cb..bd14963a 100644 --- a/src/integration/java/it/gov/innovazione/ndc/integration/BaseIntegrationTest.java +++ b/src/integration/java/it/gov/innovazione/ndc/integration/BaseIntegrationTest.java @@ -31,6 +31,7 @@ import java.util.Set; import static it.gov.innovazione.ndc.harvester.service.RepositoryUtils.asRepo; +import static it.gov.innovazione.ndc.integration.Containers.ELASTICSEARCH_PORT; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.doNothing; @@ -82,10 +83,9 @@ public void beforeEach() throws IOException { static void updateTestcontainersProperties(DynamicPropertyRegistry registry) { String url = "http://localhost:" + virtuoso.getMappedPort(Containers.VIRTUOSO_PORT); - String elasticSearchAddress = elasticsearchContainer.getHttpHostAddress(); registry.add("virtuoso.sparql", () -> url + "/sparql"); registry.add("virtuoso.sparql-graph-store", () -> url + "/sparql-graph-crud/"); - registry.add("spring.elasticsearch.rest.uris", () -> elasticSearchAddress); + registry.add("elastic.test.exposed-port", () -> elasticsearchContainer.getMappedPort(ELASTICSEARCH_PORT)); } private void dataIsHarvested() throws IOException { diff --git a/src/integration/java/it/gov/innovazione/ndc/integration/InMemoryInstanceManager.java b/src/integration/java/it/gov/innovazione/ndc/integration/InMemoryInstanceManager.java new file mode 100644 index 00000000..cd5d29f1 --- /dev/null +++ b/src/integration/java/it/gov/innovazione/ndc/integration/InMemoryInstanceManager.java @@ -0,0 +1,69 @@ +package it.gov.innovazione.ndc.integration; + +import it.gov.innovazione.ndc.harvester.model.Instance; +import it.gov.innovazione.ndc.model.harvester.Repository; +import it.gov.innovazione.ndc.service.InstanceManager; +import lombok.RequiredArgsConstructor; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Service; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Service +@Primary +@RequiredArgsConstructor +public class InMemoryInstanceManager implements InstanceManager { + + private final Map instances = new HashMap<>(); + + private Instance allInstances = Instance.PRIMARY; + + public void setAllInstances(Instance allInstances) { + this.allInstances = allInstances; + instances.replaceAll((k, v) -> allInstances); + } + + @Override + public Instance getNextOnlineInstance(String repoUrl) { + if (instances.containsKey(repoUrl)) { + return instances.get(repoUrl).switchInstance(); + } + Repository fakeRepo = Repository.builder() + .url(repoUrl) + .build(); + return getCurrentInstance(fakeRepo).switchInstance(); + } + + @Override + public Instance getNextOnlineInstance(Repository repository) { + return getCurrentInstance(repository).switchInstance(); + } + + @Override + public Instance getCurrentInstance(Repository repository) { + if (!instances.containsKey(repository.getUrl())) { + instances.put(repository.getUrl(), allInstances); + } + return instances.get(repository.getUrl()); + } + + @Override + public void switchInstances(Repository repository) { + Instance toPut = getCurrentInstance(repository).switchInstance(); + instances.put(repository.getUrl(), toPut); + } + + @Override + public List getCurrentInstances() { + return instances.entrySet().stream() + .map(entry -> RepositoryInstance.of(entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); + } + + public void switchAllInstances() { + instances.replaceAll((k, v) -> v.switchInstance()); + } +} diff --git a/src/integration/java/it/gov/innovazione/ndc/integration/RestApiIntegrationTests.java b/src/integration/java/it/gov/innovazione/ndc/integration/RestApiIntegrationTests.java index e8dbf942..98a61cf2 100644 --- a/src/integration/java/it/gov/innovazione/ndc/integration/RestApiIntegrationTests.java +++ b/src/integration/java/it/gov/innovazione/ndc/integration/RestApiIntegrationTests.java @@ -8,13 +8,17 @@ import it.gov.innovazione.ndc.harvester.SemanticAssetType; import it.gov.innovazione.ndc.model.profiles.NDC; import it.gov.innovazione.ndc.service.GithubService; +import it.gov.innovazione.ndc.service.InstanceManager; import junit.framework.AssertionFailedError; import org.apache.jena.query.QuerySolution; import org.apache.jena.query.ResultSet; import org.apache.jena.rdf.model.RDFNode; import org.apache.jena.rdfconnection.RDFConnection; import org.apache.jena.rdfconnection.RDFConnectionFactory; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.http.HttpStatus; import org.springframework.test.context.DynamicPropertyRegistry; @@ -41,12 +45,25 @@ public class RestApiIntegrationTests extends BaseIntegrationTest { private GithubService githubService; @MockBean private NdcEventPublisher ndcEventPublisher; + @Autowired + private InstanceManager instanceManager; @DynamicPropertySource static void updateDynamicPropertySource(DynamicPropertyRegistry registry) { updateTestcontainersProperties(registry); } + @BeforeEach + public void beforeSetup() { + ((InMemoryInstanceManager) instanceManager).switchAllInstances(); + } + + @AfterEach + public void afterSetup() { + ((InMemoryInstanceManager) instanceManager).switchAllInstances(); + } + + @Test void shouldBeAbleToHarvestAndSearchControlledVocabularySuccessfully() { final String assetIri = "https://w3id.org/italia/controlled-vocabulary/licences"; diff --git a/src/integration/java/it/gov/innovazione/ndc/integration/SemanticAssetMetadataRepositoryIntegrationTest.java b/src/integration/java/it/gov/innovazione/ndc/integration/SemanticAssetMetadataRepositoryIntegrationTest.java index 860350f8..591286a3 100644 --- a/src/integration/java/it/gov/innovazione/ndc/integration/SemanticAssetMetadataRepositoryIntegrationTest.java +++ b/src/integration/java/it/gov/innovazione/ndc/integration/SemanticAssetMetadataRepositoryIntegrationTest.java @@ -2,6 +2,7 @@ import it.gov.innovazione.ndc.config.ElasticConfigurator; import it.gov.innovazione.ndc.harvester.SemanticAssetType; +import it.gov.innovazione.ndc.harvester.model.Instance; import it.gov.innovazione.ndc.harvester.model.index.SemanticAssetMetadata; import it.gov.innovazione.ndc.repository.SemanticAssetMetadataRepository; import org.elasticsearch.client.RestClient; @@ -32,7 +33,9 @@ public class SemanticAssetMetadataRepositoryIntegrationTest { public static void beforeAll() { elastic.start(); elasticsearchOperations = buildElasticsearchOps(); - repository = new SemanticAssetMetadataRepository(elasticsearchOperations); + InMemoryInstanceManager instanceManager = new InMemoryInstanceManager(); + instanceManager.setAllInstances(Instance.PRIMARY); + repository = new SemanticAssetMetadataRepository(elasticsearchOperations, instanceManager); } @NotNull @@ -66,6 +69,7 @@ void shouldIndexAndSearch() { .repoUrl(repoName) .type(type) .iri(iri) + .instance(Instance.PRIMARY.name()) .build(); entries.add(entry); } @@ -76,7 +80,7 @@ void shouldIndexAndSearch() { elasticsearchOperations.indexOps(SemanticAssetMetadata.class).refresh(); - List vocabs = repository.findVocabulariesForRepoUrl("http://repo1"); + List vocabs = repository.findVocabulariesForRepoUrl("http://repo1", Instance.PRIMARY); assertThat(vocabs).hasSize(ASSET_COUNT); } } diff --git a/src/integration/java/it/gov/innovazione/ndc/integration/TripleStoreRepositoryTest.java b/src/integration/java/it/gov/innovazione/ndc/integration/TripleStoreRepositoryTest.java index 65e09664..ffe5bc81 100644 --- a/src/integration/java/it/gov/innovazione/ndc/integration/TripleStoreRepositoryTest.java +++ b/src/integration/java/it/gov/innovazione/ndc/integration/TripleStoreRepositoryTest.java @@ -30,6 +30,7 @@ public class TripleStoreRepositoryTest { private static String sparqlUrl; private static String sparqlGraphUrl; private static final String graphName = "http://www.fantasy.org/graph"; + private static final String oldGraphName = "http://old.www.fantasy.org/graph"; private static TripleStoreRepository repository; @@ -75,7 +76,7 @@ void shouldSaveGivenModelInVirtuosoTestcontainer() { "", "?o" ) - .from(graphName) + .from(oldGraphName) .build(); QueryExecution queryExecution = QueryExecutionFactory.sparqlService(sparqlUrl, findPeriodicity); @@ -105,7 +106,7 @@ void shouldSaveGivenModelWithBlankNodeInVirtuosoTestcontainer() { "", "?k" ) - .from(graphName) + .from(oldGraphName) .build(); QueryExecution queryExecution = QueryExecutionFactory.sparqlService(sparqlUrl, keywordQuery); diff --git a/src/main/java/it/gov/innovazione/ndc/config/SimpleHarvestRepositoryProcessor.java b/src/main/java/it/gov/innovazione/ndc/config/SimpleHarvestRepositoryProcessor.java index 64bbf563..e32eae61 100644 --- a/src/main/java/it/gov/innovazione/ndc/config/SimpleHarvestRepositoryProcessor.java +++ b/src/main/java/it/gov/innovazione/ndc/config/SimpleHarvestRepositoryProcessor.java @@ -10,10 +10,12 @@ import it.gov.innovazione.ndc.harvester.exception.HarvesterAlreadyInProgressException; import it.gov.innovazione.ndc.harvester.exception.HarvesterException; import it.gov.innovazione.ndc.harvester.exception.RepoContainsNdcIssueException; +import it.gov.innovazione.ndc.harvester.model.Instance; import it.gov.innovazione.ndc.harvester.service.HarvesterRunService; import it.gov.innovazione.ndc.model.harvester.HarvesterRun; import it.gov.innovazione.ndc.model.harvester.Repository; import it.gov.innovazione.ndc.service.GithubService; +import it.gov.innovazione.ndc.service.InstanceManager; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ThreadUtils; @@ -47,6 +49,7 @@ public class SimpleHarvestRepositoryProcessor { private final GithubService githubService; private final List locks = new ArrayList<>(); + private final InstanceManager instanceManager; public static List getAllRunningHarvestThreadNames() { return ThreadUtils.getAllThreads().stream() @@ -64,7 +67,10 @@ private void setThreadName(String runId, String repoId, String revision, String public void execute(String runId, Repository repository, String correlationId, String revision, boolean force, String currentUserLogin) { try { setThreadName(runId, repository.getId(), revision, "RUNNING"); - publishHarvesterStartedEvent(repository, correlationId, revision, runId, currentUserLogin); + + Instance instanceToHarvest = instanceManager.getNextOnlineInstance(repository); + + publishHarvesterStartedEvent(repository, correlationId, revision, runId, currentUserLogin, instanceToHarvest); synchronized (locks) { if (locks.contains(repository.getId() + revision)) { @@ -98,15 +104,19 @@ public void execute(String runId, Repository repository, String correlationId, S .correlationId(correlationId) .runId(runId) .currentUserId(currentUserLogin) + .instance(instanceToHarvest) .build()); verifyNoNdcIssuesInRepoIfNecessary(repository); - harvesterService.harvest(repository, revision); + harvesterService.harvest(repository, revision, instanceToHarvest); githubService.openIssueIfNecessary(); publishHarvesterSuccessfulEvent(repository, correlationId, revision, runId, currentUserLogin); + + instanceManager.switchInstances(repository); + setThreadName(runId, repository.getId(), revision, "IDLE"); } catch (HarvesterException e) { publishHarvesterFailedEvent(repository, correlationId, revision, runId, e.getHarvesterRunStatus(), e, currentUserLogin); @@ -153,7 +163,7 @@ private synchronized void verifyHarvestingIsNotInProgress(String runId, Reposito } } - public void publishHarvesterStartedEvent(Repository repository, String correlationId, String revision, String runId, String currentUserLogin) { + public void publishHarvesterStartedEvent(Repository repository, String correlationId, String revision, String runId, String currentUserLogin, Instance instance) { ndcEventPublisher.publishEvent( "harvester", "harvester.started", @@ -162,6 +172,7 @@ public void publishHarvesterStartedEvent(Repository repository, String correlati HarvesterStartedEvent.builder() .runId(runId) .repository(repository) + .instance(instance) .revision(revision) .build()); } diff --git a/src/main/java/it/gov/innovazione/ndc/eventhandler/event/ConfigService.java b/src/main/java/it/gov/innovazione/ndc/eventhandler/event/ConfigService.java index 39b3f720..6f392670 100644 --- a/src/main/java/it/gov/innovazione/ndc/eventhandler/event/ConfigService.java +++ b/src/main/java/it/gov/innovazione/ndc/eventhandler/event/ConfigService.java @@ -35,7 +35,7 @@ public Optional fromGlobal(ActualConfigService.ConfigKey key) { } } - private Optional fromRepo(ActualConfigService.ConfigKey key, String repoId) { + public Optional fromRepo(ActualConfigService.ConfigKey key, String repoId) { try { Class type = null; diff --git a/src/main/java/it/gov/innovazione/ndc/eventhandler/event/HarvesterStartedEvent.java b/src/main/java/it/gov/innovazione/ndc/eventhandler/event/HarvesterStartedEvent.java index 0a5152c7..c9c83d49 100644 --- a/src/main/java/it/gov/innovazione/ndc/eventhandler/event/HarvesterStartedEvent.java +++ b/src/main/java/it/gov/innovazione/ndc/eventhandler/event/HarvesterStartedEvent.java @@ -1,5 +1,6 @@ package it.gov.innovazione.ndc.eventhandler.event; +import it.gov.innovazione.ndc.harvester.model.Instance; import it.gov.innovazione.ndc.alerter.entities.EventCategory; import it.gov.innovazione.ndc.alerter.entities.Severity; import it.gov.innovazione.ndc.alerter.event.AlertableEvent; @@ -14,16 +15,17 @@ public class HarvesterStartedEvent implements AlertableEvent { private final String runId; private final Repository repository; + private final Instance instance; private final String revision; @Override public String getName() { - return "Run " + runId + " started"; + return "Run " + runId + " started on instance " + instance; } @Override public String getDescription() { - return "Harvester run " + runId + " started"; + return "Run " + runId + " started on instance " + instance + " for repository " + repository.getUrl() + " with revision " + revision; } @Override diff --git a/src/main/java/it/gov/innovazione/ndc/eventhandler/handler/HarvesterRunUpdatingHandler.java b/src/main/java/it/gov/innovazione/ndc/eventhandler/handler/HarvesterRunUpdatingHandler.java index 0ba6e6d1..6595b89f 100644 --- a/src/main/java/it/gov/innovazione/ndc/eventhandler/handler/HarvesterRunUpdatingHandler.java +++ b/src/main/java/it/gov/innovazione/ndc/eventhandler/handler/HarvesterRunUpdatingHandler.java @@ -65,6 +65,7 @@ private void handleHarvesterStartedEvent(NdcEventWrapper .correlationId(event.getCorrelationId()) .repositoryId(event.getPayload().getRepository().getId()) .repositoryUrl(event.getPayload().getRepository().getUrl()) + .instance(event.getPayload().getInstance().name()) .startedAt(event.getTimestamp()) .startedBy(event.getUser()) .revision(event.getPayload().getRevision()) diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/HarvesterService.java b/src/main/java/it/gov/innovazione/ndc/harvester/HarvesterService.java index 2ca171c7..8819159a 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/HarvesterService.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/HarvesterService.java @@ -2,6 +2,7 @@ import it.gov.innovazione.ndc.harvester.context.HarvestExecutionContext; import it.gov.innovazione.ndc.harvester.context.HarvestExecutionContextUtils; +import it.gov.innovazione.ndc.harvester.model.Instance; import it.gov.innovazione.ndc.harvester.model.index.RightsHolder; import it.gov.innovazione.ndc.harvester.service.RepositoryService; import it.gov.innovazione.ndc.harvester.util.FileUtils; @@ -20,6 +21,9 @@ import java.util.Objects; import java.util.Optional; +import static it.gov.innovazione.ndc.repository.TripleStoreRepository.OLD_GRAPH_PREFIX; +import static it.gov.innovazione.ndc.repository.TripleStoreRepository.ONLINE_GRAPH_PREFIX; +import static it.gov.innovazione.ndc.repository.TripleStoreRepository.TMP_GRAPH_PREFIX; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; @@ -35,11 +39,29 @@ public class HarvesterService { private final RepositoryService repositoryService; private final FileUtils fileUtils; + private static void updateContextWithMaintainers(List maintainers) { + HarvestExecutionContext context = HarvestExecutionContextUtils.getContext(); + if (Objects.isNull(context)) { + context = HarvestExecutionContext.builder() + .build(); + } + context.addMaintainers(maintainers); + } + + private static void updateContextWithRootPath(Path path) { + HarvestExecutionContext context = HarvestExecutionContextUtils.getContext(); + if (Objects.isNull(context)) { + context = HarvestExecutionContext.builder() + .build(); + } + HarvestExecutionContextUtils.setContext(context.withRootPath(path.toString())); + } + public void harvest(Repository repository) throws IOException { - harvest(repository, null); + harvest(repository, null, Instance.PRIMARY); } - public void harvest(Repository repository, String revision) throws IOException { + public void harvest(Repository repository, String revision, Instance instance) throws IOException { log.info("Processing repo {}", repository.getUrl()); Repository normalisedRepo = repository.withUrl(normaliseRepoUrl(repository.getUrl())); String repoUrl = normalisedRepo.getUrl(); @@ -48,7 +70,7 @@ public void harvest(Repository repository, String revision) throws IOException { Path path = cloneRepoToTempPath(repoUrl, revision); try { - updateContext(path); + updateContext(path, instance); harvestClonedRepo(normalisedRepo, path); } finally { agencyRepositoryService.removeClonedRepo(path); @@ -60,23 +82,19 @@ public void harvest(Repository repository, String revision) throws IOException { } } - private void updateContext(Path path) { + private void updateContext(Path path, Instance instance) { updateContextWithRootPath(path); updateContextWithMaintainers(fileUtils.getMaintainersIfPossible(path)); + updateContextWithInstance(instance); } - private static void updateContextWithMaintainers(List maintainers) { + private void updateContextWithInstance(Instance instance) { HarvestExecutionContext context = HarvestExecutionContextUtils.getContext(); - if (Objects.nonNull(context)) { - context.addMaintainers(maintainers); - } - } - - private static void updateContextWithRootPath(Path path) { - HarvestExecutionContext context = HarvestExecutionContextUtils.getContext(); - if (Objects.nonNull(context)) { - HarvestExecutionContextUtils.setContext(context.withRootPath(path.toString())); + if (Objects.isNull(context)) { + context = HarvestExecutionContext.builder() + .build(); } + HarvestExecutionContextUtils.setContext(context.withInstance(instance)); } public void clear(String repoUrl) { @@ -84,7 +102,7 @@ public void clear(String repoUrl) { repoUrl = normaliseRepoUrl(repoUrl); log.debug("Normalised repo url {}", repoUrl); try { - clearRepo(repoUrl); + clearRepoAllInstances(repoUrl); log.info("Repo {} cleared", repoUrl); } catch (Exception e) { log.error("Error while clearing {}", repoUrl, e); @@ -97,8 +115,7 @@ private String normaliseRepoUrl(String repoUrl) { } private void harvestClonedRepo(Repository repository, Path path) { - clearRepo(repository.getUrl()); - + clearRepo(repository.getUrl(), HarvestExecutionContextUtils.getContext().getInstance()); harvestSemanticAssets(repository, path); storeRightsHolders(repository); @@ -116,16 +133,26 @@ private void storeRightsHolders(Repository repository) { repositoryService.storeRightsHolders(repository, rightsHolders); } - private void clearRepo(String repoUrl) { - cleanUpWithHarvesters(repoUrl); - cleanUpTripleStore(repoUrl); - cleanUpIndexedMetadata(repoUrl); + private void clearRepo(String repoUrl, Instance instance) { + cleanUpWithHarvesters(repoUrl, instance); + cleanUpTripleStore(repoUrl, OLD_GRAPH_PREFIX); + cleanUpIndexedMetadata(repoUrl, instance); + } + + private void clearRepoAllInstances(String repoUrl) { + cleanUpWithHarvesters(repoUrl, Instance.PRIMARY); + cleanUpWithHarvesters(repoUrl, Instance.SECONDARY); + cleanUpTripleStore(repoUrl, OLD_GRAPH_PREFIX); + cleanUpTripleStore(repoUrl, TMP_GRAPH_PREFIX); + cleanUpTripleStore(repoUrl, ONLINE_GRAPH_PREFIX); + cleanUpIndexedMetadata(repoUrl, Instance.PRIMARY); + cleanUpIndexedMetadata(repoUrl, Instance.SECONDARY); } - private void cleanUpWithHarvesters(String repoUrl) { + private void cleanUpWithHarvesters(String repoUrl, Instance instance) { semanticAssetHarvesters.forEach(h -> { log.debug("Cleaning for {} before harvesting {}", h.getType(), repoUrl); - h.cleanUpBeforeHarvesting(repoUrl); + h.cleanUpBeforeHarvesting(repoUrl, instance); log.debug("Cleaned for {}", h.getType()); }); @@ -146,14 +173,14 @@ private Path cloneRepoToTempPath(String repoUrl, String revision) throws IOExcep return path; } - private void cleanUpTripleStore(String repoUrl) { + private void cleanUpTripleStore(String repoUrl, String prefix) { log.debug("Cleaning up triple store for {}", repoUrl); - tripleStoreRepository.clearExistingNamedGraph(repoUrl); + tripleStoreRepository.clearExistingNamedGraph(repoUrl, prefix); } - private void cleanUpIndexedMetadata(String repoUrl) { + private void cleanUpIndexedMetadata(String repoUrl, Instance instance) { log.debug("Cleaning up indexed metadata for {}", repoUrl); - long deletedCount = semanticAssetMetadataRepository.deleteByRepoUrl(repoUrl); + long deletedCount = semanticAssetMetadataRepository.deleteByRepoUrl(repoUrl, instance); log.debug("Deleted {} indexed metadata for {}", deletedCount, repoUrl); } } diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/SemanticAssetHarvester.java b/src/main/java/it/gov/innovazione/ndc/harvester/SemanticAssetHarvester.java index 19901815..7d340c09 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/SemanticAssetHarvester.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/SemanticAssetHarvester.java @@ -1,5 +1,6 @@ package it.gov.innovazione.ndc.harvester; +import it.gov.innovazione.ndc.harvester.model.Instance; import it.gov.innovazione.ndc.model.harvester.Repository; import java.nio.file.Path; @@ -7,7 +8,7 @@ public interface SemanticAssetHarvester { SemanticAssetType getType(); - void cleanUpBeforeHarvesting(String repoUrl); + void cleanUpBeforeHarvesting(String repoUrl, Instance instance); void harvest(Repository repository, Path rootPath); } diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/context/HarvestExecutionContext.java b/src/main/java/it/gov/innovazione/ndc/harvester/context/HarvestExecutionContext.java index d9614450..8b90e9f4 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/context/HarvestExecutionContext.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/context/HarvestExecutionContext.java @@ -1,5 +1,6 @@ package it.gov.innovazione.ndc.harvester.context; +import it.gov.innovazione.ndc.harvester.model.Instance; import it.gov.innovazione.ndc.harvester.model.index.RightsHolder; import it.gov.innovazione.ndc.model.harvester.Repository; import lombok.AccessLevel; @@ -27,6 +28,8 @@ public class HarvestExecutionContext { private final String currentUserId; @With private final String rootPath; + @With + private final Instance instance; @Singular private final List maintainers = new ArrayList<>(); @Singular diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/BaseSemanticAssetHarvester.java b/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/BaseSemanticAssetHarvester.java index e85cd08e..23186511 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/BaseSemanticAssetHarvester.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/BaseSemanticAssetHarvester.java @@ -13,6 +13,7 @@ import it.gov.innovazione.ndc.harvester.context.HarvestExecutionContextUtils; import it.gov.innovazione.ndc.harvester.exception.SinglePathProcessingException; import it.gov.innovazione.ndc.harvester.harvesters.utils.PathUtils; +import it.gov.innovazione.ndc.harvester.model.Instance; import it.gov.innovazione.ndc.harvester.model.SemanticAssetPath; import it.gov.innovazione.ndc.model.harvester.Repository; import lombok.RequiredArgsConstructor; @@ -118,7 +119,7 @@ private boolean isAnyBiggerThan(Long maxFileSizeBytes, List files) { } @Override - public void cleanUpBeforeHarvesting(String repoUrl) { + public void cleanUpBeforeHarvesting(String repoUrl, Instance instance) { // by default nothing specific } diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/ControlledVocabularyHarvester.java b/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/ControlledVocabularyHarvester.java index b115c7a2..63574e66 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/ControlledVocabularyHarvester.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/harvesters/ControlledVocabularyHarvester.java @@ -5,6 +5,7 @@ import it.gov.innovazione.ndc.harvester.AgencyRepositoryService; import it.gov.innovazione.ndc.harvester.SemanticAssetType; import it.gov.innovazione.ndc.harvester.model.CvPath; +import it.gov.innovazione.ndc.harvester.model.Instance; import it.gov.innovazione.ndc.harvester.pathprocessors.ControlledVocabularyPathProcessor; import org.springframework.stereotype.Component; @@ -33,7 +34,7 @@ protected List scanForPaths(Path rootPath) { } @Override - public void cleanUpBeforeHarvesting(String repoUrl) { - pathProcessor.dropCsvIndicesForRepo(repoUrl); + public void cleanUpBeforeHarvesting(String repoUrl, Instance instance) { + pathProcessor.dropCsvIndicesForRepo(repoUrl, instance); } } diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/model/BaseSemanticAssetModel.java b/src/main/java/it/gov/innovazione/ndc/harvester/model/BaseSemanticAssetModel.java index 7c59a725..5d923843 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/model/BaseSemanticAssetModel.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/model/BaseSemanticAssetModel.java @@ -66,22 +66,25 @@ public abstract class BaseSemanticAssetModel implements SemanticAssetModel { protected final String source; private Resource mainResource; protected final String repoUrl; + protected final Instance instance; @Getter protected final SemanticAssetModelValidationContext validationContext; - protected BaseSemanticAssetModel(Model rdfModel, String source, String repoUrl) { + protected BaseSemanticAssetModel(Model rdfModel, String source, String repoUrl, Instance instance) { this.rdfModel = rdfModel; this.source = source; this.repoUrl = repoUrl; this.validationContext = NO_VALIDATION; + this.instance = instance; } - protected BaseSemanticAssetModel(Model rdfModel, String source, String repoUrl, SemanticAssetModelValidationContext validationContext) { + protected BaseSemanticAssetModel(Model rdfModel, String source, String repoUrl, SemanticAssetModelValidationContext validationContext, Instance instance) { this.rdfModel = rdfModel; this.source = source; this.repoUrl = repoUrl; this.validationContext = validationContext; + this.instance = instance; } @Override @@ -205,6 +208,7 @@ public SemanticAssetMetadata extractMetadata() { Resource mainResource = getMainResource(); RightsHolder agencyId = getAgencyId(mainResource, validationContext); return SemanticAssetMetadata.builder() + .instance(instance.name()) .iri(mainResource.getURI()) .repoUrl(repoUrl) .rightsHolder(extractRequiredNodeSummary(mainResource, rightsHolder, FOAF.name)) diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/model/ControlledVocabularyModel.java b/src/main/java/it/gov/innovazione/ndc/harvester/model/ControlledVocabularyModel.java index 6e4da965..49b0ec44 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/model/ControlledVocabularyModel.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/model/ControlledVocabularyModel.java @@ -30,16 +30,16 @@ public class ControlledVocabularyModel extends BaseSemanticAssetModel { private String endpointUrl = ""; - public ControlledVocabularyModel(Model coreModel, String source, String repoUrl) { - super(coreModel, source, repoUrl); + public ControlledVocabularyModel(Model coreModel, String source, String repoUrl, Instance instance) { + super(coreModel, source, repoUrl, instance); } - private ControlledVocabularyModel(Model coreModel, String source, String repoUrl, SemanticAssetModelValidationContext validationContext) { - super(coreModel, source, repoUrl, validationContext); + private ControlledVocabularyModel(Model coreModel, String source, String repoUrl, SemanticAssetModelValidationContext validationContext, Instance instance) { + super(coreModel, source, repoUrl, validationContext, instance); } - public static ControlledVocabularyModel forValidation(Model coreModel, String source, String repoUrl) { - return new ControlledVocabularyModel(coreModel, source, repoUrl, SemanticAssetModelValidationContext.getForValidation()); + public static ControlledVocabularyModel forValidation(Model coreModel, String source, String repoUrl, Instance instance) { + return new ControlledVocabularyModel(coreModel, source, repoUrl, SemanticAssetModelValidationContext.getForValidation(), instance); } public String getKeyConcept() { diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/model/Instance.java b/src/main/java/it/gov/innovazione/ndc/harvester/model/Instance.java new file mode 100644 index 00000000..1e354e60 --- /dev/null +++ b/src/main/java/it/gov/innovazione/ndc/harvester/model/Instance.java @@ -0,0 +1,14 @@ +package it.gov.innovazione.ndc.harvester.model; + +public enum Instance { + PRIMARY, + SECONDARY; + + public Instance switchInstance() { + if (this == PRIMARY) { + return SECONDARY; + } else { + return PRIMARY; + } + } +} diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/model/OntologyModel.java b/src/main/java/it/gov/innovazione/ndc/harvester/model/OntologyModel.java index 46a4ae06..f6a884df 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/model/OntologyModel.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/model/OntologyModel.java @@ -23,16 +23,16 @@ public class OntologyModel extends BaseSemanticAssetModel { - public OntologyModel(Model coreModel, String source, String repoUrl) { - super(coreModel, source, repoUrl); + public OntologyModel(Model coreModel, String source, String repoUrl, Instance instance) { + super(coreModel, source, repoUrl, instance); } - private OntologyModel(Model coreModel, String source, String repoUrl, SemanticAssetModelValidationContext validationContext) { - super(coreModel, source, repoUrl, validationContext); + private OntologyModel(Model coreModel, String source, String repoUrl, SemanticAssetModelValidationContext validationContext, Instance instance) { + super(coreModel, source, repoUrl, validationContext, instance); } - public static OntologyModel forValidation(Model rdfModel, String source, String repoUrl) { - return new OntologyModel(rdfModel, source, repoUrl, SemanticAssetModelValidationContext.getForValidation()); + public static OntologyModel forValidation(Model rdfModel, String source, String repoUrl, Instance instance) { + return new OntologyModel(rdfModel, source, repoUrl, SemanticAssetModelValidationContext.getForValidation(), instance); } @Override diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/model/SchemaModel.java b/src/main/java/it/gov/innovazione/ndc/harvester/model/SchemaModel.java index c18dd274..9e266c34 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/model/SchemaModel.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/model/SchemaModel.java @@ -40,16 +40,16 @@ public class SchemaModel extends BaseSemanticAssetModel { - public SchemaModel(Model coreModel, String source, String repoUrl) { - super(coreModel, source, repoUrl); + public SchemaModel(Model coreModel, String source, String repoUrl, Instance instance) { + super(coreModel, source, repoUrl, instance); } - private SchemaModel(Model coreModel, String source, String repoUrl, SemanticAssetModelValidationContext validationContext) { - super(coreModel, source, repoUrl, validationContext); + private SchemaModel(Model coreModel, String source, String repoUrl, SemanticAssetModelValidationContext validationContext, Instance instance) { + super(coreModel, source, repoUrl, validationContext, instance); } - public static SchemaModel forValidation(Model rdfModel, String source, String repoUrl) { - return new SchemaModel(rdfModel, source, repoUrl, SemanticAssetModelValidationContext.getForValidation()); + public static SchemaModel forValidation(Model rdfModel, String source, String repoUrl, Instance instance) { + return new SchemaModel(rdfModel, source, repoUrl, SemanticAssetModelValidationContext.getForValidation(), instance); } @Override @@ -62,6 +62,7 @@ public SemanticAssetMetadata extractMetadata() { Resource mainResource = getMainResource(); RightsHolder rightsHolderObj = RightsHolderExtractor.getAgencyId(mainResource, validationContext); return SemanticAssetMetadata.builder() + .instance(instance.name()) .iri(mainResource.getURI()) .repoUrl(repoUrl) .title(LiteralExtractor.extract(mainResource, title, validationContext)) diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/model/SemanticAssetModelFactory.java b/src/main/java/it/gov/innovazione/ndc/harvester/model/SemanticAssetModelFactory.java index 9ccd9469..a3aa693d 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/model/SemanticAssetModelFactory.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/model/SemanticAssetModelFactory.java @@ -1,6 +1,7 @@ package it.gov.innovazione.ndc.harvester.model; import it.gov.innovazione.ndc.harvester.model.exception.InvalidModelException; +import it.gov.innovazione.ndc.service.InstanceManager; import lombok.RequiredArgsConstructor; import org.apache.jena.rdf.model.Model; import org.apache.jena.riot.Lang; @@ -12,6 +13,9 @@ @Component @RequiredArgsConstructor public class SemanticAssetModelFactory { + + private final InstanceManager instanceManager; + private interface ModelConstructor { T build(Model model, String source); } @@ -19,17 +23,17 @@ private interface ModelConstructor { public ControlledVocabularyModel createControlledVocabulary(String ttlFile, String repoUrl) { return loadAndBuild(ttlFile, - (coreModel, source) -> new ControlledVocabularyModel(coreModel, source, repoUrl)); + (coreModel, source) -> new ControlledVocabularyModel(coreModel, source, repoUrl, instanceManager.getNextOnlineInstance(repoUrl))); } public OntologyModel createOntology(String ttlFile, String repoUrl) { return loadAndBuild(ttlFile, - (coreModel, source) -> new OntologyModel(coreModel, source, repoUrl)); + (coreModel, source) -> new OntologyModel(coreModel, source, repoUrl, instanceManager.getNextOnlineInstance(repoUrl))); } public SchemaModel createSchema(String ttlFile, String repoUrl) { return loadAndBuild(ttlFile, - (coreModel, source) -> new SchemaModel(coreModel, source, repoUrl)); + (coreModel, source) -> new SchemaModel(coreModel, source, repoUrl, instanceManager.getNextOnlineInstance(repoUrl))); } private T loadAndBuild(String source, ModelConstructor c) { diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/model/index/SemanticAssetMetadata.java b/src/main/java/it/gov/innovazione/ndc/harvester/model/index/SemanticAssetMetadata.java index 75acdd20..31283afe 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/model/index/SemanticAssetMetadata.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/model/index/SemanticAssetMetadata.java @@ -31,7 +31,8 @@ public class SemanticAssetMetadata { private SemanticAssetType type; @Field(type = Keyword) private String repoUrl; - + @Field(type = Keyword) + private String instance; @Field(index = false, type = Keyword, normalizer = "lowercase_normalizer") private String title; diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/pathprocessors/ControlledVocabularyPathProcessor.java b/src/main/java/it/gov/innovazione/ndc/harvester/pathprocessors/ControlledVocabularyPathProcessor.java index 2dc18b73..ae7979a8 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/pathprocessors/ControlledVocabularyPathProcessor.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/pathprocessors/ControlledVocabularyPathProcessor.java @@ -4,6 +4,7 @@ import it.gov.innovazione.ndc.harvester.csv.CsvParser.CsvData; import it.gov.innovazione.ndc.harvester.model.ControlledVocabularyModel; import it.gov.innovazione.ndc.harvester.model.CvPath; +import it.gov.innovazione.ndc.harvester.model.Instance; import it.gov.innovazione.ndc.harvester.model.SemanticAssetModelFactory; import it.gov.innovazione.ndc.harvester.model.index.SemanticAssetMetadata; import it.gov.innovazione.ndc.repository.SemanticAssetMetadataRepository; @@ -63,10 +64,10 @@ private void parseAndIndexCsv(VocabularyIdentifier vocabularyIdentifier, String vocabularyDataService.indexData(vocabularyIdentifier, flatData); } - public void dropCsvIndicesForRepo(String repoUrl) { + public void dropCsvIndicesForRepo(String repoUrl, Instance instance) { log.debug("Retrieving vocab metadata for {} to drop indices", repoUrl); - List vocabs = metadataRepository.findVocabulariesForRepoUrl(repoUrl); + List vocabs = metadataRepository.findVocabulariesForRepoUrl(repoUrl, instance); if (log.isDebugEnabled()) { log.debug("Found {} vocabs with indices to drop", vocabs.size()); diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/service/ActualConfigService.java b/src/main/java/it/gov/innovazione/ndc/harvester/service/ActualConfigService.java index 5a7c8393..d778560c 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/service/ActualConfigService.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/service/ActualConfigService.java @@ -1,6 +1,7 @@ package it.gov.innovazione.ndc.harvester.service; import it.gov.innovazione.ndc.eventhandler.NdcEventPublisher; +import it.gov.innovazione.ndc.harvester.model.Instance; import it.gov.innovazione.ndc.eventhandler.event.ConfigService; import it.gov.innovazione.ndc.model.harvester.Repository; import lombok.Getter; @@ -271,7 +272,8 @@ private void removeConfigKey(ConfigKey configKey, String writtenBy, public enum ConfigKey { MAX_FILE_SIZE_BYTES("The maximum file size in bytes of a file to be harvested", Validator.IS_LONG, Parser.TO_LONG), GITHUB_ISSUER_ENABLED("Enable the GitHub issuer capability", Validator.IS_BOOLEAN, Parser.TO_BOOLEAN), - ALERTER_ENABLED("Enable the Alerter capability", Validator.IS_BOOLEAN, Parser.TO_BOOLEAN); + ALERTER_ENABLED("Enable the Alerter capability", Validator.IS_BOOLEAN, Parser.TO_BOOLEAN), + ACTIVE_INSTANCE("The atomic writer active instance", Validator.IS_INSTANCE, Parser.TO_INSTANCE); private final String description; private final Validator validator; @@ -282,7 +284,8 @@ public enum ConfigKey { @RequiredArgsConstructor private enum Validator { IS_LONG(Parser.TO_LONG), - IS_BOOLEAN(Parser.TO_BOOLEAN); + IS_BOOLEAN(Parser.TO_BOOLEAN), + IS_INSTANCE(Parser.TO_INSTANCE); private final Parser parser; @@ -302,7 +305,9 @@ public Predicate getValidator() { @RequiredArgsConstructor public enum Parser { TO_LONG(Long::parseLong), - TO_BOOLEAN(Parser::parseBoolean); + TO_BOOLEAN(Parser::parseBoolean), + TO_INSTANCE(s -> Instance.valueOf(s.toUpperCase())); + private final Function parsingFunction; private static Boolean parseBoolean(String s) { @@ -311,5 +316,5 @@ private static Boolean parseBoolean(String s) { } return Boolean.parseBoolean(trim(s)); } - } + } } diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/service/HarvesterRunService.java b/src/main/java/it/gov/innovazione/ndc/harvester/service/HarvesterRunService.java index 2227812c..d07bb48f 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/service/HarvesterRunService.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/service/HarvesterRunService.java @@ -35,21 +35,23 @@ public class HarvesterRunService { public int saveHarvesterRun(HarvesterRun harvesterRun) { String query = "INSERT INTO HARVESTER_RUN (" - + "ID, " - + "CORRELATION_ID, " - + "REPOSITORY_ID, " - + "REPOSITORY_URL, " - + "REVISION, " - + "STARTED, " - + "STARTED_BY, " - + "FINISHED, " - + "STATUS, " - + "REASON) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + + "ID, " + + "CORRELATION_ID, " + + "REPOSITORY_ID, " + + "REPOSITORY_URL, " + + "INSTANCE, " + + "REVISION, " + + "STARTED, " + + "STARTED_BY, " + + "FINISHED, " + + "STATUS, " + + "REASON) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; return jdbcTemplate.update(query, harvesterRun.getId(), harvesterRun.getCorrelationId(), harvesterRun.getRepositoryId(), harvesterRun.getRepositoryUrl(), + harvesterRun.getInstance(), harvesterRun.getRevision(), harvesterRun.getStartedAt(), harvesterRun.getStartedBy(), @@ -73,14 +75,14 @@ private boolean isMoreRecentThan(HarvesterRun harvesterRun, Long days) { private boolean isAlreadyRunning(HarvesterRun harvesterRun, Repository repository) { return ( - harvesterRun.getRepositoryId().equals(repository.getId()) - || startsWithIgnoreCase( - harvesterRun.getRepositoryUrl(), - repository.getUrl()) - || startsWithIgnoreCase( - repository.getUrl(), - harvesterRun.getRepositoryUrl())) - && harvesterRun.getStatus() == HarvesterRun.Status.RUNNING; + harvesterRun.getRepositoryId().equals(repository.getId()) + || startsWithIgnoreCase( + harvesterRun.getRepositoryUrl(), + repository.getUrl()) + || startsWithIgnoreCase( + repository.getUrl(), + harvesterRun.getRepositoryUrl())) + && harvesterRun.getStatus() == HarvesterRun.Status.RUNNING; } public boolean isHarvestingAlreadyExecuted(String repositoryId, String revision) { @@ -94,10 +96,10 @@ public boolean isHarvestingAlreadyExecuted(String repositoryId, String revision) public int updateHarvesterRun(HarvesterRun harvesterRun) { String query = "UPDATE HARVESTER_RUN SET " - + "FINISHED = ?, " - + "STATUS = ?, " - + "REASON = ? " - + "WHERE ID = ?"; + + "FINISHED = ?, " + + "STATUS = ?, " + + "REASON = ? " + + "WHERE ID = ?"; return jdbcTemplate.update(query, harvesterRun.getEndedAt(), harvesterRun.getStatus().toString(), @@ -112,24 +114,26 @@ public Stream getRecentRuns(Long days) { public List getAllRuns() { String sqlQuery = "SELECT " - + "ID, " - + "CORRELATION_ID, " - + "REPOSITORY_ID, " - + "REPOSITORY_URL, " - + "REVISION, " - + "STARTED, " - + "STARTED_BY, " - + "FINISHED, " - + "STATUS, " - + "REASON " - + "FROM HARVESTER_RUN " - + "ORDER BY STARTED DESC"; + + "ID, " + + "CORRELATION_ID, " + + "REPOSITORY_ID, " + + "REPOSITORY_URL, " + + "INSTANCE, " + + "REVISION, " + + "STARTED, " + + "STARTED_BY, " + + "FINISHED, " + + "STATUS, " + + "REASON " + + "FROM HARVESTER_RUN " + + "ORDER BY STARTED DESC"; return jdbcTemplate.query(sqlQuery, (rs, rowNum) -> HarvesterRun.builder() .id(rs.getString("ID")) .correlationId(rs.getString("CORRELATION_ID")) .repositoryId(rs.getString("REPOSITORY_ID")) .repositoryUrl(rs.getString("REPOSITORY_URL")) + .instance(rs.getString("INSTANCE")) .revision(rs.getString("REVISION")) .startedAt(getInstant(rs, "STARTED")) .startedBy(rs.getString("STARTED_BY")) @@ -172,9 +176,9 @@ private void deleteIfNecessary(HarvesterRun harvesterRun) { private boolean matchRunning(HarvesterRun harvesterRun, String threadName) { return contains(threadName, harvesterRun.getRepositoryId()) - && contains(threadName, harvesterRun.getRevision()) - && contains(threadName, harvesterRun.getId()) - && endsWithIgnoreCase(threadName, "RUNNING"); + && contains(threadName, harvesterRun.getRevision()) + && contains(threadName, harvesterRun.getId()) + && endsWithIgnoreCase(threadName, "RUNNING"); } private void delete(HarvesterRun harvesterRun) { diff --git a/src/main/java/it/gov/innovazione/ndc/model/harvester/HarvesterRun.java b/src/main/java/it/gov/innovazione/ndc/model/harvester/HarvesterRun.java index 0245c049..7ab554f4 100644 --- a/src/main/java/it/gov/innovazione/ndc/model/harvester/HarvesterRun.java +++ b/src/main/java/it/gov/innovazione/ndc/model/harvester/HarvesterRun.java @@ -14,6 +14,7 @@ public class HarvesterRun { private final String correlationId; private final String repositoryId; private final String repositoryUrl; + private final String instance; private final Instant startedAt; private final String startedBy; private final Instant endedAt; diff --git a/src/main/java/it/gov/innovazione/ndc/repository/SemanticAssetMetadataRepository.java b/src/main/java/it/gov/innovazione/ndc/repository/SemanticAssetMetadataRepository.java index d6731d50..201b81d4 100644 --- a/src/main/java/it/gov/innovazione/ndc/repository/SemanticAssetMetadataRepository.java +++ b/src/main/java/it/gov/innovazione/ndc/repository/SemanticAssetMetadataRepository.java @@ -6,7 +6,10 @@ import co.elastic.clients.elasticsearch._types.query_dsl.Query; import co.elastic.clients.elasticsearch._types.query_dsl.QueryVariant; import co.elastic.clients.elasticsearch._types.query_dsl.TermsQuery; +import it.gov.innovazione.ndc.harvester.model.Instance; import it.gov.innovazione.ndc.harvester.model.index.SemanticAssetMetadata; +import it.gov.innovazione.ndc.service.InstanceManager; +import it.gov.innovazione.ndc.service.InstanceManager.RepositoryInstance; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -26,6 +29,7 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import static it.gov.innovazione.ndc.harvester.SemanticAssetType.CONTROLLED_VOCABULARY; import static java.util.Objects.nonNull; @@ -37,6 +41,7 @@ @Slf4j public class SemanticAssetMetadataRepository { private final ElasticsearchOperations esOps; + private final InstanceManager instanceManager; public SearchPage search(String queryPattern, Set types, Set themes, Set rightsHolder, @@ -47,29 +52,66 @@ public SearchPage search(String queryPattern, Set queries.add(MatchQuery.of(mq -> mq.field("searchableText").query(queryPattern))._toQuery()); } - getTermsQuery(types, themes, rightsHolder).stream() + getQueriesForParams(types, themes, rightsHolder).stream() .map(QueryVariant::_toQuery) .forEach(queries::add); - queries.forEach(q -> log.info("Query: {}", q)); + getConditionForInstances() + .map(QueryVariant::_toQuery) + .ifPresent(queries::add); NativeQuery query = NativeQuery.builder() .withQuery(BoolQuery.of(bq -> bq.must(queries))._toQuery()) .withPageable(pageable) .build(); + log.info("Searching for assets with query: {}", query); + return searchPageFor(esOps.search(query, SemanticAssetMetadata.class), pageable); } - private List getTermsQuery(Set types, Set themes, Set rightsHolder) { + private Optional getConditionForInstances() { + List currentRepoInstances = instanceManager.getCurrentInstances(); + + List repoInstanceQueries = currentRepoInstances.stream() + .map(repositoryInstance -> getBoolQueryForRepo( + repositoryInstance.getUrl(), + repositoryInstance.getInstance())) + .map(QueryVariant::_toQuery) + .toList(); + + if (repoInstanceQueries.isEmpty()) { + return Optional.empty(); + } + + return Optional.of(BoolQuery.of(bq -> bq.should(repoInstanceQueries))); + } + + private BoolQuery getBoolQueryForRepo(String url, Instance instance) { + List termsQueries = Stream.of( + termsQuery("repoUrl", url), + termsQuery("instance", instance.name())) + .map(QueryVariant::_toQuery) + .toList(); + + return BoolQuery.of(bq -> bq.must(termsQueries)); + } + + + private List getQueriesForParams(Set types, Set themes, Set rightsHolder) { return Map.of("type", types, "themes", themes, "agencyId", rightsHolder).entrySet().stream() .filter(e -> nonNull(e.getValue())) .filter(e -> !e.getValue().isEmpty()) - .map(e -> getTermsQuery(e.getKey(), e.getValue())) + .map(e -> termsQuery(e.getKey(), e.getValue())) .toList(); } - private TermsQuery getTermsQuery(String field, Set values) { + private TermsQuery termsQuery(String field, String value) { + return termsQuery(field, Set.of(value)); + } + + + private TermsQuery termsQuery(String field, Set values) { return TermsQuery.of(t -> t.field(field).terms( terms -> terms.value(values.stream() .filter(Objects::nonNull) @@ -81,11 +123,11 @@ public Optional findByIri(String iri) { return Optional.ofNullable(esOps.get(iri, SemanticAssetMetadata.class)); } - public long deleteByRepoUrl(String repoUrl) { + public long deleteByRepoUrl(String repoUrl, Instance instance) { return esOps.delete( DeleteQuery.builder( - NativeQuery.builder().withQuery( - MatchQuery.of(mq -> mq.field("repoUrl").query(repoUrl))._toQuery()) + NativeQuery.builder() + .withQuery(getBoolQueryForRepo(repoUrl, instance)._toQuery()) .build()) .build(), SemanticAssetMetadata.class).getDeleted(); @@ -95,11 +137,12 @@ public void save(SemanticAssetMetadata metadata) { esOps.save(metadata); } - public List findVocabulariesForRepoUrl(String repoUrl) { + public List findVocabulariesForRepoUrl(String repoUrl, Instance instance) { BoolQuery boolQuery = BoolQuery.of( bq -> bq.must( List.of( termQuery("repoUrl", repoUrl)._toQuery(), + termsQuery("instance", instance.name())._toQuery(), termQuery("type", CONTROLLED_VOCABULARY.name())._toQuery()))); NativeQuery query = NativeQuery.builder() diff --git a/src/main/java/it/gov/innovazione/ndc/repository/TripleStoreRepository.java b/src/main/java/it/gov/innovazione/ndc/repository/TripleStoreRepository.java index 445643e7..5627d03e 100644 --- a/src/main/java/it/gov/innovazione/ndc/repository/TripleStoreRepository.java +++ b/src/main/java/it/gov/innovazione/ndc/repository/TripleStoreRepository.java @@ -1,5 +1,7 @@ package it.gov.innovazione.ndc.repository; +import com.apicatalog.jsonld.StringUtils; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.jena.arq.querybuilder.SelectBuilder; import org.apache.jena.atlas.web.HttpException; @@ -10,12 +12,18 @@ import org.apache.jena.update.UpdateExecution; import org.springframework.stereotype.Repository; +import java.net.URL; + import static java.lang.String.format; @Slf4j @Repository public class TripleStoreRepository { private static final String DROP_SILENT_GRAPH_WITH_LOG_ENABLE_3 = "DEFINE sql:log-enable 3%nDROP SILENT GRAPH <%s>%n"; + public static final String OLD_GRAPH_PREFIX = "old"; + public static final String TMP_GRAPH_PREFIX = "old"; + public static final String ONLINE_GRAPH_PREFIX = ""; + private static final String RENAME_GRAPH = "DEFINE sql:log-enable 3%nMOVE SILENT GRAPH <%s> to <%s>%n"; private final VirtuosoClient virtuosoClient; @@ -23,12 +31,8 @@ public TripleStoreRepository(VirtuosoClient virtuosoClient) { this.virtuosoClient = virtuosoClient; } - public void save(String graphName, Model model) { - log.info("Saving model to Virtuoso"); - try (RDFConnection connection = virtuosoClient.getConnection()) { - saveWithConnection(graphName, model, connection); - } - log.info("Model saved to Virtuoso"); + private static String getRenameCommand(String oldGraph, String newGraph) { + return format(RENAME_GRAPH, oldGraph, newGraph); } private void saveWithConnection(String graphName, Model model, RDFConnection connection) { @@ -44,12 +48,30 @@ private void saveWithConnection(String graphName, Model model, RDFConnection con } } + private static String getUpdateCommand(String repoUrl, String repoUrlPrefix) { + return format(DROP_SILENT_GRAPH_WITH_LOG_ENABLE_3, reworkRepoUrlIfNecessary(repoUrl, repoUrlPrefix)); + } + + @SneakyThrows + private static String reworkRepoUrlIfNecessary(String repoUrl, String repoUrlPrefix) { + if (StringUtils.isBlank(repoUrlPrefix)) { + return repoUrl; + } + URL url = new URL(repoUrl); + String port = url.getPort() == -1 ? "" : ":" + url.getPort(); + return url.getProtocol() + "://" + repoUrlPrefix + "." + url.getHost() + port + url.getPath(); + } + public void clearExistingNamedGraph(String repoUrl) { + clearExistingNamedGraph(repoUrl, ""); + } + + public void clearExistingNamedGraph(String repoUrl, String prefix) { try { String sparqlEndpoint = virtuosoClient.getSparqlEndpoint(); UpdateExecution .service(sparqlEndpoint) - .updateString(getUpdateCommand(repoUrl)) + .updateString(getUpdateCommand(repoUrl, prefix)) .execute(); } catch (Exception e) { log.error(format("Could not clear existing named graph! - %s", repoUrl), e); @@ -61,8 +83,33 @@ public void clearExistingNamedGraph(String repoUrl) { } } - private static String getUpdateCommand(String repoUrl) { - return format(DROP_SILENT_GRAPH_WITH_LOG_ENABLE_3, repoUrl); + public void save(String graphName, Model model) { + log.info("Saving model to Virtuoso"); + try (RDFConnection connection = virtuosoClient.getConnection()) { + saveWithConnection(reworkRepoUrlIfNecessary(graphName, OLD_GRAPH_PREFIX), model, connection); + } + log.info("Model saved to Virtuoso"); + } + + public void switchInstances(it.gov.innovazione.ndc.model.harvester.Repository repository) { + String temp = reworkRepoUrlIfNecessary(repository.getUrl(), OLD_GRAPH_PREFIX); + String temp2 = reworkRepoUrlIfNecessary(repository.getUrl(), TMP_GRAPH_PREFIX); + rename(repository.getUrl(), temp2); + rename(temp, repository.getUrl()); + rename(temp2, temp); + } + + public void rename(String oldGraph, String newGraph) { + try { + String sparqlEndpoint = virtuosoClient.getSparqlEndpoint(); + UpdateExecution + .service(sparqlEndpoint) + .updateString(getRenameCommand(oldGraph, newGraph)) + .execute(); + } catch (Exception e) { + log.error(format("Could not rename %s into %s ", oldGraph, newGraph), e); + throw new TripleStoreRepositoryException(format("Could not rename - '%s' -> '%s'", oldGraph, newGraph), e); + } } public QueryExecution select(SelectBuilder selectBuilder) { @@ -78,4 +125,5 @@ public QueryExecution select(SelectBuilder selectBuilder) { throw new TripleStoreRepositoryException(format("Could not execute select - '%s'", selectBuilder), e); } } + } diff --git a/src/main/java/it/gov/innovazione/ndc/service/DefaultInstanceManager.java b/src/main/java/it/gov/innovazione/ndc/service/DefaultInstanceManager.java new file mode 100644 index 00000000..039ff974 --- /dev/null +++ b/src/main/java/it/gov/innovazione/ndc/service/DefaultInstanceManager.java @@ -0,0 +1,56 @@ +package it.gov.innovazione.ndc.service; + +import it.gov.innovazione.ndc.eventhandler.event.ConfigService; +import it.gov.innovazione.ndc.harvester.model.Instance; +import it.gov.innovazione.ndc.harvester.service.RepositoryService; +import it.gov.innovazione.ndc.model.harvester.Repository; +import it.gov.innovazione.ndc.repository.TripleStoreRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import static it.gov.innovazione.ndc.harvester.service.ActualConfigService.ConfigKey.ACTIVE_INSTANCE; + +@Service +@RequiredArgsConstructor +public class DefaultInstanceManager implements InstanceManager { + + private final ConfigService configService; + private final RepositoryService repositoryService; + private final TripleStoreRepository tripleStoreRepository; + + public Instance getNextOnlineInstance(String repoUrl) { + Optional repository = repositoryService.findActiveRepoByUrl(repoUrl); + if (repository.isEmpty()) { + throw new IllegalArgumentException("Repository not found for url: " + repoUrl); + } + return getNextOnlineInstance(repository.get()); + } + + public Instance getNextOnlineInstance(Repository repository) { + return getCurrentInstance(repository).switchInstance(); + } + + public Instance getCurrentInstance(Repository repository) { + Optional instance = configService.fromRepo(ACTIVE_INSTANCE, repository.getId()); + return instance.orElse(Instance.PRIMARY); + } + + public void switchInstances(Repository repository) { + // switch instance on Repositories + configService.writeConfigKey(ACTIVE_INSTANCE, "system", getNextOnlineInstance(repository), repository.getId()); + + // switch instance on Virtuoso + tripleStoreRepository.switchInstances(repository); + } + + @Override + public List getCurrentInstances() { + return repositoryService.getActiveRepos().stream() + .map(repo -> RepositoryInstance.of(repo.getUrl(), getCurrentInstance(repo))) + .collect(Collectors.toList()); + } +} diff --git a/src/main/java/it/gov/innovazione/ndc/service/InstanceManager.java b/src/main/java/it/gov/innovazione/ndc/service/InstanceManager.java new file mode 100644 index 00000000..9ef30c0b --- /dev/null +++ b/src/main/java/it/gov/innovazione/ndc/service/InstanceManager.java @@ -0,0 +1,27 @@ +package it.gov.innovazione.ndc.service; + +import it.gov.innovazione.ndc.harvester.model.Instance; +import it.gov.innovazione.ndc.model.harvester.Repository; +import lombok.Data; +import lombok.RequiredArgsConstructor; + +import java.util.List; + +public interface InstanceManager { + Instance getNextOnlineInstance(String repoUrl); + + Instance getNextOnlineInstance(Repository repository); + + Instance getCurrentInstance(Repository repository); + + void switchInstances(Repository repository); + + List getCurrentInstances(); + + @Data + @RequiredArgsConstructor(staticName = "of") + class RepositoryInstance { + public final String url; + public final Instance instance; + } +} diff --git a/src/main/java/it/gov/innovazione/ndc/validator/ControlledVocabularyValidator.java b/src/main/java/it/gov/innovazione/ndc/validator/ControlledVocabularyValidator.java index d9ad58cc..bc3d6dfe 100644 --- a/src/main/java/it/gov/innovazione/ndc/validator/ControlledVocabularyValidator.java +++ b/src/main/java/it/gov/innovazione/ndc/validator/ControlledVocabularyValidator.java @@ -2,6 +2,7 @@ import it.gov.innovazione.ndc.harvester.SemanticAssetType; import it.gov.innovazione.ndc.harvester.model.ControlledVocabularyModel; +import it.gov.innovazione.ndc.harvester.model.Instance; import org.apache.jena.rdf.model.Model; import org.springframework.stereotype.Component; @@ -14,6 +15,6 @@ public ControlledVocabularyValidator() { @Override protected ControlledVocabularyModel getValidatorModel(Model rdfModel) { - return ControlledVocabularyModel.forValidation(rdfModel, null, null); + return ControlledVocabularyModel.forValidation(rdfModel, null, null, Instance.PRIMARY); } } diff --git a/src/main/java/it/gov/innovazione/ndc/validator/OntologyValidator.java b/src/main/java/it/gov/innovazione/ndc/validator/OntologyValidator.java index b6fc8a65..7a04724b 100644 --- a/src/main/java/it/gov/innovazione/ndc/validator/OntologyValidator.java +++ b/src/main/java/it/gov/innovazione/ndc/validator/OntologyValidator.java @@ -1,6 +1,7 @@ package it.gov.innovazione.ndc.validator; import it.gov.innovazione.ndc.harvester.SemanticAssetType; +import it.gov.innovazione.ndc.harvester.model.Instance; import it.gov.innovazione.ndc.harvester.model.OntologyModel; import org.apache.jena.rdf.model.Model; import org.springframework.stereotype.Component; @@ -14,6 +15,6 @@ public OntologyValidator() { @Override protected OntologyModel getValidatorModel(Model rdfModel) { - return OntologyModel.forValidation(rdfModel, null, null); + return OntologyModel.forValidation(rdfModel, null, null, Instance.PRIMARY); } } diff --git a/src/main/java/it/gov/innovazione/ndc/validator/SchemaValidator.java b/src/main/java/it/gov/innovazione/ndc/validator/SchemaValidator.java index 16516f75..559f9e37 100644 --- a/src/main/java/it/gov/innovazione/ndc/validator/SchemaValidator.java +++ b/src/main/java/it/gov/innovazione/ndc/validator/SchemaValidator.java @@ -1,6 +1,7 @@ package it.gov.innovazione.ndc.validator; import it.gov.innovazione.ndc.harvester.SemanticAssetType; +import it.gov.innovazione.ndc.harvester.model.Instance; import it.gov.innovazione.ndc.harvester.model.SchemaModel; import org.apache.jena.rdf.model.Model; import org.springframework.stereotype.Component; @@ -14,6 +15,6 @@ public SchemaValidator() { @Override protected SchemaModel getValidatorModel(Model rdfModel) { - return SchemaModel.forValidation(rdfModel, null, null); + return SchemaModel.forValidation(rdfModel, null, null, Instance.PRIMARY); } } diff --git a/src/main/resources/db/migration/V7__instance_manager.sql b/src/main/resources/db/migration/V7__instance_manager.sql new file mode 100644 index 00000000..cbd15782 --- /dev/null +++ b/src/main/resources/db/migration/V7__instance_manager.sql @@ -0,0 +1,3 @@ +alter table HARVESTER_RUN + add INSTANCE varchar(64); + diff --git a/src/test/java/it/gov/innovazione/ndc/harvester/HarvesterServiceTest.java b/src/test/java/it/gov/innovazione/ndc/harvester/HarvesterServiceTest.java index f4f7a78f..7cf90e9b 100644 --- a/src/test/java/it/gov/innovazione/ndc/harvester/HarvesterServiceTest.java +++ b/src/test/java/it/gov/innovazione/ndc/harvester/HarvesterServiceTest.java @@ -1,5 +1,6 @@ package it.gov.innovazione.ndc.harvester; +import it.gov.innovazione.ndc.harvester.model.Instance; import it.gov.innovazione.ndc.harvester.service.RepositoryService; import it.gov.innovazione.ndc.harvester.util.FileUtils; import it.gov.innovazione.ndc.model.harvester.Repository; @@ -9,7 +10,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; -import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -22,7 +22,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -87,22 +86,6 @@ void shouldGiveUpOnRepoWhenGenericExceptionIsThrown() throws IOException { verify(harvester).harvest(repo, clonedRepoPath); } - @Test - void shouldClearNamedGraphAndMetadataBeforeProcessingData() throws IOException { - String repoUrl = "someRepoUri"; - Repository repo = asRepo(repoUrl); - - when(agencyRepoService.cloneRepo(repoUrl, null)).thenReturn(clonedRepoPath); - - harvesterService.harvest(asRepo(repoUrl)); - - InOrder order = inOrder(harvester, tripleStoreRepository, metadataRepository, harvester); - order.verify(harvester).cleanUpBeforeHarvesting(repoUrl); - order.verify(tripleStoreRepository).clearExistingNamedGraph(repoUrl); - order.verify(metadataRepository).deleteByRepoUrl(repoUrl); - order.verify(harvester).harvest(repo, clonedRepoPath); - } - @Test void shouldCleanUpTemporaryFolderWithRepoAfterProcessing() throws IOException { String repoUrl = "someRepoUri"; @@ -129,19 +112,6 @@ void shouldCleanUpTemporaryFolderWithRepoAfterFailure() throws IOException { verify(agencyRepoService).removeClonedRepo(clonedRepoPath); } - @Test - void shouldClearRepoData() { - String repoUrl = "someRepoUri.git"; - String sanitizedRepoUrl = repoUrl.replace(".git", ""); - - harvesterService.clear(repoUrl); - - InOrder order = inOrder(harvester, tripleStoreRepository, metadataRepository, harvester); - order.verify(harvester).cleanUpBeforeHarvesting(sanitizedRepoUrl); - order.verify(tripleStoreRepository).clearExistingNamedGraph(sanitizedRepoUrl); - order.verify(metadataRepository).deleteByRepoUrl(sanitizedRepoUrl); - } - @Test void shouldPropagateExceptionWhileClearingRepo() { String repoUrl = "someRepoUri.git"; @@ -149,7 +119,7 @@ void shouldPropagateExceptionWhileClearingRepo() { RuntimeException exception = new RuntimeException("Something bad happened!"); - doThrow(exception).when(harvester).cleanUpBeforeHarvesting(sanitizedRepoUrl); + doThrow(exception).when(harvester).cleanUpBeforeHarvesting(sanitizedRepoUrl, Instance.PRIMARY); assertThatThrownBy(() -> harvesterService.clear(repoUrl)) .isSameAs(exception); diff --git a/src/test/java/it/gov/innovazione/ndc/harvester/harvesters/ControlledVocabularyHarvesterTest.java b/src/test/java/it/gov/innovazione/ndc/harvester/harvesters/ControlledVocabularyHarvesterTest.java index 6d0892b6..251c2819 100644 --- a/src/test/java/it/gov/innovazione/ndc/harvester/harvesters/ControlledVocabularyHarvesterTest.java +++ b/src/test/java/it/gov/innovazione/ndc/harvester/harvesters/ControlledVocabularyHarvesterTest.java @@ -3,6 +3,7 @@ import it.gov.innovazione.ndc.eventhandler.event.ConfigService; import it.gov.innovazione.ndc.harvester.AgencyRepositoryService; import it.gov.innovazione.ndc.harvester.model.CvPath; +import it.gov.innovazione.ndc.harvester.model.Instance; import it.gov.innovazione.ndc.harvester.pathprocessors.ControlledVocabularyPathProcessor; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -51,9 +52,9 @@ void shouldProcessAllScannedPaths() { void shouldCleanIndicesBeforeHarvesting() { String repoUrl = "my-repo.git"; - harvester.cleanUpBeforeHarvesting(repoUrl); + harvester.cleanUpBeforeHarvesting(repoUrl, Instance.PRIMARY); - verify(pathProcessor).dropCsvIndicesForRepo(repoUrl); + verify(pathProcessor).dropCsvIndicesForRepo(repoUrl, Instance.PRIMARY); } @Test diff --git a/src/test/java/it/gov/innovazione/ndc/harvester/model/BaseSemanticAssetModelTest.java b/src/test/java/it/gov/innovazione/ndc/harvester/model/BaseSemanticAssetModelTest.java index 1bf2266d..3e387ed7 100644 --- a/src/test/java/it/gov/innovazione/ndc/harvester/model/BaseSemanticAssetModelTest.java +++ b/src/test/java/it/gov/innovazione/ndc/harvester/model/BaseSemanticAssetModelTest.java @@ -442,7 +442,7 @@ private static class TestBaseSemanticAssetModel extends BaseSemanticAssetModel { public TestBaseSemanticAssetModel(Model coreModel, String source, String repoUrl) { - super(coreModel, source, repoUrl); + super(coreModel, source, repoUrl, Instance.PRIMARY); } @Override diff --git a/src/test/java/it/gov/innovazione/ndc/harvester/model/ControlledVocabularyModelTest.java b/src/test/java/it/gov/innovazione/ndc/harvester/model/ControlledVocabularyModelTest.java index c4df5790..f4baa91e 100644 --- a/src/test/java/it/gov/innovazione/ndc/harvester/model/ControlledVocabularyModelTest.java +++ b/src/test/java/it/gov/innovazione/ndc/harvester/model/ControlledVocabularyModelTest.java @@ -21,6 +21,7 @@ import java.util.List; import static it.gov.innovazione.ndc.harvester.SemanticAssetType.CONTROLLED_VOCABULARY; +import static it.gov.innovazione.ndc.harvester.model.Instance.PRIMARY; import static java.util.Objects.nonNull; import static org.apache.jena.rdf.model.ModelFactory.createDefaultModel; import static org.apache.jena.rdf.model.ResourceFactory.createResource; @@ -82,7 +83,7 @@ void setupMockModel() { @Test void shouldExtractMainResource() { ControlledVocabularyModel model = - new ControlledVocabularyModel(jenaModel, TTL_FILE, REPO_URL); + new ControlledVocabularyModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); Resource mainResource = model.getMainResource(); @@ -92,7 +93,7 @@ void shouldExtractMainResource() { @Test void shouldExtractMetadataWithSemanticAssetType() { ControlledVocabularyModel model = - new ControlledVocabularyModel(jenaModel, TTL_FILE, REPO_URL); + new ControlledVocabularyModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); @@ -102,7 +103,7 @@ void shouldExtractMetadataWithSemanticAssetType() { @Test void shouldValidateMetadataWithSemanticAssetType() { ControlledVocabularyModel model = - new ControlledVocabularyModel(jenaModel, TTL_FILE, REPO_URL); + new ControlledVocabularyModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetModelValidationContext semanticAssetModelValidationContext = model.validateMetadata(); @@ -119,7 +120,7 @@ void shouldFailWhenModelDoesNotContainControlledVocabulary() { conceptStatements.forEach(s -> jenaModel.remove(s)); ControlledVocabularyModel model = new ControlledVocabularyModel(jenaModel, TTL_FILE, - REPO_URL); + REPO_URL, PRIMARY); assertThatThrownBy(() -> model.getMainResource()).isInstanceOf(InvalidModelException.class); } @@ -131,7 +132,7 @@ void shouldFailWhenTtlContainsMoreThanOneControlledVocabulary() { .addProperty(RDF.type, createResource(CONTROLLED_VOCABULARY.getTypeIri())); ControlledVocabularyModel model = new ControlledVocabularyModel(jenaModel, TTL_FILE, - REPO_URL); + REPO_URL, PRIMARY); assertThatThrownBy(() -> model.getMainResource()).isInstanceOf(InvalidModelException.class); } @@ -139,7 +140,7 @@ void shouldFailWhenTtlContainsMoreThanOneControlledVocabulary() { @Test void shouldExtractKeyConcept() { ControlledVocabularyModel model = new ControlledVocabularyModel(jenaModel, TTL_FILE, - REPO_URL); + REPO_URL, PRIMARY); assertThat(model.getKeyConcept()).isEqualTo("test-concept"); } @@ -147,7 +148,7 @@ void shouldExtractKeyConcept() { @Test void shouldExtractMetadataWithDistribution() { ControlledVocabularyModel model = - new ControlledVocabularyModel(jenaModel, TTL_FILE, REPO_URL); + new ControlledVocabularyModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); @@ -164,7 +165,7 @@ void shouldComplainForTurtleDistributionWithoutUrl() { ); ControlledVocabularyModel model = - new ControlledVocabularyModel(jenaModel, TTL_FILE, REPO_URL); + new ControlledVocabularyModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); assertThatThrownBy(() -> model.extractMetadata()) .isInstanceOf(InvalidModelException.class) @@ -183,7 +184,7 @@ private void removePropertyIfExists(Statement d, Property p) { void shouldFailWhenExtractingMetadataWithOutDistribution() { jenaModel.getResource(CV_IRI).removeAll(distribution); ControlledVocabularyModel model = - new ControlledVocabularyModel(jenaModel, TTL_FILE, REPO_URL); + new ControlledVocabularyModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); assertThatThrownBy(() -> model.extractMetadata()).isInstanceOf( @@ -195,7 +196,7 @@ void shouldFailWithMissingKeyConcept() { jenaModel.getResource(CV_IRI).removeAll(NDC.keyConcept); ControlledVocabularyModel model = new ControlledVocabularyModel(jenaModel, TTL_FILE, - REPO_URL); + REPO_URL, PRIMARY); assertThatThrownBy(() -> model.getKeyConcept()).isInstanceOf(InvalidModelException.class); } @@ -206,7 +207,7 @@ void shouldFailWithInvalidKeyConcept(String keyConcept) { jenaModel.getResource(CV_IRI).getProperty(NDC.keyConcept).changeObject(keyConcept); ControlledVocabularyModel model = new ControlledVocabularyModel(jenaModel, TTL_FILE, - REPO_URL); + REPO_URL, PRIMARY); assertThatThrownBy(() -> model.getKeyConcept()).isInstanceOf(InvalidModelException.class); } @@ -219,7 +220,7 @@ void shouldFailWithMultipleKeyConcepts() { .addProperty(NDC.keyConcept, "another-concept"); ControlledVocabularyModel model = new ControlledVocabularyModel(jenaModel, TTL_FILE, - REPO_URL); + REPO_URL, PRIMARY); assertThatThrownBy(() -> model.getKeyConcept()).isInstanceOf(InvalidModelException.class); } @@ -227,7 +228,7 @@ void shouldFailWithMultipleKeyConcepts() { @Test void shouldExtractAgencyId() { ControlledVocabularyModel model = new ControlledVocabularyModel(jenaModel, TTL_FILE, - REPO_URL); + REPO_URL, PRIMARY); assertThat(model.getAgencyId().getIdentifier()).isEqualTo("agid"); } @@ -236,7 +237,7 @@ void shouldExtractAgencyId() { void shouldComplainIfRightsHolderIsUndefined() { jenaModel.getResource(CV_IRI).getProperty(rightsHolder).remove(); ControlledVocabularyModel model = new ControlledVocabularyModel(jenaModel, TTL_FILE, - REPO_URL); + REPO_URL, PRIMARY); assertThatThrownBy(() -> model.getAgencyId()) .isInstanceOf(InvalidModelException.class) @@ -247,7 +248,7 @@ void shouldComplainIfRightsHolderIsUndefined() { void shouldComplainIfRightsHolderHasNoId() { jenaModel.getResource(RIGHTS_HOLDER_IRI).getProperty(identifier).remove(); ControlledVocabularyModel model = new ControlledVocabularyModel(jenaModel, TTL_FILE, - REPO_URL); + REPO_URL, PRIMARY); assertThatThrownBy(() -> model.getAgencyId()) .isInstanceOf(InvalidModelException.class) @@ -258,7 +259,7 @@ void shouldComplainIfRightsHolderHasNoId() { @Test void shouldExtractKeyConceptMetaData() { ControlledVocabularyModel model = new ControlledVocabularyModel(jenaModel, TTL_FILE, - REPO_URL); + REPO_URL, PRIMARY); SemanticAssetMetadata semanticAssetMetadata = model.extractMetadata(); @@ -268,7 +269,7 @@ void shouldExtractKeyConceptMetaData() { @Test void shouldExtractAgencyIdMetaData() { ControlledVocabularyModel model = new ControlledVocabularyModel(jenaModel, TTL_FILE, - REPO_URL); + REPO_URL, PRIMARY); SemanticAssetMetadata semanticAssetMetadata = model.extractMetadata(); @@ -278,7 +279,7 @@ void shouldExtractAgencyIdMetaData() { @Test void shouldProvideEndpointUrlAsPartOfMetaDataAfterEnrichingModel() { ControlledVocabularyModel model = new ControlledVocabularyModel(jenaModel, TTL_FILE, - REPO_URL); + REPO_URL, PRIMARY); model.addNdcDataServiceProperties(ENDPOINT_BASE_URL); @@ -290,7 +291,7 @@ void shouldProvideEndpointUrlAsPartOfMetaDataAfterEnrichingModel() { @Test void shouldAddDataService() { ControlledVocabularyModel model = new ControlledVocabularyModel(jenaModel, TTL_FILE, - REPO_URL); + REPO_URL, PRIMARY); model.addNdcDataServiceProperties(ENDPOINT_BASE_URL); diff --git a/src/test/java/it/gov/innovazione/ndc/harvester/model/OntologyModelTest.java b/src/test/java/it/gov/innovazione/ndc/harvester/model/OntologyModelTest.java index 51e1eee2..02526cb0 100644 --- a/src/test/java/it/gov/innovazione/ndc/harvester/model/OntologyModelTest.java +++ b/src/test/java/it/gov/innovazione/ndc/harvester/model/OntologyModelTest.java @@ -15,6 +15,7 @@ import java.util.List; import static it.gov.innovazione.ndc.harvester.SemanticAssetType.ONTOLOGY; +import static it.gov.innovazione.ndc.harvester.model.Instance.PRIMARY; import static org.apache.jena.rdf.model.ModelFactory.createDefaultModel; import static org.apache.jena.rdf.model.ResourceFactory.createProperty; import static org.apache.jena.rdf.model.ResourceFactory.createResource; @@ -89,7 +90,7 @@ void setupMockModel() { @Test void shouldExtractMetadataWithSemanticAssetType() { - OntologyModel model = new OntologyModel(jenaModel, TTL_FILE, REPO_URL); + OntologyModel model = new OntologyModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); @@ -98,7 +99,7 @@ void shouldExtractMetadataWithSemanticAssetType() { @Test void shouldValidateMetadataWithSemanticAssetType() { - OntologyModel model = new OntologyModel(jenaModel, TTL_FILE, REPO_URL); + OntologyModel model = new OntologyModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetModelValidationContext semanticAssetModelValidationContext = model.validateMetadata(); @@ -108,7 +109,7 @@ void shouldValidateMetadataWithSemanticAssetType() { @Test void shouldExtractHasSemanticAssetDistributionProperty() { - OntologyModel ontologyModel = new OntologyModel(jenaModel, TTL_FILE, REPO_URL); + OntologyModel ontologyModel = new OntologyModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = ontologyModel.extractMetadata(); @@ -122,7 +123,7 @@ void shouldExtractHasSemanticAssetDistributionProperty() { void shouldFailWhenExtractingMetadataWithOutDistribution() { jenaModel.getResource(ONTOLOGY_IRI).removeAll(Admsapit.hasSemanticAssetDistribution); ControlledVocabularyModel model = - new ControlledVocabularyModel(jenaModel, TTL_FILE, REPO_URL); + new ControlledVocabularyModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); assertThatThrownBy(model::extractMetadata).isInstanceOf( InvalidModelException.class); @@ -130,7 +131,7 @@ void shouldFailWhenExtractingMetadataWithOutDistribution() { @Test void shouldExtractHasKeyClassProperty() { - OntologyModel ontologyModel = new OntologyModel(jenaModel, TTL_FILE, REPO_URL); + OntologyModel ontologyModel = new OntologyModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = ontologyModel.extractMetadata(); @@ -144,7 +145,7 @@ void shouldExtractHasKeyClassProperty() { @Test void shouldExtractMetadataWithoutHasKeyClassProperty() { jenaModel.getResource(ONTOLOGY_IRI).removeAll(Admsapit.hasKeyClass); - OntologyModel ontologyModel = new OntologyModel(jenaModel, TTL_FILE, REPO_URL); + OntologyModel ontologyModel = new OntologyModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = ontologyModel.extractMetadata(); @@ -154,7 +155,7 @@ void shouldExtractMetadataWithoutHasKeyClassProperty() { @Test void shouldExtractMetadataWithPrefix() { - OntologyModel ontologyModel = new OntologyModel(jenaModel, TTL_FILE, REPO_URL); + OntologyModel ontologyModel = new OntologyModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = ontologyModel.extractMetadata(); @@ -164,7 +165,7 @@ void shouldExtractMetadataWithPrefix() { @Test void shouldExtractMetadataWithOutPrefix() { jenaModel.getResource(ONTOLOGY_IRI).removeAll(Admsapit.prefix); - OntologyModel ontologyModel = new OntologyModel(jenaModel, TTL_FILE, REPO_URL); + OntologyModel ontologyModel = new OntologyModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = ontologyModel.extractMetadata(); @@ -173,7 +174,7 @@ void shouldExtractMetadataWithOutPrefix() { @Test void shouldExtractMetadataWithProject() { - OntologyModel ontologyModel = new OntologyModel(jenaModel, TTL_FILE, REPO_URL); + OntologyModel ontologyModel = new OntologyModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = ontologyModel.extractMetadata(); @@ -187,7 +188,7 @@ void shouldExtractMetadataWithProject() { @Test void shouldExtractMetadataWithOutProject() { jenaModel.getResource(ONTOLOGY_IRI).removeAll(Admsapit.semanticAssetInUse); - OntologyModel ontologyModel = new OntologyModel(jenaModel, TTL_FILE, REPO_URL); + OntologyModel ontologyModel = new OntologyModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = ontologyModel.extractMetadata(); diff --git a/src/test/java/it/gov/innovazione/ndc/harvester/model/SchemaModelTest.java b/src/test/java/it/gov/innovazione/ndc/harvester/model/SchemaModelTest.java index 79530f6c..200a3c5e 100644 --- a/src/test/java/it/gov/innovazione/ndc/harvester/model/SchemaModelTest.java +++ b/src/test/java/it/gov/innovazione/ndc/harvester/model/SchemaModelTest.java @@ -19,6 +19,7 @@ import java.util.List; import static it.gov.innovazione.ndc.harvester.SemanticAssetType.SCHEMA; +import static it.gov.innovazione.ndc.harvester.model.Instance.PRIMARY; import static org.apache.jena.rdf.model.ModelFactory.createDefaultModel; import static org.apache.jena.rdf.model.ResourceFactory.createLangLiteral; import static org.apache.jena.rdf.model.ResourceFactory.createResource; @@ -101,7 +102,7 @@ void setupMockModel() { @Test void shouldExtractMetadataWithIri() { - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); assertThat(metadata.getIri()).isEqualTo(SCHEMA_IRI); @@ -109,7 +110,7 @@ void shouldExtractMetadataWithIri() { @Test void shouldExtractWithRepoUrl() { - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); assertThat(metadata.getRepoUrl()).isEqualTo("http://repo"); @@ -117,7 +118,7 @@ void shouldExtractWithRepoUrl() { @Test void shouldExtractMetadataWithTitleInEnglish() { - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); assertThat(metadata.getTitle()).isEqualTo("title"); @@ -125,7 +126,7 @@ void shouldExtractMetadataWithTitleInEnglish() { @Test void shouldFailWhenExtractingMetadataWithOutTitle() { - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); jenaModel.getResource(SCHEMA_IRI).removeAll(title); assertThatThrownBy(model::extractMetadata).isInstanceOf( @@ -134,7 +135,7 @@ void shouldFailWhenExtractingMetadataWithOutTitle() { @Test void shouldExtractMetadataWithDescription() { - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); assertThat(metadata.getDescription()).isEqualTo("description"); @@ -143,7 +144,7 @@ void shouldExtractMetadataWithDescription() { @Test void shouldFailWhenExtractingMetadataWithOutDescription() { jenaModel.getResource(SCHEMA_IRI).removeAll(description); - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); assertThatThrownBy(model::extractMetadata).isInstanceOf( InvalidModelException.class); @@ -151,7 +152,7 @@ void shouldFailWhenExtractingMetadataWithOutDescription() { @Test void shouldExtractMetadataWithSemanticAssetType() { - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); @@ -160,7 +161,7 @@ void shouldExtractMetadataWithSemanticAssetType() { @Test void shouldValidateMetadataWithSemanticAssetType() { - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetModelValidationContext semanticAssetModelValidationContext = model.validateMetadata(); @@ -170,7 +171,7 @@ void shouldValidateMetadataWithSemanticAssetType() { @Test void shouldExtractMetadataWithRightsHolder() { - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); assertThat(metadata.getRightsHolder().getIri()).isEqualTo(RIGHTS_HOLDER_IRI); @@ -180,7 +181,7 @@ void shouldExtractMetadataWithRightsHolder() { @Test void shouldFailWhenExtractingMetadataWithOutRightsHolder() { jenaModel.getResource(SCHEMA_IRI).removeAll(rightsHolder); - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); assertThatThrownBy(model::extractMetadata).isInstanceOf( InvalidModelException.class); @@ -188,7 +189,7 @@ void shouldFailWhenExtractingMetadataWithOutRightsHolder() { @Test void shouldExtractMetadataWithModified() { - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); assertThat(metadata.getModifiedOn()).isEqualTo(LocalDate.of(2021, 3, 2)); @@ -197,7 +198,7 @@ void shouldExtractMetadataWithModified() { @Test void shouldFailWhenExtractingMetadataWithOutModified() { jenaModel.getResource(SCHEMA_IRI).removeAll(modified); - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); assertThat(metadata.getModifiedOn()).isNull(); @@ -205,7 +206,7 @@ void shouldFailWhenExtractingMetadataWithOutModified() { @Test void shouldExtractMetadataWithTheme() { - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); assertThat(metadata.getThemes()).containsExactly("theme"); @@ -214,7 +215,7 @@ void shouldExtractMetadataWithTheme() { @Test void shouldFailWhenExtractingMetadataWithOutTheme() { jenaModel.getResource(SCHEMA_IRI).removeAll(theme); - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); assertThatThrownBy(model::extractMetadata).isInstanceOf( InvalidModelException.class); @@ -222,7 +223,7 @@ void shouldFailWhenExtractingMetadataWithOutTheme() { @Test void shouldExtractMetadataWithDistribution() { - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); @@ -234,7 +235,7 @@ void shouldExtractMetadataWithDistribution() { @Test void shouldFailWhenExtractingMetadataWithOutDistribution() { jenaModel.getResource(SCHEMA_IRI).removeAll(distribution); - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); assertThatThrownBy(model::extractMetadata).isInstanceOf( InvalidModelException.class); @@ -242,7 +243,7 @@ void shouldFailWhenExtractingMetadataWithOutDistribution() { @Test void shouldExtractMetadataWithIssued() { - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); assertThat(metadata.getIssuedOn()).isEqualTo(LocalDate.of(2020, 1, 3)); @@ -251,7 +252,7 @@ void shouldExtractMetadataWithIssued() { @Test void shouldFailWhenExtractingMetadataWithOutIssued() { jenaModel.getResource(SCHEMA_IRI).removeAll(issued); - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); assertThat(metadata.getIssuedOn()).isNull(); @@ -259,7 +260,7 @@ void shouldFailWhenExtractingMetadataWithOutIssued() { @Test void shouldExtractMetadataWithVersionInfo() { - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); assertThat(metadata.getVersionInfo()).isEqualTo("1.0"); @@ -267,7 +268,7 @@ void shouldExtractMetadataWithVersionInfo() { @Test void shouldFailWhenExtractingMetadataWithOutVersionInfo() { - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); jenaModel.getResource(SCHEMA_IRI).removeAll(versionInfo); assertThatThrownBy(model::extractMetadata).isInstanceOf( @@ -276,7 +277,7 @@ void shouldFailWhenExtractingMetadataWithOutVersionInfo() { @Test void shouldExtractMetadataWithKeywords() { - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); assertThat(metadata.getKeywords()).containsExactlyInAnyOrder("keyword1", "keyword2"); @@ -285,7 +286,7 @@ void shouldExtractMetadataWithKeywords() { @Test void shouldExtractMetadataWithoutKeywords() { jenaModel.getResource(SCHEMA_IRI).removeAll(keyword); - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); @@ -294,7 +295,7 @@ void shouldExtractMetadataWithoutKeywords() { @Test void shouldExtractMetadataWithConformsTo() { - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); assertThat(metadata.getConformsTo()).hasSize(2); @@ -307,7 +308,7 @@ void shouldExtractMetadataWithConformsTo() { @Test void shouldExtractMetadataWithoutConformsTo() { jenaModel.getResource(SCHEMA_IRI).removeAll(conformsTo); - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); @@ -316,7 +317,7 @@ void shouldExtractMetadataWithoutConformsTo() { @Test void shouldExtractHasKeyClassProperty() { - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); @@ -330,7 +331,7 @@ void shouldExtractHasKeyClassProperty() { @Test void shouldExtractMetadataWithoutHasKeyClassProperty() { jenaModel.getResource(SCHEMA_IRI).removeAll(Admsapit.hasKeyClass); - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); @@ -339,7 +340,7 @@ void shouldExtractMetadataWithoutHasKeyClassProperty() { @Test void shouldExtractMetadataWithStatus() { - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); assertThat(metadata.getStatus()).containsExactlyInAnyOrder("catalogued", "published"); @@ -348,7 +349,7 @@ void shouldExtractMetadataWithStatus() { @Test void shouldExtractMetadataWithOutStatus() { jenaModel.getResource(SCHEMA_IRI).removeAll(Admsapit.status); - SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL); + SchemaModel model = new SchemaModel(jenaModel, TTL_FILE, REPO_URL, PRIMARY); SemanticAssetMetadata metadata = model.extractMetadata(); assertThat(metadata.getStatus()).isEmpty(); diff --git a/src/test/java/it/gov/innovazione/ndc/harvester/model/SemanticAssetModelFactoryTest.java b/src/test/java/it/gov/innovazione/ndc/harvester/model/SemanticAssetModelFactoryTest.java index 338c3f11..b16b2e62 100644 --- a/src/test/java/it/gov/innovazione/ndc/harvester/model/SemanticAssetModelFactoryTest.java +++ b/src/test/java/it/gov/innovazione/ndc/harvester/model/SemanticAssetModelFactoryTest.java @@ -1,19 +1,29 @@ package it.gov.innovazione.ndc.harvester.model; import it.gov.innovazione.ndc.harvester.model.exception.InvalidModelException; +import it.gov.innovazione.ndc.service.DefaultInstanceManager; +import it.gov.innovazione.ndc.service.InstanceManager; import org.apache.jena.rdf.model.Resource; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; @ExtendWith(MockitoExtension.class) class SemanticAssetModelFactoryTest { public static final String REPO_URL = "http://repo"; - private final SemanticAssetModelFactory factory = new SemanticAssetModelFactory(); + private SemanticAssetModelFactory factory; + + @BeforeEach + void setUp() { + InstanceManager instanceManager = mock(DefaultInstanceManager.class); + factory = new SemanticAssetModelFactory(instanceManager); + } @Test void canBuildControlledVocabularyModel() { diff --git a/src/test/java/it/gov/innovazione/ndc/harvester/pathprocessors/ControlledVocabularyPathProcessorTest.java b/src/test/java/it/gov/innovazione/ndc/harvester/pathprocessors/ControlledVocabularyPathProcessorTest.java index 93542793..ac32d60e 100644 --- a/src/test/java/it/gov/innovazione/ndc/harvester/pathprocessors/ControlledVocabularyPathProcessorTest.java +++ b/src/test/java/it/gov/innovazione/ndc/harvester/pathprocessors/ControlledVocabularyPathProcessorTest.java @@ -4,6 +4,7 @@ import it.gov.innovazione.ndc.harvester.csv.CsvParser.CsvData; import it.gov.innovazione.ndc.harvester.model.ControlledVocabularyModel; import it.gov.innovazione.ndc.harvester.model.CvPath; +import it.gov.innovazione.ndc.harvester.model.Instance; import it.gov.innovazione.ndc.harvester.model.SemanticAssetModelFactory; import it.gov.innovazione.ndc.harvester.model.index.RightsHolder; import it.gov.innovazione.ndc.harvester.model.index.SemanticAssetMetadata; @@ -123,11 +124,11 @@ void shouldDropIndicesForRepo() { String concept1 = "accomodation-ratings"; String concept2 = "education-levels"; List vocabsMetadata = buildVocabsMetadataWithAgencyAndConcepts(agencyId, List.of(concept1, concept2)); - when(metadataRepository.findVocabulariesForRepoUrl(REPO_URL)).thenReturn(vocabsMetadata); + when(metadataRepository.findVocabulariesForRepoUrl(REPO_URL, Instance.PRIMARY)).thenReturn(vocabsMetadata); - pathProcessor.dropCsvIndicesForRepo(REPO_URL); + pathProcessor.dropCsvIndicesForRepo(REPO_URL, Instance.PRIMARY); - verify(metadataRepository).findVocabulariesForRepoUrl(REPO_URL); + verify(metadataRepository).findVocabulariesForRepoUrl(REPO_URL, Instance.PRIMARY); verify(vocabularyDataService).dropIndex(new VocabularyIdentifier(agencyId, concept1)); verify(vocabularyDataService).dropIndex(new VocabularyIdentifier(agencyId, concept2)); } @@ -138,10 +139,10 @@ void shouldTryAndDropSubsequentIndicesEvenAfterFailingToDropOne() { String concept1 = "accomodation-ratings"; String concept2 = "education-levels"; List vocabsMetadata = buildVocabsMetadataWithAgencyAndConcepts(agencyId, List.of(concept1, concept2)); - when(metadataRepository.findVocabulariesForRepoUrl(REPO_URL)).thenReturn(vocabsMetadata); + when(metadataRepository.findVocabulariesForRepoUrl(REPO_URL, Instance.PRIMARY)).thenReturn(vocabsMetadata); doThrow(new RuntimeException("Could not drop index")).when(vocabularyDataService).dropIndex(new VocabularyIdentifier(agencyId, concept1)); - pathProcessor.dropCsvIndicesForRepo(REPO_URL); + pathProcessor.dropCsvIndicesForRepo(REPO_URL, Instance.PRIMARY); verify(vocabularyDataService).dropIndex(new VocabularyIdentifier(agencyId, concept1)); verify(vocabularyDataService).dropIndex(new VocabularyIdentifier(agencyId, concept2)); diff --git a/src/test/java/it/gov/innovazione/ndc/harvester/service/ActualConfigServiceTest.java b/src/test/java/it/gov/innovazione/ndc/harvester/service/ActualConfigServiceTest.java index ba17f9c4..8eca527a 100644 --- a/src/test/java/it/gov/innovazione/ndc/harvester/service/ActualConfigServiceTest.java +++ b/src/test/java/it/gov/innovazione/ndc/harvester/service/ActualConfigServiceTest.java @@ -5,6 +5,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import java.util.function.Function; import java.util.stream.Stream; import static it.gov.innovazione.ndc.harvester.service.ActualConfigService.Parser.TO_BOOLEAN; @@ -52,7 +53,8 @@ void testParseTrue(ActualConfigService.Parser parser, String validString, Object @ParameterizedTest @MethodSource("provideInvalidValues") void testParseNull(ActualConfigService.Parser parser, String invalidValue) { - assertThrows(IllegalArgumentException.class, () -> parser.getParsingFunction().apply(invalidValue)); + Function parsingFunction = parser.getParsingFunction(); + assertThrows(IllegalArgumentException.class, () -> parsingFunction.apply(invalidValue)); } } diff --git a/src/test/java/it/gov/innovazione/ndc/repository/SemanticAssetMetadataRepositoryTest.java b/src/test/java/it/gov/innovazione/ndc/repository/SemanticAssetMetadataRepositoryTest.java index 6b635786..21f44fb3 100644 --- a/src/test/java/it/gov/innovazione/ndc/repository/SemanticAssetMetadataRepositoryTest.java +++ b/src/test/java/it/gov/innovazione/ndc/repository/SemanticAssetMetadataRepositoryTest.java @@ -1,7 +1,9 @@ package it.gov.innovazione.ndc.repository; import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; +import it.gov.innovazione.ndc.harvester.model.Instance; import it.gov.innovazione.ndc.harvester.model.index.SemanticAssetMetadata; +import it.gov.innovazione.ndc.service.InstanceManager; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; @@ -35,6 +37,8 @@ class SemanticAssetMetadataRepositoryTest { @Mock private ElasticsearchOperations esOps; + @Mock + private InstanceManager instanceManager; @Mock private SearchHits searchHits; @@ -79,12 +83,12 @@ void shouldDeleteByIri() { when(esOps.delete(captor.capture(), any(Class.class))).thenReturn(deletedResponse); - long deleteCount = repository.deleteByRepoUrl("someRepoUrl"); + long deleteCount = repository.deleteByRepoUrl("someRepoUrl", Instance.PRIMARY); assertThat(deleteCount).isEqualTo(1); Query query = captor.getValue().getQuery(); assertNotNull(query); - assertEquals("Query: {\"match\":{\"repoUrl\":{\"query\":\"someRepoUrl\"}}}", + assertEquals("Query: {\"bool\":{\"must\":[{\"terms\":{\"repoUrl\":[\"someRepoUrl\"]}},{\"terms\":{\"instance\":[\"PRIMARY\"]}}]}}", Optional.of(query) .map(q -> (NativeQuery) q) .map(NativeQuery::getQuery) @@ -105,6 +109,8 @@ void shouldSave() { @Test void shouldSearchUsingQueryStringAndFiltersAndPagination() { + when(instanceManager.getCurrentInstances()).thenReturn(List.of()); + ArgumentCaptor captor = ArgumentCaptor.forClass(NativeQuery.class); when(esOps.search(captor.capture(), any(Class.class))).thenReturn(searchHits); @@ -136,6 +142,8 @@ void shouldSearchUsingQueryStringAndFiltersAndPagination() { @Test void shouldSearchWithoutFiltersAndSearchText() { + when(instanceManager.getCurrentInstances()).thenReturn(List.of()); + ArgumentCaptor captor = ArgumentCaptor.forClass(NativeQuery.class); when(esOps.search(captor.capture(), any(Class.class))).thenReturn(searchHits); diff --git a/src/test/java/it/gov/innovazione/ndc/repository/TripleStoreRepositoryTest.java b/src/test/java/it/gov/innovazione/ndc/repository/TripleStoreRepositoryTest.java index 0eaa050d..18f46ced 100644 --- a/src/test/java/it/gov/innovazione/ndc/repository/TripleStoreRepositoryTest.java +++ b/src/test/java/it/gov/innovazione/ndc/repository/TripleStoreRepositoryTest.java @@ -30,6 +30,7 @@ @ExtendWith(MockitoExtension.class) class TripleStoreRepositoryTest { private static final String REPO_URL = "http://www.repos.org/reponame"; + private static final String OLD_REPO_URL = "http://old.www.repos.org/reponame"; @Mock RDFConnection connection; @@ -49,21 +50,21 @@ void shouldConnectAndLoadModelWhenSaving() { tripleStoreRepository.save(REPO_URL, model); verify(virtuosoClient).getConnection(); - verify(connection).load(REPO_URL, model); + verify(connection).load(OLD_REPO_URL, model); } @Test void shouldThrowWhenLoadingFails() { Model model = createSimpleModel(); - doThrow(new HttpException("Something bad happened")).when(connection).load(REPO_URL, model); + doThrow(new HttpException("Something bad happened")).when(connection).load(OLD_REPO_URL, model); when(virtuosoClient.getConnection()).thenReturn(connection); assertThatThrownBy(() -> tripleStoreRepository.save(REPO_URL, model)) .isInstanceOf(TripleStoreRepositoryException.class) - .hasMessage(String.format("Could not save model to '%s'", REPO_URL)); + .hasMessage(String.format("Could not save model to '%s'", OLD_REPO_URL)); verify(virtuosoClient).getConnection(); - verify(connection).load(REPO_URL, model); + verify(connection).load(OLD_REPO_URL, model); } @Test diff --git a/src/test/java/it/gov/innovazione/ndc/validator/ValidatorTest.java b/src/test/java/it/gov/innovazione/ndc/validator/ValidatorTest.java index b1bde901..70878b7c 100644 --- a/src/test/java/it/gov/innovazione/ndc/validator/ValidatorTest.java +++ b/src/test/java/it/gov/innovazione/ndc/validator/ValidatorTest.java @@ -16,9 +16,9 @@ class ValidatorTest { @Test void assertValidatorGetModel() { Stream.of( - Pair.of(ONTOLOGY, new OntologyValidator()), - Pair.of(CONTROLLED_VOCABULARY, new ControlledVocabularyValidator()), - Pair.of(SCHEMA, new SchemaValidator())) + Pair.of(ONTOLOGY, new OntologyValidator()), + Pair.of(CONTROLLED_VOCABULARY, new ControlledVocabularyValidator()), + Pair.of(SCHEMA, new SchemaValidator())) .forEach(pair -> { assertEquals(pair.getKey(), pair.getValue().getType()); try {