diff --git a/catalog/core/catalog-core-standardframework/src/main/java/ddf/catalog/history/Historian.java b/catalog/core/catalog-core-standardframework/src/main/java/ddf/catalog/history/Historian.java index 72827841c6e4..f641cb89c14e 100644 --- a/catalog/core/catalog-core-standardframework/src/main/java/ddf/catalog/history/Historian.java +++ b/catalog/core/catalog-core-standardframework/src/main/java/ddf/catalog/history/Historian.java @@ -16,6 +16,7 @@ import static ddf.catalog.core.versioning.MetacardVersion.SKIP_VERSIONING; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import com.google.common.io.ByteSource; import ddf.catalog.content.StorageException; import ddf.catalog.content.StorageProvider; @@ -40,6 +41,7 @@ import ddf.catalog.operation.CreateResponse; import ddf.catalog.operation.DeleteResponse; import ddf.catalog.operation.Operation; +import ddf.catalog.operation.Response; import ddf.catalog.operation.SourceResponse; import ddf.catalog.operation.Update; import ddf.catalog.operation.UpdateResponse; @@ -68,6 +70,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; import java.util.function.Function; @@ -103,6 +106,20 @@ public class Historian { .or(DeletedMetacardImpl::isDeleted) .negate(); + public static final String SKIP_UPDATE_PROPERTY = + "org.codice.ddf.history.update.blacklist.metacardTypes"; + + public static final String SKIP_DELETE_PROPERTY = + "org.codice.ddf.history.deletes.blacklist.metacardTypes"; + + private final Set skipUpdateMetacardTypes = + Sets.newHashSet( + System.getProperty(SKIP_UPDATE_PROPERTY, "").replaceAll("\\s+", "").split(",")); + + private final Set skipDeleteMetacardTypes = + Sets.newHashSet( + System.getProperty(SKIP_DELETE_PROPERTY, "").replaceAll("\\s+", "").split(",")); + private List storageProviders; private List catalogProviders; @@ -163,6 +180,7 @@ public UpdateResponse version(UpdateResponse updateResponse) { updateResponse.getUpdatedMetacards().stream() .map(Update::getOldMetacard) .filter(isNotVersionNorDeleted) + .filter(this::isNotBlackListedUpdate) .collect(Collectors.toList()); if (inputMetacards.isEmpty()) { @@ -223,6 +241,7 @@ public UpdateStorageResponse version( .map(ContentItem::getMetacard) .filter(Objects::nonNull) .filter(isNotVersionNorDeleted) + .filter(this::isNotBlackListedUpdate) .collect(Collectors.toList()); if (updatedMetacards.isEmpty()) { @@ -287,6 +306,14 @@ public UpdateStorageResponse version( return updateStorageResponse; } + private boolean isNotBlackListedUpdate(Metacard metacard) { + return !skipUpdateMetacardTypes.contains(metacard.getMetacardType().getName()); + } + + private boolean isNotBlackListedDelete(Metacard metacard) { + return !skipDeleteMetacardTypes.contains(metacard.getMetacardType().getName()); + } + /** * Versions deleted {@link Metacard}s. * @@ -306,6 +333,7 @@ public DeleteResponse version(DeleteResponse deleteResponse) List originalMetacards = deleteResponse.getDeletedMetacards().stream() .filter(isNotVersionNorDeleted) + .filter(this::isNotBlackListedDelete) .collect(Collectors.toList()); if (originalMetacards.isEmpty()) { @@ -636,11 +664,12 @@ private T executeAsSystem(Callable func) { return systemSubject.execute(func); } - private boolean doSkip(@Nullable Operation op) { + private boolean doSkip(@Nullable Response response) { return !historyEnabled - || op == null + || response == null || ((boolean) - Optional.of(op) + Optional.of(response) + .map(Response::getRequest) .map(Operation::getProperties) .orElse(Collections.emptyMap()) .getOrDefault(SKIP_VERSIONING, false)); diff --git a/catalog/core/catalog-core-standardframework/src/test/java/ddf/catalog/history/HistorianTest.java b/catalog/core/catalog-core-standardframework/src/test/java/ddf/catalog/history/HistorianTest.java index e790599ce0f8..72c18f4af84c 100644 --- a/catalog/core/catalog-core-standardframework/src/test/java/ddf/catalog/history/HistorianTest.java +++ b/catalog/core/catalog-core-standardframework/src/test/java/ddf/catalog/history/HistorianTest.java @@ -13,9 +13,11 @@ */ package ddf.catalog.history; +import static ddf.catalog.data.impl.MetacardImpl.BASIC_METACARD; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; import static org.mockito.ArgumentMatchers.any; @@ -43,9 +45,11 @@ import ddf.catalog.core.versioning.impl.DeletedMetacardImpl; import ddf.catalog.core.versioning.impl.MetacardVersionImpl; import ddf.catalog.data.Metacard; +import ddf.catalog.data.MetacardType; import ddf.catalog.data.Result; import ddf.catalog.data.impl.AttributeImpl; import ddf.catalog.data.impl.MetacardImpl; +import ddf.catalog.data.impl.MetacardTypeImpl; import ddf.catalog.filter.proxy.builder.GeotoolsFilterBuilder; import ddf.catalog.operation.CreateRequest; import ddf.catalog.operation.DeleteRequest; @@ -70,6 +74,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.security.PrivilegedAction; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -77,9 +82,11 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.Callable; +import java.util.stream.Collectors; import org.apache.shiro.subject.ExecutionException; import org.codice.ddf.platform.util.uuidgenerator.UuidGenerator; import org.codice.ddf.security.Security; +import org.hamcrest.Matchers; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; @@ -94,6 +101,8 @@ public class HistorianTest { private static final String RESOURCE_URI = "content:example.com"; + private static final String METACARD_VERSION_TYPE = "metacard.version.type"; + private static final String UPDATE_DESCRIPTION = "This is an updated description."; private CatalogProvider catalogProvider; @@ -106,6 +115,9 @@ public class HistorianTest { @Before public void setup() { + System.setProperty(Historian.SKIP_UPDATE_PROPERTY, "blacklisted, no-way"); + System.setProperty(Historian.SKIP_DELETE_PROPERTY, "blacklisted,no-way"); + historian = new Historian(); uuidGenerator = mock(UuidGenerator.class); @@ -149,12 +161,13 @@ public void testUpdateResponseHistorianDisabled() @Test public void testUpdateResponseSetSkipFlag() throws SourceUnavailableException, IngestException { - Map properties = new HashMap<>(); - UpdateResponse updateResponse = createUpdateResponse(properties); + UpdateResponse updateResponse = createUpdateResponse(null); + List updateList = createUpdatedMetacardList(); + when(updateResponse.getUpdatedMetacards()).thenReturn(updateList); historian.version(updateResponse); - assertThat(properties, hasEntry(MetacardVersion.SKIP_VERSIONING, true)); + assertThat(updateResponse.getProperties(), hasEntry(MetacardVersion.SKIP_VERSIONING, true)); } @Test @@ -163,11 +176,127 @@ public void testUpdateResponseSkipProperty() throws SourceUnavailableException, properties.put(MetacardVersion.SKIP_VERSIONING, true); UpdateResponse updateResponse = createUpdateResponse(properties); + List updateList = createUpdatedMetacardList(); + when(updateResponse.getUpdatedMetacards()).thenReturn(updateList); historian.version(updateResponse); verifyZeroInteractions(catalogProvider); } + @Test + public void testUpdateResponseBlacklist() throws IngestException { + Map properties = new HashMap<>(); + UpdateResponse updateResponse = createUpdateResponse(properties); + + List updateList = + createUpdatedMetacardListTyped("allowed", "blacklisted", "right-this-way", "no-way"); + when(updateResponse.getUpdatedMetacards()).thenReturn(updateList); + + historian.version(updateResponse); + ArgumentCaptor createRequest = ArgumentCaptor.forClass(CreateRequest.class); + verify(catalogProvider).create(createRequest.capture()); + List versionedTypes = + createRequest.getValue().getMetacards().stream() + .map(metacard -> (String) metacard.getAttribute(METACARD_VERSION_TYPE).getValue()) + .collect(Collectors.toList()); + assertThat(versionedTypes, not(hasItem("blacklisted"))); + assertThat(versionedTypes, not(hasItem("no-way"))); + } + + @Test + public void testDeleteResponseBlacklist() + throws IngestException, StorageException, SourceUnavailableException { + Metacard allowedMetacard = getMetacardUpdatePairTyped("allowed").get(0); + storeMetacard(allowedMetacard); + + Metacard blacklistMetacard = getMetacardUpdatePairTyped("blacklisted").get(0); + storeMetacard(blacklistMetacard); + + List storedMetacards = new ArrayList<>(); + storedMetacards.add(allowedMetacard); + storedMetacards.add(blacklistMetacard); + // Send a delete request + DeleteStorageRequest deleteStorageRequest = + new DeleteStorageRequestImpl(storedMetacards, new HashMap<>()); + storageProvider.delete(deleteStorageRequest); + + // Version delete request + DeleteRequest deleteRequest = new DeleteRequestImpl("deleteRequest"); + DeleteResponse deleteResponse = + new DeleteResponseImpl(deleteRequest, new HashMap<>(), storedMetacards); + historian.version(deleteResponse); + + // Only the version metacard is left + List storedTypes = + storageProvider.storageMap.values().stream() + .map(ContentItem::getMetacard) + .map(metacard -> (String) metacard.getAttribute(METACARD_VERSION_TYPE).getValue()) + .collect(Collectors.toList()); + assertThat( + storedTypes.contains(allowedMetacard.getMetacardType().getName()), Matchers.is(true)); + assertThat( + storedTypes.contains(blacklistMetacard.getMetacardType().getName()), Matchers.is(false)); + } + + @Test + public void testUpdateStorageResponseBlacklistAllowed() + throws UnsupportedQueryException, SourceUnavailableException, IngestException, + URISyntaxException, StorageException { + // The metacard and updated metacard + List metacards = getMetacardUpdatePairTyped("allowed"); + + // Parameters for historian + UpdateStorageRequest storageRequest = mock(UpdateStorageRequest.class); + UpdateStorageResponse storageResponse = mock(UpdateStorageResponse.class); + UpdateResponse updateResponse = mock(UpdateResponse.class); + Update update1 = mock(Update.class); + when(update1.getOldMetacard()).thenReturn(metacards.get(0)); + when(updateResponse.getUpdatedMetacards()).thenReturn(ImmutableList.of(update1)); + + storeMetacard(metacards.get(0)); + + // send a request to update the metacard + updateMetacard(storageRequest, storageResponse, metacards.get(1)); + storageProvider.update(storageRequest); + + mockQuery(metacards.get(1)); + historian.version(storageRequest, storageResponse, updateResponse); + + // Verify that the metacard updated + Metacard update = readMetacard(metacards.get(0).getResourceURI().toString()); + + assertThat(update, equalTo(metacards.get(1))); + assertThat( + storageResponse.getUpdatedContentItems().get(0).getUri(), not(equalTo(RESOURCE_URI))); + } + + @Test + public void testUpdateStorageResponseBlacklisted() + throws UnsupportedQueryException, SourceUnavailableException, IngestException, + URISyntaxException, StorageException { + // The metacard and updated metacard + List metacards = getMetacardUpdatePairTyped("blacklisted"); + + // Parameters for historian + UpdateStorageRequest storageRequest = mock(UpdateStorageRequest.class); + UpdateStorageResponse storageResponse = mock(UpdateStorageResponse.class); + UpdateResponse updateResponse = mock(UpdateResponse.class); + Update update1 = mock(Update.class); + when(update1.getOldMetacard()).thenReturn(metacards.get(0)); + when(updateResponse.getUpdatedMetacards()).thenReturn(ImmutableList.of(update1)); + + storeMetacard(metacards.get(0)); + + // send a request to update the metacard + updateMetacard(storageRequest, storageResponse, metacards.get(1)); + storageProvider.update(storageRequest); + + mockQuery(metacards.get(1)); + historian.version(storageRequest, storageResponse, updateResponse); + + verifyZeroInteractions(updateResponse); + } + @Test public void testUpdateResponse() throws Exception { UpdateResponse updateResponse = createUpdateResponse(null); @@ -270,7 +399,7 @@ public void testUpdateStorageResponse() historian.version(storageRequest, storageResponse, updateResponse); // Verify that the metacard updated - Metacard update = readMetacard(); + Metacard update = readMetacard(null); assertThat(update, equalTo(metacards.get(1))); assertThat( @@ -350,8 +479,12 @@ public void testDeleteResponseSkipProperty() throws SourceUnavailableException, Map properties = new HashMap<>(); properties.put(MetacardVersion.SKIP_VERSIONING, true); - DeleteResponse deleteResponse = mock(DeleteResponse.class); - when(deleteResponse.getProperties()).thenReturn(properties); + DeleteRequest deleteRequest = mock(DeleteRequest.class); + when(deleteRequest.getProperties()).thenReturn(properties); + + Metacard metacard = getMetacardUpdatePair().get(0); + DeleteResponse deleteResponse = + new DeleteResponseImpl(deleteRequest, new HashMap<>(), Collections.singletonList(metacard)); historian.version(deleteResponse); verifyZeroInteractions(catalogProvider); @@ -548,15 +681,14 @@ public void testBadContentItemSize() historian.version(storageRequest, storageResponse, updateResponse); // Verify that the metacard updated - Metacard update = readMetacard(); + Metacard update = readMetacard(null); assertThat(update, equalTo(metacards.get(1))); } - private UpdateResponse createUpdateResponse(Map responseProperties) { - Map requestProperties = new HashMap<>(); - if (responseProperties == null) { - responseProperties = new HashMap<>(); + private UpdateResponse createUpdateResponse(Map requestProperties) { + if (requestProperties == null) { + requestProperties = new HashMap<>(); } UpdateResponse updateResponse = mock(UpdateResponse.class); @@ -564,7 +696,7 @@ private UpdateResponse createUpdateResponse(Map responsePr when(request.getProperties()).thenReturn(requestProperties); when(updateResponse.getRequest()).thenReturn(request); - when(updateResponse.getProperties()).thenReturn(responseProperties); + when(updateResponse.getProperties()).thenReturn(new HashMap<>()); return updateResponse; } @@ -574,6 +706,15 @@ private List createUpdatedMetacardList() { return Collections.singletonList(new UpdateImpl(metacards.get(1), metacards.get(0))); } + private List createUpdatedMetacardListTyped(String... metacardTypes) { + List updates = new ArrayList<>(); + for (String type : metacardTypes) { + List metacards = getMetacardUpdatePairTyped(type); + updates.add(new UpdateImpl(metacards.get(1), metacards.get(0))); + } + return updates; + } + private List getMetacardUpdatePair() { Metacard old = new MetacardImpl(); old.setAttribute(new AttributeImpl(Metacard.ID, METACARD_ID)); @@ -587,6 +728,22 @@ private List getMetacardUpdatePair() { return Arrays.asList(old, update); } + private List getMetacardUpdatePairTyped(String metacardType) { + MetacardType type = + new MetacardTypeImpl(metacardType, BASIC_METACARD.getAttributeDescriptors()); + + Metacard old = new MetacardImpl(type); + old.setAttribute(new AttributeImpl(Metacard.ID, METACARD_ID + metacardType)); + old.setAttribute(new AttributeImpl(Metacard.RESOURCE_URI, RESOURCE_URI + metacardType)); + + Metacard update = new MetacardImpl(type); + update.setAttribute(new AttributeImpl(Metacard.ID, METACARD_ID + metacardType)); + update.setAttribute(new AttributeImpl(Metacard.RESOURCE_URI, RESOURCE_URI + metacardType)); + update.setAttribute(new AttributeImpl(Metacard.DESCRIPTION, UPDATE_DESCRIPTION + metacardType)); + + return Arrays.asList(old, update); + } + private void mockQuery(Metacard metacard) throws UnsupportedQueryException { SourceResponse sourceResponse = mock(SourceResponse.class); Result result = mock(Result.class); @@ -614,8 +771,8 @@ private void mockQuery(Metacard metacard) throws UnsupportedQueryException { private void storeMetacard(Metacard metacard) { ContentItem item = mock(ContentItem.class); - when(item.getId()).thenReturn(METACARD_ID); - when(item.getUri()).thenReturn(RESOURCE_URI); + when(item.getId()).thenReturn(metacard.getId()); + when(item.getUri()).thenReturn(metacard.getResourceURI().toString()); when(item.getMetacard()).thenReturn(metacard); storageProvider.storageMap.put(item.getUri(), item); } @@ -630,8 +787,8 @@ private void updateMetacard( when(hasQualifier.getQualifier()).thenReturn("some-qualifier"); when(emptyQualifier.getQualifier()).thenReturn(""); - when(updatedItem.getId()).thenReturn(METACARD_ID); - when(updatedItem.getUri()).thenReturn(RESOURCE_URI); + when(updatedItem.getId()).thenReturn(update.getId()); + when(updatedItem.getUri()).thenReturn(update.getResourceURI().toString()); when(updatedItem.getMetacard()).thenReturn(update); when(request.getContentItems()).thenReturn(Collections.singletonList(updatedItem)); @@ -639,9 +796,10 @@ private void updateMetacard( .thenReturn(Arrays.asList(noMetacard, updatedItem, hasQualifier, emptyQualifier)); } - private Metacard readMetacard() throws StorageException, URISyntaxException { + private Metacard readMetacard(String customUri) throws StorageException, URISyntaxException { ReadStorageRequest request = mock(ReadStorageRequest.class); - when(request.getResourceUri()).thenReturn(new URI(RESOURCE_URI)); + when(request.getResourceUri()) + .thenReturn(customUri != null ? new URI(customUri) : new URI(RESOURCE_URI)); return storageProvider.read(request).getContentItem().getMetacard(); }