Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[2.26.x] Use UTC dates when exporting metacards in CSV format #6501

Merged
merged 1 commit into from
Jan 29, 2021
Merged

[2.26.x] Use UTC dates when exporting metacards in CSV format #6501

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions catalog/transformer/catalog-transformer-csv-common/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,17 +66,17 @@
<limit implementation="org.codice.jacoco.LenientLimit">
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.86</minimum>
<minimum>0.87</minimum>
</limit>
<limit implementation="org.codice.jacoco.LenientLimit">
<counter>BRANCH</counter>
<value>COVEREDRATIO</value>
<minimum>0.85</minimum>
<minimum>0.84</minimum>
</limit>
<limit implementation="org.codice.jacoco.LenientLimit">
<counter>COMPLEXITY</counter>
<value>COVEREDRATIO</value>
<minimum>0.84</minimum>
<minimum>0.83</minimum>
</limit>
</limits>
</rule>
Expand Down
67 changes: 65 additions & 2 deletions ...sformer-csv-common/src/main/java/ddf/catalog/transformer/csv/common/MetacardIterator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,22 +16,37 @@

import ddf.catalog.data.Attribute;
import ddf.catalog.data.AttributeDescriptor;
import ddf.catalog.data.AttributeType;
import ddf.catalog.data.AttributeType.AttributeFormat;
import ddf.catalog.data.Metacard;
import ddf.catalog.data.MetacardType;
import ddf.catalog.data.types.Core;
import java.io.Serializable;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.stream.Collectors;
import javax.xml.bind.DatatypeConverter;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* An implementation of java.util.Iterator which iterates over Metacard attribute values.
*
* @see java.util.Iterator
*/
class MetacardIterator implements Iterator<Serializable> {

private static final Logger LOGGER = LoggerFactory.getLogger(MetacardIterator.class);

private static final String MULTIVALUE_DELIMITER = "\n";

private final List<AttributeDescriptor> attributeDescriptorList;
Expand All @@ -40,6 +55,8 @@ class MetacardIterator implements Iterator<Serializable> {

private int index;

private DateTimeFormatter formatter = DateTimeFormatter.ISO_OFFSET_DATE_TIME;

/**
* @param metacard the metacard to be iterated over.
* @param attributeDescriptorList the list of attributeDescriptors used to determine which
Expand Down Expand Up @@ -67,13 +84,21 @@ public Serializable next() {

AttributeDescriptor attributeDescriptor = this.attributeDescriptorList.get(index);
Attribute attribute = metacard.getAttribute(attributeDescriptor.getName());
AttributeFormat attributeFormat = attributeDescriptor.getType().getAttributeFormat();
index++;

if (attribute != null) {
if (attributeDescriptor.isMultiValued()) {
return StringUtils.join(attribute.getValues(), MULTIVALUE_DELIMITER);
List<Serializable> convertedValues =
attribute.getValues().stream()
.map(value -> convertValue(attribute.getName(), value, attributeFormat))
.filter(Objects::nonNull)
.collect(Collectors.toList());
return StringUtils.join(convertedValues, MULTIVALUE_DELIMITER);
} else {
return attribute.getValue();
Serializable value =
convertValue(attribute.getName(), attribute.getValue(), attributeFormat);
return (value == null) ? "" : value;
}
} else if (isSourceId(attributeDescriptor) && isSourceIdSet()) {
return metacard.getSourceId();
Expand All @@ -84,6 +109,44 @@ public Serializable next() {
return "";
}

private Serializable convertValue(
String name, Serializable value, AttributeType.AttributeFormat format) {
if (value == null) {
return null;
}

switch (format) {
case DATE:
if (!(value instanceof Date)) {
LOGGER.debug(
"Dropping attribute date value {} for {} because it isn't a Date object.",
value,
name);
return null;
}
Instant instant = ((Date) value).toInstant();
ZoneId zoneId = ZoneId.of("UTC");
ZonedDateTime zonedDateTime = instant.atZone(zoneId);
return zonedDateTime.format(formatter);
case BINARY:
byte[] bytes = (byte[]) value;
return DatatypeConverter.printBase64Binary(bytes);
case BOOLEAN:
case DOUBLE:
case LONG:
case INTEGER:
case SHORT:
case STRING:
case XML:
case FLOAT:
case GEOMETRY:
return value;
case OBJECT:
default:
return null;
}
}

private boolean isMetacardTypeSet() {
return metacard.getMetacardType() != null && metacard.getMetacardType().getName() != null;
}
Expand Down
92 changes: 84 additions & 8 deletions ...mer-csv-common/src/test/java/ddf/catalog/transformer/csv/common/MetacardIteratorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,26 @@

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.isEmptyString;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import ddf.catalog.data.Attribute;
import ddf.catalog.data.AttributeDescriptor;
import ddf.catalog.data.AttributeType;
import ddf.catalog.data.AttributeType.AttributeFormat;
import ddf.catalog.data.Metacard;
import ddf.catalog.data.MetacardType;
import ddf.catalog.data.types.Core;
import java.io.Serializable;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
Expand All @@ -44,7 +52,9 @@ public class MetacardIteratorTest {
private static final String METACARDTYPE = "METACARD_TYPE";

private static final Object[][] ATTRIBUTE_DATA = {
{"attribute1", "value1"}, {"attribute2", new Integer(101)}, {"attribute3", new Double(3.14159)}
{"attribute1", AttributeFormat.STRING, "value1"},
{"attribute2", AttributeFormat.INTEGER, new Integer(101)},
{"attribute3", AttributeFormat.DOUBLE, new Double(3.14159)}
};

private static final Map<String, Serializable> METACARD_DATA_MAP = new HashMap<>();
Expand All @@ -58,10 +68,12 @@ public void setUp() {

for (Object[] entry : ATTRIBUTE_DATA) {
String attributeName = entry[0].toString();
Serializable attributeValue = (Serializable) entry[1];
AttributeFormat attributeFormat = (AttributeFormat) entry[1];
Serializable attributeValue = (Serializable) entry[2];
Attribute attribute = buildAttribute(attributeName, attributeValue);
METACARD_DATA_MAP.put(attributeName, attribute);
ATTRIBUTE_DESCRIPTOR_LIST.add(buildAttributeDescriptor(attributeName, false));
ATTRIBUTE_DESCRIPTOR_LIST.add(
buildAttributeDescriptor(attributeName, attributeFormat, false));
}

Attribute attribute = buildAttribute("skipMe", "value");
Expand All @@ -75,7 +87,7 @@ public void testColumnHeaderIterator() {

for (int i = 0; i < ATTRIBUTE_DATA.length; i++) {
assertThat(iterator.hasNext(), is(true));
assertThat(iterator.next(), is(ATTRIBUTE_DATA[i][1]));
assertThat(iterator.next(), is(ATTRIBUTE_DATA[i][2]));
}

assertThat(iterator.hasNext(), is(false));
Expand All @@ -90,7 +102,8 @@ public void testColumnHeaderIteratorWithMultivaluedAttribute() {
List<Serializable> values = Arrays.asList("value1", "value2", "value3");
Attribute attribute = buildAttribute(attributeName, values);
METACARD_DATA_MAP.put(attributeName, attribute);
ATTRIBUTE_DESCRIPTOR_LIST.add(buildAttributeDescriptor(attributeName, true));
ATTRIBUTE_DESCRIPTOR_LIST.add(
buildAttributeDescriptor(attributeName, AttributeFormat.STRING, true));

Metacard metacard = buildMetacard();
Iterator<Serializable> iterator = new MetacardIterator(metacard, ATTRIBUTE_DESCRIPTOR_LIST);
Expand All @@ -103,7 +116,8 @@ public void testSourceId() {
ATTRIBUTE_DESCRIPTOR_LIST.clear();
METACARD_DATA_MAP.clear();

ATTRIBUTE_DESCRIPTOR_LIST.add(buildAttributeDescriptor(Core.SOURCE_ID, false));
ATTRIBUTE_DESCRIPTOR_LIST.add(
buildAttributeDescriptor(Core.SOURCE_ID, AttributeFormat.STRING, false));

Metacard metacard = buildMetacard();
when(metacard.getSourceId()).thenReturn(SOURCE);
Expand All @@ -117,7 +131,8 @@ public void testMetacardType() {
ATTRIBUTE_DESCRIPTOR_LIST.clear();
METACARD_DATA_MAP.clear();

ATTRIBUTE_DESCRIPTOR_LIST.add(buildAttributeDescriptor(MetacardType.METACARD_TYPE, false));
ATTRIBUTE_DESCRIPTOR_LIST.add(
buildAttributeDescriptor(MetacardType.METACARD_TYPE, AttributeFormat.STRING, false));

Metacard metacard = buildMetacard();
MetacardType metacardType = mock(MetacardType.class);
Expand All @@ -128,6 +143,62 @@ public void testMetacardType() {
assertThat(iterator.next(), is(METACARDTYPE));
}

@Test
public void testDateIsFormattedAsUTC() {
ATTRIBUTE_DESCRIPTOR_LIST.clear();
METACARD_DATA_MAP.clear();

Instant now = Instant.now();
Attribute attribute = buildAttribute(Core.CREATED, Date.from(now));
METACARD_DATA_MAP.put(Core.CREATED, attribute);
ATTRIBUTE_DESCRIPTOR_LIST.add(
buildAttributeDescriptor(Core.CREATED, AttributeFormat.DATE, false));

Metacard metacard = buildMetacard();
Iterator<Serializable> iterator = new MetacardIterator(metacard, ATTRIBUTE_DESCRIPTOR_LIST);
assertThat(iterator.hasNext(), is(true));

String date = (String) iterator.next();
OffsetDateTime offsetDateTime = OffsetDateTime.parse(date);
assertThat(offsetDateTime.getOffset(), is(ZoneOffset.UTC));
assertThat(offsetDateTime.toInstant(), is(now));
}

@Test
public void testBinaryIsFormattedAsBase64() {
ATTRIBUTE_DESCRIPTOR_LIST.clear();
METACARD_DATA_MAP.clear();

byte[] binary = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
Attribute attribute = buildAttribute(Core.THUMBNAIL, binary);
METACARD_DATA_MAP.put(Core.THUMBNAIL, attribute);
ATTRIBUTE_DESCRIPTOR_LIST.add(
buildAttributeDescriptor(Core.THUMBNAIL, AttributeFormat.BINARY, false));

Metacard metacard = buildMetacard();
Iterator<Serializable> iterator = new MetacardIterator(metacard, ATTRIBUTE_DESCRIPTOR_LIST);

assertThat(iterator.hasNext(), is(true));
assertThat(iterator.next(), is(Base64.getEncoder().encodeToString(binary)));
}

@Test
public void testNullAttributeValue() {
ATTRIBUTE_DESCRIPTOR_LIST.clear();
METACARD_DATA_MAP.clear();

Attribute attribute = buildAttribute(Core.DESCRIPTION, (Serializable) null);
METACARD_DATA_MAP.put(Core.DESCRIPTION, attribute);
ATTRIBUTE_DESCRIPTOR_LIST.add(
buildAttributeDescriptor(Core.DESCRIPTION, AttributeFormat.STRING, false));

Metacard metacard = buildMetacard();
Iterator<Serializable> iterator = new MetacardIterator(metacard, ATTRIBUTE_DESCRIPTOR_LIST);

assertThat(iterator.hasNext(), is(true));
assertThat(iterator.next().toString(), isEmptyString());
}

@Test(expected = NoSuchElementException.class)
public void testHasNext() {
Metacard metacard = buildMetacard();
Expand All @@ -153,9 +224,14 @@ private Metacard buildMetacard() {
return metacard;
}

private AttributeDescriptor buildAttributeDescriptor(String name, boolean isMultiValued) {
private AttributeDescriptor buildAttributeDescriptor(
String name, AttributeFormat format, boolean isMultiValued) {
AttributeType attributeType = mock(AttributeType.class);
when(attributeType.getAttributeFormat()).thenReturn(format);

AttributeDescriptor attributeDescriptor = mock(AttributeDescriptor.class);
when(attributeDescriptor.getName()).thenReturn(name);
when(attributeDescriptor.getType()).thenReturn(attributeType);
when(attributeDescriptor.isMultiValued()).thenReturn(isMultiValued);
return attributeDescriptor;
}
Expand Down