Skip to content

Commit

Permalink
Use UTC dates when exporting metacards in CSV format (#6501)
Browse files Browse the repository at this point in the history
  • Loading branch information
@SmithJosh
SmithJosh authored Jan 29, 2021
1 parent f0896a7 commit 8483609
Show file tree
Hide file tree
Showing 3 changed files with 152 additions and 13 deletions.
6 changes: 3 additions & 3 deletions catalog/transformer/catalog-transformer-csv-common/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,17 +66,17 @@
<limit implementation="org.codice.jacoco.LenientLimit">
<counter>INSTRUCTION</counter>
<value>COVEREDRATIO</value>
<minimum>0.86</minimum>
<minimum>0.87</minimum>
</limit>
<limit implementation="org.codice.jacoco.LenientLimit">
<counter>BRANCH</counter>
<value>COVEREDRATIO</value>
<minimum>0.85</minimum>
<minimum>0.84</minimum>
</limit>
<limit implementation="org.codice.jacoco.LenientLimit">
<counter>COMPLEXITY</counter>
<value>COVEREDRATIO</value>
<minimum>0.84</minimum>
<minimum>0.83</minimum>
</limit>
</limits>
</rule>
Expand Down
67 changes: 65 additions & 2 deletions ...sformer-csv-common/src/main/java/ddf/catalog/transformer/csv/common/MetacardIterator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,22 +16,37 @@

import ddf.catalog.data.Attribute;
import ddf.catalog.data.AttributeDescriptor;
import ddf.catalog.data.AttributeType;
import ddf.catalog.data.AttributeType.AttributeFormat;
import ddf.catalog.data.Metacard;
import ddf.catalog.data.MetacardType;
import ddf.catalog.data.types.Core;
import java.io.Serializable;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.stream.Collectors;
import javax.xml.bind.DatatypeConverter;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* An implementation of java.util.Iterator which iterates over Metacard attribute values.
*
* @see java.util.Iterator
*/
class MetacardIterator implements Iterator<Serializable> {

private static final Logger LOGGER = LoggerFactory.getLogger(MetacardIterator.class);

private static final String MULTIVALUE_DELIMITER = "\n";

private final List<AttributeDescriptor> attributeDescriptorList;
Expand All @@ -40,6 +55,8 @@ class MetacardIterator implements Iterator<Serializable> {

private int index;

private DateTimeFormatter formatter = DateTimeFormatter.ISO_OFFSET_DATE_TIME;

/**
* @param metacard the metacard to be iterated over.
* @param attributeDescriptorList the list of attributeDescriptors used to determine which
Expand Down Expand Up @@ -67,13 +84,21 @@ public Serializable next() {

AttributeDescriptor attributeDescriptor = this.attributeDescriptorList.get(index);
Attribute attribute = metacard.getAttribute(attributeDescriptor.getName());
AttributeFormat attributeFormat = attributeDescriptor.getType().getAttributeFormat();
index++;

if (attribute != null) {
if (attributeDescriptor.isMultiValued()) {
return StringUtils.join(attribute.getValues(), MULTIVALUE_DELIMITER);
List<Serializable> convertedValues =
attribute.getValues().stream()
.map(value -> convertValue(attribute.getName(), value, attributeFormat))
.filter(Objects::nonNull)
.collect(Collectors.toList());
return StringUtils.join(convertedValues, MULTIVALUE_DELIMITER);
} else {
return attribute.getValue();
Serializable value =
convertValue(attribute.getName(), attribute.getValue(), attributeFormat);
return (value == null) ? "" : value;
}
} else if (isSourceId(attributeDescriptor) && isSourceIdSet()) {
return metacard.getSourceId();
Expand All @@ -84,6 +109,44 @@ public Serializable next() {
return "";
}

private Serializable convertValue(
String name, Serializable value, AttributeType.AttributeFormat format) {
if (value == null) {
return null;
}

switch (format) {
case DATE:
if (!(value instanceof Date)) {
LOGGER.debug(
"Dropping attribute date value {} for {} because it isn't a Date object.",
value,
name);
return null;
}
Instant instant = ((Date) value).toInstant();
ZoneId zoneId = ZoneId.of("UTC");
ZonedDateTime zonedDateTime = instant.atZone(zoneId);
return zonedDateTime.format(formatter);
case BINARY:
byte[] bytes = (byte[]) value;
return DatatypeConverter.printBase64Binary(bytes);
case BOOLEAN:
case DOUBLE:
case LONG:
case INTEGER:
case SHORT:
case STRING:
case XML:
case FLOAT:
case GEOMETRY:
return value;
case OBJECT:
default:
return null;
}
}

private boolean isMetacardTypeSet() {
return metacard.getMetacardType() != null && metacard.getMetacardType().getName() != null;
}
Expand Down
92 changes: 84 additions & 8 deletions ...mer-csv-common/src/test/java/ddf/catalog/transformer/csv/common/MetacardIteratorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,26 @@

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.isEmptyString;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

import ddf.catalog.data.Attribute;
import ddf.catalog.data.AttributeDescriptor;
import ddf.catalog.data.AttributeType;
import ddf.catalog.data.AttributeType.AttributeFormat;
import ddf.catalog.data.Metacard;
import ddf.catalog.data.MetacardType;
import ddf.catalog.data.types.Core;
import java.io.Serializable;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
Expand All @@ -44,7 +52,9 @@ public class MetacardIteratorTest {
private static final String METACARDTYPE = "METACARD_TYPE";

private static final Object[][] ATTRIBUTE_DATA = {
{"attribute1", "value1"}, {"attribute2", new Integer(101)}, {"attribute3", new Double(3.14159)}
{"attribute1", AttributeFormat.STRING, "value1"},
{"attribute2", AttributeFormat.INTEGER, new Integer(101)},
{"attribute3", AttributeFormat.DOUBLE, new Double(3.14159)}
};

private static final Map<String, Serializable> METACARD_DATA_MAP = new HashMap<>();
Expand All @@ -58,10 +68,12 @@ public void setUp() {

for (Object[] entry : ATTRIBUTE_DATA) {
String attributeName = entry[0].toString();
Serializable attributeValue = (Serializable) entry[1];
AttributeFormat attributeFormat = (AttributeFormat) entry[1];
Serializable attributeValue = (Serializable) entry[2];
Attribute attribute = buildAttribute(attributeName, attributeValue);
METACARD_DATA_MAP.put(attributeName, attribute);
ATTRIBUTE_DESCRIPTOR_LIST.add(buildAttributeDescriptor(attributeName, false));
ATTRIBUTE_DESCRIPTOR_LIST.add(
buildAttributeDescriptor(attributeName, attributeFormat, false));
}

Attribute attribute = buildAttribute("skipMe", "value");
Expand All @@ -75,7 +87,7 @@ public void testColumnHeaderIterator() {

for (int i = 0; i < ATTRIBUTE_DATA.length; i++) {
assertThat(iterator.hasNext(), is(true));
assertThat(iterator.next(), is(ATTRIBUTE_DATA[i][1]));
assertThat(iterator.next(), is(ATTRIBUTE_DATA[i][2]));
}

assertThat(iterator.hasNext(), is(false));
Expand All @@ -90,7 +102,8 @@ public void testColumnHeaderIteratorWithMultivaluedAttribute() {
List<Serializable> values = Arrays.asList("value1", "value2", "value3");
Attribute attribute = buildAttribute(attributeName, values);
METACARD_DATA_MAP.put(attributeName, attribute);
ATTRIBUTE_DESCRIPTOR_LIST.add(buildAttributeDescriptor(attributeName, true));
ATTRIBUTE_DESCRIPTOR_LIST.add(
buildAttributeDescriptor(attributeName, AttributeFormat.STRING, true));

Metacard metacard = buildMetacard();
Iterator<Serializable> iterator = new MetacardIterator(metacard, ATTRIBUTE_DESCRIPTOR_LIST);
Expand All @@ -103,7 +116,8 @@ public void testSourceId() {
ATTRIBUTE_DESCRIPTOR_LIST.clear();
METACARD_DATA_MAP.clear();

ATTRIBUTE_DESCRIPTOR_LIST.add(buildAttributeDescriptor(Core.SOURCE_ID, false));
ATTRIBUTE_DESCRIPTOR_LIST.add(
buildAttributeDescriptor(Core.SOURCE_ID, AttributeFormat.STRING, false));

Metacard metacard = buildMetacard();
when(metacard.getSourceId()).thenReturn(SOURCE);
Expand All @@ -117,7 +131,8 @@ public void testMetacardType() {
ATTRIBUTE_DESCRIPTOR_LIST.clear();
METACARD_DATA_MAP.clear();

ATTRIBUTE_DESCRIPTOR_LIST.add(buildAttributeDescriptor(MetacardType.METACARD_TYPE, false));
ATTRIBUTE_DESCRIPTOR_LIST.add(
buildAttributeDescriptor(MetacardType.METACARD_TYPE, AttributeFormat.STRING, false));

Metacard metacard = buildMetacard();
MetacardType metacardType = mock(MetacardType.class);
Expand All @@ -128,6 +143,62 @@ public void testMetacardType() {
assertThat(iterator.next(), is(METACARDTYPE));
}

@Test
public void testDateIsFormattedAsUTC() {
ATTRIBUTE_DESCRIPTOR_LIST.clear();
METACARD_DATA_MAP.clear();

Instant now = Instant.now();
Attribute attribute = buildAttribute(Core.CREATED, Date.from(now));
METACARD_DATA_MAP.put(Core.CREATED, attribute);
ATTRIBUTE_DESCRIPTOR_LIST.add(
buildAttributeDescriptor(Core.CREATED, AttributeFormat.DATE, false));

Metacard metacard = buildMetacard();
Iterator<Serializable> iterator = new MetacardIterator(metacard, ATTRIBUTE_DESCRIPTOR_LIST);
assertThat(iterator.hasNext(), is(true));

String date = (String) iterator.next();
OffsetDateTime offsetDateTime = OffsetDateTime.parse(date);
assertThat(offsetDateTime.getOffset(), is(ZoneOffset.UTC));
assertThat(offsetDateTime.toInstant(), is(now));
}

@Test
public void testBinaryIsFormattedAsBase64() {
ATTRIBUTE_DESCRIPTOR_LIST.clear();
METACARD_DATA_MAP.clear();

byte[] binary = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
Attribute attribute = buildAttribute(Core.THUMBNAIL, binary);
METACARD_DATA_MAP.put(Core.THUMBNAIL, attribute);
ATTRIBUTE_DESCRIPTOR_LIST.add(
buildAttributeDescriptor(Core.THUMBNAIL, AttributeFormat.BINARY, false));

Metacard metacard = buildMetacard();
Iterator<Serializable> iterator = new MetacardIterator(metacard, ATTRIBUTE_DESCRIPTOR_LIST);

assertThat(iterator.hasNext(), is(true));
assertThat(iterator.next(), is(Base64.getEncoder().encodeToString(binary)));
}

@Test
public void testNullAttributeValue() {
ATTRIBUTE_DESCRIPTOR_LIST.clear();
METACARD_DATA_MAP.clear();

Attribute attribute = buildAttribute(Core.DESCRIPTION, (Serializable) null);
METACARD_DATA_MAP.put(Core.DESCRIPTION, attribute);
ATTRIBUTE_DESCRIPTOR_LIST.add(
buildAttributeDescriptor(Core.DESCRIPTION, AttributeFormat.STRING, false));

Metacard metacard = buildMetacard();
Iterator<Serializable> iterator = new MetacardIterator(metacard, ATTRIBUTE_DESCRIPTOR_LIST);

assertThat(iterator.hasNext(), is(true));
assertThat(iterator.next().toString(), isEmptyString());
}

@Test(expected = NoSuchElementException.class)
public void testHasNext() {
Metacard metacard = buildMetacard();
Expand All @@ -153,9 +224,14 @@ private Metacard buildMetacard() {
return metacard;
}

private AttributeDescriptor buildAttributeDescriptor(String name, boolean isMultiValued) {
private AttributeDescriptor buildAttributeDescriptor(
String name, AttributeFormat format, boolean isMultiValued) {
AttributeType attributeType = mock(AttributeType.class);
when(attributeType.getAttributeFormat()).thenReturn(format);

AttributeDescriptor attributeDescriptor = mock(AttributeDescriptor.class);
when(attributeDescriptor.getName()).thenReturn(name);
when(attributeDescriptor.getType()).thenReturn(attributeType);
when(attributeDescriptor.isMultiValued()).thenReturn(isMultiValued);
return attributeDescriptor;
}
Expand Down

0 comments on commit 8483609

Please sign in to comment.