diff --git a/src/main/java/cli/GramiCLI.java b/src/main/java/cli/GramiCLI.java index 07854ef..10f5fba 100644 --- a/src/main/java/cli/GramiCLI.java +++ b/src/main/java/cli/GramiCLI.java @@ -8,7 +8,7 @@ import java.io.IOException; -@CommandLine.Command(description="Welcome to the CLI of GraMi - your grakn data migration tool", name = "grami", version = "0.1.0-alpha-12", mixinStandardHelpOptions = true) +@CommandLine.Command(description="Welcome to the CLI of GraMi - your grakn data migration tool", name = "grami", version = "0.1.0", mixinStandardHelpOptions = true) public class GramiCLI { public static void main(String[] args) { @@ -46,9 +46,6 @@ class MigrateCommand implements Runnable { @CommandLine.Option(names = {"-cm", "--cleanMigration"}, description = "optional - delete old schema and data and restart migration from scratch - default: continue previous migration, if exists") private boolean cleanMigration; - @CommandLine.Option(names = {"-sc", "--scope"}, description = "optional - set migration scope: 0 - apply schema only (Note: this has no effect unless you also set the cleanMigration flag to true.); 1 - migrate entities; 2 - migrate entities & relations; 3 - migrate entites, relations, & relation-with-relations; everything else defaults to 4 - migrate all (entities, relations, relation-with-relations, append-attributes") - private int scope = 4; - @Override public void run() { spec.commandLine().getOut().println("############## GraMi migration ###############"); @@ -60,24 +57,12 @@ public void run() { spec.commandLine().getOut().println("\tdatabase: " + databaseName); spec.commandLine().getOut().println("\tgrakn server: " + graknURI); spec.commandLine().getOut().println("\tdelete database and all data in it for a clean new migration?: " + cleanMigration); - spec.commandLine().getOut().println("\tmigration scope: " + scope); final MigrationConfig migrationConfig = new MigrationConfig(graknURI, databaseName, schemaFilePath, dataConfigFilePath, processorConfigFilePath); try { GraknMigrator mig = new GraknMigrator(migrationConfig, migrationStatusFilePath, cleanMigration); - - if (scope != 0 && scope != 1 && scope != 2 && scope != 3) { - scope = 4; - } - - switch (scope) { - case 0: mig.migrate(false, false, false, false); break; - case 1: mig.migrate(true, false, false, false); break; - case 2: mig.migrate(true, true, false, false); break; - case 3: mig.migrate(true, true, true, false); break; - case 4: mig.migrate(true, true, true, true); break; - } + mig.migrate(); } catch (IOException e) { e.printStackTrace(); } diff --git a/src/main/java/configuration/DataConfigEntry.java b/src/main/java/configuration/DataConfigEntry.java index 5de3eb2..8ba6fc6 100644 --- a/src/main/java/configuration/DataConfigEntry.java +++ b/src/main/java/configuration/DataConfigEntry.java @@ -13,6 +13,7 @@ public class DataConfigEntry { private DataConfigGeneratorMapping[] relationPlayers; private int batchSize; private int threads; + private Integer order; public String getDataPath() { return dataPath; @@ -46,6 +47,10 @@ public int getThreads() { return threads; } + public Integer getOrder() { + return order; + } + public ArrayList getMatchAttributes() { ArrayList matchAttributes = new ArrayList<>(); for (DataConfigEntry.DataConfigGeneratorMapping attributeMapping: getAttributes()) { diff --git a/src/main/java/configuration/ProcessorConfigEntry.java b/src/main/java/configuration/ProcessorConfigEntry.java index c8dcf27..71430b5 100644 --- a/src/main/java/configuration/ProcessorConfigEntry.java +++ b/src/main/java/configuration/ProcessorConfigEntry.java @@ -66,7 +66,7 @@ public ConceptGenerator getRelationPlayerGenerator(String key) { public HashMap getRelationRequiredPlayers() { HashMap relationPlayers = new HashMap<>(); - if (processorType.equals("relation") || processorType.equals("relation-with-relation")) { + if (processorType.equals("relation") || processorType.equals("nested-relation") || processorType.equals("attribute-relation")) { HashMap playerGenerators = getConceptGenerators().get("players"); for (Map.Entry pg: playerGenerators.entrySet()) { if (pg.getValue().isRequired()) { diff --git a/src/main/java/generator/AppendAttributeGenerator.java b/src/main/java/generator/AppendAttributeGenerator.java index 333a257..818c821 100644 --- a/src/main/java/generator/AppendAttributeGenerator.java +++ b/src/main/java/generator/AppendAttributeGenerator.java @@ -15,14 +15,15 @@ import java.util.HashMap; import java.util.Map; -import static generator.GeneratorUtil.*; +import static generator.GeneratorUtil.addAttribute; +import static generator.GeneratorUtil.malformedRow; public class AppendAttributeGenerator extends InsertGenerator { - private final DataConfigEntry dce; - private final ProcessorConfigEntry pce; private static final Logger appLogger = LogManager.getLogger("com.bayer.dt.grami"); private static final Logger dataLogger = LogManager.getLogger("com.bayer.dt.grami.data"); + private final DataConfigEntry dce; + private final ProcessorConfigEntry pce; public AppendAttributeGenerator(DataConfigEntry dataConfigEntry, ProcessorConfigEntry processorConfigEntry) { @@ -33,16 +34,17 @@ public AppendAttributeGenerator(DataConfigEntry dataConfigEntry, } public HashMap>>> graknAppendAttributeInsert(ArrayList rows, - String header) throws Exception { + String header, int rowCounter) throws Exception { HashMap>>> matchInsertPatterns = new HashMap<>(); ArrayList>> matchPatterns = new ArrayList<>(); ArrayList>> insertPatterns = new ArrayList<>(); int insertCounter = 0; - + int batchCounter = 1; for (String row : rows) { - ArrayList>> tmp = graknAppendAttributeQueryFromRow(row, header, insertCounter); + + ArrayList>> tmp = graknAppendAttributeQueryFromRow(row, header, insertCounter, rowCounter + batchCounter); if (tmp != null) { if (tmp.get(0) != null && tmp.get(1) != null) { matchPatterns.add(tmp.get(0)); @@ -50,7 +52,7 @@ public HashMap>>> graknAppendAttrib insertCounter++; } } - + batchCounter = batchCounter + 1; } matchInsertPatterns.put("match", matchPatterns); matchInsertPatterns.put("insert", insertPatterns); @@ -59,7 +61,8 @@ public HashMap>>> graknAppendAttrib public ArrayList>> graknAppendAttributeQueryFromRow(String row, String header, - int insertCounter) throws Exception { + int insertCounter, + int rowCounter) throws Exception { String fileSeparator = dce.getSeparator(); String[] rowTokens = row.split(fileSeparator); String[] columnNames = header.split(fileSeparator); @@ -77,7 +80,7 @@ public ArrayList>> graknAppendAttributeQueryFromRow(S Thing appendAttributeMatchPattern = addEntityToMatchPattern(insertCounter); for (DataConfigEntry.DataConfigGeneratorMapping generatorMappingForMatchAttribute : dce.getAttributes()) { if (generatorMappingForMatchAttribute.isMatch()) { - appendAttributeMatchPattern = addAttribute(rowTokens, appendAttributeMatchPattern, columnNames, generatorMappingForMatchAttribute, pce, generatorMappingForMatchAttribute.getPreprocessor()); + appendAttributeMatchPattern = addAttribute(rowTokens, appendAttributeMatchPattern, columnNames, rowCounter, generatorMappingForMatchAttribute, pce, generatorMappingForMatchAttribute.getPreprocessor()); } } matchPatterns.add(appendAttributeMatchPattern); @@ -87,7 +90,7 @@ public ArrayList>> graknAppendAttributeQueryFromRow(S Thing appendAttributeInsertPattern = null; for (DataConfigEntry.DataConfigGeneratorMapping generatorMappingForAppendAttribute : dce.getAttributes()) { if (!generatorMappingForAppendAttribute.isMatch()) { - appendAttributeInsertPattern = addAttribute(rowTokens, thingVar, columnNames, generatorMappingForAppendAttribute, pce, generatorMappingForAppendAttribute.getPreprocessor()); + appendAttributeInsertPattern = addAttribute(rowTokens, thingVar, rowCounter, columnNames, generatorMappingForAppendAttribute, pce, generatorMappingForAppendAttribute.getPreprocessor()); } } if (appendAttributeInsertPattern != null) { @@ -100,10 +103,10 @@ public ArrayList>> graknAppendAttributeQueryFromRow(S if (isValid(assembledPatterns)) { - appLogger.debug("valid query: <" + assembleQuery(assembledPatterns).toString() + ">"); + appLogger.debug("valid query: <" + assembleQuery(assembledPatterns) + ">"); return assembledPatterns; } else { - dataLogger.warn("in datapath <" + dce.getDataPath() + ">: skipped row b/c does not contain at least one match attribute and one insert attribute. Faulty tokenized row: " + Arrays.toString(rowTokens)); + dataLogger.warn("in datapath <" + dce.getDataPath() + ">: skipped row " + rowCounter + " b/c does not contain at least one match attribute and one insert attribute. Faulty tokenized row: " + Arrays.toString(rowTokens)); return null; } } diff --git a/src/main/java/generator/AttributeInsertGenerator.java b/src/main/java/generator/AttributeInsertGenerator.java index 3b8d649..90bd644 100644 --- a/src/main/java/generator/AttributeInsertGenerator.java +++ b/src/main/java/generator/AttributeInsertGenerator.java @@ -31,23 +31,25 @@ public AttributeInsertGenerator(DataConfigEntry dataConfigEntry, ProcessorConfig } public ArrayList> graknAttributeInsert(ArrayList rows, - String header) throws IllegalArgumentException { + String header, int rowCounter) throws IllegalArgumentException { ArrayList> patterns = new ArrayList<>(); + int batchCount = 1; for (String row : rows) { try { - ThingVariable temp = graknAttributeQueryFromRow(row, header); + ThingVariable temp = graknAttributeQueryFromRow(row, header, rowCounter + batchCount); if (temp != null) { patterns.add(temp); } } catch (Exception e) { e.printStackTrace(); } + batchCount = batchCount + 1; } return patterns; } public ThingVariable graknAttributeQueryFromRow(String row, - String header) throws Exception { + String header, int rowCounter) throws Exception { String fileSeparator = dce.getSeparator(); String[] rowTokens = row.split(fileSeparator); String[] columnNames = header.split(fileSeparator); @@ -58,7 +60,7 @@ public ThingVariable graknAttributeQueryFromRow(String row, Attribute attributeInsertStatement = null; for (DataConfigEntry.DataConfigGeneratorMapping generatorMappingForAttribute : dce.getAttributes()) { - attributeInsertStatement = addValue(rowTokens, attributeInitialStatement, columnNames, generatorMappingForAttribute, pce, generatorMappingForAttribute.getPreprocessor()); + attributeInsertStatement = addValue(rowTokens, attributeInitialStatement, rowCounter, columnNames, generatorMappingForAttribute, pce, generatorMappingForAttribute.getPreprocessor()); } if (attributeInsertStatement != null) { @@ -68,7 +70,7 @@ public ThingVariable graknAttributeQueryFromRow(String row, appLogger.debug("valid query: "); return attributeInsertStatement; } else { - dataLogger.warn("in datapath <" + dce.getDataPath() + ">: skipped row b/c does not have a proper statement or is missing required attributes. Faulty tokenized row: " + Arrays.toString(rowTokens)); + dataLogger.warn("in datapath <" + dce.getDataPath() + ">: skipped row " + rowCounter + " b/c does not have a proper statement or is missing required attributes. Faulty tokenized row: " + Arrays.toString(rowTokens)); return null; } } else { diff --git a/src/main/java/generator/EntityInsertGenerator.java b/src/main/java/generator/EntityInsertGenerator.java index fa6dc3a..b5e71c9 100644 --- a/src/main/java/generator/EntityInsertGenerator.java +++ b/src/main/java/generator/EntityInsertGenerator.java @@ -1,8 +1,5 @@ package generator; -import static generator.GeneratorUtil.malformedRow; -import static generator.GeneratorUtil.addAttribute; - import configuration.DataConfigEntry; import configuration.ProcessorConfigEntry; import graql.lang.Graql; @@ -16,12 +13,15 @@ import java.util.Arrays; import java.util.Map; +import static generator.GeneratorUtil.addAttribute; +import static generator.GeneratorUtil.malformedRow; + public class EntityInsertGenerator extends InsertGenerator { - private final DataConfigEntry dce; - private final ProcessorConfigEntry pce; private static final Logger appLogger = LogManager.getLogger("com.bayer.dt.grami"); private static final Logger dataLogger = LogManager.getLogger("com.bayer.dt.grami.data"); + private final DataConfigEntry dce; + private final ProcessorConfigEntry pce; public EntityInsertGenerator(DataConfigEntry dataConfigEntry, ProcessorConfigEntry processorConfigEntry) { super(); @@ -31,23 +31,26 @@ public EntityInsertGenerator(DataConfigEntry dataConfigEntry, ProcessorConfigEnt } public ArrayList> graknEntityInsert(ArrayList rows, - String header) throws IllegalArgumentException { + String header, int rowCounter) throws IllegalArgumentException { ArrayList> patterns = new ArrayList<>(); + int batchCount = 1; for (String row : rows) { try { - ThingVariable temp = graknEntityQueryFromRow(row, header); + ThingVariable temp = graknEntityQueryFromRow(row, header, rowCounter + batchCount); if (temp != null) { patterns.add(temp); } } catch (Exception e) { e.printStackTrace(); } + batchCount = batchCount + 1; } return patterns; } public ThingVariable graknEntityQueryFromRow(String row, - String header) throws Exception { + String header, + int rowCounter) throws Exception { String fileSeparator = dce.getSeparator(); String[] rowTokens = row.split(fileSeparator); String[] columnNames = header.split(fileSeparator); @@ -57,14 +60,14 @@ public ThingVariable graknEntityQueryFromRow(String row, Thing entityInsertStatement = addEntityToStatement(); for (DataConfigEntry.DataConfigGeneratorMapping generatorMappingForAttribute : dce.getAttributes()) { - entityInsertStatement = addAttribute(rowTokens, entityInsertStatement, columnNames, generatorMappingForAttribute, pce, generatorMappingForAttribute.getPreprocessor()); + entityInsertStatement = addAttribute(rowTokens, entityInsertStatement, columnNames, rowCounter, generatorMappingForAttribute, pce, generatorMappingForAttribute.getPreprocessor()); } if (isValid(entityInsertStatement)) { appLogger.debug("valid query: "); return entityInsertStatement; } else { - dataLogger.warn("in datapath <" + dce.getDataPath() + ">: skipped row b/c does not have a proper statement or is missing required attributes. Faulty tokenized row: " + Arrays.toString(rowTokens)); + dataLogger.warn("in datapath <" + dce.getDataPath() + ">: skipped row " + rowCounter + " b/c does not have a proper statement or is missing required attributes. Faulty tokenized row: " + Arrays.toString(rowTokens)); return null; } } diff --git a/src/main/java/generator/GeneratorUtil.java b/src/main/java/generator/GeneratorUtil.java index ad1cf80..d338efc 100644 --- a/src/main/java/generator/GeneratorUtil.java +++ b/src/main/java/generator/GeneratorUtil.java @@ -56,6 +56,7 @@ public static int[] indicesOf(String[] headerTokens, public static Attribute addValue(String[] tokens, UnboundVariable statement, + int lineNumber, String[] columnNames, DataConfigEntry.DataConfigGeneratorMapping generatorMappingForAttribute, ProcessorConfigEntry pce, @@ -74,7 +75,7 @@ public static Attribute addValue(String[] tokens, !cleanToken(tokens[columnNameIndex]).isEmpty()) { String attributeValueType = attributeGenerator.getValueType(); String cleanedToken = cleanToken(tokens[columnNameIndex]); - att = addAttributeValueOfType(statement, attributeValueType, cleanedToken, preprocessorConfig); + att = addAttributeValueOfType(statement, attributeValueType, cleanedToken, lineNumber, preprocessorConfig); } } return att; @@ -83,6 +84,7 @@ public static Attribute addValue(String[] tokens, public static Thing addAttribute(String[] tokens, Thing statement, String[] columnNames, + int lineNumber, DataConfigEntry.DataConfigGeneratorMapping generatorMappingForAttribute, ProcessorConfigEntry pce, DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig preprocessorConfig) { @@ -101,7 +103,7 @@ public static Thing addAttribute(String[] tokens, String attributeType = attributeGenerator.getAttributeType(); String attributeValueType = attributeGenerator.getValueType(); String cleanedToken = cleanToken(tokens[columnNameIndex]); - statement = cleanExplodeAdd(statement, cleanedToken, attributeType, attributeValueType, columnListSeparator, preprocessorConfig); + statement = cleanExplodeAdd(statement, cleanedToken, attributeType, attributeValueType, lineNumber, columnListSeparator, preprocessorConfig); } } return statement; @@ -110,6 +112,7 @@ public static Thing addAttribute(String[] tokens, public static Relation addAttribute(String[] tokens, Relation statement, String[] columnNames, + int lineNumber, DataConfigEntry.DataConfigGeneratorMapping generatorMappingForAttribute, ProcessorConfigEntry pce, DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig preprocessorConfig) { @@ -128,7 +131,7 @@ public static Relation addAttribute(String[] tokens, String attributeType = attributeGenerator.getAttributeType(); String attributeValueType = attributeGenerator.getValueType(); String cleanedToken = cleanToken(tokens[columnNameIndex]); - statement = cleanExplodeAdd(statement, cleanedToken, attributeType, attributeValueType, columnListSeparator, preprocessorConfig); + statement = cleanExplodeAdd(statement, cleanedToken, attributeType, attributeValueType, columnListSeparator, lineNumber, preprocessorConfig); } } return statement; @@ -136,6 +139,7 @@ public static Relation addAttribute(String[] tokens, public static Thing addAttribute(String[] tokens, UnboundVariable statement, + int lineNumber, String[] columnNames, DataConfigEntry.DataConfigGeneratorMapping generatorMappingForAttribute, ProcessorConfigEntry pce, @@ -156,7 +160,7 @@ public static Thing addAttribute(String[] tokens, String attributeType = attributeGenerator.getAttributeType(); String attributeValueType = attributeGenerator.getValueType(); String cleanedToken = cleanToken(tokens[columnNameIndex]); - returnThing = cleanExplodeAdd(statement, cleanedToken, attributeType, attributeValueType, columnListSeparator, preprocessorConfig); + returnThing = cleanExplodeAdd(statement, cleanedToken, attributeType, attributeValueType, columnListSeparator, lineNumber, preprocessorConfig); } } return returnThing; @@ -166,18 +170,19 @@ public static Thing cleanExplodeAdd(Thing statement, String cleanedToken, String conceptType, String valueType, + int lineNumber, String listSeparator, DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig preprocessorConfig) { if (listSeparator != null) { for (String exploded: cleanedToken.split(listSeparator)) { String cleanedExplodedToken = cleanToken(exploded); if (!cleanedExplodedToken.isEmpty()) { - statement = addAttributeOfColumnType(statement, conceptType, valueType, cleanedExplodedToken, preprocessorConfig); + statement = addAttributeOfColumnType(statement, conceptType, valueType, cleanedExplodedToken, lineNumber, preprocessorConfig); } } return statement; } else { - return addAttributeOfColumnType(statement, conceptType, valueType, cleanedToken, preprocessorConfig); + return addAttributeOfColumnType(statement, conceptType, valueType, cleanedToken, lineNumber, preprocessorConfig); } } @@ -186,17 +191,18 @@ public static Relation cleanExplodeAdd(Relation statement, String conceptType, String valueType, String listSeparator, + int lineNumber, DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig preprocessorConfig) { if (listSeparator != null) { for (String exploded: cleanedToken.split(listSeparator)) { String cleanedExplodedToken = cleanToken(exploded); if (!cleanedExplodedToken.isEmpty()) { - statement = addAttributeOfColumnType(statement, conceptType, valueType, cleanedExplodedToken, preprocessorConfig); + statement = addAttributeOfColumnType(statement, conceptType, valueType, cleanedExplodedToken, lineNumber, preprocessorConfig); } } return statement; } else { - return addAttributeOfColumnType(statement, conceptType, valueType, cleanedToken, preprocessorConfig); + return addAttributeOfColumnType(statement, conceptType, valueType, cleanedToken, lineNumber, preprocessorConfig); } } @@ -205,6 +211,7 @@ public static Thing cleanExplodeAdd(UnboundVariable statement, String conceptType, String valueType, String listSeparator, + int lineNumber, DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig preprocessorConfig) { Thing returnThing = null; if (listSeparator != null) { @@ -213,16 +220,16 @@ public static Thing cleanExplodeAdd(UnboundVariable statement, String cleanedExplodedToken = cleanToken(exploded); if (!cleanedExplodedToken.isEmpty()) { if (count == 0) { - returnThing = addAttributeOfColumnType(statement, conceptType, valueType, cleanedExplodedToken, preprocessorConfig); + returnThing = addAttributeOfColumnType(statement, conceptType, valueType, cleanedExplodedToken, lineNumber, preprocessorConfig); } else { - returnThing = addAttributeOfColumnType(returnThing, conceptType, valueType, cleanedExplodedToken, preprocessorConfig); + returnThing = addAttributeOfColumnType(returnThing, conceptType, valueType, cleanedExplodedToken, lineNumber, preprocessorConfig); } count++; } } return returnThing; } else { - return addAttributeOfColumnType(statement, conceptType, valueType, cleanedToken, preprocessorConfig); + return addAttributeOfColumnType(statement, conceptType, valueType, cleanedToken, lineNumber, preprocessorConfig); } } @@ -230,6 +237,7 @@ public static Thing addAttributeOfColumnType(Thing statement, String conceptType, String valueType, String cleanedValue, + int lineNumber, DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig preprocessorConfig) { if (preprocessorConfig != null) { cleanedValue = applyPreprocessor(cleanedValue, preprocessorConfig); @@ -243,16 +251,14 @@ public static Thing addAttributeOfColumnType(Thing statement, try { statement = statement.has(conceptType, Integer.parseInt(cleanedValue)); } catch (NumberFormatException numberFormatException) { - dataLogger.warn(String.format("current row has column of type for variable < %s > with non- value - skipping column", conceptType)); - dataLogger.warn(numberFormatException.getMessage()); + dataLogger.warn(String.format("row < %s > has column of type for variable < %s > with non- value - skipping column - < %s >", lineNumber, conceptType, numberFormatException.getMessage())); } break; case "double": try { statement = statement.has(conceptType, Double.parseDouble(cleanedValue)); } catch (NumberFormatException numberFormatException) { - dataLogger.warn(String.format("current row has column of type for variable < %s > with non- value - skipping column", conceptType)); - dataLogger.warn(numberFormatException.getMessage()); + dataLogger.warn(String.format("row < %s > has column of type for variable < %s > with non- value - skipping column - < %s >", lineNumber, conceptType, numberFormatException.getMessage())); } break; case "boolean": @@ -261,7 +267,7 @@ public static Thing addAttributeOfColumnType(Thing statement, } else if (cleanedValue.toLowerCase().equals("false")) { statement = statement.has(conceptType, false); } else { - dataLogger.warn(String.format("current row has column of type for variable < %s > with non- value - skipping column", conceptType)); + dataLogger.warn(String.format("row < %s > has column of type for variable < %s > with non- value - skipping column", lineNumber, conceptType)); } break; case "datetime": @@ -278,12 +284,11 @@ public static Thing addAttributeOfColumnType(Thing statement, statement = statement.has(conceptType, dateTime); } } catch (DateTimeException dateTimeException) { - dataLogger.warn(String.format("current row has column of type for variable < %s > with non- datetime value: ", conceptType)); - dataLogger.warn(dateTimeException.getMessage()); + dataLogger.warn(String.format("row < %s > has column of type for variable < %s > with non- datetime value: - < %s >", lineNumber, conceptType, dateTimeException.getMessage())); } break; default: - dataLogger.warn("column type not valid - must be either: string, long, double, boolean, or datetime"); + dataLogger.warn("row < %s > column type not valid - must be either: string, long, double, boolean, or datetime"); break; } return statement; @@ -293,6 +298,7 @@ public static Relation addAttributeOfColumnType(Relation statement, String conceptType, String valueType, String cleanedValue, + int lineNumber, DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig preprocessorConfig) { if (preprocessorConfig != null) { cleanedValue = applyPreprocessor(cleanedValue, preprocessorConfig); @@ -306,16 +312,14 @@ public static Relation addAttributeOfColumnType(Relation statement, try { statement = statement.has(conceptType, Integer.parseInt(cleanedValue)); } catch (NumberFormatException numberFormatException) { - dataLogger.warn(String.format("current row has column of type for variable < %s > with non- value - skipping column", conceptType)); - dataLogger.warn(numberFormatException.getMessage()); + dataLogger.warn(String.format("row < %s > has column of type for variable < %s > with non- value - skipping column - < %s >", lineNumber, conceptType, numberFormatException.getMessage())); } break; case "double": try { statement = statement.has(conceptType, Double.parseDouble(cleanedValue)); } catch (NumberFormatException numberFormatException) { - dataLogger.warn(String.format("current row has column of type for variable < %s > with non- value - skipping column", conceptType)); - dataLogger.warn(numberFormatException.getMessage()); + dataLogger.warn(String.format("row < %s > has column of type for variable < %s > with non- value - skipping column - < %s >", lineNumber, conceptType, numberFormatException.getMessage())); } break; case "boolean": @@ -324,7 +328,7 @@ public static Relation addAttributeOfColumnType(Relation statement, } else if (cleanedValue.toLowerCase().equals("false")) { statement = statement.has(conceptType, false); } else { - dataLogger.warn(String.format("current row has column of type for variable < %s > with non- value - skipping column", conceptType)); + dataLogger.warn(String.format("row < %s > has column of type for variable < %s > with non- value - skipping column", lineNumber, conceptType)); } break; case "datetime": @@ -341,12 +345,11 @@ public static Relation addAttributeOfColumnType(Relation statement, statement = statement.has(conceptType, dateTime); } } catch (DateTimeException dateTimeException) { - dataLogger.warn(String.format("current row has column of type for variable < %s > with non- datetime value: ", conceptType)); - dataLogger.warn(dateTimeException.getMessage()); + dataLogger.warn(String.format("row < %s > has column of type for variable < %s > with non- datetime value: - < %s >", lineNumber, conceptType, dateTimeException.getMessage())); } break; default: - dataLogger.warn("column type not valid - must be either: string, long, double, boolean, or datetime"); + dataLogger.warn(String.format("row < %s > column type not valid - must be either: string, long, double, boolean, or datetime", lineNumber)); break; } return statement; @@ -356,6 +359,7 @@ public static Thing addAttributeOfColumnType(UnboundVariable statement, String conceptType, String valueType, String cleanedValue, + int lineNumber, DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig preprocessorConfig) { if (preprocessorConfig != null) { cleanedValue = applyPreprocessor(cleanedValue, preprocessorConfig); @@ -371,16 +375,14 @@ public static Thing addAttributeOfColumnType(UnboundVariable statement, try { returnThing = statement.has(conceptType, Integer.parseInt(cleanedValue)); } catch (NumberFormatException numberFormatException) { - dataLogger.warn(String.format("current row has column of type for variable < %s > with non- value - skipping column", conceptType)); - dataLogger.warn(numberFormatException.getMessage()); + dataLogger.warn(String.format("row < %s > has column of type for variable < %s > with non- value - skipping column - < %s >", lineNumber, conceptType, numberFormatException.getMessage())); } break; case "double": try { returnThing = statement.has(conceptType, Double.parseDouble(cleanedValue)); } catch (NumberFormatException numberFormatException) { - dataLogger.warn(String.format("current row has column of type for variable < %s > with non- value - skipping column", conceptType)); - dataLogger.warn(numberFormatException.getMessage()); + dataLogger.warn(String.format("row < %s > has column of type for variable < %s > with non- value - skipping column - < %s >", lineNumber, conceptType, numberFormatException.getMessage())); } break; case "boolean": @@ -389,7 +391,7 @@ public static Thing addAttributeOfColumnType(UnboundVariable statement, } else if (cleanedValue.toLowerCase().equals("false")) { returnThing = statement.has(conceptType, false); } else { - dataLogger.warn(String.format("current row has column of type for variable < %s > with non- value - skipping column", conceptType)); + dataLogger.warn(String.format("row < %s > has column of type for variable < %s > with non- value - skipping column", lineNumber, conceptType)); } break; case "datetime": @@ -406,12 +408,11 @@ public static Thing addAttributeOfColumnType(UnboundVariable statement, returnThing = statement.has(conceptType, dateTime); } } catch (DateTimeException dateTimeException) { - dataLogger.warn(String.format("current row has column of type for variable < %s > with non- datetime value: ", conceptType)); - dataLogger.warn(dateTimeException.getMessage()); + dataLogger.warn(String.format("row < %s > has column of type for variable < %s > with non- datetime value: - < %s >", lineNumber, conceptType, dateTimeException.getMessage())); } break; default: - dataLogger.warn("column type not valid - must be either: string, long, double, boolean, or datetime"); + dataLogger.warn(String.format("row < %s > column type not valid - must be either: string, long, double, boolean, or datetime", lineNumber)); break; } return returnThing; @@ -420,6 +421,7 @@ public static Thing addAttributeOfColumnType(UnboundVariable statement, public static Attribute addAttributeValueOfType(UnboundVariable unboundVar, String valueType, String cleanedValue, + int lineNumber, DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig preprocessorConfig) { if (preprocessorConfig != null) { cleanedValue = applyPreprocessor(cleanedValue, preprocessorConfig); @@ -434,16 +436,14 @@ public static Attribute addAttributeValueOfType(UnboundVariable unboundVar, try { att = unboundVar.eq(Integer.parseInt(cleanedValue)); } catch (NumberFormatException numberFormatException) { - dataLogger.warn("current row has column of type with non- value - skipping column"); - dataLogger.warn(numberFormatException.getMessage()); + dataLogger.warn(String.format("row < %s > has column of type with non- value - skipping column - < %s >", lineNumber, numberFormatException.getMessage())); } break; case "double": try { att = unboundVar.eq(Double.parseDouble(cleanedValue)); } catch (NumberFormatException numberFormatException) { - dataLogger.warn("current row has column of type with non- value - skipping column"); - dataLogger.warn(numberFormatException.getMessage()); + dataLogger.warn(String.format("row < %s > has column of type with non- value - skipping column - < %s >", lineNumber, numberFormatException.getMessage())); } break; case "boolean": @@ -452,7 +452,7 @@ public static Attribute addAttributeValueOfType(UnboundVariable unboundVar, } else if (cleanedValue.toLowerCase().equals("false")) { att = unboundVar.eq( false); } else { - dataLogger.warn("current row has column of type with non- value - skipping column"); + dataLogger.warn(String.format("row < %s > has column of type with non- value - skipping column", lineNumber)); } break; case "datetime": @@ -469,12 +469,11 @@ public static Attribute addAttributeValueOfType(UnboundVariable unboundVar, att = unboundVar.eq(dateTime); } } catch (DateTimeException dateTimeException) { - dataLogger.warn("current row has column of type with non- datetime value: "); - dataLogger.warn(dateTimeException.getMessage()); + dataLogger.warn(String.format("row < %s > has column of type with non- datetime value: - < %s >", lineNumber, dateTimeException.getMessage())); } break; default: - dataLogger.warn("column type not valid - must be either: string, long, double, boolean, or datetime"); + dataLogger.warn(String.format("row < %s > column type not valid - must be either: string, long, double, boolean, or datetime", lineNumber)); break; } return att; diff --git a/src/main/java/generator/InsertGenerator.java b/src/main/java/generator/InsertGenerator.java index 19a1ece..42cb921 100644 --- a/src/main/java/generator/InsertGenerator.java +++ b/src/main/java/generator/InsertGenerator.java @@ -6,8 +6,20 @@ import java.util.HashMap; public abstract class InsertGenerator { - public ArrayList> graknEntityInsert(ArrayList rows, String header) { return null; }; - public ArrayList> graknAttributeInsert(ArrayList rows, String header) { return null; }; - public HashMap>>> graknRelationInsert(ArrayList rows, String header) throws Exception { return null; }; - public HashMap>>> graknAppendAttributeInsert(ArrayList rows, String header) throws Exception { return null; }; + public ArrayList> graknEntityInsert(ArrayList rows, String header, int rowCounter) { + return null; + } + + public ArrayList> graknAttributeInsert(ArrayList rows, String header, int rowCounter) { + return null; + } + + public HashMap>>> graknRelationInsert(ArrayList rows, String header, int rowCounter) throws Exception { + return null; + } + + public HashMap>>> graknAppendAttributeInsert(ArrayList rows, String header, int rowCounter) throws Exception { + return null; + } + } diff --git a/src/main/java/generator/NestedRelationInsertGenerator.java b/src/main/java/generator/NestedRelationInsertGenerator.java new file mode 100644 index 0000000..0353693 --- /dev/null +++ b/src/main/java/generator/NestedRelationInsertGenerator.java @@ -0,0 +1,17 @@ +package generator; + +import configuration.DataConfigEntry; +import configuration.ProcessorConfigEntry; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class NestedRelationInsertGenerator extends RelationInsertGenerator { + + private static final Logger appLogger = LogManager.getLogger("com.bayer.dt.grami"); + + public NestedRelationInsertGenerator(DataConfigEntry dce, ProcessorConfigEntry processorConfigEntry) { + super(dce, processorConfigEntry); + appLogger.debug("Creating NestedRelationInsertGenerator for " + pce.getProcessor() + " of type " + pce.getProcessorType()); + } + +} \ No newline at end of file diff --git a/src/main/java/generator/RelationInsertGenerator.java b/src/main/java/generator/RelationInsertGenerator.java index 76777fd..42fda2a 100644 --- a/src/main/java/generator/RelationInsertGenerator.java +++ b/src/main/java/generator/RelationInsertGenerator.java @@ -1,27 +1,27 @@ package generator; -import static generator.GeneratorUtil.*; - import configuration.DataConfigEntry; import configuration.ProcessorConfigEntry; import graql.lang.Graql; import graql.lang.pattern.Pattern; import graql.lang.pattern.variable.ThingVariable; -import graql.lang.pattern.variable.ThingVariable.Thing; -import graql.lang.pattern.variable.ThingVariable.Relation; import graql.lang.pattern.variable.ThingVariable.Attribute; +import graql.lang.pattern.variable.ThingVariable.Relation; +import graql.lang.pattern.variable.ThingVariable.Thing; import graql.lang.pattern.variable.UnboundVariable; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.*; +import static generator.GeneratorUtil.*; + public class RelationInsertGenerator extends InsertGenerator { - public final DataConfigEntry dce; - public final ProcessorConfigEntry pce; private static final Logger appLogger = LogManager.getLogger("com.bayer.dt.grami"); private static final Logger dataLogger = LogManager.getLogger("com.bayer.dt.grami.data"); + public final DataConfigEntry dce; + public final ProcessorConfigEntry pce; public RelationInsertGenerator(DataConfigEntry dce, ProcessorConfigEntry processorConfigEntry) { super(); @@ -30,16 +30,16 @@ public RelationInsertGenerator(DataConfigEntry dce, ProcessorConfigEntry process appLogger.debug("Creating RelationInsertGenerator for " + pce.getProcessor() + " of type " + pce.getProcessorType()); } - public HashMap>>> graknRelationInsert(ArrayList rows, String header) throws Exception { - HashMap>>> matchInsertStatements = new HashMap<>(); + public HashMap>>> graknRelationInsert(ArrayList rows, String header, int rowCounter) throws Exception { + HashMap>>> matchInsertStatements = new HashMap<>(); ArrayList>> matchStatements = new ArrayList<>(); ArrayList>> insertStatements = new ArrayList<>(); int insertCounter = 0; - + int batchCounter = 1; for (String row : rows) { - ArrayList>> tmp = graknRelationshipQueryFromRow(row, header, insertCounter); + ArrayList>> tmp = graknRelationshipQueryFromRow(row, header, insertCounter, rowCounter + batchCounter); if (tmp != null) { if (tmp.get(0) != null && tmp.get(1) != null) { matchStatements.add(tmp.get(0)); @@ -47,21 +47,21 @@ public HashMap>>> graknRelationInse insertCounter++; } } - + batchCounter = batchCounter + 1; } matchInsertStatements.put("match", matchStatements); matchInsertStatements.put("insert", insertStatements); return matchInsertStatements; } - public ArrayList>> graknRelationshipQueryFromRow(String row, String header, int insertCounter) throws Exception { + public ArrayList>> graknRelationshipQueryFromRow(String row, String header, int insertCounter, int rowCounter) throws Exception { String fileSeparator = dce.getSeparator(); String[] rowTokens = row.split(fileSeparator); String[] columnNames = header.split(fileSeparator); appLogger.debug("processing tokenized row: " + Arrays.toString(rowTokens)); GeneratorUtil.malformedRow(row, rowTokens, columnNames.length); - ArrayList> miStatements = new ArrayList<>(createPlayerMatchAndInsert(rowTokens, columnNames, insertCounter)); + ArrayList> miStatements = new ArrayList<>(createPlayerMatchAndInsert(rowTokens, columnNames, insertCounter, rowCounter)); if (miStatements.size() >= 1) { ArrayList> matchStatements = new ArrayList<>(miStatements.subList(0, miStatements.size() - 1)); @@ -73,7 +73,7 @@ public ArrayList>> graknRelationshipQueryFromRow(Stri if (dce.getAttributes() != null) { for (DataConfigEntry.DataConfigGeneratorMapping generatorMappingForAttribute : dce.getAttributes()) { - assembledInsertStatement = addAttribute(rowTokens, assembledInsertStatement, columnNames, generatorMappingForAttribute, pce, generatorMappingForAttribute.getPreprocessor()); + assembledInsertStatement = addAttribute(rowTokens, assembledInsertStatement, columnNames, rowCounter, generatorMappingForAttribute, pce, generatorMappingForAttribute.getPreprocessor()); } } insertStatements.add(assembledInsertStatement); @@ -86,11 +86,11 @@ public ArrayList>> graknRelationshipQueryFromRow(Stri appLogger.debug("valid query: <" + assembleQuery(assembledStatements) + ">"); return assembledStatements; } else { - dataLogger.warn("in datapath <" + dce.getDataPath() + ">: skipped row b/c does not have a proper statement or is missing required players or attributes. Faulty tokenized row: " + Arrays.toString(rowTokens)); + dataLogger.warn("in datapath <" + dce.getDataPath() + ">: skipped row " + rowCounter + " b/c does not have a proper statement or is missing required players or attributes. Faulty tokenized row: " + Arrays.toString(rowTokens)); return null; } } else { - dataLogger.warn("in datapath <" + dce.getDataPath() + ">: skipped row b/c has 0 players. Faulty tokenized row: " + Arrays.toString(rowTokens)); + dataLogger.warn("in datapath <" + dce.getDataPath() + ">: skipped row " + rowCounter + " b/c has 0 players. Faulty tokenized row: " + Arrays.toString(rowTokens)); return null; } } else { @@ -110,7 +110,7 @@ private String assembleQuery(ArrayList>> queries) { return ret.toString(); } - private Collection> createPlayerMatchAndInsert(String[] rowTokens, String[] columnNames, int insertCounter) { + private Collection> createPlayerMatchAndInsert(String[] rowTokens, String[] columnNames, int insertCounter, int rowCounter) { ArrayList> players = new ArrayList<>(); UnboundVariable relVariable = Graql.var("rel"); ArrayList> relationStrings = new ArrayList<>(); @@ -138,9 +138,9 @@ private Collection> createPlayerMatchAndInsert(String String playerVariable = playerGenerator.getPlayerType() + "-" + playerCounter; String playerRole = playerGenerator.getRoleType(); if (playerGenerator.getUniquePlayerId().contains("_attribute_player_")) { - players.add(createAttributePlayerMatchStatement(currentExplodedCleanedToken, playerGenerator, playerVariable, generatorMappingForPlayer.getPreprocessor())); + players.add(createAttributePlayerMatchStatement(currentExplodedCleanedToken, rowCounter, playerGenerator, playerVariable, generatorMappingForPlayer.getPreprocessor())); } else { - players.add(createEntityPlayerMatchStatement(currentExplodedCleanedToken, playerGenerator, playerVariable, generatorMappingForPlayer.getPreprocessor())); + players.add(createEntityPlayerMatchStatement(currentExplodedCleanedToken, rowCounter, playerGenerator, playerVariable, generatorMappingForPlayer.getPreprocessor())); } ArrayList rel = new ArrayList<>(); rel.add(playerRole); @@ -153,9 +153,9 @@ private Collection> createPlayerMatchAndInsert(String String playerVariable = playerGenerator.getPlayerType() + "-" + playerCounter; String playerRole = playerGenerator.getRoleType(); if (playerGenerator.getUniquePlayerId().contains("_attribute_player_")) { - players.add(createAttributePlayerMatchStatement(currentCleanedToken, playerGenerator, playerVariable, generatorMappingForPlayer.getPreprocessor())); + players.add(createAttributePlayerMatchStatement(currentCleanedToken, rowCounter, playerGenerator, playerVariable, generatorMappingForPlayer.getPreprocessor())); } else { - players.add(createEntityPlayerMatchStatement(currentCleanedToken, playerGenerator, playerVariable, generatorMappingForPlayer.getPreprocessor())); + players.add(createEntityPlayerMatchStatement(currentCleanedToken, rowCounter, playerGenerator, playerVariable, generatorMappingForPlayer.getPreprocessor())); } ArrayList rel = new ArrayList<>(); rel.add(playerRole); @@ -191,7 +191,7 @@ private Collection> createPlayerMatchAndInsert(String // String playerVariable = playerGenerator.getPlayerType() + "-" + playerCounter + "-" + insertCounter; String playerVariable = playerGenerator.getPlayerType() + "-" + playerCounter; String playerRole = playerGenerator.getRoleType(); - players.add(createRelationPlayerMatchStatementByAttribute(currentExplodedCleanedToken, playerGenerator, generatorMappingForRelationPlayer, playerVariable)); + players.add(createRelationPlayerMatchStatementByAttribute(currentExplodedCleanedToken, rowCounter, playerGenerator, generatorMappingForRelationPlayer, playerVariable)); ArrayList rel = new ArrayList<>(); rel.add(playerRole); rel.add(playerVariable); @@ -203,7 +203,7 @@ private Collection> createPlayerMatchAndInsert(String // String playerVariable = playerGenerator.getPlayerType() + "-" + playerCounter + "-" + insertCounter; String playerVariable = playerGenerator.getPlayerType() + "-" + playerCounter; String playerRole = playerGenerator.getRoleType(); - players.add(createRelationPlayerMatchStatementByAttribute(currentCleanedToken, playerGenerator, generatorMappingForRelationPlayer, playerVariable)); + players.add(createRelationPlayerMatchStatementByAttribute(currentCleanedToken, rowCounter, playerGenerator, generatorMappingForRelationPlayer, playerVariable)); ArrayList rel = new ArrayList<>(); rel.add(playerRole); rel.add(playerVariable); @@ -226,7 +226,7 @@ private Collection> createPlayerMatchAndInsert(String // String playerVariable = playerGenerator.getPlayerType() + "-" + playerCounter + "-" + insertCounter; String playerVariable = playerGenerator.getPlayerType() + "-" + playerCounter; String playerRole = playerGenerator.getRoleType(); - players.addAll(createRelationPlayerMatchStatementByPlayers(rowTokens, columnNameIndices, playerGenerator, generatorMappingForRelationPlayer, playerVariable, insertCounter)); + players.addAll(createRelationPlayerMatchStatementByPlayers(rowTokens, rowCounter, columnNameIndices, playerGenerator, generatorMappingForRelationPlayer, playerVariable, insertCounter)); ArrayList rel = new ArrayList<>(); rel.add(playerRole); rel.add(playerVariable); @@ -251,6 +251,7 @@ private Collection> createPlayerMatchAndInsert(String } private ThingVariable createEntityPlayerMatchStatement(String cleanedToken, + int rowCounter, ProcessorConfigEntry.ConceptGenerator playerGenerator, String playerVariable, DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig preprocessorConfig) { @@ -259,21 +260,23 @@ private ThingVariable createEntityPlayerMatchStatement(String cleanedToken, .isa(playerGenerator.getPlayerType()); String attributeType = playerGenerator.getUniquePlayerId(); String attributeValueType = playerGenerator.getIdValueType(); - ms = addAttributeOfColumnType(ms, attributeType, attributeValueType, cleanedToken, preprocessorConfig); + ms = addAttributeOfColumnType(ms, attributeType, attributeValueType, cleanedToken, rowCounter, preprocessorConfig); return ms; } private ThingVariable createAttributePlayerMatchStatement(String cleanedToken, - ProcessorConfigEntry.ConceptGenerator playerGenerator, - String playerVariable, - DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig preprocessorConfig) { + int rowCounter, + ProcessorConfigEntry.ConceptGenerator playerGenerator, + String playerVariable, + DataConfigEntry.DataConfigGeneratorMapping.PreprocessorConfig preprocessorConfig) { UnboundVariable uv = Graql.var(playerVariable); - Attribute ms = addAttributeValueOfType(uv, playerGenerator.getIdValueType(), cleanedToken, preprocessorConfig); + Attribute ms = addAttributeValueOfType(uv, playerGenerator.getIdValueType(), cleanedToken, rowCounter, preprocessorConfig); ms = ms.isa(playerGenerator.getPlayerType()); return ms; } private ThingVariable createRelationPlayerMatchStatementByAttribute(String cleanedToken, + int rowCounter, ProcessorConfigEntry.ConceptGenerator playerGenerator, DataConfigEntry.DataConfigGeneratorMapping dcm, String playerVariable) { @@ -282,11 +285,11 @@ private ThingVariable createRelationPlayerMatchStatementByAttribute(String clean .isa(playerGenerator.getPlayerType()); String attributeType = playerGenerator.getMatchByAttribute().get(dcm.getMatchByAttribute()).getAttributeType(); String attributeValueType = playerGenerator.getMatchByAttribute().get(dcm.getMatchByAttribute()).getValueType(); - ms = addAttributeOfColumnType(ms, attributeType, attributeValueType, cleanedToken, dcm.getPreprocessor()); + ms = addAttributeOfColumnType(ms, attributeType, attributeValueType, cleanedToken, rowCounter, dcm.getPreprocessor()); return ms; } - private ArrayList> createRelationPlayerMatchStatementByPlayers(String[] rowTokens, int[] columnNameIndices, ProcessorConfigEntry.ConceptGenerator playerGenerator, DataConfigEntry.DataConfigGeneratorMapping dcm, String playerVariable, int insertCounter) { + private ArrayList> createRelationPlayerMatchStatementByPlayers(String[] rowTokens, int rowCounter, int[] columnNameIndices, ProcessorConfigEntry.ConceptGenerator playerGenerator, DataConfigEntry.DataConfigGeneratorMapping dcm, String playerVariable, int insertCounter) { ArrayList> assembledMatchStatements = new ArrayList<>(); UnboundVariable relVariable = Graql.var(playerVariable); ArrayList> relationStrings = new ArrayList<>(); @@ -303,7 +306,7 @@ private ArrayList> createRelationPlayerMatchStatementByPlayers( String relationPlayerPlayerAttributeValueType = playerGenerator.getMatchByPlayer().get(dcm.getMatchByPlayers()[i]).getIdValueType(); Thing relationPlayerCurrentPlayerMatchStatement = Graql.var(relationPlayerPlayerVariable).isa(relationPlayerPlayerType); - relationPlayerCurrentPlayerMatchStatement = addAttributeOfColumnType(relationPlayerCurrentPlayerMatchStatement, relationPlayerPlayerAttributeType, relationPlayerPlayerAttributeValueType, cleanedToken, dcm.getPreprocessor()); + relationPlayerCurrentPlayerMatchStatement = addAttributeOfColumnType(relationPlayerCurrentPlayerMatchStatement, relationPlayerPlayerAttributeType, relationPlayerPlayerAttributeValueType, cleanedToken, rowCounter, dcm.getPreprocessor()); assembledMatchStatements.add(relationPlayerCurrentPlayerMatchStatement); // here add the matched player to the relation statement (i.e.: (role: $variable)): diff --git a/src/main/java/generator/RelationWithRelationInsertGenerator.java b/src/main/java/generator/RelationWithRelationInsertGenerator.java deleted file mode 100644 index fd0cae9..0000000 --- a/src/main/java/generator/RelationWithRelationInsertGenerator.java +++ /dev/null @@ -1,18 +0,0 @@ -package generator; - -import configuration.DataConfigEntry; -import configuration.ProcessorConfigEntry; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class RelationWithRelationInsertGenerator extends RelationInsertGenerator { - - private static final Logger appLogger = LogManager.getLogger("com.bayer.dt.grami"); - private static final Logger dataLogger = LogManager.getLogger("com.bayer.dt.grami.data"); - - public RelationWithRelationInsertGenerator(DataConfigEntry dce, ProcessorConfigEntry processorConfigEntry) { - super(dce, processorConfigEntry); - appLogger.debug("Creating RelationWithRelationInsertGenerator for " + pce.getProcessor() + " of type " + pce.getProcessorType()); - } - -} \ No newline at end of file diff --git a/src/main/java/migrator/EntryMigrationConfig.java b/src/main/java/migrator/EntryMigrationConfig.java new file mode 100644 index 0000000..15979e4 --- /dev/null +++ b/src/main/java/migrator/EntryMigrationConfig.java @@ -0,0 +1,42 @@ +package migrator; + +import configuration.DataConfigEntry; +import configuration.ProcessorConfigEntry; +import generator.InsertGenerator; + +public class EntryMigrationConfig { + private final DataConfigEntry dce; + private final ProcessorConfigEntry pce; + private final String migrationStatusKey; + private final Integer migratedRows; + private final InsertGenerator insertGenerator; + + public EntryMigrationConfig(DataConfigEntry dce, ProcessorConfigEntry pce, String migrationStatusKey, Integer migratedRows, InsertGenerator insertGenerator) { + this.dce = dce; + this.pce = pce; + this.migrationStatusKey = migrationStatusKey; + this.migratedRows = migratedRows; + this.insertGenerator = insertGenerator; + } + + + public DataConfigEntry getDce() { + return dce; + } + + public ProcessorConfigEntry getPce() { + return pce; + } + + public String getMigrationStatusKey() { + return migrationStatusKey; + } + + public Integer getMigratedRows() { + return migratedRows; + } + + public InsertGenerator getInsertGenerator() { + return insertGenerator; + } +} diff --git a/src/main/java/migrator/GraknMigrator.java b/src/main/java/migrator/GraknMigrator.java index 03015da..16fdf89 100644 --- a/src/main/java/migrator/GraknMigrator.java +++ b/src/main/java/migrator/GraknMigrator.java @@ -2,32 +2,37 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; -import configuration.*; +import configuration.DataConfigEntry; +import configuration.MigrationConfig; +import configuration.MigrationStatus; +import configuration.ProcessorConfigEntry; import generator.*; import grakn.client.api.GraknClient; import grakn.client.api.GraknSession; import graql.lang.pattern.variable.ThingVariable; -import loader.DataLoader; import insert.GraknInserter; +import loader.DataLoader; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.io.*; import java.lang.reflect.Type; import java.math.RoundingMode; import java.text.DecimalFormat; import java.util.ArrayList; import java.util.HashMap; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import java.util.List; public class GraknMigrator { + private static final Logger appLogger = LogManager.getLogger("com.bayer.dt.grami"); private final HashMap dataConfig; private final String migrationStatePath; - private boolean cleanAndMigrate = false; - private HashMap migrationStatus; private final GraknInserter graknInserter; private final MigrationConfig migrationConfig; - private static final Logger appLogger = LogManager.getLogger("com.bayer.dt.grami"); + private boolean cleanAndMigrate = false; + private HashMap migrationStatus; public GraknMigrator(MigrationConfig migrationConfig, String migrationStatePath) { @@ -51,7 +56,7 @@ public GraknMigrator(MigrationConfig migrationConfig, } } - public void migrate(boolean migrateEntities, boolean migrateRelations, boolean migrateRelationRelations, boolean migrateAppendAttributes) throws IOException { + public void migrate() throws IOException { initializeMigrationStatus(); GraknClient client = graknInserter.getClient(); @@ -64,62 +69,115 @@ public void migrate(boolean migrateEntities, boolean migrateRelations, boolean m } GraknSession dataSession = graknInserter.getDataSession(client); - migrateThingsInOrder(dataSession, migrateEntities, migrateRelations, migrateRelationRelations, migrateAppendAttributes); + migrateThingsInOrder(dataSession); dataSession.close(); client.close(); appLogger.info("GraMi is finished migrating your stuff!"); } - private void migrateThingsInOrder(GraknSession session, boolean migrateEntities, boolean migrateRelations, boolean migrateRelationRelations, boolean migrateAppendAttributes) throws IOException { + private void migrateThingsInOrder(GraknSession session) throws IOException { + List migrationBatch = new ArrayList<>(); + // independent attributes appLogger.info("migrating independent attributes..."); - getStatusAndMigrate(session, "attribute"); + migrationBatch = getEntryMigrationConfigsByProcessorType("attribute"); + for (EntryMigrationConfig conf : migrationBatch) { + appLogger.info("starting migration for: [" + conf.getMigrationStatusKey() + "]"); + batchDataBuildQueriesAndInsert(conf, session); + updateMigrationStatusIsCompleted(conf.getMigrationStatusKey()); + appLogger.info("migration for: [" + conf.getMigrationStatusKey() + "] is completed"); + } appLogger.info("migration of independent attributes completed"); - if (migrateEntities) { - appLogger.info("migrating entities..."); - getStatusAndMigrate(session, "entity"); - appLogger.info("migration of entities completed"); + // entities + appLogger.info("migrating entities..."); + migrationBatch = getEntryMigrationConfigsByProcessorType("entity"); + for (EntryMigrationConfig conf : migrationBatch) { + appLogger.info("starting migration for: [" + conf.getMigrationStatusKey() + "]"); + batchDataBuildQueriesAndInsert(conf, session); + updateMigrationStatusIsCompleted(conf.getMigrationStatusKey()); + appLogger.info("migration for: [" + conf.getMigrationStatusKey() + "] is completed"); + } + appLogger.info("migration of entities completed"); + + // relations containing only entities + appLogger.info("migrating entity-relations..."); + migrationBatch = getEntryMigrationConfigsByProcessorType("relation"); + for (EntryMigrationConfig conf : migrationBatch) { + appLogger.info("starting migration for: [" + conf.getMigrationStatusKey() + "]"); + batchDataBuildQueriesAndInsert(conf, session); + updateMigrationStatusIsCompleted(conf.getMigrationStatusKey()); + appLogger.info("migration for: [" + conf.getMigrationStatusKey() + "] is completed"); } - if (migrateEntities && migrateRelations) { - appLogger.info("migrating relations..."); - getStatusAndMigrate(session, "relation"); - appLogger.info("migration of relations completed"); + appLogger.info("migration of entity-relations completed"); + + // nested relations + appLogger.info("migrating nested-relations..."); + migrationBatch = getEntryMigrationConfigsByProcessorType("nested-relation"); + for (EntryMigrationConfig conf : migrationBatch) { + appLogger.info("starting migration for: [" + conf.getMigrationStatusKey() + "]"); + batchDataBuildQueriesAndInsert(conf, session); + updateMigrationStatusIsCompleted(conf.getMigrationStatusKey()); + appLogger.info("migration for: [" + conf.getMigrationStatusKey() + "] is completed"); } - if (migrateEntities && migrateRelations && migrateRelationRelations) { - appLogger.info("migrating relation-with-relations..."); - getStatusAndMigrate(session, "relation-with-relation"); - appLogger.info("migration of relation-with-relations completed"); + appLogger.info("migration of nested-relations completed"); + + // append attributes + appLogger.info("migrating append-attribute..."); + migrationBatch = getEntryMigrationConfigsByProcessorType("append-attribute"); + for (EntryMigrationConfig conf : migrationBatch) { + appLogger.info("starting migration for: [" + conf.getMigrationStatusKey() + "]"); + batchDataBuildQueriesAndInsert(conf, session); + updateMigrationStatusIsCompleted(conf.getMigrationStatusKey()); + appLogger.info("migration for: [" + conf.getMigrationStatusKey() + "] is completed"); } - if (migrateEntities && migrateRelations && migrateRelationRelations && migrateAppendAttributes) { - appLogger.info("migrating append-attribute..."); - getStatusAndMigrate(session, "append-attribute"); - appLogger.info("migration of append-attribute completed"); - } //TODO: relations with attributes have to come at end + appLogger.info("migration of append-attribute completed"); + + + // attribute relations + appLogger.info("migrating attribute-relations..."); + migrationBatch = getEntryMigrationConfigsByProcessorType("attribute-relation"); + for (EntryMigrationConfig conf : migrationBatch) { + appLogger.info("starting migration for: [" + conf.getMigrationStatusKey() + "]"); + batchDataBuildQueriesAndInsert(conf, session); + updateMigrationStatusIsCompleted(conf.getMigrationStatusKey()); + appLogger.info("migration for: [" + conf.getMigrationStatusKey() + "] is completed"); + } + appLogger.info("migration of attribute-relations completed"); + + // dependency migrations (go by number, low to high) +// appLogger.info("migrating dependency-migrations..."); +// getStatusAndMigrateByDependency(session); +// appLogger.info("migration of attribute-relations completed"); + } - private void getStatusAndMigrate(GraknSession session, String processorType) throws IOException { + private List getEntryMigrationConfigsByProcessorType(String processorType) { + List entries = new ArrayList<>(); for (String dcEntryKey : dataConfig.keySet()) { DataConfigEntry dce = dataConfig.get(dcEntryKey); - String currentProcessor = dce.getProcessor(); - if(isOfProcessorType(currentProcessor, processorType)){ - appLogger.info("migrating [" + dcEntryKey + "]..."); - String migrationStatusKey = dcEntryKey + "-" + dce.getDataPath(); - if (migrationStatus != null && migrationStatus.get(migrationStatusKey) != null) { + ProcessorConfigEntry pce = getProcessorConfigEntry(dce.getProcessor()); + String migrationStatusKey = dcEntryKey + "-" + dce.getDataPath(); + + if (isOfProcessorType(dce.getProcessor(), processorType) && dce.getOrder() == null) { + if (migrationStatus != null && migrationStatus.get(migrationStatusKey) != null) { // previous migration present appLogger.info("previous migration status found for: [" + migrationStatusKey + "]"); if (!migrationStatus.get(migrationStatusKey).isCompleted()) { appLogger.info(migrationStatusKey + " not completely migrated yet, rows already migrated: " + migrationStatus.get(migrationStatusKey).getMigratedRows()); - getGeneratorAndInsert(session, dce, migrationStatus.get(migrationStatusKey).getMigratedRows(), migrationStatusKey); - } else { + EntryMigrationConfig entry = new EntryMigrationConfig(dce, pce, migrationStatusKey, migrationStatus.get(migrationStatusKey).getMigratedRows(), getProcessor(dce)); + entries.add(entry); + } else { // migration already completed appLogger.info(migrationStatusKey + " is already completely migrated - moving on..."); } - } else { + } else { // no previous migration appLogger.info("nothing previously migrated for [" + migrationStatusKey + "] - starting with row 0"); - getGeneratorAndInsert(session, dce, 0, migrationStatusKey); + EntryMigrationConfig entry = new EntryMigrationConfig(dce, pce, migrationStatusKey, 0, getProcessor(dce)); + entries.add(entry); } } } + return entries; } private boolean isOfProcessorType(String key, String conceptType) { @@ -131,19 +189,20 @@ private boolean isOfProcessorType(String key, String conceptType) { return false; } - private void getGeneratorAndInsert(GraknSession session, DataConfigEntry dce, int skipRows, String migrationStatusKey) throws IOException { - // choose insert generator - InsertGenerator gen = getProcessor(dce); - - writeThingToGrakn(dce, gen, session, skipRows, migrationStatusKey); - updateMigrationStatusIsCompleted(migrationStatusKey); + private ProcessorConfigEntry getProcessorConfigEntry(String key) { + for (ProcessorConfigEntry gce : migrationConfig.getProcessorConfig().get("processors")) { + if (gce.getProcessor().equals(key)) { + return gce; + } + } + return null; } - private void writeThingToGrakn(DataConfigEntry dce, InsertGenerator gen, GraknSession session, int skipLines, String migrationStatusKey) { + private void batchDataBuildQueriesAndInsert(EntryMigrationConfig conf, GraknSession session) { - appLogger.info("inserting using " + dce.getThreads() + " threads" + " with thread commit size of " + dce.getBatchSize() + " rows"); + appLogger.info("inserting using " + conf.getDce().getThreads() + " threads" + " with thread commit size of " + conf.getDce().getBatchSize() + " rows"); - InputStream entityStream = DataLoader.getInputStream(dce.getDataPath()); + InputStream entityStream = DataLoader.getInputStream(conf.getDce().getDataPath()); String header = ""; ArrayList rows = new ArrayList<>(); String line; @@ -162,29 +221,29 @@ private void writeThingToGrakn(DataConfigEntry dce, InsertGenerator gen, GraknSe } // skip over all lines already added into KG - if (totalRecordCounter <= skipLines) { + if (totalRecordCounter <= conf.getMigratedRows()) { continue; } // insert Batch once chunk size is reached rows.add(line); batchSizeCounter++; // if (batchSizeCounter == dce.getBatchSize()) { - if (batchSizeCounter == dce.getBatchSize() * dce.getThreads()) { + if (batchSizeCounter == conf.getDce().getBatchSize() * conf.getDce().getThreads()) { System.out.print("+"); System.out.flush(); - writeThing(dce, gen, session, rows, batchSizeCounter, header, migrationStatusKey); + buildQueriesAndInsert(conf, session, rows, batchSizeCounter, totalRecordCounter, header); batchSizeCounter = 0; rows.clear(); } // logging if (totalRecordCounter % 50000 == 0) { System.out.println(); - appLogger.info("processed " + totalRecordCounter/1000 + "k rows"); + appLogger.info("processed " + totalRecordCounter / 1000 + "k rows"); } } //insert the rest when loop exits with less than batch size if (!rows.isEmpty()) { - writeThing(dce, gen, session, rows, batchSizeCounter, header, migrationStatusKey); + buildQueriesAndInsert(conf, session, rows, batchSizeCounter, totalRecordCounter, header); if (totalRecordCounter % 50000 != 0) { System.out.println(); } @@ -201,31 +260,34 @@ private void writeThingToGrakn(DataConfigEntry dce, InsertGenerator gen, GraknSe } - - private void writeThing(DataConfigEntry dce, InsertGenerator gen, GraknSession session, ArrayList rows, int lineCounter, String header, String migrationStatusKey) throws IOException { - int threads = dce.getThreads(); + private void buildQueriesAndInsert(EntryMigrationConfig conf, GraknSession session, ArrayList rows, int lineCounter, int rowCounter, String header) throws IOException { + int threads = conf.getDce().getThreads(); try { - if (isOfProcessorType(dce.getProcessor(), "entity")) { - ArrayList> insertStatements = gen.graknEntityInsert(rows, header); + if (isOfProcessorType(conf.getDce().getProcessor(), "entity")) { + ArrayList> insertStatements = conf.getInsertGenerator().graknEntityInsert(rows, header, rowCounter - lineCounter); appLogger.trace("number of generated insert Statements: " + insertStatements.size()); - graknInserter.insertThreadedInserting(insertStatements, session, threads, dce.getBatchSize()); - } else if (isOfProcessorType(dce.getProcessor(), "relation") || - isOfProcessorType(dce.getProcessor(), "relation-with-relation")) { - HashMap>>> statements = gen.graknRelationInsert(rows, header); + graknInserter.insertThreadedInserting(insertStatements, session, threads, conf.getDce().getBatchSize()); + + } else if (isOfProcessorType(conf.getDce().getProcessor(), "relation") || + isOfProcessorType(conf.getDce().getProcessor(), "nested-relation") || + isOfProcessorType(conf.getDce().getProcessor(), "attribute-relation") ) { + HashMap>>> statements = conf.getInsertGenerator().graknRelationInsert(rows, header, rowCounter - lineCounter); appLogger.trace("number of generated insert Statements: " + statements.get("match").size()); - graknInserter.matchInsertThreadedInserting(statements, session, threads, dce.getBatchSize()); - } else if (isOfProcessorType(dce.getProcessor(), "append-attribute")) { - HashMap>>> statements = gen.graknAppendAttributeInsert(rows, header); + graknInserter.matchInsertThreadedInserting(statements, session, threads, conf.getDce().getBatchSize()); + + } else if (isOfProcessorType(conf.getDce().getProcessor(), "append-attribute")) { + HashMap>>> statements = conf.getInsertGenerator().graknAppendAttributeInsert(rows, header, rowCounter - lineCounter); appLogger.trace("number of generated insert Statements: " + statements.get("match").size()); - graknInserter.matchInsertThreadedInserting(statements, session, threads, dce.getBatchSize()); - } else if (isOfProcessorType(dce.getProcessor(), "attribute")) { - ArrayList> statements = gen.graknAttributeInsert(rows, header); + graknInserter.matchInsertThreadedInserting(statements, session, threads, conf.getDce().getBatchSize()); + + } else if (isOfProcessorType(conf.getDce().getProcessor(), "attribute")) { + ArrayList> statements = conf.getInsertGenerator().graknAttributeInsert(rows, header, rowCounter - lineCounter); appLogger.trace("number of generated insert Statements: " + statements.size()); - graknInserter.insertThreadedInserting(statements, session, threads, dce.getBatchSize()); + graknInserter.insertThreadedInserting(statements, session, threads, conf.getDce().getBatchSize()); } else { - throw new IllegalArgumentException("the processor <" + dce.getProcessor() + "> is not known"); + throw new IllegalArgumentException("the processor <" + conf.getDce().getProcessor() + "> is not known - please check your processor config"); } - updateMigrationStatusMigratedRows(dce, lineCounter, migrationStatusKey); + updateMigrationStatusMigratedRows(conf, lineCounter); } catch (Exception ee) { ee.printStackTrace(); } @@ -239,29 +301,30 @@ private void initializeMigrationStatus() { BufferedReader bufferedReader; try { bufferedReader = new BufferedReader(new FileReader(migrationStatePath)); - Type MigrationStatusType = new TypeToken>() {}.getType(); + Type MigrationStatusType = new TypeToken>() { + }.getType(); migrationStatus = new Gson().fromJson(bufferedReader, MigrationStatusType); } catch (FileNotFoundException e) { e.printStackTrace(); } } - private void updateMigrationStatusMigratedRows(DataConfigEntry dce, int lineCounter, String migrationStatusKey) throws IOException { + private void updateMigrationStatusMigratedRows(EntryMigrationConfig conf, int lineCounter) throws IOException { try { - ProcessorConfigEntry pce = getGenFromGenConfig(dce.getProcessor(), migrationConfig.getProcessorConfig()); Gson gson = new Gson(); - Type MigrationStatusMapType = new TypeToken>(){}.getType(); + Type MigrationStatusMapType = new TypeToken>() { + }.getType(); if (migrationStatus != null) { - if (migrationStatus.get(migrationStatusKey) != null) { //updating an existing entry - int updatedMigratedRows = migrationStatus.get(migrationStatusKey).getMigratedRows() + lineCounter; - migrationStatus.get(migrationStatusKey).setMigratedRows(updatedMigratedRows); + if (migrationStatus.get(conf.getMigrationStatusKey()) != null) { //updating an existing entry + int updatedMigratedRows = migrationStatus.get(conf.getMigrationStatusKey()).getMigratedRows() + lineCounter; + migrationStatus.get(conf.getMigrationStatusKey()).setMigratedRows(updatedMigratedRows); } else { // writing new entry - migrationStatus.put(migrationStatusKey, new MigrationStatus(pce.getSchemaType(), false, lineCounter)); + migrationStatus.put(conf.getMigrationStatusKey(), new MigrationStatus(conf.getPce().getSchemaType(), false, lineCounter)); } } else { //writing very first entry (i.e. file was empty) migrationStatus = new HashMap<>(); - migrationStatus.put(migrationStatusKey, new MigrationStatus(pce.getSchemaType(), false, lineCounter)); + migrationStatus.put(conf.getMigrationStatusKey(), new MigrationStatus(conf.getPce().getSchemaType(), false, lineCounter)); } // update file @@ -276,7 +339,8 @@ private void updateMigrationStatusMigratedRows(DataConfigEntry dce, int lineCoun private void updateMigrationStatusIsCompleted(String migrationStatusKey) throws IOException { try { Gson gson = new Gson(); - Type MigrationStatusMapType = new TypeToken>(){}.getType(); + Type MigrationStatusMapType = new TypeToken>() { + }.getType(); migrationStatus.get(migrationStatusKey).setCompleted(true); FileWriter fw = new FileWriter(migrationStatePath); gson.toJson(migrationStatus, MigrationStatusMapType, fw); @@ -295,9 +359,12 @@ private InsertGenerator getProcessor(DataConfigEntry dce) { } else if (gce != null && gce.getProcessorType().equals("relation")) { appLogger.debug("selected generator: " + gce.getProcessor() + " of type: " + gce.getProcessorType() + " based on dataConfig.generator: " + dce.getProcessor()); return new RelationInsertGenerator(dce, gce); - } else if (gce != null && gce.getProcessorType().equals("relation-with-relation")) { + } else if (gce != null && gce.getProcessorType().equals("nested-relation")) { appLogger.debug("selected generator: " + gce.getProcessor() + " of type: " + gce.getProcessorType() + " based on dataConfig.generator: " + dce.getProcessor()); - return new RelationWithRelationInsertGenerator(dce, gce); + return new NestedRelationInsertGenerator(dce, gce); + } else if (gce != null && gce.getProcessorType().equals("attribute-relation")) { + appLogger.debug("selected generator: " + gce.getProcessor() + " of type: " + gce.getProcessorType() + " based on dataConfig.generator: " + dce.getProcessor()); + return new RelationInsertGenerator(dce, gce); } else if (gce != null && gce.getProcessorType().equals("append-attribute")) { appLogger.debug("selected generator: " + gce.getProcessor() + " of type: " + gce.getProcessorType() + " based on dataConfig.generator: " + dce.getProcessor()); return new AppendAttributeGenerator(dce, gce); diff --git a/src/test/java/generator/AttributeInsertGeneratorTest.java b/src/test/java/generator/AttributeInsertGeneratorTest.java index 4dfc5c7..00979c6 100644 --- a/src/test/java/generator/AttributeInsertGeneratorTest.java +++ b/src/test/java/generator/AttributeInsertGeneratorTest.java @@ -29,7 +29,7 @@ public void graknAttributeQueryFromRowTest() { String header = rows.get(0); rows = new ArrayList<>(rows.subList(1, rows.size())); - ArrayList> result = testAttributeInsertGenerator.graknAttributeInsert(rows, header); + ArrayList> result = testAttributeInsertGenerator.graknAttributeInsert(rows, header, 1); String tc0 = "$a \"yes\" isa is-in-use"; Assert.assertEquals(tc0, result.get(0).toString()); diff --git a/src/test/java/generator/EntityInsertGeneratorTest.java b/src/test/java/generator/EntityInsertGeneratorTest.java index 2ab5298..60fef11 100644 --- a/src/test/java/generator/EntityInsertGeneratorTest.java +++ b/src/test/java/generator/EntityInsertGeneratorTest.java @@ -33,7 +33,7 @@ public void graknEntityQueryFromRowTest() { String header = rows.get(0); rows = new ArrayList<>(rows.subList(1, rows.size())); - ArrayList > result = testEntityInsertGenerator.graknEntityInsert(rows, header); + ArrayList > result = testEntityInsertGenerator.graknEntityInsert(rows, header, 1); String tc0 = "$e isa entity1, has entity1-id \"entity1id0\", has entity1-name \"entity1name0\", has entity1-exp \"entity1id0exp0\""; Assert.assertEquals(tc0, result.get(0).toString()); @@ -108,7 +108,7 @@ public void graknEntityQueryFromRowWithBoolAndDoubleTest() { String header = rows.get(0); rows = new ArrayList<>(rows.subList(1, rows.size())); - ArrayList> result = testEntityInsertGenerator.graknEntityInsert(rows, header); + ArrayList> result = testEntityInsertGenerator.graknEntityInsert(rows, header, 1); String tc0 = "$e isa entity2, has entity2-id \"entity2id0\", has entity2-bool true, has entity2-double 0.0"; Assert.assertEquals(tc0, result.get(0).toString()); @@ -155,7 +155,7 @@ public void graknEntityQueryFromRowWithLongTest() { String header = rows.get(0); rows = new ArrayList<>(rows.subList(1, rows.size())); - ArrayList> result = testEntityInsertGenerator.graknEntityInsert(rows, header); + ArrayList> result = testEntityInsertGenerator.graknEntityInsert(rows, header, 1); String tc0 = "$e isa entity3, has entity3-id \"entity3id0\", has entity3-int 0"; Assert.assertEquals(tc0, result.get(0).toString()); diff --git a/src/test/java/generator/RelationInsertGeneratorTest.java b/src/test/java/generator/RelationInsertGeneratorTest.java index b5dbcfa..ce9ade4 100644 --- a/src/test/java/generator/RelationInsertGeneratorTest.java +++ b/src/test/java/generator/RelationInsertGeneratorTest.java @@ -31,7 +31,7 @@ public void graknRelationQueryFromRowTest() throws Exception { String header = rows.get(0); rows = new ArrayList<>(rows.subList(1, rows.size())); - HashMap>>> result = testRelationInsertGenerator.graknRelationInsert(rows, header); + HashMap>>> result = testRelationInsertGenerator.graknRelationInsert(rows, header, 1); // test all there String tc2m = "$entity1-0 isa entity1, has entity1-id \"entity1id1\";$entity2-1 isa entity2, has entity2-id \"entity2id1\";$entity3-2 isa entity3, has entity3-id \"entity3id1\";"; diff --git a/src/test/java/migrator/MigrationTest.java b/src/test/java/migrator/MigrationTest.java index 5048961..ce6a17d 100644 --- a/src/test/java/migrator/MigrationTest.java +++ b/src/test/java/migrator/MigrationTest.java @@ -35,7 +35,7 @@ public void migrateGenericTestsTest() throws IOException { MigrationConfig migrationConfig = new MigrationConfig(graknURI,databaseName, asp, adcp, gcp); GraknMigrator mig = new GraknMigrator(migrationConfig, msp, true); - mig.migrate(true, true, true,true); + mig.migrate(); } @Test @@ -49,14 +49,14 @@ public void migratePhoneCallsTest() throws IOException { MigrationConfig migrationConfig = new MigrationConfig(graknURI,databaseName, asp, adcp, gcp); GraknMigrator mig = new GraknMigrator(migrationConfig, msp, true); - mig.migrate(true, true, true,true); + mig.migrate(); GraknInserter gi = new GraknInserter(graknURI.split(":")[0], graknURI.split(":")[1], asp, databaseName); GraknClient client = gi.getClient(); GraknSession session = gi.getDataSession(client); testEntities(session); testRelations(session); - testRelationWithRelations(session); + testNestedRelations(session); testAppendAttribute(session); testAttributes(session); testAttributeRelation(session); @@ -137,7 +137,7 @@ public void testRelations(GraknSession session) { read.close(); } - public void testRelationWithRelations(GraknSession session) { + public void testNestedRelations(GraknSession session) { // query specific communication-channel and count the number of past calls (single past-call): GraknTransaction read = session.transaction(GraknTransaction.Type.READ); @@ -283,7 +283,7 @@ public void issue10Test() throws IOException { MigrationConfig migrationConfig = new MigrationConfig(graknURI,databaseName, asp, adcp, gcp); GraknMigrator mig = new GraknMigrator(migrationConfig, msp, true); - mig.migrate(true, true, true,true); + mig.migrate(); } } diff --git a/src/test/java/migrator/SchemaUpdaterTest.java b/src/test/java/migrator/SchemaUpdaterTest.java index 23f3a57..82b515b 100644 --- a/src/test/java/migrator/SchemaUpdaterTest.java +++ b/src/test/java/migrator/SchemaUpdaterTest.java @@ -32,7 +32,7 @@ public void updateSchemaTest() throws IOException { MigrationConfig migrationConfig = new MigrationConfig(graknURI,databaseName, asp, adcp, gcp); GraknMigrator mig = new GraknMigrator(migrationConfig, msp, true); - mig.migrate(true, true, true,true); + mig.migrate(); GraknInserter gi = new GraknInserter(graknURI.split(":")[0], graknURI.split(":")[1], asp, databaseName); diff --git a/src/test/resources/genericTests/migrationStatus-test.json b/src/test/resources/genericTests/migrationStatus-test.json index 21f8d94..e69de29 100644 --- a/src/test/resources/genericTests/migrationStatus-test.json +++ b/src/test/resources/genericTests/migrationStatus-test.json @@ -1 +0,0 @@ -{"entity3-src/test/resources/genericTests/entity3-test-data.tsv":{"conceptName":"entity3","isCompleted":true,"migratedRows":12},"entity2-src/test/resources/genericTests/entity2-test-data.tsv":{"conceptName":"entity2","isCompleted":true,"migratedRows":12},"rel1-src/test/resources/genericTests/rel1-test-data.tsv":{"conceptName":"rel1","isCompleted":true,"migratedRows":44},"entity1-src/test/resources/genericTests/entity1-test-data.tsv":{"conceptName":"entity1","isCompleted":true,"migratedRows":32}} \ No newline at end of file diff --git a/src/test/resources/phone-calls/call.csv b/src/test/resources/phone-calls/call.csv index fe86398..f2b1596 100644 --- a/src/test/resources/phone-calls/call.csv +++ b/src/test/resources/phone-calls/call.csv @@ -16,7 +16,7 @@ caller_id,callee_id,started_at,duration +81 308 988 7153,+351 515 605 7915,2018-09-23T22:23:25,336 +81 308 988 7153,+351 515 605 7915,2018-09-24T22:23:25,332 +81 308 988 7153,+351 515 605 7915,2018-09-25T22:23:25,331 -+261 860 539 4754,+351 272 414 6570,2018-09-26T05:34:19,405 ++261 860 539 4754,+351 272 414 6570,2018-09-26T05:34:19,lala +62 107 530 7500,+263 498 495 0617,2018-09-25T22:58:02,5665 +54 398 559 0423,+86 892 682 0628,2018-09-23T08:55:18,822 +7 690 597 4443,+54 398 559 0423,2018-09-25T09:10:25,8494 @@ -112,7 +112,7 @@ caller_id,callee_id,started_at,duration +63 815 962 6097,+63 808 497 1769,2018-09-17T07:29:13,1036 +54 398 559 0423,+81 308 988 7153,2018-09-27T08:22:32,6468 +370 351 224 5176,+33 614 339 0298,2018-09-17T01:58:04,77 -+63 815 962 6097,+263 498 495 0617,2018-09-19T23:16:49,2519 ++63 815 962 6097,+263 498 495 0617,2018-09-19T23:16:49,lulu +81 746 154 2598,+1 254 875 4647,2018-09-15T07:02:32,914 +62 107 530 7500,+81 308 988 7153,2018-09-22T04:57:20,4455 +54 398 559 0423,+62 107 530 7500,2018-09-28T01:57:20,5272 diff --git a/src/test/resources/phone-calls/processorConfig.json b/src/test/resources/phone-calls/processorConfig.json index 997656a..3f4ae3a 100644 --- a/src/test/resources/phone-calls/processorConfig.json +++ b/src/test/resources/phone-calls/processorConfig.json @@ -113,7 +113,7 @@ }, { "processor": "communication-channel", - "processorType": "relation-with-relation", + "processorType": "nested-relation", "schemaType": "communication-channel", "conceptGenerators": { "players": { @@ -225,7 +225,7 @@ }, { "processor": "in-use", - "processorType": "relation", + "processorType": "attribute-relation", "schemaType": "in-use", "conceptGenerators": { "players": {