From 63ea59c9a4776c12124e13fc5e73bda80333cd80 Mon Sep 17 00:00:00 2001 From: BenediktKleppmann Date: Tue, 24 Jul 2018 10:53:39 +0200 Subject: [PATCH] changes for final release --- .../app/controllers/ExtendTable.java | 77 ++++++++++++------- .../UnconstrainedSearch.java | 35 +++++++-- 2 files changed, 80 insertions(+), 32 deletions(-) diff --git a/DS4DM-Webservice/DS4DM_webservice/app/controllers/ExtendTable.java b/DS4DM-Webservice/DS4DM_webservice/app/controllers/ExtendTable.java index ed172ed..e167dc3 100644 --- a/DS4DM-Webservice/DS4DM_webservice/app/controllers/ExtendTable.java +++ b/DS4DM-Webservice/DS4DM_webservice/app/controllers/ExtendTable.java @@ -140,6 +140,7 @@ //import model.QueryTable; import play.mvc.Controller; import tests.CsvTableParser_keepCase; +import tests.SaveTableToCsv; import uploadTable.FindCorrespondences; import uploadTable.TableIndexer; import uploadTable.additionalWinterClasses.MatchableTableColumn; @@ -613,25 +614,35 @@ public void test2(String repository) throws IOException, org.apache.lucene.query public Result test3(){ + try{ +// Process process = Runtime.getRuntime().exec("python public/exampleData/temp_tables/correlation_based_filtering.py 0.1 \"Museum\" \"Visitors 2010\""); + +// ProcessBuilder builder = new ProcessBuilder("python", "public/exampleData/temp_tables/correlation_based_filtering.py", "0.1", "Museum", "Visitors 2010"); +// Process process = builder.start(); + + Process process = Runtime.getRuntime().exec(new String[] { "bash", "-c", "python public/exampleData/temp_tables/correlation_based_filtering.py 0.1 \"Museum\" \"Visitors 2010\"" }); + Scanner scanner = new Scanner(process.getInputStream()); + + while (scanner.hasNext()) { + System.out.println(scanner.nextLine()); + } + process.waitFor(); + System.out.println("line3"); + System.out.println(String.valueOf(process.exitValue())); + + int len; + if ((len = process.getErrorStream().available()) > 0) { + byte[] buf = new byte[len]; + process.getErrorStream().read(buf); + out.println("Command error:\t\""+new String(buf)+"\""); + } - uploadTable.Table fusedTable = new uploadTable.Table(); - String[][] relation = new String[3][4]; - relation[0][0] = "0 "; - relation[1][0] = "1 "; - relation[2][0] = "2 "; - relation[0][1] = ""; - relation[1][1] = ""; - relation[2][1] = ""; - relation[0][2] = null; - relation[1][2] = null; - relation[0][3] = "0 "; - relation[1][3] = "1 "; - relation[2][3] = "2 "; + - fusedTable.setRelation(relation); - Gson gson = new Gson(); - String return_string = gson.toJson(fusedTable, uploadTable.Table.class); - return ok(return_string); + + + return ok("done"); + } catch (Exception e){e.printStackTrace();return ok("exception");} } @@ -655,7 +666,13 @@ public Result test3(){ */ public Result correlationBasedSearch(String repositoryName) throws IOException, InterruptedException{ - System.out.println("Unconstrianed Search...."); + + try{ + FileOutputStream fileOutputStream = new FileOutputStream(new File("public/exampleData/test3.txt")); + System.setOut(new PrintStream(fileOutputStream)); + }catch (FileNotFoundException e){e.printStackTrace();} + System.out.println("correlationBasedSearch...."); + if (repositoryName == null || repositoryName=="") repositoryName = "DefaultRepository"; @@ -684,9 +701,12 @@ public Result correlationBasedSearch(String repositoryName) throws IOException, /******************************************************* * Unconstrianed Search *******************************************************/ - System.out.println("Unconstrianed Search"); + System.out.println("Unconstrianed Search..."); Table fused = UnconstrainedSearch.getFusedTable(queryTableObject, repositoryName); - + + // TESTING vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv +// SaveTableToCsv.save(fused, "fused"); + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /******************************************************* * save the fused-table to csv @@ -731,18 +751,23 @@ public Result correlationBasedSearch(String repositoryName) throws IOException, Double minimumCorrelation = queryTableObject.getMinimumCorrelation(); if (minimumCorrelation == null){minimumCorrelation = 0.4;} - Process process = Runtime.getRuntime().exec("python public/exampleData/temp_tables/correlation_based_filtering.py " + String.valueOf(minimumCorrelation) + " " + fused.getSubjectColumn().getHeader() + " " + queryTableObject.getCorrelationAttribute()); + +// Process process = Runtime.getRuntime().exec("python public/exampleData/temp_tables/correlation_based_filtering.py " + String.valueOf(minimumCorrelation) + " \"" + fused.getSubjectColumn().getHeader() + "\" \"" + queryTableObject.getCorrelationAttribute() + "\""); +// Process process = new ProcessBuilder("python", "public/exampleData/temp_tables/correlation_based_filtering.py", String.valueOf(minimumCorrelation), fused.getSubjectColumn().getHeader(), queryTableObject.getCorrelationAttribute()).start(); + + Process process = Runtime.getRuntime().exec(new String[] { "bash", "-c", "python public/exampleData/temp_tables/correlation_based_filtering.py " + String.valueOf(minimumCorrelation) + " '" + fused.getSubjectColumn().getHeader() + "' '" + queryTableObject.getCorrelationAttribute() + "'" }); + Scanner scanner = new Scanner(process.getInputStream()); PrintWriter out = new PrintWriter("public/exampleData/temp_tables/test.txt"); - out.println("public/exampleData/temp_tables/correlation_based_filtering.py 0.4"); - out.println("line2"); + System.out.println("python public/exampleData/temp_tables/correlation_based_filtering.py " + String.valueOf(minimumCorrelation) + " '" + fused.getSubjectColumn().getHeader() + "' '" + queryTableObject.getCorrelationAttribute() + "'" ); + System.out.println("line2"); while (scanner.hasNext()) { - out.println(scanner.nextLine()); + System.out.println(scanner.nextLine()); } process.waitFor(); - out.println("line3"); - out.println(String.valueOf(process.exitValue())); + System.out.println("line3"); + System.out.println(String.valueOf(process.exitValue())); int len; if ((len = process.getErrorStream().available()) > 0) { diff --git a/DS4DM-Webservice/DS4DM_webservice/app/unconstrainedSearch/UnconstrainedSearch.java b/DS4DM-Webservice/DS4DM_webservice/app/unconstrainedSearch/UnconstrainedSearch.java index 37d658a..c0999ad 100644 --- a/DS4DM-Webservice/DS4DM_webservice/app/unconstrainedSearch/UnconstrainedSearch.java +++ b/DS4DM-Webservice/DS4DM_webservice/app/unconstrainedSearch/UnconstrainedSearch.java @@ -12,6 +12,7 @@ package unconstrainedSearch; import java.io.File; +import java.io.FileWriter; import java.io.IOException; import java.util.Arrays; import java.util.Collection; @@ -22,6 +23,8 @@ import java.util.Map; import java.util.Set; +import com.opencsv.CSVWriter; + import de.uni_mannheim.informatik.dws.winter.matching.aggregators.TopKCorrespondencesAggregator; import de.uni_mannheim.informatik.dws.winter.matching.rules.AggregateBySecondRecordRule; import de.uni_mannheim.informatik.dws.winter.matching.rules.FlattenAggregatedCorrespondencesRule; @@ -46,14 +49,14 @@ public class UnconstrainedSearch { - public static Table getFusedTable(model.QueryTable queryTableObject, String repositoryName){ + public static Table getFusedTable(model.QueryTable queryTableObject, String repositoryName) throws IOException{ queryTableObject.saveToFile("public/exampleData/temp_query_table.csv"); CsvTableParser csvtableparser = new CsvTableParser(); Table queryTable = csvtableparser.parseTable(new File("public/exampleData/temp_query_table.csv")); // TESTING vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - SaveTableToCsv.save(queryTable, "queryTable"); +// SaveTableToCsv.save(queryTable, "queryTable"); // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Table fused = null; @@ -85,9 +88,16 @@ public static Table getFusedTable(model.QueryTable queryTableObject, String repo List foundTablenames = SearchForTables.search(queryTableObject, keyColumnIndex, repositoryName); System.out.println("Unconstrianed Search7.1"); + + if(foundTablenames!=null && foundTablenames.size()>0) { System.out.println("Unconstrianed Search7.2"); +// TESTING vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + + CSVWriter csvwriter = new CSVWriter(new FileWriter("/home/bkleppma/ds4dm_webservice/DS4DM/DS4DM_webservice/public/exampleData/testTable.csv"), ';'); +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + // load the tables from the search result Collection tables = new LinkedList<>(); int foundTableId = 1; @@ -97,9 +107,18 @@ public static Table getFusedTable(model.QueryTable queryTableObject, String repo foundTable.setTableId(foundTableId++); tablesById.put(foundTable.getTableId(), foundTable); tables.add(foundTable); + // TESTING vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + String[] new_row = {foundTablename}; + csvwriter.writeNext(new_row); + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ } } +// TESTING vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + csvwriter.close(); +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + // add the query table to the search results // it will match the query table perfectly and make sure that all records are kept in the final result // even if no other table contained a certain record @@ -108,6 +127,9 @@ public static Table getFusedTable(model.QueryTable queryTableObject, String repo tablesById.put(queryTable.getTableId(), queryTable); tables.add(queryTable); + + + /******************************************************* * SCHEMA MATCHING *******************************************************/ @@ -119,7 +141,7 @@ public static Table getFusedTable(model.QueryTable queryTableObject, String repo FusibleDataSet tablesDS = loader.createTablesDataSet(tables); // TESTING vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - SaveTableToCsv.save(queryDS, "queryDS"); +// SaveTableToCsv.save(queryDS, "queryDS"); // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ // run schema matching @@ -173,8 +195,9 @@ public static Table getFusedTable(model.QueryTable queryTableObject, String repo queryDS = loader.createQueryDataSet(queryConsolidated); tablesDS = loader.createQueryDataSet(tablesConsolidated); + // TESTING vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - SaveTableToCsv.save(queryDS, "queryDS2"); +// SaveTableToCsv.save(queryDS, "queryDS2"); // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ // run identity resolution @@ -203,7 +226,7 @@ public static Table getFusedTable(model.QueryTable queryTableObject, String repo fused = fuser.fuseTables(queryConsolidated, queryDS, tablesDS, recordCorrespondences); // TESTING vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - SaveTableToCsv.save(fused, "fused"); +// SaveTableToCsv.save(fused, "fused"); // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Double minimumDensity = queryTableObject.getMinimumDensity(); @@ -211,7 +234,7 @@ public static Table getFusedTable(model.QueryTable queryTableObject, String repo fused = consolidator.removeSparseColumns(fused, minimumDensity); // remove columns that are mostly NULL // TESTING vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - SaveTableToCsv.save(fused, "fused2"); +// SaveTableToCsv.save(fused, "fused2"); // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ } catch (Exception e) {e.printStackTrace();}