Skip to content

Commit

Permalink
changes for final release
Browse files Browse the repository at this point in the history
  • Loading branch information
@BenediktKleppmann
BenediktKleppmann committed Jul 24, 2018
1 parent b55d490 commit 63ea59c
Show file tree
Hide file tree
Showing 2 changed files with 80 additions and 32 deletions.
77 changes: 51 additions & 26 deletions DS4DM-Webservice/DS4DM_webservice/app/controllers/ExtendTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@
//import model.QueryTable;
import play.mvc.Controller;
import tests.CsvTableParser_keepCase;
import tests.SaveTableToCsv;
import uploadTable.FindCorrespondences;
import uploadTable.TableIndexer;
import uploadTable.additionalWinterClasses.MatchableTableColumn;
Expand Down Expand Up @@ -613,25 +614,35 @@ public void test2(String repository) throws IOException, org.apache.lucene.query


public Result test3(){
try{
// Process process = Runtime.getRuntime().exec("python public/exampleData/temp_tables/correlation_based_filtering.py 0.1 \"Museum\" \"Visitors 2010\"");

// ProcessBuilder builder = new ProcessBuilder("python", "public/exampleData/temp_tables/correlation_based_filtering.py", "0.1", "Museum", "Visitors 2010");
// Process process = builder.start();

Process process = Runtime.getRuntime().exec(new String[] { "bash", "-c", "python public/exampleData/temp_tables/correlation_based_filtering.py 0.1 \"Museum\" \"Visitors 2010\"" });
Scanner scanner = new Scanner(process.getInputStream());

while (scanner.hasNext()) {
System.out.println(scanner.nextLine());
}
process.waitFor();
System.out.println("line3");
System.out.println(String.valueOf(process.exitValue()));

int len;
if ((len = process.getErrorStream().available()) > 0) {
byte[] buf = new byte[len];
process.getErrorStream().read(buf);
out.println("Command error:\t\""+new String(buf)+"\"");
}

uploadTable.Table fusedTable = new uploadTable.Table();
String[][] relation = new String[3][4];
relation[0][0] = "0 ";
relation[1][0] = "1 ";
relation[2][0] = "2 ";
relation[0][1] = "";
relation[1][1] = "";
relation[2][1] = "";
relation[0][2] = null;
relation[1][2] = null;
relation[0][3] = "0 ";
relation[1][3] = "1 ";
relation[2][3] = "2 ";


fusedTable.setRelation(relation);
Gson gson = new Gson();
String return_string = gson.toJson(fusedTable, uploadTable.Table.class);
return ok(return_string);

return ok("done");
} catch (Exception e){e.printStackTrace();return ok("exception");}
}


Expand All @@ -655,7 +666,13 @@ public Result test3(){
*/

public Result correlationBasedSearch(String repositoryName) throws IOException, InterruptedException{
System.out.println("Unconstrianed Search....");

try{
FileOutputStream fileOutputStream = new FileOutputStream(new File("public/exampleData/test3.txt"));
System.setOut(new PrintStream(fileOutputStream));
}catch (FileNotFoundException e){e.printStackTrace();}
System.out.println("correlationBasedSearch....");

if (repositoryName == null || repositoryName=="") repositoryName = "DefaultRepository";


Expand Down Expand Up @@ -684,9 +701,12 @@ public Result correlationBasedSearch(String repositoryName) throws IOException,
/*******************************************************
* Unconstrianed Search
*******************************************************/
System.out.println("Unconstrianed Search");
System.out.println("Unconstrianed Search...");
Table fused = UnconstrainedSearch.getFusedTable(queryTableObject, repositoryName);


// TESTING vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
// SaveTableToCsv.save(fused, "fused");
// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

/*******************************************************
* save the fused-table to csv
Expand Down Expand Up @@ -731,18 +751,23 @@ public Result correlationBasedSearch(String repositoryName) throws IOException,
Double minimumCorrelation = queryTableObject.getMinimumCorrelation();
if (minimumCorrelation == null){minimumCorrelation = 0.4;}

Process process = Runtime.getRuntime().exec("python public/exampleData/temp_tables/correlation_based_filtering.py " + String.valueOf(minimumCorrelation) + " " + fused.getSubjectColumn().getHeader() + " " + queryTableObject.getCorrelationAttribute());

// Process process = Runtime.getRuntime().exec("python public/exampleData/temp_tables/correlation_based_filtering.py " + String.valueOf(minimumCorrelation) + " \"" + fused.getSubjectColumn().getHeader() + "\" \"" + queryTableObject.getCorrelationAttribute() + "\"");
// Process process = new ProcessBuilder("python", "public/exampleData/temp_tables/correlation_based_filtering.py", String.valueOf(minimumCorrelation), fused.getSubjectColumn().getHeader(), queryTableObject.getCorrelationAttribute()).start();

Process process = Runtime.getRuntime().exec(new String[] { "bash", "-c", "python public/exampleData/temp_tables/correlation_based_filtering.py " + String.valueOf(minimumCorrelation) + " '" + fused.getSubjectColumn().getHeader() + "' '" + queryTableObject.getCorrelationAttribute() + "'" });

Scanner scanner = new Scanner(process.getInputStream());

PrintWriter out = new PrintWriter("public/exampleData/temp_tables/test.txt");
out.println("public/exampleData/temp_tables/correlation_based_filtering.py 0.4");
out.println("line2");
System.out.println("python public/exampleData/temp_tables/correlation_based_filtering.py " + String.valueOf(minimumCorrelation) + " '" + fused.getSubjectColumn().getHeader() + "' '" + queryTableObject.getCorrelationAttribute() + "'" );
System.out.println("line2");
while (scanner.hasNext()) {
out.println(scanner.nextLine());
System.out.println(scanner.nextLine());
}
process.waitFor();
out.println("line3");
out.println(String.valueOf(process.exitValue()));
System.out.println("line3");
System.out.println(String.valueOf(process.exitValue()));

int len;
if ((len = process.getErrorStream().available()) > 0) {
Expand Down
35 changes: 29 additions & 6 deletions DS4DM-Webservice/DS4DM_webservice/app/unconstrainedSearch/UnconstrainedSearch.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
package unconstrainedSearch;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
Expand All @@ -22,6 +23,8 @@
import java.util.Map;
import java.util.Set;

import com.opencsv.CSVWriter;

import de.uni_mannheim.informatik.dws.winter.matching.aggregators.TopKCorrespondencesAggregator;
import de.uni_mannheim.informatik.dws.winter.matching.rules.AggregateBySecondRecordRule;
import de.uni_mannheim.informatik.dws.winter.matching.rules.FlattenAggregatedCorrespondencesRule;
Expand All @@ -46,14 +49,14 @@
public class UnconstrainedSearch {


public static Table getFusedTable(model.QueryTable queryTableObject, String repositoryName){
public static Table getFusedTable(model.QueryTable queryTableObject, String repositoryName) throws IOException{

queryTableObject.saveToFile("public/exampleData/temp_query_table.csv");
CsvTableParser csvtableparser = new CsvTableParser();
Table queryTable = csvtableparser.parseTable(new File("public/exampleData/temp_query_table.csv"));

// TESTING vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
SaveTableToCsv.save(queryTable, "queryTable");
// SaveTableToCsv.save(queryTable, "queryTable");
// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

Table fused = null;
Expand Down Expand Up @@ -85,9 +88,16 @@ public static Table getFusedTable(model.QueryTable queryTableObject, String repo
List<String> foundTablenames = SearchForTables.search(queryTableObject, keyColumnIndex, repositoryName);
System.out.println("Unconstrianed Search7.1");



if(foundTablenames!=null && foundTablenames.size()>0) {
System.out.println("Unconstrianed Search7.2");

// TESTING vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv

CSVWriter csvwriter = new CSVWriter(new FileWriter("/home/bkleppma/ds4dm_webservice/DS4DM/DS4DM_webservice/public/exampleData/testTable.csv"), ';');
// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

// load the tables from the search result
Collection<Table> tables = new LinkedList<>();
int foundTableId = 1;
Expand All @@ -97,9 +107,18 @@ public static Table getFusedTable(model.QueryTable queryTableObject, String repo
foundTable.setTableId(foundTableId++);
tablesById.put(foundTable.getTableId(), foundTable);
tables.add(foundTable);
// TESTING vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
String[] new_row = {foundTablename};
csvwriter.writeNext(new_row);
// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
}
}

// TESTING vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
csvwriter.close();
// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^


// add the query table to the search results
// it will match the query table perfectly and make sure that all records are kept in the final result
// even if no other table contained a certain record
Expand All @@ -108,6 +127,9 @@ public static Table getFusedTable(model.QueryTable queryTableObject, String repo
tablesById.put(queryTable.getTableId(), queryTable);
tables.add(queryTable);




/*******************************************************
* SCHEMA MATCHING
*******************************************************/
Expand All @@ -119,7 +141,7 @@ public static Table getFusedTable(model.QueryTable queryTableObject, String repo
FusibleDataSet<MatchableTableRow, MatchableTableColumn> tablesDS = loader.createTablesDataSet(tables);

// TESTING vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
SaveTableToCsv.save(queryDS, "queryDS");
// SaveTableToCsv.save(queryDS, "queryDS");
// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

// run schema matching
Expand Down Expand Up @@ -173,8 +195,9 @@ public static Table getFusedTable(model.QueryTable queryTableObject, String repo
queryDS = loader.createQueryDataSet(queryConsolidated);
tablesDS = loader.createQueryDataSet(tablesConsolidated);


// TESTING vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
SaveTableToCsv.save(queryDS, "queryDS2");
// SaveTableToCsv.save(queryDS, "queryDS2");
// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

// run identity resolution
Expand Down Expand Up @@ -203,15 +226,15 @@ public static Table getFusedTable(model.QueryTable queryTableObject, String repo
fused = fuser.fuseTables(queryConsolidated, queryDS, tablesDS, recordCorrespondences);

// TESTING vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
SaveTableToCsv.save(fused, "fused");
// SaveTableToCsv.save(fused, "fused");
// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

Double minimumDensity = queryTableObject.getMinimumDensity();
if (minimumDensity==null){minimumDensity= 0.6;}
fused = consolidator.removeSparseColumns(fused, minimumDensity); // remove columns that are mostly NULL

// TESTING vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
SaveTableToCsv.save(fused, "fused2");
// SaveTableToCsv.save(fused, "fused2");
// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

} catch (Exception e) {e.printStackTrace();}
Expand Down

0 comments on commit 63ea59c

Please sign in to comment.