diff --git a/code/ArgumentIR/Resources/views/proposal-info.html b/code/ArgumentIR/Resources/views/proposal-info.html index 1b4c3c7..150eb0a 100644 --- a/code/ArgumentIR/Resources/views/proposal-info.html +++ b/code/ArgumentIR/Resources/views/proposal-info.html @@ -2,13 +2,13 @@
$TITLE$
- $DATE$ • Comments: $NUM_COMMENTS$ • Supports: $NUM_SUPPORTS$
- Proposal code: $CODE$ + Proposal $CODE$ of $DATE$
+ Arguments: $NUM_ARGUMENTS$ • Comments: $NUM_COMMENTS$ • Supports: $NUM_SUPPORTS$

$SUMMARY$


- Districts: $DISTRICTS$
- Categories: $CATEGORIES$
- Topics: $TOPICS$
- Link: $URL$
+ - Districts: $DISTRICTS$
+ - Categories: $CATEGORIES$
+ - Topics: $TOPICS$
+

Link: $URL$

Comments:

$COMMENTS$
diff --git a/code/ArgumentIR/src/es/uam/irg/decidemadrid/db/MongoDbManager.java b/code/ArgumentIR/src/es/uam/irg/decidemadrid/db/MongoDbManager.java index fc33c59..3d11995 100644 --- a/code/ArgumentIR/src/es/uam/irg/decidemadrid/db/MongoDbManager.java +++ b/code/ArgumentIR/src/es/uam/irg/decidemadrid/db/MongoDbManager.java @@ -23,8 +23,10 @@ import com.mongodb.client.MongoDatabase; import com.mongodb.client.model.Filters; import com.mongodb.client.model.UpdateOptions; +import es.uam.irg.nlp.am.arguments.Argument; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -40,16 +42,16 @@ public class MongoDbManager { // Public constants + public static final String DB_COLLECTION = "annotations"; public static final String DB_NAME = "decide_madrid_2019_09"; public static final int DB_PORT = 27017; public static final String DB_SERVER = "localhost"; - public static final String DB_COLLECTION = "annotations"; private static final String NO_TOPIC = "-"; // Private connector object + private String collName; private MongoDatabase db; private MongoClient mongoClient; - private String collName; /** * Manager constructor. @@ -72,15 +74,15 @@ public MongoDbManager(String client, int port, String database, String collectio } /** - * - * @param setup + * + * @param setup */ public MongoDbManager(Map setup) { String client = setup.get("db_server").toString(); int port = Integer.parseInt(setup.get("db_port").toString()); String database = setup.get("db_name").toString(); String collection = setup.get("db_collection").toString(); - + this.mongoClient = new MongoClient(client, port); this.db = mongoClient.getDatabase(database); this.collName = collection; @@ -124,6 +126,42 @@ public List getDocumentsByFilter(String topic, Integer[] customProposa return docs; } + /** + * + * @param maxTreeLevel + * @return + */ + public Map> selectProposalArguments(int maxTreeLevel) { + Map> arguments = new HashMap<>(); + int proposalId; + Argument argument; + + try { + + // Query documents + MongoCollection collection = db.getCollection(collName); + FindIterable cursor = collection.find(); + + for (Iterator it = cursor.iterator(); it.hasNext();) { + Document doc = it.next(); + argument = new Argument(doc); + proposalId = argument.getProposalId(); + + if (argument.getTreeLevel() <= maxTreeLevel) { + if (!arguments.containsKey(proposalId)) { + arguments.put(proposalId, new ArrayList<>()); + } + arguments.get(proposalId).add(argument); + } + } + + } catch (Exception ex) { + System.err.println("MongoDB error: " + ex.getMessage()); + } + + return arguments; + } + /** * * @param doc diff --git a/code/ArgumentIR/src/es/uam/irg/ir/ArgumentIR.java b/code/ArgumentIR/src/es/uam/irg/ir/ArgumentIR.java index 6fd8031..10e43ad 100644 --- a/code/ArgumentIR/src/es/uam/irg/ir/ArgumentIR.java +++ b/code/ArgumentIR/src/es/uam/irg/ir/ArgumentIR.java @@ -18,6 +18,7 @@ package es.uam.irg.ir; import es.uam.irg.ir.gui.ArgumentIRForm; +import es.uam.irg.utils.FunctionUtils; /** * Program main class. @@ -46,12 +47,12 @@ private static void showWinform() { /* Create and display the form */ java.awt.EventQueue.invokeLater(() -> { - System.out.println(">> ARG-IR BEGINS"); + FunctionUtils.printWithDatestamp(">> ARG-IR BEGINS"); ArgumentIRForm form = new ArgumentIRForm(); form.setVisible(true); - System.out.println(">> ARG-IR ENDS"); + FunctionUtils.printWithDatestamp(">> ARG-IR ENDS"); }); } diff --git a/code/ArgumentIR/src/es/uam/irg/ir/InfoRetriever.java b/code/ArgumentIR/src/es/uam/irg/ir/InfoRetriever.java index 45c1499..26dfae7 100644 --- a/code/ArgumentIR/src/es/uam/irg/ir/InfoRetriever.java +++ b/code/ArgumentIR/src/es/uam/irg/ir/InfoRetriever.java @@ -100,37 +100,35 @@ public void createIndex(Map proposals, Map queryData(String querystr, int hitsPerPage, String reRankBy) { + public List queryData(String querystr, int hitsPerPage) { List docList = new ArrayList<>(); try { // The "title" arg specifies the default field to use when no field is explicitly specified in the query Query q = new QueryParser("title", analyzer).parse(querystr); - // Search - IndexReader reader = DirectoryReader.open(index); - IndexSearcher searcher = new IndexSearcher(reader); - TopDocs docs = searcher.search(q, hitsPerPage); - ScoreDoc[] hits = docs.scoreDocs; - - // Store results - for (int i = 0; i < hits.length; ++i) { - int docId = hits[i].doc; - Document doc = searcher.doc(docId); - int proposalId = Integer.parseInt(doc.get("id")); - docList.add(proposalId); + // Search within the index + try ( IndexReader reader = DirectoryReader.open(index)) { + IndexSearcher searcher = new IndexSearcher(reader); + TopDocs docs = searcher.search(q, hitsPerPage); + ScoreDoc[] hits = docs.scoreDocs; + + // Store results + for (int i = 0; i < hits.length; ++i) { + int docId = hits[i].doc; + Document doc = searcher.doc(docId); + int proposalId = Integer.parseInt(doc.get("id")); + docList.add(proposalId); + } } - // Reader can only be closed when there is no need to access the documents any more. - reader.close(); - } catch (ParseException | IOException ex) { Logger.getLogger(InfoRetriever.class.getName()).log(Level.SEVERE, null, ex); } diff --git a/code/ArgumentIR/src/es/uam/irg/ir/gui/ArgumentIRModel.java b/code/ArgumentIR/src/es/uam/irg/ir/gui/ArgumentIRModel.java index e5b0af1..32e8073 100644 --- a/code/ArgumentIR/src/es/uam/irg/ir/gui/ArgumentIRModel.java +++ b/code/ArgumentIR/src/es/uam/irg/ir/gui/ArgumentIRModel.java @@ -18,12 +18,15 @@ package es.uam.irg.ir.gui; import es.uam.irg.decidemadrid.db.DMDBManager; +import es.uam.irg.decidemadrid.db.MongoDbManager; import es.uam.irg.decidemadrid.entities.DMComment; import es.uam.irg.decidemadrid.entities.DMCommentTree; import es.uam.irg.decidemadrid.entities.DMProposal; import es.uam.irg.decidemadrid.entities.DMProposalSummary; import es.uam.irg.ir.InfoRetriever; +import es.uam.irg.nlp.am.arguments.Argument; import es.uam.irg.utils.FunctionUtils; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -36,6 +39,7 @@ public class ArgumentIRModel { // Class constants + private static final int MAX_TREE_LEVEL = 2; private static final boolean VERBOSE = true; // Class objects @@ -44,6 +48,7 @@ public class ArgumentIRModel { private final Map msqlSetup; // Class data variables + private Map> proposalArguments; private Map> proposalCommentTrees; private Map proposalComments; private Map proposalSummaries; @@ -84,11 +89,15 @@ public String getQueryResult(String query, int nTop, String reRankBy) { } else { // Query data long start = System.nanoTime(); - List docList = this.retriever.queryData(query, nTop, reRankBy); + List ids = this.retriever.queryData(query, nTop); long finish = System.nanoTime(); double timeElapsed = (finish - start) / 1000000; - int nReports = docList.size(); - System.out.println(">> Found " + nReports + " hits in " + timeElapsed + " ms"); + int nReports = ids.size(); + FunctionUtils.printWithDatestamp(">> Found " + nReports + " hits in " + timeElapsed + " ms"); + + // Rerank data + List docList = sortResults(ids, reRankBy); + FunctionUtils.printWithDatestamp(">> Data reranked by: " + reRankBy); // Create user report if (docList.size() > 0) { @@ -97,13 +106,16 @@ public String getQueryResult(String query, int nTop, String reRankBy) { DMProposal proposal; DMProposalSummary summary; List commentTrees; + List arguments; // Format data for (int docId : docList) { proposal = proposals.get(docId); summary = proposalSummaries.get(docId); commentTrees = proposalCommentTrees.get(docId); - report = this.formatter.getProposalInfoReport(proposal, summary, commentTrees, proposalComments); + arguments = proposalArguments.get(docId); + + report = this.formatter.getProposalInfoReport(proposal, summary, commentTrees, proposalComments, arguments); body.append(report); } @@ -118,7 +130,7 @@ public String getQueryResult(String query, int nTop, String reRankBy) { * */ private void createIndex() { - System.out.println(">> Creating Lucene index..."); + FunctionUtils.printWithDatestamp(">> Creating Lucene index"); this.retriever = new InfoRetriever(); this.retriever.createIndex(proposals, proposalSummaries); } @@ -130,9 +142,10 @@ private void loadData() { proposals = new HashMap<>(); proposalComments = new HashMap<>(); - // Connecting to databse + // Connecting to databases and fetching data try { - System.out.println(">> Loading data..."); + FunctionUtils.printWithDatestamp(">> Creating connections"); + DMDBManager dbManager = null; if (msqlSetup != null && msqlSetup.size() == 4) { dbManager = new DMDBManager(msqlSetup); @@ -140,6 +153,15 @@ private void loadData() { dbManager = new DMDBManager(); } + MongoDbManager mngManager = null; + if (mdbSetup != null && mdbSetup.size() == 4) { + mngManager = new MongoDbManager(mdbSetup); + } else { + mngManager = new MongoDbManager(); + } + + FunctionUtils.printWithDatestamp(">> Loading data"); + // Get proposals proposals = dbManager.selectProposals(); @@ -152,6 +174,9 @@ private void loadData() { // Get comments trees proposalCommentTrees = dbManager.selectCommentTrees(); + // Get arguments data + proposalArguments = mngManager.selectProposalArguments(MAX_TREE_LEVEL); + } catch (Exception ex) { Logger.getLogger(InfoRetriever.class.getName()).log(Level.SEVERE, null, ex); } @@ -159,9 +184,39 @@ private void loadData() { // Show results if (VERBOSE) { System.out.println(" - Number of proposals: " + proposals.size()); - System.out.println(" - Number of proposal summaries: " + proposals.size()); + System.out.println(" - Number of proposal summaries: " + proposalSummaries.size()); System.out.println(" - Number of comments: " + proposalComments.size()); + System.out.println(" - Number of comment trees: " + proposalCommentTrees.size()); + System.out.println(" - Number of arguments: " + proposalArguments.size()); + } + } + + /** + * + * @param docs + * @param reRankBy + * @return + */ + private List sortResults(List ids, String reRankBy) { + List docList = new ArrayList<>(); + + if (reRankBy.equals("Nothing")) { + docList.addAll(ids); + + } else if (reRankBy.equals("Arguments")) { + Map argsByProp = new HashMap<>(); + int nArgs; + + for (int id : ids) { + nArgs = (proposalArguments.containsKey(id) ? proposalArguments.get(id).size() : 0); + argsByProp.put(id, nArgs); + } + + argsByProp = FunctionUtils.sortMapByValue(argsByProp); + docList.addAll(argsByProp.keySet()); } + + return docList; } } diff --git a/code/ArgumentIR/src/es/uam/irg/ir/gui/ReportFormatter.java b/code/ArgumentIR/src/es/uam/irg/ir/gui/ReportFormatter.java index 26c2138..e77661c 100644 --- a/code/ArgumentIR/src/es/uam/irg/ir/gui/ReportFormatter.java +++ b/code/ArgumentIR/src/es/uam/irg/ir/gui/ReportFormatter.java @@ -22,6 +22,7 @@ import es.uam.irg.decidemadrid.entities.DMProposal; import es.uam.irg.decidemadrid.entities.DMProposalSummary; import es.uam.irg.io.IOManager; +import es.uam.irg.nlp.am.arguments.Argument; import java.text.DecimalFormat; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -93,17 +94,19 @@ public String getNoValidQueryReport() { * @param summary * @param commentTrees * @param comments + * @param arguments * @return */ - public String getProposalInfoReport(DMProposal proposal, DMProposalSummary summary, List commentTrees, Map comments) { + public String getProposalInfoReport(DMProposal proposal, DMProposalSummary summary, List commentTrees, Map comments, List arguments) { String report = reports.get("PROPOSAL_INFO"); StringBuilder body = new StringBuilder(); - report = report.replace("$TITLE$", proposal.getTitle()); + report = report.replace("$TITLE$", proposal.getTitle().toUpperCase()); + report = report.replace("$CODE$", proposal.getCode()); report = report.replace("$DATE$", proposal.getDate()); + report = report.replace("$NUM_ARGUMENTS$", "" + (arguments != null ? arguments.size() : 0)); report = report.replace("$NUM_COMMENTS$", "" + proposal.getNumComments()); report = report.replace("$NUM_SUPPORTS$", "" + proposal.getNumSupports()); - report = report.replace("$CODE$", proposal.getCode()); report = report.replace("$CATEGORIES$", summary.getCategories()); report = report.replace("$DISTRICTS$", summary.getDistricts()); report = report.replace("$TOPICS$", summary.getTopics()); diff --git a/code/ArgumentIR/src/es/uam/irg/utils/FunctionUtils.java b/code/ArgumentIR/src/es/uam/irg/utils/FunctionUtils.java index b34802f..f1550e3 100644 --- a/code/ArgumentIR/src/es/uam/irg/utils/FunctionUtils.java +++ b/code/ArgumentIR/src/es/uam/irg/utils/FunctionUtils.java @@ -19,9 +19,12 @@ //import es.uam.irg.io.IOManager; import es.uam.irg.io.IOManager; +import java.text.DateFormat; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; +import java.util.Date; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -37,6 +40,9 @@ public class FunctionUtils { private static final String MDB_SETUP_FILEPATH = "Resources/config/mdb_setup.yaml"; private static final String MSQL_SETUP_FILEPATH = "Resources/config/msql_setup.yaml"; + // Class variables + private static DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); + /** * * @param @@ -86,7 +92,7 @@ public static Map getDatabaseConfiguration(String dbType) { return setup; } - + /** * * @param filename @@ -119,13 +125,21 @@ public static T[] getSubArray(T[] array, int startIx, int endIndex) throws E return newArray; } + /** + * + * @param msg + */ + public static void printWithDatestamp(String msg) { + System.out.println(msg + " - " + dateFormat.format(new Date())); + } + /** * * @param map * @return */ - public static Map sortMapByValue(Map map) { - LinkedHashMap reverseSortedMap = new LinkedHashMap<>(); + public static Map sortMapByValue(Map map) { + LinkedHashMap reverseSortedMap = new LinkedHashMap<>(); // Use Comparator.reverseOrder() for reverse ordering map.entrySet()