add pagination to results table (#96)

* Add pagination to results table
CERT-Polska · Apr 20, 2020 · fe49b19 · fe49b19
1 parent cfc00f9
commit fe49b19
Show file tree

Hide file tree

Showing 6 changed files with 89 additions and 37 deletions.
diff --git a/src/daemon.py b/src/daemon.py
@@ -162,18 +162,19 @@ def execute_yara(job: JobId, files: List[str]) -> None:
         return
 
     rule = compile_yara(job)
-
+    len_matches = 0
     for sample in files:
         try:
             matches = rule.match(sample)
             if matches:
+                len_matches += 1
                 update_metadata(job, sample, [r.rule for r in matches])
         except yara.Error:
             logging.exception(f"Yara failed to check file {sample}")
         except FileNotFoundError:
             logging.exception(f"Failed to open file for yara check: {sample}")
 
-    db.update_job(job, len(files))
+    db.update_job(job, len(files), len_matches)
 
 
 def execute_search(job_id: JobId) -> None:

diff --git a/src/db.py b/src/db.py
@@ -127,12 +127,16 @@ def set_job_to_processing(
                 "status": "processing",
                 "iterator": iterator,
                 "files_processed": 0,
+                "files_matched": 0,
                 "total_files": file_count,
             },
         )
 
-    def update_job(self, job: JobId, files_processed: int) -> None:
+    def update_job(
+        self, job: JobId, files_processed: int, files_matched: int
+    ) -> None:
         self.redis.hincrby(job.key, "files_processed", files_processed)
+        self.redis.hincrby(job.key, "files_matched", files_matched)
 
     def set_job_to_parsing(self, job: JobId) -> None:
         """ Sets the job status to parsing """
@@ -168,8 +172,9 @@ def get_job(self, job: JobId) -> JobSchema:
             raw_yara=data.get("raw_yara", "ERROR"),
             submitted=data.get("submitted", 0),
             priority=data.get("priority", "medium"),
-            files_processed=data.get("files_processed", 0),
-            total_files=data.get("total_files", 0),
+            files_processed=int(data.get("files_processed", 0)),
+            files_matched=int(data.get("files_matched", 0)),
+            total_files=int(data.get("total_files", 0)),
             iterator=data.get("iterator", None),
             taint=data.get("taint", None),
         )

diff --git a/src/mqueryfront/package.json b/src/mqueryfront/package.json
@@ -14,7 +14,8 @@
     "react-dom": "^16.7.0",
     "react-router-dom": "^4.3.1",
     "react-scripts": "1.1.4",
-    "webpack-dev-server": "^3.1.14"
+    "webpack-dev-server": "^3.1.14",
+    "react-js-pagination": "^3.0.3"
   },
   "scripts": {
     "start": "react-scripts start",

diff --git a/src/mqueryfront/src/QueryPage.js b/src/mqueryfront/src/QueryPage.js
@@ -23,6 +23,7 @@ class QueryPage extends Component {
             queryPlan: null,
             queryError: null,
             datasets: {},
+            activePage: 1,
         };
 
         this.updateQhash = this.updateQhash.bind(this);
@@ -78,12 +79,14 @@ class QueryPage extends Component {
             qhash: newQhash,
             matches: [],
             job: [],
+            activePage: 1,
         });
-        this.loadMatches();
+        this.loadJob();
     }
 
-    loadMatches() {
-        const LIMIT = 50;
+    loadJob() {
+        const LIMIT = 20;
+        let OFFSET = (this.state.activePage - 1) * 20;
 
         if (!this.state.qhash) {
             return;
@@ -95,40 +98,48 @@ class QueryPage extends Component {
                     "/matches/" +
                     this.state.qhash +
                     "?offset=" +
-                    this.state.matches.length +
+                    OFFSET +
                     "&limit=" +
                     LIMIT
             )
             .then((response) => {
-                let newShouldRequest = true;
-
-                if (
-                    ["done", "cancelled", "failed", "expired"].indexOf(
-                        response.data.job.status
-                    ) !== -1
-                ) {
-                    if (!response.data.matches.length) {
-                        newShouldRequest = false;
-                    }
-                }
-
+                let job = response.data.job;
                 this.setState({
-                    matches: [...this.state.matches, ...response.data.matches],
-                    job: response.data.job,
+                    job: job,
+                    matches: response.data.matches,
                 });
-
-                if (newShouldRequest) {
-                    let nextTimeout =
-                        response.data.matches.length >= LIMIT ? 50 : 1000;
-                    this.timeout = setTimeout(
-                        () => this.loadMatches(),
-                        nextTimeout
-                    );
+                let doneStatuses = ["cancelled", "failed", "expired"];
+                let isDone = doneStatuses.indexOf(job.status) !== -1;
+                let processedAll = job.files_processed >= job.total_files;
+                if (isDone || processedAll) {
+                    return;
                 }
-            })
-            .catch(() => {
+                this.timeout = setTimeout(() => this.loadJob(), 1000);
+            });
+    }
+
+    callbackResultsActivePage = (pageNumber) => {
+        this.setState({ activePage: pageNumber }, () => {
+            this.loadMatches();
+        });
+    };
+
+    loadMatches() {
+        const LIMIT = 20;
+        let OFFSET = (this.state.activePage - 1) * 20;
+        axios
+            .get(
+                API_URL +
+                    "/matches/" +
+                    this.state.qhash +
+                    "?offset=" +
+                    OFFSET +
+                    "&limit=" +
+                    LIMIT
+            )
+            .then((response) => {
                 this.setState({
-                    shouldRequest: false,
+                    matches: response.data.matches,
                 });
             });
     }
@@ -169,6 +180,7 @@ class QueryPage extends Component {
                 queryError={this.state.queryError}
             />
         );
+
         var queryResults = (
             <div>
                 <button
@@ -183,6 +195,7 @@ class QueryPage extends Component {
                     qhash={this.state.qhash}
                     job={this.state.job}
                     matches={this.state.matches}
+                    parentCallback={this.callbackResultsActivePage}
                 />
             </div>
         );

diff --git a/src/mqueryfront/src/QueryResultsStatus.js b/src/mqueryfront/src/QueryResultsStatus.js
@@ -1,6 +1,7 @@
 import React, { Component } from "react";
 import axios from "axios/index";
 import { API_URL } from "./config";
+import Pagination from "react-js-pagination";
 
 function MatchItem(props) {
     const metadata = Object.values(props.meta).map((v) => (
@@ -63,13 +64,27 @@ class QueryResultsStatus extends Component {
     constructor(props) {
         super(props);
 
+        this.state = {
+            activePage: 1,
+            itemsPerPage: 20,
+        };
+
         this.handleCancelJob = this.handleCancelJob.bind(this);
     }
 
     handleCancelJob() {
         axios.delete(API_URL + "/job/" + this.props.qhash);
     }
 
+    sendResultsActivePage = (pageNumber) => {
+        this.props.parentCallback(pageNumber);
+    };
+
+    handlePageChange(pageNumber) {
+        this.setState({ activePage: pageNumber });
+        this.sendResultsActivePage(pageNumber);
+    }
+
     renderSwitchStatus(status) {
         switch (status) {
             case "done":
@@ -86,6 +101,12 @@ class QueryResultsStatus extends Component {
         }
     }
 
+    componentDidUpdate(prevProps) {
+        if (prevProps.qhash !== this.props.qhash) {
+            this.setState({ activePage: 1 });
+        }
+    }
+
     render() {
         if (this.props.job && this.props.job.error) {
             return (
@@ -142,12 +163,11 @@ class QueryResultsStatus extends Component {
             cancel = <span />;
         }
 
-        const lenMatches = this.props.matches.length;
+        const lenMatches = this.props.job.files_matched;
 
         if (this.props.job.status === "expired") {
             return ReturnExpiredJob(this.props.job.error);
         }
-
         let results = <div />;
 
         if (lenMatches === 0 && this.props.job.status === "done") {
@@ -164,6 +184,17 @@ class QueryResultsStatus extends Component {
                         </thead>
                         <tbody>{matches}</tbody>
                     </table>
+                    {lenMatches > 0 && (
+                        <Pagination
+                            activePage={this.state.activePage}
+                            itemsCountPerPage={this.state.itemsPerPage}
+                            totalItemsCount={lenMatches}
+                            pageRangeDisplayed={5}
+                            onChange={this.handlePageChange.bind(this)}
+                            itemClass="page-item"
+                            linkClass="page-link"
+                        />
+                    )}
                 </div>
             );
         }

diff --git a/src/schema.py b/src/schema.py
@@ -13,6 +13,7 @@ class JobSchema(BaseModel):
     submitted: int
     priority: str
     files_processed: int
+    files_matched: int
     total_files: int
     iterator: Optional[str]
     taint: Optional[str]