From fe49b19f572996b85f63fae4b2854f1f5db7d9a9 Mon Sep 17 00:00:00 2001
From: DJK <63305564+KrzyWie@users.noreply.github.com>
Date: Mon, 20 Apr 2020 16:04:08 +0200
Subject: [PATCH] add pagination to results table (#96)
* Add pagination to results table
---
src/daemon.py | 5 +-
src/db.py | 11 +++-
src/mqueryfront/package.json | 3 +-
src/mqueryfront/src/QueryPage.js | 71 ++++++++++++++---------
src/mqueryfront/src/QueryResultsStatus.js | 35 ++++++++++-
src/schema.py | 1 +
6 files changed, 89 insertions(+), 37 deletions(-)
diff --git a/src/daemon.py b/src/daemon.py
index da6ebefd..77adc61c 100644
--- a/src/daemon.py
+++ b/src/daemon.py
@@ -162,18 +162,19 @@ def execute_yara(job: JobId, files: List[str]) -> None:
return
rule = compile_yara(job)
-
+ len_matches = 0
for sample in files:
try:
matches = rule.match(sample)
if matches:
+ len_matches += 1
update_metadata(job, sample, [r.rule for r in matches])
except yara.Error:
logging.exception(f"Yara failed to check file {sample}")
except FileNotFoundError:
logging.exception(f"Failed to open file for yara check: {sample}")
- db.update_job(job, len(files))
+ db.update_job(job, len(files), len_matches)
def execute_search(job_id: JobId) -> None:
diff --git a/src/db.py b/src/db.py
index 99b8dcd2..bd57a586 100644
--- a/src/db.py
+++ b/src/db.py
@@ -127,12 +127,16 @@ def set_job_to_processing(
"status": "processing",
"iterator": iterator,
"files_processed": 0,
+ "files_matched": 0,
"total_files": file_count,
},
)
- def update_job(self, job: JobId, files_processed: int) -> None:
+ def update_job(
+ self, job: JobId, files_processed: int, files_matched: int
+ ) -> None:
self.redis.hincrby(job.key, "files_processed", files_processed)
+ self.redis.hincrby(job.key, "files_matched", files_matched)
def set_job_to_parsing(self, job: JobId) -> None:
""" Sets the job status to parsing """
@@ -168,8 +172,9 @@ def get_job(self, job: JobId) -> JobSchema:
raw_yara=data.get("raw_yara", "ERROR"),
submitted=data.get("submitted", 0),
priority=data.get("priority", "medium"),
- files_processed=data.get("files_processed", 0),
- total_files=data.get("total_files", 0),
+ files_processed=int(data.get("files_processed", 0)),
+ files_matched=int(data.get("files_matched", 0)),
+ total_files=int(data.get("total_files", 0)),
iterator=data.get("iterator", None),
taint=data.get("taint", None),
)
diff --git a/src/mqueryfront/package.json b/src/mqueryfront/package.json
index 3019815c..636fc17f 100644
--- a/src/mqueryfront/package.json
+++ b/src/mqueryfront/package.json
@@ -14,7 +14,8 @@
"react-dom": "^16.7.0",
"react-router-dom": "^4.3.1",
"react-scripts": "1.1.4",
- "webpack-dev-server": "^3.1.14"
+ "webpack-dev-server": "^3.1.14",
+ "react-js-pagination": "^3.0.3"
},
"scripts": {
"start": "react-scripts start",
diff --git a/src/mqueryfront/src/QueryPage.js b/src/mqueryfront/src/QueryPage.js
index 31b0b4d2..2db52a16 100644
--- a/src/mqueryfront/src/QueryPage.js
+++ b/src/mqueryfront/src/QueryPage.js
@@ -23,6 +23,7 @@ class QueryPage extends Component {
queryPlan: null,
queryError: null,
datasets: {},
+ activePage: 1,
};
this.updateQhash = this.updateQhash.bind(this);
@@ -78,12 +79,14 @@ class QueryPage extends Component {
qhash: newQhash,
matches: [],
job: [],
+ activePage: 1,
});
- this.loadMatches();
+ this.loadJob();
}
- loadMatches() {
- const LIMIT = 50;
+ loadJob() {
+ const LIMIT = 20;
+ let OFFSET = (this.state.activePage - 1) * 20;
if (!this.state.qhash) {
return;
@@ -95,40 +98,48 @@ class QueryPage extends Component {
"/matches/" +
this.state.qhash +
"?offset=" +
- this.state.matches.length +
+ OFFSET +
"&limit=" +
LIMIT
)
.then((response) => {
- let newShouldRequest = true;
-
- if (
- ["done", "cancelled", "failed", "expired"].indexOf(
- response.data.job.status
- ) !== -1
- ) {
- if (!response.data.matches.length) {
- newShouldRequest = false;
- }
- }
-
+ let job = response.data.job;
this.setState({
- matches: [...this.state.matches, ...response.data.matches],
- job: response.data.job,
+ job: job,
+ matches: response.data.matches,
});
-
- if (newShouldRequest) {
- let nextTimeout =
- response.data.matches.length >= LIMIT ? 50 : 1000;
- this.timeout = setTimeout(
- () => this.loadMatches(),
- nextTimeout
- );
+ let doneStatuses = ["cancelled", "failed", "expired"];
+ let isDone = doneStatuses.indexOf(job.status) !== -1;
+ let processedAll = job.files_processed >= job.total_files;
+ if (isDone || processedAll) {
+ return;
}
- })
- .catch(() => {
+ this.timeout = setTimeout(() => this.loadJob(), 1000);
+ });
+ }
+
+ callbackResultsActivePage = (pageNumber) => {
+ this.setState({ activePage: pageNumber }, () => {
+ this.loadMatches();
+ });
+ };
+
+ loadMatches() {
+ const LIMIT = 20;
+ let OFFSET = (this.state.activePage - 1) * 20;
+ axios
+ .get(
+ API_URL +
+ "/matches/" +
+ this.state.qhash +
+ "?offset=" +
+ OFFSET +
+ "&limit=" +
+ LIMIT
+ )
+ .then((response) => {
this.setState({
- shouldRequest: false,
+ matches: response.data.matches,
});
});
}
@@ -169,6 +180,7 @@ class QueryPage extends Component {
queryError={this.state.queryError}
/>
);
+
var queryResults = (
);
diff --git a/src/mqueryfront/src/QueryResultsStatus.js b/src/mqueryfront/src/QueryResultsStatus.js
index a989b217..1c7e6805 100644
--- a/src/mqueryfront/src/QueryResultsStatus.js
+++ b/src/mqueryfront/src/QueryResultsStatus.js
@@ -1,6 +1,7 @@
import React, { Component } from "react";
import axios from "axios/index";
import { API_URL } from "./config";
+import Pagination from "react-js-pagination";
function MatchItem(props) {
const metadata = Object.values(props.meta).map((v) => (
@@ -63,6 +64,11 @@ class QueryResultsStatus extends Component {
constructor(props) {
super(props);
+ this.state = {
+ activePage: 1,
+ itemsPerPage: 20,
+ };
+
this.handleCancelJob = this.handleCancelJob.bind(this);
}
@@ -70,6 +76,15 @@ class QueryResultsStatus extends Component {
axios.delete(API_URL + "/job/" + this.props.qhash);
}
+ sendResultsActivePage = (pageNumber) => {
+ this.props.parentCallback(pageNumber);
+ };
+
+ handlePageChange(pageNumber) {
+ this.setState({ activePage: pageNumber });
+ this.sendResultsActivePage(pageNumber);
+ }
+
renderSwitchStatus(status) {
switch (status) {
case "done":
@@ -86,6 +101,12 @@ class QueryResultsStatus extends Component {
}
}
+ componentDidUpdate(prevProps) {
+ if (prevProps.qhash !== this.props.qhash) {
+ this.setState({ activePage: 1 });
+ }
+ }
+
render() {
if (this.props.job && this.props.job.error) {
return (
@@ -142,12 +163,11 @@ class QueryResultsStatus extends Component {
cancel = ;
}
- const lenMatches = this.props.matches.length;
+ const lenMatches = this.props.job.files_matched;
if (this.props.job.status === "expired") {
return ReturnExpiredJob(this.props.job.error);
}
-
let results = ;
if (lenMatches === 0 && this.props.job.status === "done") {
@@ -164,6 +184,17 @@ class QueryResultsStatus extends Component {
{matches}
+ {lenMatches > 0 && (
+
+ )}
);
}
diff --git a/src/schema.py b/src/schema.py
index 0a3bba83..badbe7e1 100644
--- a/src/schema.py
+++ b/src/schema.py
@@ -13,6 +13,7 @@ class JobSchema(BaseModel):
submitted: int
priority: str
files_processed: int
+ files_matched: int
total_files: int
iterator: Optional[str]
taint: Optional[str]