diff --git a/client/src/hooks/getMentions.js b/client/src/hooks/getMentions.js
index 79b09a9..830d5d0 100644
--- a/client/src/hooks/getMentions.js
+++ b/client/src/hooks/getMentions.js
@@ -3,7 +3,7 @@ import axios from "axios";
export const getMentions = async (dispatch, keyword, platforms, page, sort) => {
const platformsArray = Object.keys(platforms).filter((key) => platforms[key]);
- let platformsString = platformsArray.join();
+ const platformsString = platformsArray.join();
try {
const url = `/api/mentions?platforms=${platformsString}${
@@ -11,7 +11,6 @@ export const getMentions = async (dispatch, keyword, platforms, page, sort) => {
}&page=${page}&sort=${sort}`;
const res = await axios.get(url);
-
return res.data;
} catch (err) {
dispatch({
diff --git a/client/src/layout/Mention.js b/client/src/layout/Mention.js
index 2ba6267..3daea2e 100644
--- a/client/src/layout/Mention.js
+++ b/client/src/layout/Mention.js
@@ -1,4 +1,4 @@
-import React from "react";
+import React, { useContext } from "react";
import {
Typography,
Card,
@@ -9,7 +9,8 @@ import {
import { makeStyles } from "@material-ui/core/styles";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { faSmile } from "@fortawesome/free-regular-svg-icons";
-import redditLogo from "../utils/images/reddit-logo.png"
+import redditLogo from "../utils/images/reddit-logo.png";
+import { UserContext } from "../context/user";
const useStyles = makeStyles((theme) => ({
root: {
@@ -68,19 +69,19 @@ const useStyles = makeStyles((theme) => ({
},
}));
- const image = (image) => {
- if (image === "default" || image === "self") {
- return redditLogo;
- } else {
- return image;
- }
- };
-
+const image = (image) => {
+ if (image === "default" || image === "self") {
+ return redditLogo;
+ } else {
+ return image;
+ }
+};
const Mention = ({ mention }) => {
const classes = useStyles();
const keyword = "DolphinCorp";
- const regex = new RegExp(`${keyword}`, "i");
+ const { searchTerm } = useContext(UserContext);
+ const regex = new RegExp(`${searchTerm}`, "i");
const indexK = mention.title.search(regex);
return (
@@ -94,8 +95,8 @@ const Mention = ({ mention }) => {
{/* {mention.title.substring(0, indexK)}
- {keyword}
- {mention.title.substring(indexK + keyword.length)} */}
+ {searchTerm}
+ {mention.title.substring(indexK + searchTerm.length)} */}
{mention.title}
diff --git a/client/src/pages/HomePage.js b/client/src/pages/HomePage.js
index 8b72170..7a5f126 100644
--- a/client/src/pages/HomePage.js
+++ b/client/src/pages/HomePage.js
@@ -7,6 +7,7 @@ import SortToggle from "../layout/SortToggle";
import { makeStyles } from "@material-ui/core/styles";
import Spinner from "../layout/Spinner";
import { getMentions } from "../hooks/getMentions";
+import InfiniteScroll from "react-infinite-scroller";
const useStyles = makeStyles((theme) => ({
box: {
@@ -23,6 +24,7 @@ const HomePage = () => {
const [switching, setSwitching] = useState(false);
const [currentPage, setCurrentPage] = useState(1);
const [sort, setSort] = useState("date");
+
const { dispatch, error, searchTerm, user } = useContext(UserContext);
const loadMore = async () => {
@@ -60,7 +62,7 @@ const HomePage = () => {
setCurrentPage(1);
})
.catch((err) => alert("Cookie expired. Please log in again"));
- }, [searchTerm, user.platforms]);
+ }, [searchTerm, user.platforms, dispatch]);
if (mentionDatas === null) return ;
@@ -97,6 +99,7 @@ const HomePage = () => {
error={error}
/>
)}
+
diff --git a/client/src/pages/PagesWrapper.js b/client/src/pages/PagesWrapper.js
index c985279..08a11b0 100644
--- a/client/src/pages/PagesWrapper.js
+++ b/client/src/pages/PagesWrapper.js
@@ -7,7 +7,7 @@ const PagesWrapper = ({ children }) => {
useEffect(() => {
authenticate(dispatch);
- }, [isAuthenticated]);
+ }, [isAuthenticated, dispatch]);
return <>{children};
};
diff --git a/client/src/pages/PrivateRoute.js b/client/src/pages/PrivateRoute.js
index 69528e3..2330ef9 100644
--- a/client/src/pages/PrivateRoute.js
+++ b/client/src/pages/PrivateRoute.js
@@ -1,4 +1,4 @@
-import React, { useContext, useEffect } from "react";
+import React, { useContext } from "react";
import { Route, Redirect } from "react-router-dom";
import { UserContext } from "../context/user";
diff --git a/server/app.js b/server/app.js
index 027d48a..a7b3817 100644
--- a/server/app.js
+++ b/server/app.js
@@ -12,6 +12,7 @@ const pingRouter = require("./routes/ping");
const mentionRouter = require("./routes/mentionRoutes");
const userRouter = require("./routes/userRoutes");
const mongoose = require("mongoose");
+const { connectRedis, handleTaskQueues } = require("./utils/jobHandler");
const connectDB = async () => {
try {
@@ -31,6 +32,10 @@ const connectDB = async () => {
connectDB();
+connectRedis(); // if error, try start redis server in background
+
+handleTaskQueues();
+
const { json, urlencoded } = express;
var app = express();
@@ -53,37 +58,4 @@ app.use(notFound);
// error handler
app.use(errorHandler);
-// test redis
-// to start redis server in background
-// run "redis-server --daemonize yes"
-// to stop the server running "redis-cli shutdown"
-const redis = require("redis");
-const client = redis.createClient({
- host: process.env.REDIS_HOST,
- port: process.env.REDIS_PORT,
-});
-client.on("error", (err) => {
- console.log("Error " + err);
-});
-
-client.on("ready", function () {
- console.log("redis is running");
-});
-
-// test bull
-var Queue = require("bull");
-
-const myFirstQueue = new Queue("my-first-queue", {
- redis: { port: process.env.REDIS_PORT, host: process.env.REDIS_HOST },
-});
-
-// job producer
-const job = myFirstQueue.add();
-
-// job consumer
-myFirstQueue.process(function (job, done) {
- // call done when finished
- done(console.log(`Job with id ${job.id} has been completed`));
-});
-
module.exports = app;
diff --git a/server/controllers/mentionController.js b/server/controllers/mentionController.js
index ef906d5..56f8cc5 100644
--- a/server/controllers/mentionController.js
+++ b/server/controllers/mentionController.js
@@ -42,10 +42,17 @@ const getMentions = async (req, res) => {
// Fetching Mentions pertaining to selected platforms
// Sorting handled by MongoDB
-
const fetchMentions = async (array, sorting) => {
const results = await Mention.find({
- $or: [...getPlatformsObject(array)],
+ $and: [
+ {
+ $or: [
+ { title: { $regex: req.user.name, $options: "i" } },
+ { content: { $regex: req.user.name, $options: "i" } },
+ ],
+ },
+ { $or: [...getPlatformsObject(array)] },
+ ],
}).sort(getSortOption(sorting));
return results;
diff --git a/server/controllers/userController.js b/server/controllers/userController.js
index ade4ccf..c7f095a 100644
--- a/server/controllers/userController.js
+++ b/server/controllers/userController.js
@@ -32,10 +32,6 @@ const signUp = async (req, res) => {
secure: false, // should be true in Production !
});
- // Added for Co-op Midterm Presentation
- // To be handled later on by BullMQ
- await addMentionsToDB(user.name);
-
res.status(201).json({
_id: user._id,
name: user.name,
@@ -66,10 +62,6 @@ const signIn = async (req, res) => {
secure: false, // should be true in Production !
});
- // Added for Co-op Midterm Presentation
- // To be handled later on by BullMQ
- await addMentionsToDB(user.name);
-
res.status(201).json({
_id: user._id,
name: user.name,
diff --git a/server/dump.rdb b/server/dump.rdb
new file mode 100644
index 0000000..18c3b4e
Binary files /dev/null and b/server/dump.rdb differ
diff --git a/server/models/mentionModel.js b/server/models/mentionModel.js
index 145d328..53ca084 100644
--- a/server/models/mentionModel.js
+++ b/server/models/mentionModel.js
@@ -25,6 +25,10 @@ const mentionSchema = mongoose.Schema({
type: Number,
default: 0,
},
+ url: {
+ type: String,
+ required: true,
+ },
});
module.exports = mongoose.model("Mention", mentionSchema);
diff --git a/server/models/userModel.js b/server/models/userModel.js
index efe05e0..26e50fe 100644
--- a/server/models/userModel.js
+++ b/server/models/userModel.js
@@ -44,4 +44,8 @@ userSchema.pre("save", async function (next) {
this.password = await bcrypt.hash(this.password, salt);
});
+userSchema.statics.getAllCompanies = async function () {
+ return await this.distinct("name");
+};
+
module.exports = mongoose.model("User", userSchema);
diff --git a/server/utils/jobHandler.js b/server/utils/jobHandler.js
new file mode 100644
index 0000000..e16542e
--- /dev/null
+++ b/server/utils/jobHandler.js
@@ -0,0 +1,71 @@
+const redis = require("redis");
+const Queue = require("bull");
+const User = require("../models/userModel");
+const Mention = require("../models/mentionModel");
+const { addMentionsToDB } = require("./scraper");
+
+const connectRedis = () => {
+ // to start redis server in background
+ // run "redis-server --daemonize yes"
+ // to stop the server running "redis-cli shutdown"
+ const client = redis.createClient({
+ host: process.env.REDIS_HOST,
+ port: process.env.REDIS_PORT,
+ });
+ client.on("error", (err) => {
+ console.log("Error " + err);
+ });
+
+ client.on("ready", function () {
+ console.log("redis is running");
+ });
+};
+
+const handleTaskQueues = () => {
+ const companiesQueue = new Queue("queue-for-getting-componies", {
+ redis: { port: process.env.REDIS_PORT, host: process.env.REDIS_HOST },
+ });
+
+ const scrapingQueue = new Queue("queue-for-scraping", {
+ redis: { port: process.env.REDIS_PORT, host: process.env.REDIS_HOST },
+ });
+ // job producer, repeated every 15 mins
+ companiesQueue.add(
+ {},
+ {
+ repeat: { cron: "*/15 * * * *" },
+ delay: 2000,
+ jobId: "repeatCompaniesUpdate",
+ }
+ );
+
+ // to remove existedQueue use
+ // companiesQueue.empty();
+
+ // job consumer
+ companiesQueue.process(async function (job, done) {
+ try {
+ User.getAllCompanies().then((companies) =>
+ companies.forEach((company) => {
+ for (var platform in User.schema.obj.platforms) {
+ scrapingQueue.add({ company: company, platform: platform });
+ }
+ })
+ );
+ done(new Error("error adding companies to queue"));
+ } catch (err) {
+ console.log(err);
+ }
+ });
+
+ scrapingQueue.process(async function (job, done) {
+ try {
+ await addMentionsToDB(job.data.company, job.data.platform);
+ done(new Error(`${job.data.company} and ${job.data.platform} error`));
+ } catch (err) {
+ console.log(err);
+ }
+ });
+};
+
+module.exports = { connectRedis, handleTaskQueues };
diff --git a/server/utils/reddit.js b/server/utils/reddit.js
index af87826..1ed5f8a 100644
--- a/server/utils/reddit.js
+++ b/server/utils/reddit.js
@@ -16,16 +16,16 @@ const searchRecursive = async (term, after = "", posts = []) => {
try {
const res = await axios.get(url);
-
if (res.status === 200) {
- res.data.data.children.forEach((post) => {
+ res.data.data.children.forEach(async (post) => {
posts.push({
- content: post.data.url,
+ content: post.data.selftext || post.data.url,
title: post.data.title,
platform: "reddit",
image: image(post.data.thumbnail),
date: new Date(post.data.created * 1000),
popularity: post.data.ups,
+ url: post.data.url,
});
});
if (res.data.data.after) {
diff --git a/server/utils/scraper.js b/server/utils/scraper.js
index 20f2646..332a4a1 100644
--- a/server/utils/scraper.js
+++ b/server/utils/scraper.js
@@ -1,13 +1,34 @@
-const Mention = require("../models/mentionModel"); // Added for Co-op Midterm Presentation
-const { searchRecursive } = require("../utils/reddit"); // Added for Co-op Midterm Presentation
+const Mention = require("../models/mentionModel");
+const { searchRecursive } = require("../utils/reddit");
-// Added for Co-op Midterm Presentation
-// To be handled later on by BullMQ
-const addMentionsToDB = async (company) => {
+// To be handled by BullMQ
+const addMentionsToDB = async (company, platform) => {
try {
- const posts = await searchRecursive(company);
- await Mention.deleteMany();
- await Mention.insertMany(posts);
+ switch (platform) {
+ case "reddit":
+ const posts = await searchRecursive(company);
+ posts.forEach(async (post) => {
+ const bool = await Mention.exists({ url: post.url });
+ if (!bool) {
+ await Mention.create(post);
+ }
+ });
+ break;
+ case "twitter":
+ break;
+ case "facebook":
+ break;
+ case "amazon":
+ break;
+ case "forbes":
+ break;
+ case "shopify":
+ break;
+ case "businessinsider":
+ break;
+ default:
+ console.log(`oops! ${platform} is not inside our platform`);
+ }
} catch (error) {
console.log(error.message);
}