Merge branch 'dev' into reorder-switch

client/src/hooks/getMentions.js

@@ -3,15 +3,14 @@ import axios from "axios";
 export const getMentions = async (dispatch, keyword, platforms, page, sort) => {
   const platformsArray = Object.keys(platforms).filter((key) => platforms[key]);
 
-  let platformsString = platformsArray.join();
+  const platformsString = platformsArray.join();
 
   try {
     const url = `/api/mentions?platforms=${platformsString}${
       keyword ? "&keyword=" + keyword : ""
     }&page=${page}&sort=${sort}`;
 
     const res = await axios.get(url);
-
     return res.data;
   } catch (err) {
     dispatch({

client/src/layout/Mention.js

@@ -1,4 +1,4 @@
-import React from "react";
+import React, { useContext } from "react";
 import {
   Typography,
   Card,
@@ -9,7 +9,8 @@ import {
 import { makeStyles } from "@material-ui/core/styles";
 import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
 import { faSmile } from "@fortawesome/free-regular-svg-icons";
-import redditLogo from "../utils/images/reddit-logo.png"
+import redditLogo from "../utils/images/reddit-logo.png";
+import { UserContext } from "../context/user";
 
 const useStyles = makeStyles((theme) => ({
   root: {
@@ -68,19 +69,19 @@ const useStyles = makeStyles((theme) => ({
   },
 }));
 
-  const image = (image) => {
-    if (image === "default" || image === "self") {
-      return redditLogo;
-    } else {
-      return image;
-    }
-  };
-
+const image = (image) => {
+  if (image === "default" || image === "self") {
+    return redditLogo;
+  } else {
+    return image;
+  }
+};
 
 const Mention = ({ mention }) => {
   const classes = useStyles();
   const keyword = "DolphinCorp";
-  const regex = new RegExp(`${keyword}`, "i");
+  const { searchTerm } = useContext(UserContext);
+  const regex = new RegExp(`${searchTerm}`, "i");
   const indexK = mention.title.search(regex);
 
   return (
@@ -94,8 +95,8 @@ const Mention = ({ mention }) => {
         <Box component="div" className={classes.titleBox}>
           <Typography variant="h6" gutterBottom>
             {/* {mention.title.substring(0, indexK)}
-            <span style={{ color: "#536dfe" }}>{keyword}</span>
-            {mention.title.substring(indexK + keyword.length)} */}
+            <span style={{ color: "#536dfe" }}>{searchTerm}</span>
+            {mention.title.substring(indexK + searchTerm.length)} */}
             {mention.title}
           </Typography>
         </Box>

client/src/pages/HomePage.js

@@ -7,6 +7,7 @@ import SortToggle from "../layout/SortToggle";
 import { makeStyles } from "@material-ui/core/styles";
 import Spinner from "../layout/Spinner";
 import { getMentions } from "../hooks/getMentions";
+import InfiniteScroll from "react-infinite-scroller";
 
 const useStyles = makeStyles((theme) => ({
   box: {
@@ -23,6 +24,7 @@ const HomePage = () => {
   const [switching, setSwitching] = useState(false);
   const [currentPage, setCurrentPage] = useState(1);
   const [sort, setSort] = useState("date");
+
   const { dispatch, error, searchTerm, user } = useContext(UserContext);
 
   const loadMore = async () => {
@@ -60,7 +62,7 @@ const HomePage = () => {
         setCurrentPage(1);
       })
       .catch((err) => alert("Cookie expired. Please log in again"));
-  }, [searchTerm, user.platforms]);
+  }, [searchTerm, user.platforms, dispatch]);
 
   if (mentionDatas === null) return <Spinner />;
 
@@ -97,6 +99,7 @@ const HomePage = () => {
               error={error}
             />
           )}
+
         </Grid>
         <Grid item xs={2} style={{ backgroundColor: "#FAFBFF" }}></Grid>
       </Grid>

client/src/pages/PagesWrapper.js

@@ -7,7 +7,7 @@ const PagesWrapper = ({ children }) => {
 
   useEffect(() => {
     authenticate(dispatch);
-  }, [isAuthenticated]);
+  }, [isAuthenticated, dispatch]);
 
   return <>{children}</>;
 };

client/src/pages/PrivateRoute.js

@@ -1,4 +1,4 @@
-import React, { useContext, useEffect } from "react";
+import React, { useContext } from "react";
 import { Route, Redirect } from "react-router-dom";
 import { UserContext } from "../context/user";
 

server/app.js

@@ -12,6 +12,7 @@ const pingRouter = require("./routes/ping");
 const mentionRouter = require("./routes/mentionRoutes");
 const userRouter = require("./routes/userRoutes");
 const mongoose = require("mongoose");
+const { connectRedis, handleTaskQueues } = require("./utils/jobHandler");
 
 const connectDB = async () => {
   try {
@@ -31,6 +32,10 @@ const connectDB = async () => {
 
 connectDB();
 
+connectRedis(); // if error, try start redis server in background
+
+handleTaskQueues();
+
 const { json, urlencoded } = express;
 
 var app = express();
@@ -53,37 +58,4 @@ app.use(notFound);
 // error handler
 app.use(errorHandler);
 
-// test redis
-// to start redis server in background
-// run "redis-server --daemonize yes"
-// to stop the server running "redis-cli shutdown"
-const redis = require("redis");
-const client = redis.createClient({
-  host: process.env.REDIS_HOST,
-  port: process.env.REDIS_PORT,
-});
-client.on("error", (err) => {
-  console.log("Error " + err);
-});
-
-client.on("ready", function () {
-  console.log("redis is running");
-});
-
-// test bull
-var Queue = require("bull");
-
-const myFirstQueue = new Queue("my-first-queue", {
-  redis: { port: process.env.REDIS_PORT, host: process.env.REDIS_HOST },
-});
-
-// job producer
-const job = myFirstQueue.add();
-
-// job consumer
-myFirstQueue.process(function (job, done) {
-  // call done when finished
-  done(console.log(`Job with id ${job.id} has been completed`));
-});
-
 module.exports = app;

server/controllers/mentionController.js

@@ -42,10 +42,17 @@ const getMentions = async (req, res) => {
 
       // Fetching Mentions pertaining to selected platforms
       // Sorting handled by MongoDB
-
       const fetchMentions = async (array, sorting) => {
         const results = await Mention.find({
-          $or: [...getPlatformsObject(array)],
+          $and: [
+            {
+              $or: [
+                { title: { $regex: req.user.name, $options: "i" } },
+                { content: { $regex: req.user.name, $options: "i" } },
+              ],
+            },
+            { $or: [...getPlatformsObject(array)] },
+          ],
         }).sort(getSortOption(sorting));
 
         return results;

server/controllers/userController.js

@@ -32,10 +32,6 @@ const signUp = async (req, res) => {
         secure: false, // should be true in Production !
       });
 
-      // Added for Co-op Midterm Presentation
-      // To be handled later on by BullMQ
-      await addMentionsToDB(user.name);
-
       res.status(201).json({
         _id: user._id,
         name: user.name,
@@ -66,10 +62,6 @@ const signIn = async (req, res) => {
       secure: false, // should be true in Production !
     });
 
-    // Added for Co-op Midterm Presentation
-    // To be handled later on by BullMQ
-    await addMentionsToDB(user.name);
-
     res.status(201).json({
       _id: user._id,
       name: user.name,

server/models/mentionModel.js

@@ -25,6 +25,10 @@ const mentionSchema = mongoose.Schema({
     type: Number,
     default: 0,
   },
+  url: {
+    type: String,
+    required: true,
+  },
 });
 
 module.exports = mongoose.model("Mention", mentionSchema);

server/models/userModel.js

@@ -44,4 +44,8 @@ userSchema.pre("save", async function (next) {
   this.password = await bcrypt.hash(this.password, salt);
 });
 
+userSchema.statics.getAllCompanies = async function () {
+  return await this.distinct("name");
+};
+
 module.exports = mongoose.model("User", userSchema);

server/utils/jobHandler.js

@@ -0,0 +1,71 @@
+const redis = require("redis");
+const Queue = require("bull");
+const User = require("../models/userModel");
+const Mention = require("../models/mentionModel");
+const { addMentionsToDB } = require("./scraper");
+
+const connectRedis = () => {
+  // to start redis server in background
+  // run "redis-server --daemonize yes"
+  // to stop the server running "redis-cli shutdown"
+  const client = redis.createClient({
+    host: process.env.REDIS_HOST,
+    port: process.env.REDIS_PORT,
+  });
+  client.on("error", (err) => {
+    console.log("Error " + err);
+  });
+
+  client.on("ready", function () {
+    console.log("redis is running");
+  });
+};
+
+const handleTaskQueues = () => {
+  const companiesQueue = new Queue("queue-for-getting-componies", {
+    redis: { port: process.env.REDIS_PORT, host: process.env.REDIS_HOST },
+  });
+
+  const scrapingQueue = new Queue("queue-for-scraping", {
+    redis: { port: process.env.REDIS_PORT, host: process.env.REDIS_HOST },
+  });
+  // job producer, repeated every 15 mins
+  companiesQueue.add(
+    {},
+    {
+      repeat: { cron: "*/15 * * * *" },
+      delay: 2000,
+      jobId: "repeatCompaniesUpdate",
+    }
+  );
+
+  // to remove existedQueue use
+  // companiesQueue.empty();
+
+  // job consumer
+  companiesQueue.process(async function (job, done) {
+    try {
+      User.getAllCompanies().then((companies) =>
+        companies.forEach((company) => {
+          for (var platform in User.schema.obj.platforms) {
+            scrapingQueue.add({ company: company, platform: platform });
+          }
+        })
+      );
+      done(new Error("error adding companies to queue"));
+    } catch (err) {
+      console.log(err);
+    }
+  });
+
+  scrapingQueue.process(async function (job, done) {
+    try {
+      await addMentionsToDB(job.data.company, job.data.platform);
+      done(new Error(`${job.data.company} and ${job.data.platform} error`));
+    } catch (err) {
+      console.log(err);
+    }
+  });
+};
+
+module.exports = { connectRedis, handleTaskQueues };

server/utils/reddit.js

@@ -16,16 +16,16 @@ const searchRecursive = async (term, after = "", posts = []) => {
 
   try {
     const res = await axios.get(url);
-
     if (res.status === 200) {
-      res.data.data.children.forEach((post) => {
+      res.data.data.children.forEach(async (post) => {
         posts.push({
-          content: post.data.url,
+          content: post.data.selftext || post.data.url,
           title: post.data.title,
           platform: "reddit",
           image: image(post.data.thumbnail),
           date: new Date(post.data.created * 1000),
           popularity: post.data.ups,
+          url: post.data.url,
         });
       });
       if (res.data.data.after) {

server/utils/scraper.js

@@ -1,13 +1,34 @@
-const Mention = require("../models/mentionModel"); // Added for Co-op Midterm Presentation
-const { searchRecursive } = require("../utils/reddit"); // Added for Co-op Midterm Presentation
+const Mention = require("../models/mentionModel");
+const { searchRecursive } = require("../utils/reddit");
 
-// Added for Co-op Midterm Presentation
-// To be handled later on by BullMQ
-const addMentionsToDB = async (company) => {
+// To be handled by BullMQ
+const addMentionsToDB = async (company, platform) => {
   try {
-    const posts = await searchRecursive(company);
-    await Mention.deleteMany();
-    await Mention.insertMany(posts);
+    switch (platform) {
+      case "reddit":
+        const posts = await searchRecursive(company);
+        posts.forEach(async (post) => {
+          const bool = await Mention.exists({ url: post.url });
+          if (!bool) {
+            await Mention.create(post);
+          }
+        });
+        break;
+      case "twitter":
+        break;
+      case "facebook":
+        break;
+      case "amazon":
+        break;
+      case "forbes":
+        break;
+      case "shopify":
+        break;
+      case "businessinsider":
+        break;
+      default:
+        console.log(`oops! ${platform} is not inside our platform`);
+    }
   } catch (error) {
     console.log(error.message);
   }