Merge pull request #58 from CheatSOL/develop

FEAT: 4주차 develop 브랜치 main 브랜치로 병합

.github/workflows/main.yml

@@ -43,6 +43,12 @@ jobs:
           echo "KIS_APP_KEY=${{ secrets.KIS_APP_KEY}}" >> .env
           echo "KIS_SECRECT_KEY=${{ secrets.KIS_SECRECT_KEY }}" >> .env
           echo "PORT=${{ secrets.PORT }}" >> .env
+          echo "INSTA_UID_KEY=${{ secrets.INSTA_UID_KEY }}" >> .env
+          echo "INSTA_KEY=${{ secrets.INSTA_KEY }}" >> .env
+          echo "NAVER_ID=${{ secrets.NAVER_ID }}" >> .env
+          echo "NAVER_SECRET=${{ secrets.NAVER_SECRET }}" >> .env
+          echo "YOUTUBE_KEY=${{ secrets.YOUTUBE_KEY }}" >> .env
+          echo "INSTA_IP=${{ secrets.INSTA_IP }}" >> .env
 
       # Build step removed since there is no build script
 
@@ -54,7 +60,7 @@ jobs:
           key: ${{ secrets.EC2_KEY }}
           port: 22
           source: "."
-          target: "/home/ec2-user/CheatSOL-BE"
+          target: "/home/ubuntu/CheatSOL-BE"
 
       - name: Execute remote commands via SSH
         uses: appleboy/ssh-action@master
@@ -66,18 +72,14 @@ jobs:
           script: |
             # Install Node.js and pm2 on Amazon Linux
             curl -sL https://rpm.nodesource.com/setup_20.x | sudo bash -
-            sudo yum install -y nodejs
-            sudo npm install -g pm2
+            sudo apt-get update -y
+            sudo apt-get install -y nodejs
+            sudo apt-get install -g pm2
             # Navigate to the application directory
-            cd /home/ec2-user/CheatSOL-BE
-
-            # Install Python and pip
-            sudo yum install -y python3
-            sudo alternatives --set python /usr/bin/python3
-            sudo yum install -y python3-pip
+            cd /home/ubuntu/CheatSOL-BE
 
             # Install project dependencies
-            npm install --production
+            npm install
             pip install -r requirements.txt
             # Restart or start the application with pm2
             pm2 stop all

app.js

@@ -3,41 +3,52 @@ var express = require("express");
 var path = require("path");
 var cookieParser = require("cookie-parser");
 var logger = require("morgan");
-const {wsdata}=require("./src/utils/WSPrice");
+const axios = require("axios");
+const cheerio = require("cheerio");
+
+// 라우터 추가
+
+const { wsdata } = require("./src/utils/WSPrice");
 var indexRouter = require("./src/routes/index");
 var companyRouter = require("./src/routes/company");
-var googleRouter = require("./src/routes/google");
-var stockInfoRouter = require("./src/routes/stock.info.detail");
+var socialRouter = require("./src/routes/social");
 const db = require("./src/models/DB");
-const http=require('http');
+var keywordRouter = require("./src/routes/keyword"); //연관검색어 router
+var socialChartRouter = require("./src/routes/socialChart");
+var stockInfoRouter = require("./src/routes/stock.info.detail");
+var googleNewsRouter = require("./src/routes/google-news");
+var youTubeNewsRouter = require("./src/routes/youtube-news");
+var naverNewsRouter = require("./src/routes/naver-news");
+var instaNewsRouter = require("./src/routes/instagram-news");
+const http = require("http");
 var app = express();
 app.use(logger("dev"));
 app.use(express.json());
 app.use(express.urlencoded({ extended: false }));
 app.use(cookieParser());
 const server = http.createServer(app);
-const WebSocket = require('ws');
+const WebSocket = require("ws");
 const wss = new WebSocket.Server({ server });
-wss.on('connection',  async function connection(ws) {
-  console.log('새로운 WebSocket 클라이언트가 연결되었습니다.');
+wss.on("connection", async function connection(ws) {
+  console.log("새로운 WebSocket 클라이언트가 연결되었습니다.");
   let id;
-  ws.on('message', (message) => {
+  ws.on("message", (message) => {
     const data = JSON.parse(message);
-    console.log('Received data:', data);
+    console.log("Received data:", data);
     id = data.id;
     if (id) {
-        wsdata(ws, id);  // id가 설정된 후에 wsdata 호출
+      wsdata(ws, id); // id가 설정된 후에 wsdata 호출
     } else {
-        console.error('ID is undefined');
+      console.error("ID is undefined");
     }
-});
+  });
 
-  ws.on('close', function close() {
-      console.log('WebSocket 연결이 종료되었습니다.');
+  ws.on("close", function close() {
+    console.log("WebSocket 연결이 종료되었습니다.");
   });
 });
 server.listen(3002, () => {
-  console.log('서버가 3002번 포트에서 실행 중입니다.');
+  console.log("서버가 3002번 포트에서 실행 중입니다.");
 });
 
 db.sequelize
@@ -55,10 +66,17 @@ db.sequelize
     console.error("Unable to connect to the database:", err);
   });
 
+// 라우터 url 설정
 app.use("/api", indexRouter);
 app.use("/api/company", companyRouter);
-app.use("/api/trends", googleRouter);
+app.use("/api/social", socialRouter);
+app.use("/api/keyword", keywordRouter);
+app.use("/api/trends", socialChartRouter);
 app.use("/api/stockInfo", stockInfoRouter);
+app.use("/api/news/google", googleNewsRouter);
+app.use("/api/news/youtube", youTubeNewsRouter);
+app.use("/api/news/naver", naverNewsRouter);
+app.use("/api/news/instagram", instaNewsRouter);
 
 // catch 404 and forward to error handler
 app.use(function (req, res, next) {

bin/www

@@ -94,4 +94,4 @@ function onListening() {
     ? 'pipe ' + addr
     : 'port ' + addr.port;
   debug('Listening on ' + bind);
-}
+}

package.json

@@ -19,11 +19,13 @@
     "http-errors": "~1.6.3",
     "https": "^1.0.0",
     "morgan": "~1.9.1",
-    "nodemon": "^3.1.3",
+    "nodemon": "^3.1.4",
     "pg": "^8.12.0",
     "pg-hstore": "^2.3.4",
     "python-shell": "^5.0.0",
+    "rss-parser": "^3.13.0",
     "sequelize": "^6.37.3",
-    "ws": "^8.17.1"
+    "ws": "^8.17.1",
+    "youtube-node": "^1.3.3"
   }
 }

requirements.txt

@@ -1,6 +1,5 @@
-psycopg2-binary==2.9.9
 gensim==4.3.2
 pandas==2.2.2
-psycopg2==2.9.9
+psycopg2-binary==2.9.9
 python-dotenv==1.0.1
-scipy==1.10.0
+scipy==1.12.0

src/controllers/CacheController.js

@@ -0,0 +1,57 @@
+const db = require("../models/DB");
+
+async function getCache(keyword, social, period) {
+  try {
+    const cacheData = await db.Cache.findOne({
+      where: {
+        keyword: keyword,
+        social: social,
+        period: period,
+      },
+      order: [["updatedAt", "DESC"]],
+    });
+
+    return cacheData;
+  } catch (error) {
+    console.log("cacheError", error);
+    return null;
+  }
+}
+
+async function setCache(keyword, social, period, data) {
+  await db.Cache.create({
+    keyword: keyword,
+    social: social,
+    period: period,
+    data: data,
+  });
+}
+
+async function updateCache(keyword, social, period, data) {
+  await db.Cache.update(
+    {
+      data: data,
+      updatedAt: Date.now(),
+    },
+    {
+      where: {
+        keyword: keyword,
+        social: social,
+        period: period,
+      },
+    }
+  );
+}
+
+function isExpired(cache) {
+  if (cache === null || cache === undefined) return true;
+  const givenDate = new Date(cache.updatedAt);
+
+  const expirationDate = new Date(givenDate.getTime() + 24 * 60 * 60 * 1000);
+
+  const currentDate = new Date();
+
+  return expirationDate < currentDate;
+}
+
+module.exports = { getCache, setCache, updateCache, isExpired };

src/controllers/NewsCrawling.js

@@ -1,8 +1,8 @@
-const axios=require('axios');
-const cheerio=require('cheerio');
-const { Model, Op } = require('sequelize');
-const { CompanyNews, Company } = require('../models/DB');
-const fetchNews = require('../utils/NaverStockNews');
+const axios = require("axios");
+const cheerio = require("cheerio");
+const { Model, Op } = require("sequelize");
+const { CompanyNews, Company } = require("../models/DB");
+const fetchNews = require("../utils/NaverStockNews");
 
 async function fetchNewsContent(link) {
   try {
@@ -32,7 +32,6 @@ function delay(ms) {
 async function saveNewsToDatabase(id, newsItems) {
   const newsData = [];
 
-  //console.log("newsItems", newsItems);
   for (let items of newsItems) {
     for (let item of items) {
       const link = `https://n.news.naver.com/article/${item.officeId}/${item.articleId}`;

src/controllers/social/instagram.js

@@ -0,0 +1,140 @@
+const axios = require("axios");
+const cheerio = require("cheerio");
+
+function formatInstagramStyle(input) {
+  let num = input[0];
+  let percentage = input[1] * 100;
+
+  // 숫자를 적절한 단위로 축약
+  function formatNumber(number) {
+    if (number >= 1e6) {
+      return (number / 1e6).toFixed(1) + "M";
+    } else if (number >= 1e3) {
+      return (number / 1e3).toFixed(1) + "K";
+    } else {
+      return number.toString();
+    }
+  }
+
+  // 백분율로 변환
+  let formattedNum = formatNumber(num);
+  let formattedPercentage = percentage.toFixed(0) + "%";
+
+  return [formattedNum, formattedPercentage];
+}
+
+const formatDate = function formatDate(originalDate) {
+  let year = parseInt(originalDate.slice(2, 4));
+  let month = parseInt(originalDate.slice(5, 7));
+  month++;
+  let ans;
+  if (month > 12) {
+    month = 1;
+    year++;
+  }
+  if (month < 10) {
+    ans = year + ".0" + month;
+  } else {
+    ans = year + "." + month;
+  }
+
+  return ans;
+};
+const getTagId = async function convertWordToTagId(word) {
+  try {
+    const { data } = await axios.get(
+      `
+      https://moana.mediance.co.kr/v1/instagram-tags/find?keyword=${encodeURI(
+        word
+      )}&uid=${process.env.INSTA_UID_KEY}&ip=${process.env.INSTA_IP}`,
+      {
+        headers: {
+          Authorization: process.env.INSTA_KEY,
+        },
+      }
+    );
+    console.log("Daa:", data);
+    return data.id;
+  } catch (error) {
+    console.error("Error fetching tag ID:", error);
+  }
+};
+
+const getTags = async function getHotHashTags(word) {
+  try {
+    const { data } = await axios.get(
+      `https://moana.mediance.co.kr/v1/instagram-tags/find?keyword=${encodeURI(
+        word
+      )}&ip=${process.env.INSTA_IP}&uid=${process.env.INSTA_UID_KEY}`,
+      {
+        headers: {
+          Authorization: process.env.INSTA_KEY,
+        },
+      }
+    );
+    // console.log("xdata);/
+    return data.instagramTag.instagramTagTree.engagementTags;
+  } catch (error) {
+    console.error("Error fetching hotTags data:", error);
+  }
+};
+const getTrend = async function getTrendWithTagId(id) {
+  try {
+    const { data } = await axios.get(
+      `https://moana.mediance.co.kr/v1/instagram-tags/${id}/series-summary`,
+      {
+        headers: {
+          Authorization: process.env.INSTA_KEY,
+        },
+      }
+    );
+
+    const trend = data.data.map((item) => {
+      return { date: formatDate(item.statOn), posts: item.postCount };
+    });
+
+    return trend;
+  } catch (error) {
+    console.error("Error fetching trend data:", error);
+  }
+};
+
+const getTagInfo = async function getTagInfo(id) {
+  try {
+    const { data } = await axios.get(
+      `https://moana.mediance.co.kr/v1/instagram-tags/${id}/summary?ip=${process.env.INSTA_IP}}`,
+      {
+        headers: {
+          Authorization: process.env.INSTA_KEY,
+        },
+      }
+    );
+
+    return formatInstagramStyle([
+      data.postCount,
+      data.instagramTag.instagramTagStat.engagementAvg /
+        data.instagramTag.instagramTagStat.occupyTimeAvg,
+    ]);
+  } catch (error) {
+    console.error("Error fetching or parsing data:", error);
+    return null;
+  }
+};
+const getInstagramInfo = async function scrapingInstagramSocialInfo(word) {
+  try {
+    const id = await getTagId(word);
+    if (id) {
+      const trendData = await getTrend(id);
+      const topTags = (await getTags(word)).slice(0, 3);
+      const tagInfo = await getTagInfo(id);
+
+      return { id, trendData, topTags, tagInfo };
+    } else {
+      console.log("No ID found for the given word.");
+    }
+  } catch (error) {
+    console.error("Error in instagramInfo function:", error);
+  }
+};
+
+module.exports = { getInstagramInfo };