Skip to content

Commit

Permalink
fix(route/xueqiu): fix xueqiu user & stock comments
Browse files Browse the repository at this point in the history
  • Loading branch information
cztchoice committed Dec 13, 2024
1 parent 352cac9 commit ce805c2
Show file tree
Hide file tree
Showing 3 changed files with 48 additions and 41 deletions.
38 changes: 28 additions & 10 deletions lib/routes/xueqiu/cookies.ts
Original file line number Diff line number Diff line change
@@ -1,25 +1,43 @@
import cache from '@/utils/cache';
import { config } from '@/config';
import puppeteer from '@/utils/puppeteer';
import { getCookies } from '@/utils/puppeteer-utils';
import { getCookies, setCookies } from '@/utils/puppeteer-utils';

export const parseToken = (link: string) =>
cache.tryGet(
'xueqiu:token',
async () => {
const browser = await puppeteer({ stealth: true });
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
});
await page.goto(link, {
waitUntil: 'domcontentloaded',
});
const page = await getPage(link, null);
await page.evaluate(() => document.documentElement.innerHTML);
const cookies = await getCookies(page);

return cookies;
},
config.cache.routeExpire,
false
);

export const getPage = async (url: string, cookie: string | Record<string, any> | null = null) => {
const browser = await puppeteer({ stealth: true });
const page = await browser.newPage();
await page.setRequestInterception(true);

page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
});

if (cookie) {
await setCookies(page, cookie, 'xueqiu.com');
}
return page;
};

export const getJson = async (url: string, cookie: string | Record<string, any> | null = null) => {
const page = await getPage(url, cookie);

const data = await page.goto(url, {
waitUntil: 'domcontentloaded',
});
const res = await data?.json();
return res;
};
26 changes: 12 additions & 14 deletions lib/routes/xueqiu/stock-comments.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,10 @@ import { getCurrentPath } from '@/utils/helpers';
const __dirname = getCurrentPath(import.meta.url);

import cache from '@/utils/cache';
import got from '@/utils/got';
import { load } from 'cheerio';
import { art } from '@/utils/render';
import path from 'node:path';
import { parseDate } from '@/utils/parse-date';
import { getJson, getPage } from '@/routes/xueqiu/cookies';
import sanitizeHtml from 'sanitize-html';

export const route: Route = {
Expand All @@ -17,7 +16,7 @@ export const route: Route = {
parameters: { id: '股票代码(需要带上交易所)' },
features: {
requireConfig: false,
requirePuppeteer: false,
requirePuppeteer: true,
antiCrawler: false,
supportBT: false,
supportPodcast: false,
Expand All @@ -36,22 +35,21 @@ export const route: Route = {
async function handler(ctx) {
const id = ctx.req.param('id');

const res = await got({
method: 'get',
url: `https://xueqiu.com/query/v1/symbol/search/status?u=11111&count=100&comment=0&symbol=${id}&source=all&sort=time`,
});
const url = `https://xueqiu.com/query/v1/symbol/search/status?u=11111&count=100&comment=0&symbol=${id}&source=all&sort=time`;

const res = await getJson(url);

// 获取stock_name
const stock_name = await cache.tryGet(`stock_name_${id}`, async () => {
const res = await got({
method: 'get',
url: `https://xueqiu.com/S/${id}`,
});
const $ = load(res.data); // 使用 cheerio 加载返回的 HTML
return $('.stock-name').text().split('(')[0];
const page = await getPage(`https://xueqiu.com/S/${id}`);
await page.waitForSelector('.stock-name');

// 获取文本并处理
const stockName = await page.$eval('.stock-name', (element) => (element.textContent ? element.textContent.split('(')[0].trim() : ''));
return stockName;
});

const data = res.data.list;
const data = res.list;
return {
title: `${id} ${stock_name} - 评论`,
link: `https://xueqiu.com/S/${id}`,
Expand Down
25 changes: 8 additions & 17 deletions lib/routes/xueqiu/user.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
import { Route } from '@/types';
import cache from '@/utils/cache';
import got from '@/utils/got';
import queryString from 'query-string';
import { parseDate } from '@/utils/parse-date';
import sanitizeHtml from 'sanitize-html';
import { parseToken } from '@/routes/xueqiu/cookies';
import { parseToken, getJson } from '@/routes/xueqiu/cookies';

const rootUrl = 'https://xueqiu.com';

Expand All @@ -15,7 +14,7 @@ export const route: Route = {
parameters: { id: '用户 id, 可在用户主页 URL 中找到', type: '动态的类型, 不填则默认全部' },
features: {
requireConfig: false,
requirePuppeteer: false,
requirePuppeteer: true,
antiCrawler: false,
supportBT: false,
supportPodcast: false,
Expand Down Expand Up @@ -50,20 +49,12 @@ async function handler(ctx) {

const link = `${rootUrl}/u/${id}`;
const token = await parseToken(link);
const res2 = await got({
method: 'get',
url: `${rootUrl}/v4/statuses/user_timeline.json`,
searchParams: queryString.stringify({
user_id: id,
type,
source,
}),
headers: {
Cookie: token,
Referer: link,
},
});
const data = res2.data.statuses.filter((s) => s.mark !== 1); // 去除置顶动态

const url = `${rootUrl}/v4/statuses/user_timeline.json?user_id=${id}&type=${type}&source=${source}`;

const res2 = await getJson(url, token);

const data = res2.statuses.filter((s) => s.mark !== 1); // 去除置顶动态

const items = await Promise.all(
data.map((item) =>
Expand Down

0 comments on commit ce805c2

Please sign in to comment.