From 15ec12b0a3a95716243bfd06f6d41959bff7c36e Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Sat, 5 Dec 2020 23:22:58 -0500 Subject: [PATCH] feat(notification): add price to links (#1209) fix(store): label selection ordering and pricing fix(nvidia): deprecation notice removed outside of usa fix(amazon): maxPrice selector Resolves #1188 Resolves #673 Resolves #1187 --- package-lock.json | 67 +++++++++++++++++++------- package.json | 2 +- src/__test__/notification-test.ts | 1 + src/config.ts | 2 +- src/logger.ts | 11 ++--- src/notification/discord.ts | 62 ++++++++++++++++-------- src/store/includes-labels.ts | 21 ++++---- src/store/lookup.ts | 79 +++++++++++++++---------------- src/store/model/amazon-ca.ts | 3 +- src/store/model/amazon-es.ts | 3 +- src/store/model/amazon-fr.ts | 2 +- src/store/model/amazon-it.ts | 2 +- src/store/model/amazon-nl.ts | 2 +- src/store/model/amazon-uk.ts | 2 +- src/store/model/amazon.ts | 6 +-- src/store/model/index.ts | 3 +- src/store/model/store.ts | 1 + 17 files changed, 157 insertions(+), 112 deletions(-) diff --git a/package-lock.json b/package-lock.json index 972fa0f7af..9963776250 100644 --- a/package-lock.json +++ b/package-lock.json @@ -395,6 +395,21 @@ "kuler": "^2.0.0" } }, + "@discordjs/collection": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/@discordjs/collection/-/collection-0.1.6.tgz", + "integrity": "sha512-utRNxnd9kSS2qhyivo9lMlt5qgAUasH2gb7BEOn6p0efFh24gjGomHzWKMAPn2hEReOPQZCJaRKoURwRotKucQ==" + }, + "@discordjs/form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@discordjs/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-ZfFsbgEXW71Rw/6EtBdrP5VxBJy4dthyC0tpQKGKmYFImlmmrykO14Za+BiIVduwjte0jXEBlhSKf0MWbFp9Eg==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + }, "@eslint/eslintrc": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.2.1.tgz", @@ -1224,6 +1239,14 @@ "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", "dev": true }, + "abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "requires": { + "event-target-shim": "^5.0.0" + } + }, "acorn": { "version": "8.0.4", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.0.4.tgz", @@ -2747,24 +2770,25 @@ "path-type": "^4.0.0" } }, - "discord-webhook-node": { - "version": "1.1.8", - "resolved": "https://registry.npmjs.org/discord-webhook-node/-/discord-webhook-node-1.1.8.tgz", - "integrity": "sha512-3u0rrwywwYGc6HrgYirN/9gkBYqmdpvReyQjapoXARAHi0P0fIyf3W5tS5i3U3cc7e44E+e7dIHYUeec7yWaug==", + "discord.js": { + "version": "12.5.1", + "resolved": "https://registry.npmjs.org/discord.js/-/discord.js-12.5.1.tgz", + "integrity": "sha512-VwZkVaUAIOB9mKdca0I5MefPMTQJTNg0qdgi1huF3iwsFwJ0L5s/Y69AQe+iPmjuV6j9rtKoG0Ta0n9vgEIL6w==", "requires": { - "form-data": "^3.0.0", - "node-fetch": "^2.6.0" + "@discordjs/collection": "^0.1.6", + "@discordjs/form-data": "^3.0.1", + "abort-controller": "^3.0.0", + "node-fetch": "^2.6.1", + "prism-media": "^1.2.2", + "setimmediate": "^1.0.5", + "tweetnacl": "^1.0.3", + "ws": "^7.3.1" }, "dependencies": { - "form-data": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz", - "integrity": "sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - } + "tweetnacl": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-1.0.3.tgz", + "integrity": "sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw==" } } }, @@ -3712,6 +3736,11 @@ "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", "dev": true }, + "event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" + }, "eventemitter3": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz", @@ -8781,6 +8810,11 @@ "fast-diff": "^1.1.2" } }, + "prism-media": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/prism-media/-/prism-media-1.2.3.tgz", + "integrity": "sha512-fSrR66n0l6roW9Rx4rSLMyTPTjRTiXy5RVqDOurACQ6si1rKHHKDU5gwBJoCsIV0R3o9gi+K50akl/qyw1C74A==" + }, "process": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", @@ -9751,8 +9785,7 @@ "setimmediate": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", - "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=", - "dev": true + "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=" }, "sha.js": { "version": "2.4.11", diff --git a/package.json b/package.json index 07d4dd4999..a07934e34e 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "@slack/web-api": "^5.14.0", "chalk": "^4.1.0", "cheerio": "^1.0.0-rc.3", - "discord-webhook-node": "^1.1.8", + "discord.js": "^12.5.1", "dotenv": "^8.2.0", "messaging-api-telegram": "^1.0.1", "mqtt": "^4.2.6", diff --git a/src/__test__/notification-test.ts b/src/__test__/notification-test.ts index 57f4f407ce..a8b9fd09da 100644 --- a/src/__test__/notification-test.ts +++ b/src/__test__/notification-test.ts @@ -5,6 +5,7 @@ const link: Link = { brand: 'test:brand', cartUrl: 'https://www.example.com/cartUrl', model: 'test:model', + price: 100, series: 'test:series', url: 'https://www.example.com/url' }; diff --git a/src/config.ts b/src/config.ts index 826c0f2ab9..ec6ca7d7a5 100644 --- a/src/config.ts +++ b/src/config.ts @@ -180,7 +180,7 @@ const notifications = { desktop: process.env.DESKTOP_NOTIFICATIONS === 'true', discord: { notifyGroup: envOrArray(process.env.DISCORD_NOTIFY_GROUP), - webHookUrl: envOrArray(process.env.DISCORD_WEB_HOOK) + webhooks: envOrArray(process.env.DISCORD_WEB_HOOK) }, email: { password: envOrString(process.env.EMAIL_PASSWORD), diff --git a/src/logger.ts b/src/logger.ts index 8d621f07dd..477855a77a 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -118,11 +118,9 @@ export const Print = { return `ℹ ${buildProductString(link, store)} :: IN STOCK, WAITING`; }, - // eslint-disable-next-line max-params maxPrice( link: Link, store: Store, - price: number, maxPrice: number, color?: boolean ): string { @@ -131,14 +129,13 @@ export const Print = { '✖ ' + buildProductString(link, store, true) + ' :: ' + - chalk.yellow(`PRICE ${price} EXCEEDS LIMIT ${maxPrice}`) + chalk.yellow(`PRICE ${link.price ?? ''} EXCEEDS LIMIT ${maxPrice}`) ); } - return `✖ ${buildProductString( - link, - store - )} :: PRICE ${price} EXCEEDS LIMIT ${maxPrice}`; + return `✖ ${buildProductString(link, store)} :: PRICE ${ + link.price ?? '' + } EXCEEDS LIMIT ${maxPrice}`; }, message( message: string, diff --git a/src/notification/discord.ts b/src/notification/discord.ts index 758ce517a4..3626619650 100644 --- a/src/notification/discord.ts +++ b/src/notification/discord.ts @@ -1,41 +1,63 @@ import {Link, Store} from '../store/model'; -import {MessageBuilder, Webhook} from 'discord-webhook-node'; +import Discord from 'discord.js'; import {config} from '../config'; import {logger} from '../logger'; const discord = config.notifications.discord; -const hooks = discord.webHookUrl; -const notifyGroup = discord.notifyGroup; +const {notifyGroup, webhooks} = discord; + +function getIdAndToken(webhook: string) { + const match = /.*\/webhooks\/(\d+)\/(.+)/.exec(webhook); + + if (!match) { + throw new Error('could not get discord webhook'); + } + + return { + id: match[1], + token: match[2] + }; +} export function sendDiscordMessage(link: Link, store: Store) { - if (discord.webHookUrl.length > 0) { + if (webhooks.length > 0) { logger.debug('↗ sending discord message'); (async () => { try { - const embed = new MessageBuilder(); - embed.setTitle('Stock Notification'); - if (link.cartUrl) - embed.addField('Add To Cart Link', link.cartUrl, true); - embed.addField('Product Page', link.url, true); + const embed = new Discord.MessageEmbed() + .setTitle('_**Stock alert!**_') + .setDescription( + '> provided by [streetmerchant](https://github.com/jef/streetmerchant) with :heart:' + ) + .setThumbnail( + 'https://raw.githubusercontent.com/jef/streetmerchant/main/media/streetmerchant-square.png' + ) + .setColor('#52b788') + .setTimestamp(); + embed.addField('Store', store.name, true); + if (link.price) embed.addField('Price', `$${link.price}`, true); + embed.addField('Product Page', link.url); + if (link.cartUrl) embed.addField('Add to Cart', link.cartUrl); embed.addField('Brand', link.brand, true); - embed.addField('Series', link.series, true); embed.addField('Model', link.model, true); - - if (notifyGroup) { - embed.setText(notifyGroup.join(' ')); - } - - embed.setColor(0x76b900); - embed.setTimestamp(); + embed.addField('Series', link.series, true); const promises = []; - for (const hook of hooks) { - promises.push(new Webhook(hook).send(embed)); + for (const webhook of webhooks) { + const {id, token} = getIdAndToken(webhook); + const client = new Discord.WebhookClient(id, token); + promises.push({ + client, + message: client.send(notifyGroup.join(' '), { + embeds: [embed], + username: 'streetmerchant' + }) + }); } - await Promise.all(promises); + (await Promise.all(promises)).forEach(({client}) => client.destroy()); logger.info('✔ discord message sent'); } catch (error: unknown) { diff --git a/src/store/includes-labels.ts b/src/store/includes-labels.ts index 1361ce4bdf..9d3b8543aa 100644 --- a/src/store/includes-labels.ts +++ b/src/store/includes-labels.ts @@ -116,27 +116,22 @@ export function includesLabels( ); } -export async function cardPrice( +export async function getPrice( page: Page, query: Pricing, - max: number, options: Selector ): Promise { - if (!max || max === -1) { - return null; - } - const selector = {...options, selector: query.container}; - const cardPrice = await extractPageContents(page, selector); + const priceString = await extractPageContents(page, selector); - if (cardPrice) { - const priceSeperator = query.euroFormat ? /\./g : /,/g; - const cardpriceNumber = Number.parseFloat( - cardPrice.replace(priceSeperator, '').match(/\d+/g)!.join('.') + if (priceString) { + const priceSeparator = query.euroFormat ? /\./g : /,/g; + const price = Number.parseFloat( + priceString.replace(priceSeparator, '').match(/\d+/g)!.join('.') ); - logger.debug(`Raw card price: ${cardPrice} | Limit: ${max}`); - return cardpriceNumber > max ? cardpriceNumber : null; + logger.debug('received price', price); + return price; } return null; diff --git a/src/store/lookup.ts b/src/store/lookup.ts index 275557a609..5b3298508d 100644 --- a/src/store/lookup.ts +++ b/src/store/lookup.ts @@ -1,7 +1,7 @@ import {Browser, Page, PageEventObj, Request, Response} from 'puppeteer'; import {Link, Store, getStores} from './model'; import {Print, logger} from '../logger'; -import {Selector, cardPrice, pageIncludesLabels} from './includes-labels'; +import {Selector, getPrice, pageIncludesLabels} from './includes-labels'; import { closePage, delay, @@ -303,6 +303,43 @@ async function lookupCardInStock(store: Store, page: Page, link: Link) { type: 'textContent' }; + if (store.labels.captcha) { + if (await pageIncludesLabels(page, store.labels.captcha, baseOptions)) { + logger.warn(Print.captcha(link, store, true)); + await delay(getSleepTime(store)); + return false; + } + } + + if (store.labels.bannedSeller) { + if ( + await pageIncludesLabels(page, store.labels.bannedSeller, baseOptions) + ) { + logger.warn(Print.bannedSeller(link, store, true)); + return false; + } + } + + if (store.labels.maxPrice) { + const maxPrice = config.store.maxPrice.series[link.series]; + + link.price = await getPrice(page, store.labels.maxPrice, baseOptions); + + if (link.price && link.price > maxPrice && maxPrice > 0) { + logger.info(Print.maxPrice(link, store, maxPrice, true)); + return false; + } + } + + // Fixme: currently causing issues + // Do API inventory validation in realtime (no cache) if available + // if ( + // store.realTimeInventoryLookup !== undefined && + // link.itemNumber !== undefined + // ) { + // return store.realTimeInventoryLookup(link.itemNumber); + // } + if (store.labels.inStock) { const options = { ...baseOptions, @@ -336,46 +373,6 @@ async function lookupCardInStock(store: Store, page: Page, link: Link) { } } - if (store.labels.bannedSeller) { - if ( - await pageIncludesLabels(page, store.labels.bannedSeller, baseOptions) - ) { - logger.warn(Print.bannedSeller(link, store, true)); - return false; - } - } - - if (store.labels.maxPrice) { - const price = await cardPrice( - page, - store.labels.maxPrice, - config.store.maxPrice.series[link.series], - baseOptions - ); - const maxPrice = config.store.maxPrice.series[link.series]; - if (price) { - logger.info(Print.maxPrice(link, store, price, maxPrice, true)); - return false; - } - } - - if (store.labels.captcha) { - if (await pageIncludesLabels(page, store.labels.captcha, baseOptions)) { - logger.warn(Print.captcha(link, store, true)); - await delay(getSleepTime(store)); - return false; - } - } - - // Fixme: currently causing issues - // Do API inventory validation in realtime (no cache) if available - // if ( - // store.realTimeInventoryLookup !== undefined && - // link.itemNumber !== undefined - // ) { - // return store.realTimeInventoryLookup(link.itemNumber); - // } - return true; } diff --git a/src/store/model/amazon-ca.ts b/src/store/model/amazon-ca.ts index 307cba2a97..680130083b 100644 --- a/src/store/model/amazon-ca.ts +++ b/src/store/model/amazon-ca.ts @@ -11,8 +11,7 @@ export const AmazonCa: Store = { text: ['add to cart'] }, maxPrice: { - container: 'span[class*="PriceString"]', - euroFormat: false + container: '#priceblock_ourprice' } }, links: [ diff --git a/src/store/model/amazon-es.ts b/src/store/model/amazon-es.ts index ae61c6d629..5a1ee033b4 100644 --- a/src/store/model/amazon-es.ts +++ b/src/store/model/amazon-es.ts @@ -12,8 +12,7 @@ export const AmazonEs: Store = { text: ['añadir a la cesta'] }, maxPrice: { - container: 'span[class*="PriceString"]', - euroFormat: true + container: '#priceblock_ourprice' }, outOfStock: [ { diff --git a/src/store/model/amazon-fr.ts b/src/store/model/amazon-fr.ts index 1c1a5000ec..e2ffd4e2a5 100644 --- a/src/store/model/amazon-fr.ts +++ b/src/store/model/amazon-fr.ts @@ -12,7 +12,7 @@ export const AmazonFr: Store = { text: ['ajouter au panier'] }, maxPrice: { - container: 'span[class*="PriceString"]', + container: '#priceblock_ourprice', euroFormat: true }, outOfStock: [ diff --git a/src/store/model/amazon-it.ts b/src/store/model/amazon-it.ts index 66cdbc51ba..09f383e26c 100644 --- a/src/store/model/amazon-it.ts +++ b/src/store/model/amazon-it.ts @@ -12,7 +12,7 @@ export const AmazonIt: Store = { text: ['Aggiungi al carrello'] }, maxPrice: { - container: 'span[class*="PriceString"]' + container: '#priceblock_ourprice' } }, links: [ diff --git a/src/store/model/amazon-nl.ts b/src/store/model/amazon-nl.ts index e6e352a634..c7a13096c8 100644 --- a/src/store/model/amazon-nl.ts +++ b/src/store/model/amazon-nl.ts @@ -16,7 +16,7 @@ export const AmazonNl: Store = { ] }, maxPrice: { - container: 'span[class*="PriceString"]', + container: '#priceblock_ourprice', euroFormat: true }, outOfStock: [ diff --git a/src/store/model/amazon-uk.ts b/src/store/model/amazon-uk.ts index fa2ec05aa4..35c0c4d799 100644 --- a/src/store/model/amazon-uk.ts +++ b/src/store/model/amazon-uk.ts @@ -14,7 +14,7 @@ export const AmazonUk: Store = { text: ['in stock'] }, maxPrice: { - container: 'span[class*="PriceString"]' + container: '#priceblock_ourprice' }, outOfStock: [ { diff --git a/src/store/model/amazon.ts b/src/store/model/amazon.ts index 43ad0d74e7..10b16cbe8c 100644 --- a/src/store/model/amazon.ts +++ b/src/store/model/amazon.ts @@ -18,17 +18,17 @@ export const Amazon: Store = { } ], maxPrice: { - container: '#price_inside_buybox' + container: '#priceblock_ourprice' } }, links: [ { brand: 'test:brand', cartUrl: - 'https://www.amazon.com/gp/aws/cart/add.html?ASIN.1=B07TDN1SC5&Quantity.1=1', + 'https://www.amazon.com/gp/aws/cart/add.html?ASIN.1=B083248S3B&Quantity.1=1', model: 'test:model', series: 'test:series', - url: 'https://www.amazon.com/dp/B07TDN1SC5' + url: 'https://www.amazon.com/dp/B083248S3B' }, { brand: 'asus', diff --git a/src/store/model/index.ts b/src/store/model/index.ts index bb05f1d476..ed915efa48 100644 --- a/src/store/model/index.ts +++ b/src/store/model/index.ts @@ -232,7 +232,8 @@ function warnIfStoreDeprecated(store: Store) { switch (store.name) { case 'nvidia': case 'nvidia-api': - logger.warn(`${store.name} is deprecated in favor of bestbuy`); + if (config.store.country === 'usa') + logger.warn(`${store.name} is deprecated in favor of bestbuy`); break; case 'evga': logger.warn( diff --git a/src/store/model/store.ts b/src/store/model/store.ts index 00e38524da..57feb8db74 100644 --- a/src/store/model/store.ts +++ b/src/store/model/store.ts @@ -137,6 +137,7 @@ export type Link = { labels?: Labels; model: Model; openCartAction?: (browser: Browser) => Promise; + price?: number | null; series: Series; screenshot?: string; url: string;