diff --git a/__mocks__/data.ts b/__mocks__/data.ts index 81226cb..9fc5652 100644 --- a/__mocks__/data.ts +++ b/__mocks__/data.ts @@ -22,6 +22,7 @@ export const dummyKeywords = [ lastUpdated: '2022-11-15T10:49:53.113', added: '2022-11-11T10:01:06.951', position: 19, + volume: 10000, history: { '2022-11-11': 21, '2022-11-12': 24, @@ -45,6 +46,7 @@ export const dummyKeywords = [ lastUpdated: '2022-11-15T10:49:53.119', added: '2022-11-15T10:01:06.951', position: 29, + volume: 1200, history: { '2022-11-11': 33, '2022-11-12': 34, diff --git a/components/common/ChartSlim.tsx b/components/common/ChartSlim.tsx index 3c6dad2..8f126f1 100644 --- a/components/common/ChartSlim.tsx +++ b/components/common/ChartSlim.tsx @@ -36,7 +36,7 @@ const ChartSlim = ({ labels, sreies, noMaxLimit = false }:ChartProps) => { }, }; - return
+ return
{ scDataType = 'threeDays', } = props; const { - keyword, domain, ID, city, position, url = '', lastUpdated, country, sticky, history = {}, updating = false, lastUpdateError = false, + keyword, domain, ID, city, position, url = '', lastUpdated, country, sticky, history = {}, updating = false, lastUpdateError = false, volume, } = keywordData; const [showOptions, setShowOptions] = useState(false); const [showPositionError, setPositionError] = useState(false); @@ -99,7 +100,7 @@ const Keyword = (props: KeywordProps) => { showKeywordDetails()}> {keyword}{city ? ` (${city})` : ''} @@ -131,12 +132,18 @@ const Keyword = (props: KeywordProps) => { {chartData.labels.length > 0 && (
showKeywordDetails()}>
)} + +
diff --git a/components/keywords/KeywordFilter.tsx b/components/keywords/KeywordFilter.tsx index 288a0e7..966f39f 100644 --- a/components/keywords/KeywordFilter.tsx +++ b/components/keywords/KeywordFilter.tsx @@ -76,6 +76,8 @@ const KeywordFilters = (props: KeywordFilterProps) => { { value: 'date_desc', label: 'Oldest' }, { value: 'alpha_asc', label: 'Alphabetically(A-Z)' }, { value: 'alpha_desc', label: 'Alphabetically(Z-A)' }, + { value: 'vol_asc', label: 'Lowest Search Volume' }, + { value: 'vol_desc', label: 'Highest Search Volume' }, ]; if (integratedConsole) { sortOptionChoices.push({ value: 'imp_desc', label: `Most Viewed${isConsole ? ' (Default)' : ''}` }); @@ -170,8 +172,8 @@ const KeywordFilters = (props: KeywordFilterProps) => { {sortOptions && (
    + className='sort_options mt-2 border absolute w-48 min-w-[0] right-0 rounded-lg + max-h-96 bg-white z-[9999] overflow-y-auto styled-scrollbar'> {sortOptionChoices.map((sortOption) => { return
  • { Position Best - History (7d) + History (7d) + Volume URL - Updated + Updated {showSCData && (
    {/* Search Console */} diff --git a/components/settings/AdWordsSettings.tsx b/components/settings/AdWordsSettings.tsx index 12c4e98..d1bbcfa 100644 --- a/components/settings/AdWordsSettings.tsx +++ b/components/settings/AdWordsSettings.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import { useTestAdwordsIntegration } from '../../services/adwords'; +import { useMutateKeywordsVolume, useTestAdwordsIntegration } from '../../services/adwords'; import Icon from '../common/Icon'; import SecretField from '../common/SecretField'; @@ -24,7 +24,10 @@ const AdWordsSettings = ({ settings, settingsError, updateSettings, performUpdat } = settings || {}; const { mutate: testAdWordsIntegration, isLoading: isTesting } = useTestAdwordsIntegration(); + const { mutate: getAllVolumeData, isLoading: isUpdatingVolume } = useMutateKeywordsVolume(); + const cloudProjectIntegrated = adwords_client_id && adwords_client_secret && adwords_refresh_token; + const hasAllCredentials = adwords_client_id && adwords_client_secret && adwords_refresh_token && adwords_developer_token && adwords_account_id; const udpateAndAuthenticate = () => { if (adwords_client_id && adwords_client_secret) { @@ -40,11 +43,17 @@ const AdWordsSettings = ({ settings, settingsError, updateSettings, performUpdat }; const testIntegration = () => { - if (adwords_client_id && adwords_client_secret && adwords_refresh_token && adwords_developer_token && adwords_account_id) { + if (hasAllCredentials) { testAdWordsIntegration({ developer_token: adwords_developer_token, account_id: adwords_account_id }); } }; + const updateVolumeData = () => { + if (hasAllCredentials) { + getAllVolumeData({ domain: 'all' }); + } + }; + return (
    @@ -98,9 +107,9 @@ const AdWordsSettings = ({ settings, settingsError, updateSettings, performUpdat
    +
    + {!hasAllCredentials &&
    } +

    Update Keyword Volume Data

    +
    +
    +

    Update Volume data for all your Tracked Keywords.

    +
    +
    + +
    +
    +

    Integrate Google Adwords to get Keyword Ideas & Search Volume.{' '}

    diff --git a/database/migrations/1709217223856-add-keyword-volume-field copy.js b/database/migrations/1709217223856-add-keyword-volume-field copy.js new file mode 100644 index 0000000..19b3564 --- /dev/null +++ b/database/migrations/1709217223856-add-keyword-volume-field copy.js @@ -0,0 +1,35 @@ +// Migration: Adds volume field to the keyword table. + +// CLI Migration +module.exports = { + up: async (queryInterface, Sequelize) => { + return queryInterface.sequelize.transaction(async (t) => { + try { + const keywordTableDefinition = await queryInterface.describeTable('keyword'); + if (keywordTableDefinition) { + if (!keywordTableDefinition.volume) { + await queryInterface.addColumn('keyword', 'volume', { + type: Sequelize.DataTypes.STRING, allowNull: false, defaultValue: 0, + }, { transaction: t }); + } + } + } catch (error) { + console.log('error :', error); + } + }); + }, + down: (queryInterface) => { + return queryInterface.sequelize.transaction(async (t) => { + try { + const keywordTableDefinition = await queryInterface.describeTable('keyword'); + if (keywordTableDefinition) { + if (keywordTableDefinition.volume) { + await queryInterface.removeColumn('keyword', 'volume', { transaction: t }); + } + } + } catch (error) { + console.log('error :', error); + } + }); + }, +}; diff --git a/database/models/keyword.ts b/database/models/keyword.ts index 352e8a8..884c304 100644 --- a/database/models/keyword.ts +++ b/database/models/keyword.ts @@ -47,6 +47,9 @@ class Keyword extends Model { @Column({ type: DataType.STRING, allowNull: true, defaultValue: JSON.stringify([]) }) history!: string; + @Column({ type: DataType.INTEGER, allowNull: false, defaultValue: 0 }) + volume!: number; + @Column({ type: DataType.STRING, allowNull: true, defaultValue: JSON.stringify([]) }) url!: string; diff --git a/pages/api/keywords.ts b/pages/api/keywords.ts index 7139e63..a02a317 100644 --- a/pages/api/keywords.ts +++ b/pages/api/keywords.ts @@ -7,6 +7,7 @@ import verifyUser from '../../utils/verifyUser'; import parseKeywords from '../../utils/parseKeywords'; import { integrateKeywordSCData, readLocalSCData } from '../../utils/searchConsole'; import refreshAndUpdateKeywords from '../../utils/refresh'; +import { getKeywordsVolume, updateKeywordsVolumeData } from '../../utils/adwords'; type KeywordsGetResponse = { keywords?: KeywordType[], @@ -103,8 +104,20 @@ const addKeywords = async (req: NextApiRequest, res: NextApiResponse el.get({ plain: true })); const keywordsParsed: KeywordType[] = parseKeywords(formattedkeywords); + + // Queue the SERP Scraping Process const settings = await getAppSettings(); - refreshAndUpdateKeywords(newKeywords, settings); // Queue the SERP Scraping Process + refreshAndUpdateKeywords(newKeywords, settings); + + // Update the Keyword Volume + const { adwords_account_id, adwords_client_id, adwords_client_secret, adwords_developer_token } = settings; + if (adwords_account_id && adwords_client_id && adwords_client_secret && adwords_developer_token) { + const keywordsVolumeData = await getKeywordsVolume(keywordsParsed); + if (keywordsVolumeData.volumes !== false) { + await updateKeywordsVolumeData(keywordsVolumeData.volumes); + } + } + return res.status(201).json({ keywords: keywordsParsed }); } catch (error) { console.log('[ERROR] Adding New Keywords ', error); diff --git a/pages/api/refresh.ts b/pages/api/refresh.ts index 222f599..19e8d9b 100644 --- a/pages/api/refresh.ts +++ b/pages/api/refresh.ts @@ -93,6 +93,7 @@ const getKeywordSearchResults = async (req: NextApiRequest, res: NextApiResponse country: req.query.country as string, domain: '', lastUpdated: '', + volume: 0, added: '', position: 111, sticky: false, diff --git a/pages/api/volume.ts b/pages/api/volume.ts new file mode 100644 index 0000000..a2aabcd --- /dev/null +++ b/pages/api/volume.ts @@ -0,0 +1,67 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { Op } from 'sequelize'; +import db from '../../database/database'; +import Keyword from '../../database/models/keyword'; +import verifyUser from '../../utils/verifyUser'; +import parseKeywords from '../../utils/parseKeywords'; +import { getKeywordsVolume, updateKeywordsVolumeData } from '../../utils/adwords'; + +type KeywordsRefreshRes = { + keywords?: KeywordType[] + error?: string|null, +} + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + await db.sync(); + const authorized = verifyUser(req, res); + if (authorized !== 'authorized') { + return res.status(401).json({ error: authorized }); + } + if (req.method === 'POST') { + return updatekeywordVolume(req, res); + } + return res.status(502).json({ error: 'Unrecognized Route.' }); +} + +const updatekeywordVolume = async (req: NextApiRequest, res: NextApiResponse) => { + const { keywords = [], domain = '', update = true } = req.body || {}; + if (keywords.length === 0 && !domain) { + return res.status(400).json({ error: 'Please provide keyword Ids or a domain name.' }); + } + + try { + let keywordsToSend: KeywordType[] = []; + if (keywords.length > 0) { + const allKeywords:Keyword[] = await Keyword.findAll({ where: { ID: { [Op.in]: keywords } } }); + keywordsToSend = parseKeywords(allKeywords.map((e) => e.get({ plain: true }))); + } + if (domain) { + // const allDomain = domain === 'all'; + // const allKeywords:Keyword[] = allDomain ? await Keyword.findAll() : await Keyword.findAll(allDomain ? {} : { where: { domain } }); + // keywordsToSend = parseKeywords(allKeywords.map((e) => e.get({ plain: true }))); + } + + if (keywordsToSend.length > 0) { + const keywordsVolumeData = await getKeywordsVolume(keywordsToSend); + // console.log('keywordsVolumeData :', keywordsVolumeData); + if (keywordsVolumeData.error) { + return res.status(400).json({ keywords: [], error: keywordsVolumeData.error }); + } + if (keywordsVolumeData.volumes !== false) { + if (update) { + const updated = await updateKeywordsVolumeData(keywordsVolumeData.volumes); + if (updated) { + return res.status(200).json({ keywords }); + } + } + } else { + return res.status(400).json({ error: 'Error Fetching Keywords Volume Data from Google Adwords' }); + } + } + + return res.status(400).json({ keywords: [], error: 'Error Updating Keywords Volume data' }); + } catch (error) { + console.log('[Error] updating keywords Volume Data: ', error); + return res.status(400).json({ error: 'Error Updating Keywords Volume data' }); + } +}; diff --git a/services/adwords.tsx b/services/adwords.tsx index 732b7fe..dd5b7ef 100644 --- a/services/adwords.tsx +++ b/services/adwords.tsx @@ -99,3 +99,30 @@ export function useMutateFavKeywordIdeas(router:NextRouter, onSuccess?: Function }, }); } + +export function useMutateKeywordsVolume(onSuccess?: Function) { + return useMutation(async (data:Record) => { + const headers = new Headers({ 'Content-Type': 'application/json', Accept: 'application/json' }); + const fetchOpts = { method: 'POST', headers, body: JSON.stringify({ ...data }) }; + const res = await fetch(`${window.location.origin}/api/volume`, fetchOpts); + if (res.status >= 400 && res.status < 600) { + const errorData = await res.json(); + throw new Error(errorData?.error ? errorData.error : 'Bad response from server'); + } + return res.json(); + }, { + onSuccess: async (data) => { + toast('Keyword Volume Data Loaded Successfully! Reloading Page...', { icon: '✔️' }); + if (onSuccess) { + onSuccess(false); + } + setTimeout(() => { + window.location.reload(); + }, 3000); + }, + onError: (error) => { + console.log('Error Loading Keyword Volume Data!!!', error); + toast('Error Loading Keyword Volume Data', { icon: '⚠️' }); + }, + }); +} diff --git a/tailwind.config.js b/tailwind.config.js index 05981e0..76a83a0 100644 --- a/tailwind.config.js +++ b/tailwind.config.js @@ -16,6 +16,7 @@ module.exports = { 'w-[240px]', 'min-w-[270px]', 'min-w-[180px]', + 'max-w-[180px]', ], theme: { extend: {}, diff --git a/types.d.ts b/types.d.ts index af1fcac..7edbab4 100644 --- a/types.d.ts +++ b/types.d.ts @@ -32,6 +32,7 @@ type KeywordType = { lastUpdated: string, added: string, position: number, + volume: number, sticky: boolean, history: KeywordHistory, lastResult: KeywordLastResult[], diff --git a/utils/adwords.ts b/utils/adwords.ts index ea93ba7..a2b83c7 100644 --- a/utils/adwords.ts +++ b/utils/adwords.ts @@ -1,6 +1,7 @@ import { readFile, writeFile } from 'fs/promises'; import Cryptr from 'cryptr'; import TTLCache from '@isaacs/ttlcache'; +import { setTimeout as sleep } from 'timers/promises'; import Keyword from '../database/models/keyword'; import parseKeywords from './parseKeywords'; import countries from './countries'; @@ -8,15 +9,17 @@ import { readLocalSCData } from './searchConsole'; const memoryCache = new TTLCache({ max: 10000 }); +type keywordIdeasMetrics = { + competition: IdeaKeyword['competition'], + monthlySearchVolumes: {month : string, year : string, monthlySearches : string}[], + avgMonthlySearches: string, + competitionIndex: string, + lowTopOfPageBidMicros: string, + highTopOfPageBidMicros: string + } + type keywordIdeasResponseItem = { - keywordIdeaMetrics: { - competition: IdeaKeyword['competition'], - monthlySearchVolumes: {month : string, year : string, monthlySearches : string}[], - avgMonthlySearches: string, - competitionIndex: string, - lowTopOfPageBidMicros: string, - highTopOfPageBidMicros: string - }, + keywordIdeaMetrics: keywordIdeasMetrics, text: string, keywordAnnotations: Object }; @@ -250,6 +253,130 @@ const extractAdwordskeywordIdeas = (keywordIdeas:keywordIdeasResponseItem[], opt return keywords.sort((a: IdeaKeyword, b: IdeaKeyword) => (b.avgMonthlySearches > a.avgMonthlySearches ? 1 : -1)); }; +/** + * Retrieves keyword search volumes from Google Adwords API based on provided keywords and their countries. + * @param {KeywordType[]} keywords - The keywords that you want to get the search volume data for. + * @returns returns a Promise that resolves to an object with a `volumes` and error `proprties`. + * The `volumes` propery which outputs `false` if the request fails and outputs the volume data in `{[keywordID]: volume}` object if succeeds. + * The `error` porperty that outputs the error message if any. + */ +export const getKeywordsVolume = async (keywords: KeywordType[]): Promise<{error?: string, volumes: false | Record}> => { + const credentials = await getAdwordsCredentials(); + if (!credentials) { return { error: 'Cannot Load Adwords Credentials', volumes: false }; } + const { client_id, client_secret, developer_token, account_id } = credentials; + if (!client_id || !client_secret || !developer_token || !account_id) { + return { error: 'Adwords Not Integrated Properly', volumes: false }; + } + + // Generate Access Token + let accessToken = ''; + const cachedAccessToken:string|false|undefined = memoryCache.get('adwords_token'); + if (cachedAccessToken && !test) { + accessToken = cachedAccessToken; + } else { + accessToken = await getAdwordsAccessToken(credentials); + memoryCache.delete('adwords_token'); + memoryCache.set('adwords_token', accessToken, { ttl: 3300000 }); + } + const fetchedKeywords:Record = {}; + + if (accessToken) { + // Group keywords based on their country. + const keywordRequests: Record = {}; + keywords.forEach((kw) => { + const kwCountry = kw.country; + if (keywordRequests[kwCountry]) { + keywordRequests[kwCountry].push(kw); + } else { + keywordRequests[kwCountry] = [kw]; + } + }); + + // Send Requests to adwords based on grouped countries. + // Since adwords does not allow sending country data for each keyword we are making requests for. + for (const country in keywordRequests) { + if (Object.hasOwn(keywordRequests, country) && keywordRequests[country].length > 0) { + try { + // API: https://developers.google.com/google-ads/api/rest/reference/rest/v16/customers/generateKeywordHistoricalMetrics + const customerID = account_id.replaceAll('-', ''); + const geoTargetConstants = countries[country][3]; // '2840'; + const reqKeywords = keywordRequests[country].map((kw) => kw.keyword); + const reqPayload: Record = { + keywords: [...new Set(reqKeywords)], + geoTargetConstants: `geoTargetConstants/${geoTargetConstants}`, + // language: `languageConstants/${language}`, + }; + const resp = await fetch(`https://googleads.googleapis.com/v16/customers/${customerID}:generateKeywordHistoricalMetrics`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'developer-token': developer_token, + Authorization: `Bearer ${accessToken}`, + 'login-customer-id': customerID, + }, + body: JSON.stringify(reqPayload), + }); + const ideaData = await resp.json(); + + if (resp.status !== 200) { + console.log('[ERROR] Adwords Volume Request Response :', ideaData?.error?.details[0]?.errors[0]?.message); + console.log('Response from AdWords :', JSON.stringify(ideaData, null, 2)); + } + + if (ideaData?.results) { + if (Array.isArray(ideaData.results) && ideaData.results.length > 0) { + const volumeDataObj:Map = new Map(); + ideaData.results.forEach((item:{ keywordMetrics: keywordIdeasMetrics, text: string }) => { + const kwVol = item?.keywordMetrics?.avgMonthlySearches; + volumeDataObj.set(`${country}:${item.text}`, kwVol ? parseInt(kwVol, 10) : 0); + }); + + keywordRequests[country].forEach((keyword) => { + const keywordKey = `${keyword.country}:${keyword.keyword}`; + if (volumeDataObj.has(keywordKey)) { + const volume = volumeDataObj.get(keywordKey); + if (volume !== undefined) { + fetchedKeywords[keyword.ID] = volume; + } + } + }); + // console.log('fetchedKeywords :', fetchedKeywords); + } + } + } catch (error) { + console.log('[ERROR] Fetching Keyword Volume from Adwords :', error); + } + if (Object.keys(keywordRequests).length > 1) { + await sleep(7000); + } + } + } + } + + return { volumes: fetchedKeywords }; +}; + +/** + * Updates volume data for keywords in the Keywords database using async/await and error handling. + * @param {false | Record} volumesData - The `volumesData` parameter can either be `false` or an object containing + * keyword IDs as keys and corresponding volume data as values. + * @returns returns a Promise that resolves to `true` if `volumesData` is not `false` else it returns `false`. + */ +export const updateKeywordsVolumeData = async (volumesData: false | Record) => { + if (volumesData === false) { return false; } + + Object.keys(volumesData).forEach(async (keywordID) => { + const keyID = parseInt(keywordID, 10); + const volumeData = volumesData && volumesData[keyID] ? volumesData[keyID] : 0; + try { + await Keyword.update({ volume: volumeData }, { where: { ID: keyID } }); + } catch (error) { + console.log(''); + } + }); + return true; +}; + /** * The function `getLocalKeywordIdeas` reads keyword ideas data from a local JSON file based on a domain slug and returns it as a Promise. * @param {string} domain - The `domain` parameter is the domain slug for which the keyword Ideas are fetched. diff --git a/utils/client/sortFilter.ts b/utils/client/sortFilter.ts index 80204eb..aef8d73 100644 --- a/utils/client/sortFilter.ts +++ b/utils/client/sortFilter.ts @@ -28,6 +28,12 @@ export const sortKeywords = (theKeywords:KeywordType[], sortBy:string, scDataTyp case 'alpha_desc': sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => (a.keyword > b.keyword ? 1 : -1)); break; + case 'vol_asc': + sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => (b.volume - a.volume)); + break; + case 'vol_desc': + sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => (a.volume - b.volume)); + break; case 'imp_desc': if (scDataType) { sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => {