Skip to content

Commit

Permalink
feat: Adds keyword search volume data feature for tracked keywords.
Browse files Browse the repository at this point in the history
- Adds a volume field in the keyword table.
- Adds a button in the Adwords Integration screen to update all the tracked keywords.
- When a new keyword is added, the volume data is automatically fetched.
- Adds ability to sort keywords based on search volume.
  • Loading branch information
towfiqi committed Mar 1, 2024
1 parent 4d15989 commit 2a1fc0e
Show file tree
Hide file tree
Showing 16 changed files with 341 additions and 21 deletions.
2 changes: 2 additions & 0 deletions __mocks__/data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ export const dummyKeywords = [
lastUpdated: '2022-11-15T10:49:53.113',
added: '2022-11-11T10:01:06.951',
position: 19,
volume: 10000,
history: {
'2022-11-11': 21,
'2022-11-12': 24,
Expand All @@ -45,6 +46,7 @@ export const dummyKeywords = [
lastUpdated: '2022-11-15T10:49:53.119',
added: '2022-11-15T10:01:06.951',
position: 29,
volume: 1200,
history: {
'2022-11-11': 33,
'2022-11-12': 34,
Expand Down
2 changes: 1 addition & 1 deletion components/common/ChartSlim.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ const ChartSlim = ({ labels, sreies, noMaxLimit = false }:ChartProps) => {
},
};

return <div className='w-[100px] h-[30px] rounded border border-gray-200'>
return <div className='w-[80px] h-[30px] rounded border border-gray-200'>
<Line
datasetIdKey='XXX'
options={options}
Expand Down
13 changes: 10 additions & 3 deletions components/keywords/Keyword.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import countries from '../../utils/countries';
import ChartSlim from '../common/ChartSlim';
import KeywordPosition from './KeywordPosition';
import { generateTheChartData } from '../../utils/client/generateChartData';
import { formattedNum } from '../../utils/client/helpers';

type KeywordProps = {
keywordData: KeywordType,
Expand Down Expand Up @@ -40,7 +41,7 @@ const Keyword = (props: KeywordProps) => {
scDataType = 'threeDays',
} = props;
const {
keyword, domain, ID, city, position, url = '', lastUpdated, country, sticky, history = {}, updating = false, lastUpdateError = false,
keyword, domain, ID, city, position, url = '', lastUpdated, country, sticky, history = {}, updating = false, lastUpdateError = false, volume,
} = keywordData;
const [showOptions, setShowOptions] = useState(false);
const [showPositionError, setPositionError] = useState(false);
Expand Down Expand Up @@ -99,7 +100,7 @@ const Keyword = (props: KeywordProps) => {
<Icon type="check" size={10} />
</button>
<a
className='py-2 hover:text-blue-600 lg:flex lg:items-center lg:w-full lg:max-w-[200px]'
className={`py-2 hover:text-blue-600 lg:flex lg:items-center lg:w-full ${showSCData ? 'lg:max-w-[180px]' : 'lg:max-w-[240px]'}`}
onClick={() => showKeywordDetails()}>
<span className={`fflag fflag-${country} w-[18px] h-[12px] mr-2`} title={countries[country][0]} />
<span className=' text-ellipsis overflow-hidden whitespace-nowrap w-[calc(100%-30px)]'>{keyword}{city ? ` (${city})` : ''}</span>
Expand Down Expand Up @@ -131,12 +132,18 @@ const Keyword = (props: KeywordProps) => {

{chartData.labels.length > 0 && (
<div
className='hidden basis-32 grow-0 cursor-pointer lg:block'
className='hidden basis-20 grow-0 cursor-pointer lg:block'
onClick={() => showKeywordDetails()}>
<ChartSlim labels={chartData.labels} sreies={chartData.sreies} />
</div>
)}

<div
className={`keyword_best hidden bg-[#f8f9ff] w-fit min-w-[50px] h-12 p-2 text-base mt-[-20px] rounded right-5 lg:relative lg:block
lg:bg-transparent lg:w-auto lg:h-auto lg:mt-0 lg:p-0 lg:text-sm lg:flex-1 lg:basis-24 lg:grow-0 lg:right-0 text-center`}>
{formattedNum(volume)}
</div>

<div
className={`keyword_url inline-block mt-4 mr-5 ml-5 lg:flex-1 text-gray-400 lg:m-0 max-w-[70px]
overflow-hidden text-ellipsis whitespace-nowrap lg:max-w-none lg:pr-5`}>
Expand Down
6 changes: 4 additions & 2 deletions components/keywords/KeywordFilter.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,8 @@ const KeywordFilters = (props: KeywordFilterProps) => {
{ value: 'date_desc', label: 'Oldest' },
{ value: 'alpha_asc', label: 'Alphabetically(A-Z)' },
{ value: 'alpha_desc', label: 'Alphabetically(Z-A)' },
{ value: 'vol_asc', label: 'Lowest Search Volume' },
{ value: 'vol_desc', label: 'Highest Search Volume' },
];
if (integratedConsole) {
sortOptionChoices.push({ value: 'imp_desc', label: `Most Viewed${isConsole ? ' (Default)' : ''}` });
Expand Down Expand Up @@ -170,8 +172,8 @@ const KeywordFilters = (props: KeywordFilterProps) => {
{sortOptions && (
<ul
data-testid="sort_options"
className='sort_options mt-2 border absolute min-w-[0] right-0 rounded-lg
max-h-96 bg-white z-[9999] w-44 overflow-y-auto styled-scrollbar'>
className='sort_options mt-2 border absolute w-48 min-w-[0] right-0 rounded-lg
max-h-96 bg-white z-[9999] overflow-y-auto styled-scrollbar'>
{sortOptionChoices.map((sortOption) => {
return <li
key={sortOption.value}
Expand Down
5 changes: 3 additions & 2 deletions components/keywords/KeywordsTable.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -158,9 +158,10 @@ const KeywordsTable = (props: KeywordsTableProps) => {
</span>
<span className='domKeywords_head_position flex-1 basis-24 grow-0 text-center'>Position</span>
<span className='domKeywords_head_best flex-1 basis-16 grow-0 text-center'>Best</span>
<span className='domKeywords_head_history flex-1 basis-32 grow-0 '>History (7d)</span>
<span className='domKeywords_head_history flex-1 basis-20 grow-0'>History (7d)</span>
<span className='domKeywords_head_volume flex-1 basis-24 grow-0 text-center'>Volume</span>
<span className='domKeywords_head_url flex-1'>URL</span>
<span className='domKeywords_head_updated flex-1'>Updated</span>
<span className='domKeywords_head_updated flex-1 relative left-3'>Updated</span>
{showSCData && (
<div className='domKeywords_head_sc flex-1 min-w-[170px] mr-7 text-center'>
{/* Search Console */}
Expand Down
35 changes: 31 additions & 4 deletions components/settings/AdWordsSettings.tsx
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import React from 'react';
import { useTestAdwordsIntegration } from '../../services/adwords';
import { useMutateKeywordsVolume, useTestAdwordsIntegration } from '../../services/adwords';
import Icon from '../common/Icon';
import SecretField from '../common/SecretField';

Expand All @@ -24,7 +24,10 @@ const AdWordsSettings = ({ settings, settingsError, updateSettings, performUpdat
} = settings || {};

const { mutate: testAdWordsIntegration, isLoading: isTesting } = useTestAdwordsIntegration();
const { mutate: getAllVolumeData, isLoading: isUpdatingVolume } = useMutateKeywordsVolume();

const cloudProjectIntegrated = adwords_client_id && adwords_client_secret && adwords_refresh_token;
const hasAllCredentials = adwords_client_id && adwords_client_secret && adwords_refresh_token && adwords_developer_token && adwords_account_id;

const udpateAndAuthenticate = () => {
if (adwords_client_id && adwords_client_secret) {
Expand All @@ -40,11 +43,17 @@ const AdWordsSettings = ({ settings, settingsError, updateSettings, performUpdat
};

const testIntegration = () => {
if (adwords_client_id && adwords_client_secret && adwords_refresh_token && adwords_developer_token && adwords_account_id) {
if (hasAllCredentials) {
testAdWordsIntegration({ developer_token: adwords_developer_token, account_id: adwords_account_id });
}
};

const updateVolumeData = () => {
if (hasAllCredentials) {
getAllVolumeData({ domain: 'all' });
}
};

return (
<div>
<div>
Expand Down Expand Up @@ -98,16 +107,34 @@ const AdWordsSettings = ({ settings, settingsError, updateSettings, performUpdat
<div className="settings__section__input mb-4 flex justify-between items-center w-full">
<button
className={`py-2 px-5 w-full text-sm font-semibold rounded bg-indigo-50 text-blue-700 border border-indigo-100
${adwords_client_id && adwords_client_secret && adwords_refresh_token ? 'cursor-pointer' : 'cursor-not-allowed opacity-40'}
${hasAllCredentials ? 'cursor-pointer' : 'cursor-not-allowed opacity-40'}
hover:bg-blue-700 hover:text-white transition`}
title='Insert All the data in the above fields to Authenticate'
title={hasAllCredentials ? '' : 'Insert All the data in the above fields to Test the Integration'}
onClick={testIntegration}>
{isTesting && <Icon type='loading' />}
<Icon type='adwords' size={14} /> Test AdWords Integration
</button>
</div>
</div>
</div>
<div className='mt-4 mb-4 border-b border-gray-100 pt-4 pb-0 relative'>
{!hasAllCredentials && <div className=' absolute w-full h-full z-50' />}
<h4 className=' mb-3 font-semibold text-blue-700'>Update Keyword Volume Data</h4>
<div className={!hasAllCredentials ? 'opacity-40' : ''}>
<div className="settings__section__input mb-4 flex justify-between items-center w-full">
<p>Update Volume data for all your Tracked Keywords.</p>
</div>
<div className="settings__section__input mb-4 flex justify-between items-center w-full">
<button
className={`py-2 px-5 w-full text-sm font-semibold rounded bg-indigo-50 text-blue-700 border border-indigo-100
${hasAllCredentials ? 'cursor-pointer' : 'cursor-not-allowed opacity-40'}
hover:bg-blue-700 hover:text-white transition`}
onClick={updateVolumeData}>
<Icon type={isUpdatingVolume ? 'loading' : 'reload'} size={isUpdatingVolume ? 16 : 12} /> Update Keywords Volume
</button>
</div>
</div>
</div>
<p className='mb-4 text-xs'>
<a target='_blank' rel='noreferrer' href='https://docs.serpbear.com/keyword-research' className=' underline text-blue-600'>Integrate Google Adwords</a> to get Keyword Ideas & Search Volume.{' '}
</p>
Expand Down
35 changes: 35 additions & 0 deletions database/migrations/1709217223856-add-keyword-volume-field copy.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
// Migration: Adds volume field to the keyword table.

// CLI Migration
module.exports = {
up: async (queryInterface, Sequelize) => {
return queryInterface.sequelize.transaction(async (t) => {
try {
const keywordTableDefinition = await queryInterface.describeTable('keyword');
if (keywordTableDefinition) {
if (!keywordTableDefinition.volume) {
await queryInterface.addColumn('keyword', 'volume', {
type: Sequelize.DataTypes.STRING, allowNull: false, defaultValue: 0,
}, { transaction: t });
}
}
} catch (error) {
console.log('error :', error);
}
});
},
down: (queryInterface) => {
return queryInterface.sequelize.transaction(async (t) => {
try {
const keywordTableDefinition = await queryInterface.describeTable('keyword');
if (keywordTableDefinition) {
if (keywordTableDefinition.volume) {
await queryInterface.removeColumn('keyword', 'volume', { transaction: t });
}
}
} catch (error) {
console.log('error :', error);
}
});
},
};
3 changes: 3 additions & 0 deletions database/models/keyword.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,9 @@ class Keyword extends Model {
@Column({ type: DataType.STRING, allowNull: true, defaultValue: JSON.stringify([]) })
history!: string;

@Column({ type: DataType.INTEGER, allowNull: false, defaultValue: 0 })
volume!: number;

@Column({ type: DataType.STRING, allowNull: true, defaultValue: JSON.stringify([]) })
url!: string;

Expand Down
15 changes: 14 additions & 1 deletion pages/api/keywords.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import verifyUser from '../../utils/verifyUser';
import parseKeywords from '../../utils/parseKeywords';
import { integrateKeywordSCData, readLocalSCData } from '../../utils/searchConsole';
import refreshAndUpdateKeywords from '../../utils/refresh';
import { getKeywordsVolume, updateKeywordsVolumeData } from '../../utils/adwords';

type KeywordsGetResponse = {
keywords?: KeywordType[],
Expand Down Expand Up @@ -103,8 +104,20 @@ const addKeywords = async (req: NextApiRequest, res: NextApiResponse<KeywordsGet
const newKeywords:Keyword[] = await Keyword.bulkCreate(keywordsToAdd);
const formattedkeywords = newKeywords.map((el) => el.get({ plain: true }));
const keywordsParsed: KeywordType[] = parseKeywords(formattedkeywords);

// Queue the SERP Scraping Process
const settings = await getAppSettings();
refreshAndUpdateKeywords(newKeywords, settings); // Queue the SERP Scraping Process
refreshAndUpdateKeywords(newKeywords, settings);

// Update the Keyword Volume
const { adwords_account_id, adwords_client_id, adwords_client_secret, adwords_developer_token } = settings;
if (adwords_account_id && adwords_client_id && adwords_client_secret && adwords_developer_token) {
const keywordsVolumeData = await getKeywordsVolume(keywordsParsed);
if (keywordsVolumeData.volumes !== false) {
await updateKeywordsVolumeData(keywordsVolumeData.volumes);
}
}

return res.status(201).json({ keywords: keywordsParsed });
} catch (error) {
console.log('[ERROR] Adding New Keywords ', error);
Expand Down
1 change: 1 addition & 0 deletions pages/api/refresh.ts
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ const getKeywordSearchResults = async (req: NextApiRequest, res: NextApiResponse
country: req.query.country as string,
domain: '',
lastUpdated: '',
volume: 0,
added: '',
position: 111,
sticky: false,
Expand Down
67 changes: 67 additions & 0 deletions pages/api/volume.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { Op } from 'sequelize';
import db from '../../database/database';
import Keyword from '../../database/models/keyword';
import verifyUser from '../../utils/verifyUser';
import parseKeywords from '../../utils/parseKeywords';
import { getKeywordsVolume, updateKeywordsVolumeData } from '../../utils/adwords';

type KeywordsRefreshRes = {
keywords?: KeywordType[]
error?: string|null,
}

export default async function handler(req: NextApiRequest, res: NextApiResponse) {
await db.sync();
const authorized = verifyUser(req, res);
if (authorized !== 'authorized') {
return res.status(401).json({ error: authorized });
}
if (req.method === 'POST') {
return updatekeywordVolume(req, res);
}
return res.status(502).json({ error: 'Unrecognized Route.' });
}

const updatekeywordVolume = async (req: NextApiRequest, res: NextApiResponse<KeywordsRefreshRes>) => {
const { keywords = [], domain = '', update = true } = req.body || {};
if (keywords.length === 0 && !domain) {
return res.status(400).json({ error: 'Please provide keyword Ids or a domain name.' });
}

try {
let keywordsToSend: KeywordType[] = [];
if (keywords.length > 0) {
const allKeywords:Keyword[] = await Keyword.findAll({ where: { ID: { [Op.in]: keywords } } });
keywordsToSend = parseKeywords(allKeywords.map((e) => e.get({ plain: true })));
}
if (domain) {
// const allDomain = domain === 'all';
// const allKeywords:Keyword[] = allDomain ? await Keyword.findAll() : await Keyword.findAll(allDomain ? {} : { where: { domain } });
// keywordsToSend = parseKeywords(allKeywords.map((e) => e.get({ plain: true })));
}

if (keywordsToSend.length > 0) {
const keywordsVolumeData = await getKeywordsVolume(keywordsToSend);
// console.log('keywordsVolumeData :', keywordsVolumeData);
if (keywordsVolumeData.error) {
return res.status(400).json({ keywords: [], error: keywordsVolumeData.error });
}
if (keywordsVolumeData.volumes !== false) {
if (update) {
const updated = await updateKeywordsVolumeData(keywordsVolumeData.volumes);
if (updated) {
return res.status(200).json({ keywords });
}
}
} else {
return res.status(400).json({ error: 'Error Fetching Keywords Volume Data from Google Adwords' });
}
}

return res.status(400).json({ keywords: [], error: 'Error Updating Keywords Volume data' });
} catch (error) {
console.log('[Error] updating keywords Volume Data: ', error);
return res.status(400).json({ error: 'Error Updating Keywords Volume data' });
}
};
27 changes: 27 additions & 0 deletions services/adwords.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -99,3 +99,30 @@ export function useMutateFavKeywordIdeas(router:NextRouter, onSuccess?: Function
},
});
}

export function useMutateKeywordsVolume(onSuccess?: Function) {
return useMutation(async (data:Record<string, any>) => {
const headers = new Headers({ 'Content-Type': 'application/json', Accept: 'application/json' });
const fetchOpts = { method: 'POST', headers, body: JSON.stringify({ ...data }) };
const res = await fetch(`${window.location.origin}/api/volume`, fetchOpts);
if (res.status >= 400 && res.status < 600) {
const errorData = await res.json();
throw new Error(errorData?.error ? errorData.error : 'Bad response from server');
}
return res.json();
}, {
onSuccess: async (data) => {
toast('Keyword Volume Data Loaded Successfully! Reloading Page...', { icon: '✔️' });
if (onSuccess) {
onSuccess(false);
}
setTimeout(() => {
window.location.reload();
}, 3000);
},
onError: (error) => {
console.log('Error Loading Keyword Volume Data!!!', error);
toast('Error Loading Keyword Volume Data', { icon: '⚠️' });
},
});
}
1 change: 1 addition & 0 deletions tailwind.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ module.exports = {
'w-[240px]',
'min-w-[270px]',
'min-w-[180px]',
'max-w-[180px]',
],
theme: {
extend: {},
Expand Down
1 change: 1 addition & 0 deletions types.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ type KeywordType = {
lastUpdated: string,
added: string,
position: number,
volume: number,
sticky: boolean,
history: KeywordHistory,
lastResult: KeywordLastResult[],
Expand Down
Loading

0 comments on commit 2a1fc0e

Please sign in to comment.