diff --git a/src/middleware/stats.ts b/src/middleware/stats.ts index 19a66185a..a941b60e7 100644 --- a/src/middleware/stats.ts +++ b/src/middleware/stats.ts @@ -37,10 +37,6 @@ export class StatsMiddleware { countDbUsers = wrapAsRequestHandler(() => this.statsService.countDbUsers()) - countGithubSites = wrapAsRequestHandler(() => - this.statsService.countGithubSites() - ) - countMigratedSites = wrapAsRequestHandler(() => this.statsService.countMigratedSites() ) diff --git a/src/routes/v2/authenticated/__tests__/Sites.spec.ts b/src/routes/v2/authenticated/__tests__/Sites.spec.ts index c90bf704b..24e72c8d9 100644 --- a/src/routes/v2/authenticated/__tests__/Sites.spec.ts +++ b/src/routes/v2/authenticated/__tests__/Sites.spec.ts @@ -35,7 +35,6 @@ describe("Sites Router", () => { } const mockStatsMiddleware = { - countGithubSites: jest.fn(), countMigratedSites: jest.fn(), } diff --git a/src/services/api/AxiosInstance.ts b/src/services/api/AxiosInstance.ts index 000ab6d34..37259e9ef 100644 --- a/src/services/api/AxiosInstance.ts +++ b/src/services/api/AxiosInstance.ts @@ -1,6 +1,6 @@ import axios, { AxiosRequestConfig, AxiosResponse } from "axios" import { setupCache } from "axios-cache-interceptor" -import { err } from "neverthrow" +import _ from "lodash" import { config } from "@config/config" @@ -11,6 +11,27 @@ import tracer from "@utils/tracer" import { customHeaderInterpreter } from "@root/utils/headerInterpreter" import { tokenServiceInstance } from "@services/db/TokenService" +import { statsService } from "../infra/StatsService" + +const GITHUB_EXPERIMENTAL_TRIAL_SITES = ["pa-corp"] + +const REPOS_SUBSTRING = "repos/isomerpages" +const extractRepoNameFromGithubUrl = (url: string): string => { + const idx = url.search(REPOS_SUBSTRING) + // NOTE: Should not hit here because we check that the url contains the site already + if (idx === -1) return "" + const ignoredLength = REPOS_SUBSTRING.length + return _.takeWhile( + url.slice(idx + ignoredLength + 1), + (char) => char !== "/" + ).join("") +} + +const getIsEmailUserFromAuthMessage = ( + authMessage?: string | number | boolean +): boolean => + !authMessage || authMessage === "token " || authMessage === "token undefined" + // Env vars const GITHUB_ORG_NAME = config.get("github.orgName") @@ -22,10 +43,7 @@ const requestFormatter = async (axiosConfig: AxiosRequestConfig) => { // If accessToken is missing, authMessage is `token ` // NOTE: This also implies that the user has not provided // their own github token and hence, are email login users. - const isEmailLoginUser = - !authMessage || - authMessage === "token " || - authMessage === "token undefined" + const isEmailLoginUser = getIsEmailUserFromAuthMessage(authMessage) if (isEmailLoginUser) { const accessToken = await tokenServiceInstance.getAccessToken() @@ -73,6 +91,21 @@ const respHandler = (response: AxiosResponse) => { return response } +const githubApiInterceptor = (resp: AxiosResponse) => { + const fullUrl = `${resp.config.baseURL || ""}${resp.config.url || ""}` + if ( + resp.status !== 304 && + _.some(GITHUB_EXPERIMENTAL_TRIAL_SITES, (site) => fullUrl.includes(site)) && + resp.config.method + ) { + statsService.incrementGithubApiCall( + resp.config.method, + extractRepoNameFromGithubUrl(fullUrl) + ) + } + return resp +} + const isomerRepoAxiosInstance = setupCache( axios.create({ baseURL: `https://api.github.com/repos/${GITHUB_ORG_NAME}/`, @@ -85,9 +118,11 @@ const isomerRepoAxiosInstance = setupCache( ) isomerRepoAxiosInstance.interceptors.request.use(requestFormatter) isomerRepoAxiosInstance.interceptors.response.use(respHandler) +isomerRepoAxiosInstance.interceptors.response.use(githubApiInterceptor) const genericGitHubAxiosInstance = axios.create() genericGitHubAxiosInstance.interceptors.request.use(requestFormatter) genericGitHubAxiosInstance.interceptors.response.use(respHandler) +genericGitHubAxiosInstance.interceptors.response.use(githubApiInterceptor) export { isomerRepoAxiosInstance, genericGitHubAxiosInstance } diff --git a/src/services/infra/StatsService.ts b/src/services/infra/StatsService.ts index 51d8d9840..e625f3047 100644 --- a/src/services/infra/StatsService.ts +++ b/src/services/infra/StatsService.ts @@ -1,22 +1,14 @@ /* eslint-disable import/prefer-default-export */ +import { Method } from "axios" import StatsDClient, { StatsD } from "hot-shots" -import _ from "lodash" import { ModelStatic } from "sequelize" import { config } from "@config/config" -import { - Versions, - GH_MAX_REPO_COUNT, - GITHUB_ORG_REPOS_ENDPOINT, - ISOMERPAGES_REPO_PAGE_COUNT, - VersionNumber, -} from "@constants/index" +import { Versions, VersionNumber } from "@constants/index" import { AccessToken, Site, User } from "@root/database/models" -import { genericGitHubAxiosInstance } from "../api/AxiosInstance" - export class StatsService { private readonly statsD: StatsD @@ -60,30 +52,6 @@ export class StatsService { }) } - countGithubSites = async () => { - const accessToken = await this.accessTokenRepo.findOne() - // NOTE: Cannot submit metrics if we are unable to get said metric - if (!accessToken) return - - const sitesArr = await Promise.all( - _.fill(Array(ISOMERPAGES_REPO_PAGE_COUNT), null) - .map((__, idx) => ({ - per_page: GH_MAX_REPO_COUNT, - sort: "full_name", - page: idx + 1, - })) - .map((params) => - genericGitHubAxiosInstance - .get(GITHUB_ORG_REPOS_ENDPOINT, { - params, - }) - .then(({ data }) => data) - ) - ) - - this.statsD.distribution("sites.github.all", sitesArr.flat().length, 1) - } - countMigratedSites = async () => { const numMigratedSites = await this.sitesRepo.count({ where: { @@ -105,6 +73,15 @@ export class StatsService { version: Versions.V2, }) } + + incrementGithubApiCall = (method: Method, site: string) => { + this.statsD.increment("users.github.api", { + site, + // NOTE: Allowed to pass in lowercase, + // standardised to uppercase for consistency + method: method.toUpperCase(), + }) + } } const statsDClient = new StatsDClient({