feat(stats): track gh calls

src/middleware/stats.ts

@@ -37,10 +37,6 @@ export class StatsMiddleware {
 
   countDbUsers = wrapAsRequestHandler(() => this.statsService.countDbUsers())
 
-  countGithubSites = wrapAsRequestHandler(() =>
-    this.statsService.countGithubSites()
-  )
-
   countMigratedSites = wrapAsRequestHandler(() =>
     this.statsService.countMigratedSites()
   )

src/routes/v2/authenticated/__tests__/Sites.spec.ts

@@ -35,7 +35,6 @@ describe("Sites Router", () => {
   }
 
   const mockStatsMiddleware = {
-    countGithubSites: jest.fn(),
     countMigratedSites: jest.fn(),
   }
 

src/services/api/AxiosInstance.ts

@@ -1,6 +1,6 @@
 import axios, { AxiosRequestConfig, AxiosResponse } from "axios"
 import { setupCache } from "axios-cache-interceptor"
-import { err } from "neverthrow"
+import _ from "lodash"
 
 import { config } from "@config/config"
 
@@ -11,6 +11,27 @@ import tracer from "@utils/tracer"
 import { customHeaderInterpreter } from "@root/utils/headerInterpreter"
 import { tokenServiceInstance } from "@services/db/TokenService"
 
+import { statsService } from "../infra/StatsService"
+
+const GITHUB_EXPERIMENTAL_TRIAL_SITES = ["pa-corp"]
+
+const REPOS_SUBSTRING = "repos/isomerpages"
+const extractRepoNameFromGithubUrl = (url: string): string => {
+  const idx = url.search(REPOS_SUBSTRING)
+  // NOTE: Should not hit here because we check that the url contains the site already
+  if (idx === -1) return ""
+  const ignoredLength = REPOS_SUBSTRING.length
+  return _.takeWhile(
+    url.slice(idx + ignoredLength + 1),
+    (char) => char !== "/"
+  ).join("")
+}
+
+const getIsEmailUserFromAuthMessage = (
+  authMessage?: string | number | boolean
+): boolean =>
+  !authMessage || authMessage === "token " || authMessage === "token undefined"
+
 // Env vars
 const GITHUB_ORG_NAME = config.get("github.orgName")
 
@@ -22,10 +43,7 @@ const requestFormatter = async (axiosConfig: AxiosRequestConfig) => {
   // If accessToken is missing, authMessage is `token `
   // NOTE: This also implies that the user has not provided
   // their own github token and hence, are email login users.
-  const isEmailLoginUser =
-    !authMessage ||
-    authMessage === "token " ||
-    authMessage === "token undefined"
+  const isEmailLoginUser = getIsEmailUserFromAuthMessage(authMessage)
 
   if (isEmailLoginUser) {
     const accessToken = await tokenServiceInstance.getAccessToken()
@@ -73,6 +91,21 @@ const respHandler = (response: AxiosResponse) => {
   return response
 }
 
+const githubApiInterceptor = (resp: AxiosResponse) => {
+  const fullUrl = `${resp.config.baseURL || ""}${resp.config.url || ""}`
+  if (
+    resp.status !== 304 &&
+    _.some(GITHUB_EXPERIMENTAL_TRIAL_SITES, (site) => fullUrl.includes(site)) &&
+    resp.config.method
+  ) {
+    statsService.incrementGithubApiCall(
+      resp.config.method,
+      extractRepoNameFromGithubUrl(fullUrl)
+    )
+  }
+  return resp
+}
+
 const isomerRepoAxiosInstance = setupCache(
   axios.create({
     baseURL: `https://api.github.com/repos/${GITHUB_ORG_NAME}/`,
@@ -85,9 +118,11 @@ const isomerRepoAxiosInstance = setupCache(
 )
 isomerRepoAxiosInstance.interceptors.request.use(requestFormatter)
 isomerRepoAxiosInstance.interceptors.response.use(respHandler)
+isomerRepoAxiosInstance.interceptors.response.use(githubApiInterceptor)
 
 const genericGitHubAxiosInstance = axios.create()
 genericGitHubAxiosInstance.interceptors.request.use(requestFormatter)
 genericGitHubAxiosInstance.interceptors.response.use(respHandler)
+genericGitHubAxiosInstance.interceptors.response.use(githubApiInterceptor)
 
 export { isomerRepoAxiosInstance, genericGitHubAxiosInstance }

src/services/infra/StatsService.ts

@@ -1,22 +1,14 @@
 /* eslint-disable import/prefer-default-export */
+import { Method } from "axios"
 import StatsDClient, { StatsD } from "hot-shots"
-import _ from "lodash"
 import { ModelStatic } from "sequelize"
 
 import { config } from "@config/config"
 
-import {
-  Versions,
-  GH_MAX_REPO_COUNT,
-  GITHUB_ORG_REPOS_ENDPOINT,
-  ISOMERPAGES_REPO_PAGE_COUNT,
-  VersionNumber,
-} from "@constants/index"
+import { Versions, VersionNumber } from "@constants/index"
 
 import { AccessToken, Site, User } from "@root/database/models"
 
-import { genericGitHubAxiosInstance } from "../api/AxiosInstance"
-
 export class StatsService {
   private readonly statsD: StatsD
 
@@ -60,30 +52,6 @@ export class StatsService {
     })
   }
 
-  countGithubSites = async () => {
-    const accessToken = await this.accessTokenRepo.findOne()
-    // NOTE: Cannot submit metrics if we are unable to get said metric
-    if (!accessToken) return
-
-    const sitesArr = await Promise.all(
-      _.fill(Array(ISOMERPAGES_REPO_PAGE_COUNT), null)
-        .map((__, idx) => ({
-          per_page: GH_MAX_REPO_COUNT,
-          sort: "full_name",
-          page: idx + 1,
-        }))
-        .map((params) =>
-          genericGitHubAxiosInstance
-            .get<unknown[]>(GITHUB_ORG_REPOS_ENDPOINT, {
-              params,
-            })
-            .then(({ data }) => data)
-        )
-    )
-
-    this.statsD.distribution("sites.github.all", sitesArr.flat().length, 1)
-  }
-
   countMigratedSites = async () => {
     const numMigratedSites = await this.sitesRepo.count({
       where: {
@@ -105,6 +73,15 @@ export class StatsService {
       version: Versions.V2,
     })
   }
+
+  incrementGithubApiCall = (method: Method, site: string) => {
+    this.statsD.increment("users.github.api", {
+      site,
+      // NOTE: Allowed to pass in lowercase,
+      // standardised to uppercase for consistency
+      method: method.toUpperCase(),
+    })
+  }
 }
 
 const statsDClient = new StatsDClient({