From 3311a26c9d0b8328c957f4906568425c325ff9a2 Mon Sep 17 00:00:00 2001 From: Andrew Nguyen <52666982+Andrewnt219@users.noreply.github.com> Date: Tue, 1 Feb 2022 09:49:21 -0500 Subject: [PATCH] feat: add octokit to dashboard --- pnpm-lock.yaml | 86 +++++++++++++++++++++++++++ src/api/status/env.local | 2 + src/api/status/package.json | 8 ++- src/api/status/src/js/github-stats.js | 79 ++++++++++++------------ src/api/status/src/js/octokit.js | 31 ++++++++++ 5 files changed, 161 insertions(+), 45 deletions(-) create mode 100644 src/api/status/src/js/octokit.js diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3d99591635..bfeaad6285 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -291,6 +291,9 @@ importers: src/api/status: specifiers: + '@octokit/core': 3.5.1 + '@octokit/plugin-retry': 3.0.9 + '@octokit/plugin-throttling': 3.5.2 '@senecacdot/satellite': ^1.x env-cmd: 10.1.0 express: 4.17.2 @@ -300,6 +303,9 @@ importers: sass: 1.45.2 vite: 2.7.13 dependencies: + '@octokit/core': 3.5.1 + '@octokit/plugin-retry': 3.0.9 + '@octokit/plugin-throttling': 3.5.2_@octokit+core@3.5.1 '@senecacdot/satellite': 1.17.0 express: 4.17.2 express-handlebars: 6.0.2 @@ -3559,10 +3565,65 @@ packages: dev: true optional: true + /@octokit/auth-token/2.5.0: + resolution: {integrity: sha512-r5FVUJCOLl19AxiuZD2VRZ/ORjp/4IN98Of6YJoJOkY75CIBuYfmiNHGrDwXr+aLGG55igl9QrxX3hbiXlLb+g==} + dependencies: + '@octokit/types': 6.34.0 + dev: false + + /@octokit/core/3.5.1: + resolution: {integrity: sha512-omncwpLVxMP+GLpLPgeGJBF6IWJFjXDS5flY5VbppePYX9XehevbDykRH9PdCdvqt9TS5AOTiDide7h0qrkHjw==} + dependencies: + '@octokit/auth-token': 2.5.0 + '@octokit/graphql': 4.8.0 + '@octokit/request': 5.6.3 + '@octokit/request-error': 2.1.0 + '@octokit/types': 6.34.0 + before-after-hook: 2.2.2 + universal-user-agent: 6.0.0 + transitivePeerDependencies: + - encoding + dev: false + + /@octokit/endpoint/6.0.12: + resolution: {integrity: sha512-lF3puPwkQWGfkMClXb4k/eUT/nZKQfxinRWJrdZaJO85Dqwo/G0yOC434Jr2ojwafWJMYqFGFa5ms4jJUgujdA==} + dependencies: + '@octokit/types': 6.34.0 + is-plain-object: 5.0.0 + universal-user-agent: 6.0.0 + dev: false + + /@octokit/graphql/4.8.0: + resolution: {integrity: sha512-0gv+qLSBLKF0z8TKaSKTsS39scVKF9dbMxJpj3U0vC7wjNWFuIpL/z76Qe2fiuCbDRcJSavkXsVtMS6/dtQQsg==} + dependencies: + '@octokit/request': 5.6.3 + '@octokit/types': 6.34.0 + universal-user-agent: 6.0.0 + transitivePeerDependencies: + - encoding + dev: false + /@octokit/openapi-types/11.2.0: resolution: {integrity: sha512-PBsVO+15KSlGmiI8QAzaqvsNlZlrDlyAJYcrXBCvVUxCp7VnXjkwPoFHgjEJXx3WF9BAwkA6nfCUA7i9sODzKA==} dev: false + /@octokit/plugin-retry/3.0.9: + resolution: {integrity: sha512-r+fArdP5+TG6l1Rv/C9hVoty6tldw6cE2pRHNGmFPdyfrc696R6JjrQ3d7HdVqGwuzfyrcaLAKD7K8TX8aehUQ==} + dependencies: + '@octokit/types': 6.34.0 + bottleneck: 2.19.5 + dev: false + + /@octokit/plugin-throttling/3.5.2_@octokit+core@3.5.1: + resolution: {integrity: sha512-Eu7kfJxU8vmHqWGNszWpg+GVp2tnAfax3XQV5CkYPEE69C+KvInJXW9WajgSeW+cxYe0UVdouzCtcreGNuJo7A==} + peerDependencies: + '@octokit/core': ^3.5.0 + dependencies: + '@octokit/core': 3.5.1 + '@octokit/types': 6.34.0 + bottleneck: 2.19.5 + dev: false + /@octokit/request-error/2.1.0: resolution: {integrity: sha512-1VIvgXxs9WHSjicsRwq8PlR2LR2x6DwsJAaFgzdi0JfJoGSO8mYI/cHJQ+9FbN21aa+DrgNLnwObmyeSC8Rmpg==} dependencies: @@ -3571,6 +3632,19 @@ packages: once: 1.4.0 dev: false + /@octokit/request/5.6.3: + resolution: {integrity: sha512-bFJl0I1KVc9jYTe9tdGGpAMPy32dLBXXo1dS/YwSCTL/2nd9XeHsY616RE3HPXDVk+a+dBuzyz5YdlXwcDTr2A==} + dependencies: + '@octokit/endpoint': 6.0.12 + '@octokit/request-error': 2.1.0 + '@octokit/types': 6.34.0 + is-plain-object: 5.0.0 + node-fetch: 2.6.7 + universal-user-agent: 6.0.0 + transitivePeerDependencies: + - encoding + dev: false + /@octokit/types/6.34.0: resolution: {integrity: sha512-s1zLBjWhdEI2zwaoSgyOFoKSl109CUcVBCc7biPJ3aAf6LGLU6szDvi31JPU7bxfla2lqfhjbbg/5DdFNxOwHw==} dependencies: @@ -5160,6 +5234,10 @@ packages: tweetnacl: 0.14.5 dev: true + /before-after-hook/2.2.2: + resolution: {integrity: sha512-3pZEU3NT5BFUo/AD5ERPWOgQOCZITni6iavr5AUw5AUwQjMlI0kzu5btnyD39AF0gUEsDPwJT+oY1ORBJijPjQ==} + dev: false + /big-integer/1.6.51: resolution: {integrity: sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg==} engines: {node: '>=0.6'} @@ -5246,6 +5324,10 @@ packages: resolution: {integrity: sha1-aN/1++YMUes3cl6p4+0xDcwed24=} dev: false + /bottleneck/2.19.5: + resolution: {integrity: sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==} + dev: false + /boxen/4.2.0: resolution: {integrity: sha512-eB4uT9RGzg2odpER62bBwSLvUeGC+WbRjjyyFhGsKnc8wp/m0+hQsMUvUe3H2V0D5vw0nBdO1hCJoZo5mKeuIQ==} engines: {node: '>=8'} @@ -15312,6 +15394,10 @@ packages: - supports-color dev: true + /universal-user-agent/6.0.0: + resolution: {integrity: sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==} + dev: false + /universalify/0.1.2: resolution: {integrity: sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==} engines: {node: '>= 4.0.0'} diff --git a/src/api/status/env.local b/src/api/status/env.local index 0f2567ebe3..faca80f836 100644 --- a/src/api/status/env.local +++ b/src/api/status/env.local @@ -3,3 +3,5 @@ PATH_PREFIX=/v1/status WEB_URL=http://localhost POSTS_URL=http://localhost/v1/posts MOCK_REDIS=1 +# github access token for octokit +GITHUB_TOKEN= diff --git a/src/api/status/package.json b/src/api/status/package.json index ee343984bf..c09c517e02 100644 --- a/src/api/status/package.json +++ b/src/api/status/package.json @@ -20,13 +20,15 @@ }, "homepage": "https://github.com/Seneca-CDOT/telescope#readme", "dependencies": { + "@octokit/core": "3.5.1", + "@octokit/plugin-retry": "3.0.9", + "@octokit/plugin-throttling": "3.5.2", "@senecacdot/satellite": "^1.x", "express": "4.17.2", "express-handlebars": "6.0.2", + "npm-run-all": "4.1.5", "sass": "1.45.2", - "vite": "2.7.13", - "npm-run-all": "4.1.5" - + "vite": "2.7.13" }, "engines": { "node": ">=12.0.0" diff --git a/src/api/status/src/js/github-stats.js b/src/api/status/src/js/github-stats.js index f0747b8b56..e5f3690826 100644 --- a/src/api/status/src/js/github-stats.js +++ b/src/api/status/src/js/github-stats.js @@ -1,55 +1,50 @@ -const { fetch } = require('@senecacdot/satellite'); - +const { logger } = require('@senecacdot/satellite'); +const octokit = require('./octokit'); const redis = require('../redis'); const cacheTime = 60 * 10; // 10 mins of cache time -const fetchGitHubApi = (owner, repo, path) => - fetch(`https://api.github.com/repos/${owner}/${repo}/${path}`, { - headers: { Accept: 'application/vnd.github.v3+json' }, - }); - const getStatsParticipation = async (owner, repo) => { // get weekly commits for last year: https://docs.github.com/en/rest/reference/repos#get-the-weekly-commit-count - const res = await fetchGitHubApi(owner, repo, 'stats/participation'); - const participationResponse = await res.json(); - if (!res.ok) { - throw new Error(`[Code ${res.status}] - ${participationResponse.message}`); - } + const participationResponse = await octokit.request( + 'GET /repos/{owner}/{repo}/stats/participation', + { + owner, + repo, + } + ); return { weeklyCommits: { - commits: participationResponse.all[participationResponse.all.length - 1], + commits: participationResponse.data.all[participationResponse.data.all.length - 1], }, yearlyCommits: { - commits: participationResponse.all.reduce((a, b) => a + b), + commits: participationResponse.data.all.reduce((a, b) => a + b), }, }; }; const getStatsCodeFrequency = async (owner, repo) => { // get weekly commits activity: https://docs.github.com/en/rest/reference/repos#get-the-weekly-commit-activity - const res = await fetchGitHubApi(owner, repo, 'stats/code_frequency'); - const codeFrequencyResponse = await res.json(); - - if (!res.ok) { - throw new Error(`[Code ${res.status}] - ${codeFrequencyResponse.message}`); - } - - const [, linesAdded, linesRemoved] = codeFrequencyResponse[codeFrequencyResponse.length - 1]; + const codeFrequencyResponse = await octokit.request( + 'GET /repos/{owner}/{repo}/stats/code_frequency', + { + owner, + repo, + } + ); + const [, linesAdded, linesRemoved] = + codeFrequencyResponse.data[codeFrequencyResponse.data.length - 1]; return { weeklyCommits: { linesAdded, linesRemoved } }; }; const getCommitsInfo = async (owner, repo) => { // get latest author from commits list: https://docs.github.com/en/rest/reference/repos#list-commits - const res = await fetchGitHubApi(owner, repo, 'commits'); - const commitResponse = await res.json(); - - if (!res.ok) { - throw new Error(`[Code ${res.status}] - ${commitResponse.message}`); - } - - const lastCommitResponse = commitResponse[0]; + const commitResponse = await octokit.request('GET /repos/{owner}/{repo}/commits', { + owner, + repo, + }); + const lastCommitResponse = commitResponse.data[0]; return { avatar: lastCommitResponse.author.avatar_url || '', @@ -63,13 +58,12 @@ const getCommitsInfo = async (owner, repo) => { const getContributorsInfo = async (owner, repo) => { // get total contributors: https://docs.github.com/en/rest/reference/repos#list-repository-contributors - const res = await fetchGitHubApi(owner, repo, 'contributors?per_page=1'); - - if (res.status >= 400) { - throw new Error(`[Code ${res.status}] - ${(await res.json()).message}`); - } - - const contributorsResponse = await res.headers.get('link'); + const { headers } = await octokit.request('GET /repos/{owner}/{repo}/contributors', { + owner, + repo, + per_page: 1, + }); + const contributorsResponse = headers.link; const [, totalContributors] = contributorsResponse.match(/.*"next".*&page=([0-9]*).*"last".*/); return { @@ -78,7 +72,6 @@ const getContributorsInfo = async (owner, repo) => { }; module.exports = async function getGitHubData(owner, repo) { - let githubData = {}; try { const cached = await redis.get(`github:info-${repo}`); if (cached) { @@ -93,7 +86,7 @@ module.exports = async function getGitHubData(owner, repo) { getContributorsInfo(owner, repo), ]); - githubData = { + const githubData = { weeklyCommits: { ...statsParticipation.weeklyCommits, ...statsCodeFrequency.weeklyCommits }, yearlyCommits: { ...statsParticipation.yearlyCommits }, ...commitsInfo, @@ -101,8 +94,10 @@ module.exports = async function getGitHubData(owner, repo) { }; await redis.set(`github:info-${repo}`, JSON.stringify(githubData), 'EX', cacheTime); - } catch (err) { - console.error(err); + return githubData; + } catch (error) { + logger.warn({ error }, 'Fail to fetch github data'); } - return githubData; + + return {}; }; diff --git a/src/api/status/src/js/octokit.js b/src/api/status/src/js/octokit.js new file mode 100644 index 0000000000..7b3ce9e8f2 --- /dev/null +++ b/src/api/status/src/js/octokit.js @@ -0,0 +1,31 @@ +const { Octokit } = require('@octokit/core'); +const { retry } = require('@octokit/plugin-retry'); +const { throttling } = require('@octokit/plugin-throttling'); + +const MyOctokit = Octokit.plugin(retry, throttling); +const octokit = new MyOctokit({ + auth: process.env.GITHUB_TOKEN, + // options for throttling plugin https://github.com/octokit/plugin-throttling.js + throttle: { + onRateLimit: (retryAfter, options, octokitClient) => { + octokitClient.log.warn( + `Request quota exhausted for request ${options.method} ${options.url}` + ); + + if (options.request.retryCount === 0) { + // only retries once + octokitClient.log.info(`Retrying after ${retryAfter} seconds!`); + // Return true to automatically retry the request after retryAfter seconds. + return true; + } + + return false; + }, + onAbuseLimit: (retryAfter, options, octokitClient) => { + // does not retry, only logs a warning + octokitClient.log.warn(`Abuse detected for request ${options.method} ${options.url}`); + }, + }, +}); + +module.exports = octokit;