feat: call db to retrieve permits instead of parsing github comments

build/esbuild-config.ts

@@ -1,5 +1,6 @@
+import * as dotenv from "dotenv";
 import esbuild from "esbuild";
-import { invertColors } from "./plugins/invert-colors";
+
 const typescriptEntries = [
   "static/scripts/rewards/index.ts",
   "static/scripts/audit-report/audit.ts",
@@ -24,4 +25,19 @@ export let esBuildContext = {
     ".svg": "dataurl",
   },
   outdir: "static/out",
+  define: createEnvDefines(["SUPABASE_URL", "SUPABASE_KEY"]),
 } as esbuild.BuildOptions;
+
+function createEnvDefines(variableNames: string[]): Record<string, string> {
+  const defines: Record<string, string> = {};
+  dotenv.config();
+  for (const name of variableNames) {
+    const envVar = process.env[name];
+    if (envVar !== undefined) {
+      defines[`process.env.${name}`] = JSON.stringify(envVar);
+    } else {
+      throw new Error(`Missing environment variable: ${name}`);
+    }
+  }
+  return defines;
+}

package.json

@@ -17,6 +17,7 @@
     "@octokit/plugin-throttling": "^5.1.0",
     "@octokit/rest": "^19.0.7",
     "@sinclair/typebox": "^0.31.28",
+    "@supabase/supabase-js": "2.39.6",
     "@types/libsodium-wrappers": "^0.7.10",
     "@types/node": "^18.15.11",
     "@uniswap/permit2-sdk": "^1.2.0",

static/scripts/audit-report/audit.ts

@@ -1,15 +1,16 @@
-import { ethers } from "ethers";
-import { Octokit } from "@octokit/rest";
 import { throttling } from "@octokit/plugin-throttling";
+import { Octokit } from "@octokit/rest";
+import { createClient } from "@supabase/supabase-js";
 import axios from "axios";
+import { ethers } from "ethers";
 import GoDB from "godb";
 import { permit2Abi } from "../rewards/abis";
-import { ObserverKeys, ElemInterface, QuickImport, StandardInterface, TxData, GoDBSchema, GitHubUrlParts, ChainScanResult, SavedData } from "./types";
 import { Chain, ChainScan, DatabaseName, NULL_HASH, NULL_ID } from "./constants";
 import {
   getCurrency,
   getGitHubUrlPartsArray,
   getRandomAPIKey,
+  getRandomRpcUrl,
   getOptimalRPC,
   isValidUrl,
   parseRepoUrl,
@@ -18,8 +19,11 @@ import {
   RateLimitOptions,
   secondaryRateLimitHandler,
 } from "./helpers";
+import { ChainScanResult, ElemInterface, GitHubUrlParts, GoDBSchema, ObserverKeys, QuickImport, SavedData, StandardInterface, TxData } from "./types";
 import { getTxInfo } from "./utils/getTransaction";
 
+const supabase = createClient(process.env.SUPABASE_URL!, process.env.SUPABASE_KEY!);
+
 const rateOctokit = Octokit.plugin(throttling);
 
 let BOT_WALLET_ADDRESS = "";
@@ -292,129 +296,6 @@ class QueueSet {
 const updateQueue = new smartQueue();
 const rpcQueue = new QueueSet();
 
-const commentFetcher = async () => {
-  if (isComment) {
-    const commentIntervalID = setInterval(async () => {
-      clearInterval(commentIntervalID);
-      try {
-        if (issueList.length !== 0) {
-          const octokit = new Octokit({
-            auth: GITHUB_PAT,
-            throttle: {
-              onRateLimit: (retryAfter, options) => {
-                return primaryRateLimitHandler(retryAfter, options as RateLimitOptions);
-              },
-              onSecondaryRateLimit: (retryAfter, options) => {
-                return secondaryRateLimitHandler(retryAfter, options as RateLimitOptions);
-              },
-            },
-          });
-          let [owner, repo] = parseRepoUrl(issueList[0].html_url);
-          const { data } = await octokit.rest.issues.listComments({
-            owner,
-            repo,
-            issue_number: issueList[0].number,
-            per_page: offset,
-            page: commentPageNumber,
-          });
-          if (data.length === 0) {
-            commentPageNumber = 1;
-            await issueList.shift();
-            if (issueList.length > 0) {
-              commentFetcher();
-            } else {
-              isComment = false;
-              finishedQueue.mutate("isComment", true);
-            }
-          } else {
-            let isFound = false;
-            for (let comment of data) {
-              if (comment.user && comment.user.node_id === botNodeId && comment.body) {
-                const match = comment.body.match(urlRegex);
-                if (match && isValidUrl(match[1])) {
-                  const url = new URL(match[1]);
-                  const params = new URLSearchParams(url.search);
-                  const base64Payload = params.get("claim");
-                  let network = getCurrency(comment.body); // Might change it to `const claimNetwork = params.get("network");` later because previous permits are missing network query
-                  if (base64Payload) {
-                    const {
-                      owner: ownerAddress,
-                      signature,
-                      permit: {
-                        deadline,
-                        nonce,
-                        permitted: { amount, token },
-                      },
-                      transferDetails: { to },
-                    } = JSON.parse(window.atob(base64Payload)) as TxData;
-                    await updateQueue.add(signature, {
-                      k: signature,
-                      t: "git",
-                      c: {
-                        nonce,
-                        owner: ownerAddress,
-                        token,
-                        amount,
-                        to,
-                        deadline,
-                        signature,
-                      },
-                      s: {
-                        git: {
-                          issue_title: issueList[0].title,
-                          issue_number: issueList[0].number,
-                          owner,
-                          repo,
-                          bounty_hunter: {
-                            name: issueList[0].assignee ? issueList[0].assignee.login : issueList[0].assignees[0]?.login,
-                            url: issueList[0].assignee ? issueList[0].assignee.html_url : issueList[0].assignees[0]?.html_url,
-                          },
-                        },
-                        ether: undefined,
-                        network: network as string,
-                      },
-                    });
-                    isFound = true;
-                  }
-                } else {
-                  console.log("URL not found, skipping");
-                }
-              }
-            }
-
-            if (isFound) {
-              commentPageNumber = 1;
-              await issueList.shift();
-            } else {
-              commentPageNumber++;
-            }
-
-            if (issueList.length > 0) {
-              commentFetcher();
-            } else {
-              isComment = false;
-              finishedQueue.mutate("isComment", true);
-            }
-          }
-        } else {
-          isComment = false;
-          finishedQueue.mutate("isComment", true);
-        }
-      } catch (error) {
-        console.error(error);
-        finishedQueue.raise();
-        await issueList.shift();
-        if (issueList.length > 0) {
-          commentFetcher();
-        } else {
-          isComment = false;
-          finishedQueue.mutate("isComment", true);
-        }
-      }
-    }, GIT_INTERVAL);
-  }
-};
-
 const gitFetcher = async (repoUrls: GitHubUrlParts[]) => {
   if (isGit) {
     const octokit = new rateOctokit({
@@ -432,45 +313,26 @@ const gitFetcher = async (repoUrls: GitHubUrlParts[]) => {
     const getIssuesForRepo = async (owner: string, repo: string) => {
       const issueList: any[] = [];
 
-      while (true) {
-        try {
-          const { data } = await octokit.rest.issues.listForRepo({
-            owner,
-            repo,
-            state: "closed",
-            per_page: offset,
-            page: gitPageNumber,
-          });
-
-          if (data.length === 0) break;
-
-          const issues = data.filter(issue => !issue.pull_request && issue.comments > 0);
-          if (issues.length > 0) {
-            if (!lastGitID) {
-              lastGitID = issues[0].number;
-            }
-            let iEF = true;
-            for (let i of issues) {
-              if (i.number !== gitID) {
-                await issueList.push(i);
-              } else {
-                iEF = false;
-                break;
-              }
-            }
-
-            if (iEF) {
-              gitPageNumber++;
-            } else {
-              break;
-            }
-          } else {
-            break;
+      try {
+        const { data: gitData } = await octokit.rest.repos.get({
+          owner,
+          repo,
+        });
+        if (!gitData.organization) {
+          console.warn(`${owner}/${repo} is not an organization, skipping.`);
+        } else {
+          const { data } = await supabase
+            .from("permits")
+            .select("*, locations(*), users(*, wallets(*)), tokens(*)")
+            .eq("locations.organization_id", gitData.organization?.id)
+            .not("locations", "is", null);
+          if (data) {
+            issueList.push(...data.map(d => ({ ...d, owner, repo })));
           }
-        } catch (error: any) {
-          console.error(error);
-          throw error;
         }
+      } catch (error: any) {
+        console.error(error);
+        throw error;
       }
 
       return issueList;
@@ -483,14 +345,56 @@ const gitFetcher = async (repoUrls: GitHubUrlParts[]) => {
       for (let i = 0; i < allIssues.length; i++) {
         const issues = allIssues[i];
         issueList.push(...issues);
-        console.log(`Fetched ${issues.length} issues for repository ${repoUrls[i].owner}/${repoUrls[i].repo}`);
+        console.log(`Fetched ${issues.length} permits for repository ${repoUrls[i].owner}/${repoUrls[i].repo}`);
       }
 
       isGit = false;
       finishedQueue.mutate("isGit", true);
-      commentFetcher();
+      for (const issue of issueList) {
+        const { data: userData } = await octokit.request("GET /user/:id", { id: issue.locations.user_id });
+        const { data } = await supabase.from("locations").select("*").eq("issue_id", issue.locations.issue_id).single();
+        const lastSlashIndex = data.node_url.lastIndexOf("/");
+        const hashIndex = data.node_url.lastIndexOf("#") || data.node_url.length;
+        const issueNumber = Number(data.node_url.substring(lastSlashIndex + 1, hashIndex));
+        const { data: issueData } = await octokit.rest.issues.get({
+          issue_number: issueNumber,
+          owner: issue.owner,
+          repo: issue.repo,
+        });
+        updateQueue.add(issue.signature, {
+          c: {
+            amount: issue.amount,
+            deadline: issue.deadline,
+            nonce: issue.nonce,
+            owner: issue.owner,
+            signature: issue.signature,
+            to: issue.users.wallets.address,
+            token: issue.tokens.address,
+          },
+          k: issue.signature,
+          s: {
+            ether: undefined,
+            git: {
+              bounty_hunter: {
+                name: userData.login,
+                url: userData.html_url,
+              },
+              issue_number: issueData.number,
+              issue_title: issueData.title,
+              owner: issue.owner,
+              repo: issue.repo,
+            },
+            network: getCurrency(issue.network_id)!,
+          },
+          t: "git",
+        });
+      }
+      isComment = false;
+      finishedQueue.mutate("isComment", true);
     } catch (error: any) {
       console.error("Error fetching issues:", error);
+      isComment = false;
+      finishedQueue.mutate("isComment", true);
     }
   }
 };
@@ -567,7 +471,6 @@ const rpcFetcher = async () => {
         const data = await rpcQueue.read();
         if (data) {
           const { hash, chain } = data;
-
           const providerUrl = await getOptimalRPC(chain);
           const txInfo = await getTxInfo(hash, providerUrl, chain);
 

static/scripts/audit-report/helpers.ts

@@ -177,10 +177,10 @@ export const isValidUrl = (urlString: string) => {
   }
 };
 
-export const getCurrency = (comment: string) => {
-  if (comment.includes("WXDAI")) {
+export const getCurrency = (id: number) => {
+  if (id === 100) {
     return Chain.Gnosis;
-  } else if (comment.includes("DAI")) {
+  } else if (id === 1) {
     return Chain.Ethereum;
   }
   return null;