Migrage planet feed list to supabase script

tools/migrate/README.md

@@ -1,24 +1,42 @@
 # Planet CDOT Feed List migration tool
 
-This tool downloads all users from the wiki page and dumps them into a JSON file.
+This tool downloads all users from the wiki page and inserts them into supabase db or dumps them into a JSON file
 
-## Customization
+## Writing the feed list to JSON
 
-In `migrage.js` you can find the following two variables: `FEED_URL` and `FILE_NAME`.
+In `to_json.js` you can find the following two variables: `FEED_URL` and `FILE_NAME`.
 
 - `FEED_URL` points to the current location of the [Planet CDOT Feed List](https://wiki.cdot.senecacollege.ca/wiki/Planet_CDOT_Feed_List#Feeds)
 - `FILE_NAME` allows users to specify the desired filename of the output file
 
-## Install Dependencies
+## Migrating to Supabase
 
+Migrating to Supabase requires creating a `supabase` client with admin rights.
+
+`to_supabase.js` gets `SUPABASE_URL` and `SERVICE_ROLE_KEY` of the current supabase setup from environment variables.
+
+To start, copy the example env var from `env.example`:
+
+```bash
+# on Linux/macOS
+cp env.example .env
+# on Windows
+copy env.example .env
 ```
-cd src/tools/migrate
-npm install
+
+## Install dependencies
+
+```bash
+cd tools/migrate
+pnpm install
 ```
 
 ## Usage
 
-```
-cd src/tools/migrate
-npm start
+```bash
+cd tools/migrate
+# To JSON
+pnpm to_json
+# To Supabase
+pnpm to_supabase
 ```

tools/migrate/env.example

@@ -0,0 +1,7 @@
+# Environment variables needed to connect to supabase-db
+# `cp env.example .env` on Linux/macOS, or `copy env.example .env` on Windows
+
+
+SUPABASE_URL=http://localhost/v1/supabase
+
+SERVICE_ROLE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q

tools/migrate/migrate.js → tools/migrate/feed.js

@@ -1,10 +1,8 @@
 const fetch = require('node-fetch');
 const jsdom = require('jsdom');
-const fs = require('fs');
 
 const { JSDOM } = jsdom;
 const FEED_URL = 'https://wiki.cdot.senecacollege.ca/wiki/Planet_CDOT_Feed_List';
-const FILE = 'legacy_users.json';
 
 const getWikiText = async (url) => {
   try {
@@ -18,7 +16,15 @@ const getWikiText = async (url) => {
   }
 };
 
-(async () => {
+/**
+ * @typedef {{firstName:string,lastName:string,feed: string }} PlanetUser
+ */
+
+/**
+ * Parse all users from the planet feed list hosted at https://wiki.cdot.senecacollege.ca/wiki/Planet_CDOT_Feed_List
+ * @return {PromiseLike<PlanetUser[]>} - A list valid parsed users
+ */
+const parsePlanetFeedList = async () => {
   let wikiText;
 
   // Try to fetch the feed list from 'FEED_URL'
@@ -38,30 +44,30 @@ const getWikiText = async (url) => {
   let lastName;
   let feed;
   const users = [];
+  const uniqueUrls = new Set();
 
   // Iterate through all lines and find url/name pairs, then parse them.
   lines.forEach((line, index) => {
     if (!commentRegex.test(line) && line.startsWith('[')) {
       feed = line.replace(/[[\]']/g, '');
 
       try {
-        new URL(feed);
-        // If the URL is valid, continue
+        // eslint-disable-next-line no-new
+        new URL(feed); // If the URL is valid, continue
         [firstName, lastName] = lines[index + 1].replace(/^\s*name\s*=\s*/, '').split(' ');
-        users.push({ firstName, lastName, feed });
+
+        if (!uniqueUrls.has(feed)) {
+          users.push({ firstName, lastName, feed });
+          uniqueUrls.add(feed);
+        }
       } catch {
         // If the URL is invalid, display error message
         console.error(`Skipping invalid wiki feed url ${feed} for author ${firstName} ${lastName}`);
       }
     }
   });
 
-  try {
-    fs.writeFileSync(`${FILE}`, JSON.stringify(users));
-    console.log(
-      `Processed ${users.length} records. Legacy users were successfully written to file: ${FILE}.`
-    );
-  } catch (err) {
-    console.error(err);
-  }
-})();
+  return users;
+};
+
+module.exports = { getWikiText, parsePlanetFeedList };

tools/migrate/package.json

@@ -2,15 +2,18 @@
   "author": "",
   "private": true,
   "dependencies": {
+    "@supabase/supabase-js": "1.29.4",
+    "dotenv": "10.0.0",
     "jsdom": "18.1.1",
     "node-fetch": "2.6.7"
   },
-  "description": "Migrates users from the planet feed wiki list to a JSON file.",
+  "description": "Migrates users from the planet feed wiki list to a JSON file or supabase-db",
   "license": "ISC",
   "main": "migrate.js",
   "name": "migrate",
   "scripts": {
-    "start": "node migrate.js"
+    "to_json": "node to_json.js",
+    "to_supabase": "node to_supabase.js"
   },
-  "version": "1.0.0"
+  "version": "1.1.0"
 }

tools/migrate/to_json.js

@@ -0,0 +1,18 @@
+const fs = require('fs');
+
+const FILE = 'legacy_users.json';
+
+const { parsePlanetFeedList } = require('./feed');
+
+(async () => {
+  const planetUsers = await parsePlanetFeedList();
+
+  try {
+    fs.writeFileSync(`${FILE}`, JSON.stringify(planetUsers));
+    console.log(
+      `Processed ${planetUsers.length} records. Legacy users were successfully written to file: ${FILE}.`
+    );
+  } catch (err) {
+    console.error(err);
+  }
+})();

tools/migrate/to_supabase.js

@@ -0,0 +1,38 @@
+const { createClient } = require('@supabase/supabase-js');
+const { parsePlanetFeedList } = require('./feed');
+
+require('dotenv').config();
+
+const { SUPABASE_URL, SERVICE_ROLE_KEY } = process.env;
+
+(async () => {
+  if (!SUPABASE_URL || !SERVICE_ROLE_KEY) {
+    console.error('Environment variables are not set to migrate feeds to supabase');
+    console.error('SUPBASE_URL or SERVICE_ROLE_KEY is missing');
+    process.exit(1);
+  }
+
+  const supabase = createClient(SUPABASE_URL, SERVICE_ROLE_KEY);
+  const planetUsers = await parsePlanetFeedList();
+
+  const feeds = planetUsers.map(({ feed, firstName, lastName }) => ({
+    url: feed,
+    wiki_author_name: `${firstName} ${lastName}`,
+    type: 'blog',
+  }));
+
+  const { error, count } = await supabase.from('feeds').upsert(feeds, {
+    // Replace the existing url if exists
+    onConflict: 'url',
+    count: 'exact',
+    returning: 'minimal',
+  });
+
+  if (error) {
+    console.error('Failed to migrage planet feed list to Supabase');
+    console.error(`Supabase Error: `, error);
+    process.exit(1);
+  }
+
+  console.log(`Migrated ${count} users to supabase-db`);
+})();