Change parser to work with supabase (#3363)

- Remove wiki-feed-parser, add logic to get feeds from supabase
- Update e2e test to reflect on the change
- Fix Supabase configs, remove secrets from env.local
- Use database staging

docker/development.yml

@@ -90,6 +90,11 @@ services:
       context: ../src/api/parser
       cache_from:
         - docker.cdot.systems/parser:buildcache
+    environment:
+      # In development and testing, the SSO service needs to contact the Supabase
+      # service directly via Docker vs through the http://localhost/v1/supabase domain.
+      # Using staging database
+      - SUPABASE_URL=https://dev.api.telescope.cdot.systems/v1/supabase
     depends_on:
       - elasticsearch
       - traefik

docker/docker-compose.yml

@@ -196,6 +196,8 @@ services:
       - ELASTIC_MAX_RESULTS_PER_PAGE
       - ELASTIC_URL=http://elasticsearch
       - ELASTIC_PORT=9200
+      - SERVICE_ROLE_KEY
+      - SUPABASE_URL
       # Satellite authentication/authorization support
       - JWT_ISSUER
       - JWT_AUDIENCE

src/api/parser/env.local

@@ -32,3 +32,12 @@ FEED_QUEUE_DELAY_MS=60000
 FEED_QUEUE_PARALLEL_WORKERS=1
 
 PARSER_PORT = 10000
+
+################################################################################
+# Supabase Services
+################################################################################
+
+# Supabase Secrets
+# Using staging database
+#SUPABASE_URL=http://localhost/v1/supabase
+SUPABASE_URL=https://dev.supabase.telescope.cdot.systems/

src/api/parser/package.json

@@ -21,6 +21,7 @@
     "@bull-board/api": "3.10.3",
     "@bull-board/express": "3.10.3",
     "@senecacdot/satellite": "^1.27.0",
+    "@supabase/supabase-js": "1.29.4",
     "bull": "3.29.3",
     "clean-whitespace": "0.1.2",
     "highlight.js": "11.4.0",

src/api/parser/src/parser.js

@@ -1,7 +1,7 @@
 const { logger } = require('@senecacdot/satellite');
 const { feedQueue } = require('./feed/queue');
-const getWikiFeeds = require('./utils/wiki-feed-parser');
 const Feed = require('./data/feed');
+const { getAllFeeds } = require('./utils/supabase');
 
 /**
  * Adds the feed to the database if necessary, or gets a more complete
@@ -58,8 +58,8 @@ const processFeeds = (feeds) => {
  */
 const processAllFeeds = async () => {
   try {
-    // Get an Array of Feed objects from the wiki feed list
-    const feeds = await getWikiFeeds();
+    // Get an Array of Feed objects from the wiki feed list and Redis
+    const feeds = await getAllFeeds();
     // Process these feeds into the database and feed queue
     await processFeeds(feeds);
   } catch (err) {

src/api/parser/src/utils/supabase.js

@@ -0,0 +1,30 @@
+const { logger } = require('@senecacdot/satellite');
+const { createClient } = require('@supabase/supabase-js');
+
+const { SUPABASE_URL, SERVICE_ROLE_KEY } = process.env;
+
+const supabase = createClient(SUPABASE_URL, SERVICE_ROLE_KEY);
+
+if (!SUPABASE_URL || !SERVICE_ROLE_KEY) {
+  logger.error('SUPBASE_URL or SERVICE_ROLE_KEY is missing');
+  process.exit(1);
+}
+
+module.exports = {
+  async getAllFeeds() {
+    const { data, error } = await supabase
+      .from('feeds')
+      .select('wiki_author_name, url, telescope_profiles (display_name)');
+
+    if (error) {
+      logger.error({ error });
+      throw Error(error.message, "can't fetch feeds from supabase");
+    }
+
+    return data.map((feed) => ({
+      // Prefer the a user's display name if present, fallback to wiki name otherwise
+      author: feed.display_name || feed.wiki_author_name,
+      url: feed.url,
+    }));
+  },
+};

src/api/parser/test/e2e/parser-flow.test.js

@@ -4,11 +4,14 @@ const normalizeUrl = require('normalize-url');
 const { loadFeedsIntoQueue, invalidateFeed } = require('../../src/parser');
 const feedWorker = require('../../src/feed/worker');
 const { feedQueue } = require('../../src/feed/queue');
-const getWikiFeeds = require('../../src/utils/wiki-feed-parser');
+const { getAllFeeds } = require('../../src/utils/supabase');
 
 const urlToId = (url) => hash(normalizeUrl(url));
 
-jest.mock('../../src/utils/wiki-feed-parser', () => jest.fn());
+jest.mock('../../src/utils/supabase', () => ({
+  ...jest.requireActual('../../src/utils/supabase'),
+  getAllFeeds: jest.fn(),
+}));
 
 const fetchData = async (url) => {
   const res = await fetch(url);
@@ -36,9 +39,10 @@ const processFeeds = () => {
 
 let satellite;
 
-beforeAll(() => {
+beforeAll(async () => {
   satellite = new Satellite();
-  return processFeeds(); // start the feed queue for e2e test
+  await feedQueue.empty(); // remove jobs from the queue
+  await processFeeds(); // start the feed queue for e2e test
 });
 
 afterAll(() => Promise.all([feedQueue.close(), satellite.stop()]));
@@ -55,7 +59,7 @@ describe("Testing parser service's flow", () => {
         url: 'http://localhost:8888/feed.xml',
       },
     ];
-    getWikiFeeds.mockImplementation(() => Promise.resolve(valid)); // mock getWikiFeeds to return what feeds we want to test
+    getAllFeeds.mockImplementation(() => Promise.resolve(valid));
     loadFeedsIntoQueue();
     await waitForDrained();
 
@@ -83,7 +87,7 @@ describe("Testing parser service's flow", () => {
         url: 'https://janehasinvalidfeed.com/feed',
       },
     ];
-    getWikiFeeds.mockImplementation(() => Promise.resolve(invalid));
+    getAllFeeds.mockImplementation(() => Promise.resolve(invalid));
     loadFeedsIntoQueue();
     await waitForDrained();