Fetch statics from github

packages/api/package.json

@@ -10,7 +10,7 @@
     "lint:check": "prettier-standard --check",
     "dev": "nodemon",
     "start": "node dist/index.js",
-    "test": "jest",
+    "test": "jest --runInBand",
     "prepare": "yarn codegen",
     "docker:build": "docker build . --no-cache -t witnet/data-feeds-explorer-api --stream"
   },
@@ -35,12 +35,14 @@
   },
   "dependencies": {
     "apollo-server": "^2.22.1",
+    "axios": "^0.26.1",
     "dataloader": "2.0.0",
     "dotenv": "^10.0.0",
     "graphql": "^15.5.0",
     "jest": "^27.0.1",
     "migrate-mongo": "^8.2.3",
     "mongodb": "^3.6.9",
+    "node-cache": "^5.1.2",
     "web3": "^1.3.6",
     "web3-utils": "^1.3.6"
   }

packages/api/src/fetchSvgs.ts

@@ -0,0 +1,42 @@
+import axios from 'axios'
+import { removeRepeatedElements } from './utils'
+
+export const DEFAULT_SVG =
+  '<svg class="icon-size" width="34" height="34" viewBox="0 0 34 34" fill="none" xmlns="http://www.w3.org/2000/svg"><circle cx="17" cy="17" r="17" fill="#C4C4C4"/><path d="M18.066 20.785h-2.988a9.951 9.951 0 0 1 .164-1.816c.11-.516.293-.98.551-1.395a6.466 6.466 0 0 1 1.055-1.23c.367-.328.687-.64.96-.938.274-.297.489-.601.645-.914.156-.32.235-.676.235-1.066 0-.453-.07-.828-.211-1.125a1.428 1.428 0 0 0-.61-.692c-.265-.156-.601-.234-1.008-.234-.336 0-.648.074-.937.223-.29.14-.527.359-.715.656-.18.297-.273.687-.281 1.172h-3.399c.024-1.07.27-1.953.739-2.649a4.252 4.252 0 0 1 1.91-1.558c.797-.344 1.691-.516 2.683-.516 1.094 0 2.032.18 2.813.54.781.35 1.379.87 1.793 1.558.414.68.62 1.508.62 2.484 0 .68-.132 1.285-.398 1.817a6.21 6.21 0 0 1-1.043 1.464c-.43.454-.902.922-1.417 1.407-.446.398-.75.816-.915 1.254-.156.437-.238.957-.246 1.558Zm-3.34 3.621c0-.5.172-.914.516-1.242.344-.336.805-.504 1.383-.504.57 0 1.027.168 1.371.504.352.328.527.742.527 1.242 0 .485-.175.895-.527 1.23-.344.337-.8.505-1.371.505-.578 0-1.04-.168-1.383-.504a1.655 1.655 0 0 1-.515-1.23Z" fill="#fff"/></svg>'
+
+const STATIC_LOGOS_SVG_URL =
+  'https://raw.githubusercontent.com/witnet/data-feeds-explorer/main/packages/ui/assets/svg/'
+
+export async function fetchSvgs (
+  networksToFetch: Array<string>
+): Promise<{ [key: string]: string }> {
+  const networksWithoutRepeated = removeRepeatedElements(networksToFetch)
+  const logosUrls = networksWithoutRepeated.map(
+    (networkToFetch: string) => `${STATIC_LOGOS_SVG_URL}${networkToFetch}.svg`
+  )
+
+  // Fetch all logos from github
+  const promises = logosUrls.map(url => axios.get(url))
+  return new Promise(resolve => {
+    Promise.allSettled(promises).then(results => {
+      const svgs = results.map((result, index) => {
+        if (result.status === 'rejected') {
+          console.log(`Error fetching logo from: ${logosUrls[index]}`)
+          return DEFAULT_SVG
+        }
+
+        return result.value.data
+      })
+
+      const svgByName = networksWithoutRepeated.reduce(
+        (acc, val, index) => ({
+          ...acc,
+          [val]: svgs[index]
+        }),
+        {}
+      )
+
+      resolve(svgByName)
+    })
+  })
+}

packages/api/src/index.ts

@@ -1,45 +1,60 @@
 require('dotenv/config')
 
-import fs from 'fs'
-import path from 'path'
 import Web3 from 'web3'
 import { MongoManager } from './database'
 import { FeedRepository } from './repository/Feed'
 import { ResultRequestRepository } from './repository/ResultRequest'
 import { createServer } from './server'
-import {
-  FeedInfo,
-  FeedInfoConfig,
-  Repositories,
-  RouterDataFeedsConfig,
-  NetworksConfig
-} from './types'
+import { Repositories, RouterDataFeedsConfig, NetworksConfig } from './types'
 import { Web3Middleware } from './web3Middleware/index'
-import { normalizeConfig, normalizeNetworkConfig } from './utils/index'
-import dataFeedsRouterConfig from './dataFeedsRouter.json'
+import { normalizeNetworkConfig } from './utils/index'
+import {
+  normalizeAndValidateDataFeedConfig,
+  fetchDataFeedsRouterConfig
+} from './readDataFeeds'
+import { SvgCache } from './svgCache'
 
 async function main () {
+  const svgCache = new SvgCache()
   const mongoManager = new MongoManager()
   const db = await mongoManager.start()
-  const dataFeeds = readDataFeeds()
-  const networksConfig = readNetworks()
+  const dataFeedsRouterConfig: RouterDataFeedsConfig = await fetchDataFeedsRouterConfig()
+  const dataFeeds = normalizeAndValidateDataFeedConfig(dataFeedsRouterConfig)
+  const networksConfigPartial: Array<Omit<
+    NetworksConfig,
+    'logo'
+  >> = normalizeNetworkConfig(dataFeedsRouterConfig)
+
+  const logosToFetch = networksConfigPartial.map(
+    (networksConfig: NetworksConfig) => {
+      return networksConfig.chain.toLowerCase()
+    }
+  )
+
+  const networksLogos: { [key: string]: string } = await svgCache.getMany(
+    logosToFetch
+  )
+
+  const networksConfig = networksConfigPartial.map((networksConfig, index) => ({
+    ...networksConfig,
+    logo: networksLogos[logosToFetch[index]]
+  }))
 
   const repositories: Repositories = {
     feedRepository: new FeedRepository(dataFeeds),
     resultRequestRepository: new ResultRequestRepository(db, dataFeeds)
   }
-  const config = {
-    dataFeedsConfig: dataFeeds,
-    networksConfig: networksConfig
-  }
 
   const web3Middleware = new Web3Middleware(
     { repositories, Web3: Web3 },
     dataFeeds
   )
   web3Middleware.listen()
 
-  const server = await createServer(repositories, config)
+  const server = await createServer(repositories, svgCache, {
+    dataFeedsConfig: dataFeeds,
+    networksConfig
+  })
 
   server
     .listen({ host: '0.0.0.0', port: process.env.SERVER_PORT })
@@ -48,85 +63,4 @@ async function main () {
     })
 }
 
-export function readNetworks (): Array<NetworksConfig> {
-  return normalizeNetworkConfig(dataFeedsRouterConfig as RouterDataFeedsConfig)
-}
-
-export function readDataFeeds (): Array<FeedInfo> {
-  const dataFeeds: Array<Omit<
-    FeedInfoConfig,
-    'abi' | 'routerAbi'
-  >> = normalizeConfig(dataFeedsRouterConfig as RouterDataFeedsConfig)
-
-  // Throw and error if config file is not valid
-  validateDataFeeds(dataFeeds)
-
-  return dataFeeds.map(dataFeed => ({
-    ...dataFeed,
-    routerAbi: JSON.parse(
-      fs.readFileSync(
-        path.resolve(
-          process.env.DATA_FEED_ROUTER_ABI_PATH ||
-            './src/abi/PriceFeedRouter.json'
-        ),
-        'utf-8'
-      )
-    ),
-    abi: JSON.parse(
-      fs.readFileSync(
-        path.resolve(
-          process.env.DATA_FEED_ABI_PATH || './src/abi/PriceFeed.json'
-        ),
-        'utf-8'
-      )
-    )
-  }))
-}
-
-// Throw an error if a field is missing in the data feed config file
-function validateDataFeeds (
-  dataFeeds: Array<Omit<FeedInfoConfig, 'abi' | 'routerAbi'>>
-) {
-  const expectedFields = [
-    'feedFullName',
-    'id',
-    'address',
-    'contractId',
-    'routerAddress',
-    'network',
-    'networkName',
-    'chain',
-    'name',
-    'label',
-    'pollingPeriod',
-    'color',
-    'blockExplorer',
-    'deviation',
-    'heartbeat',
-    'finality'
-  ]
-
-  dataFeeds.forEach((feedInfoConfig, index) => {
-    expectedFields.forEach(field => {
-      // Validate nested keys in a field
-      field.split('.').reduce((acc, val) => {
-        // Throw error if the key is not found or has a falsy value
-        if (!(val in acc) || !acc[val]) {
-          throw new Error(
-            `Missing field ${field} in index ${index} in data feed config file`
-          )
-        } else {
-          // Throw error if not validated new fields are added in the config file
-          if (Object.keys(feedInfoConfig).length !== expectedFields.length) {
-            throw new Error(
-              `There are more fields in the feed config than expected`
-            )
-          }
-          return acc[val]
-        }
-      }, feedInfoConfig)
-    })
-  })
-}
-
 main()

packages/api/src/loaders/index.ts

@@ -1,11 +1,14 @@
 import DataLoader from 'dataloader'
+import { SvgCache } from '../svgCache'
 import { Repositories, ResultRequestDbObjectNormalized } from '../types'
 
 export class Loaders {
   repositories: Repositories
+  svgCache: SvgCache
 
-  constructor (repositories: Repositories) {
+  constructor (repositories: Repositories, svgCache: SvgCache) {
     this.repositories = repositories
+    this.svgCache = svgCache
   }
   // returns a loader that fetches data using the given function
   private genericLoader<T> (load: (filter) => T) {
@@ -30,6 +33,7 @@ export class Loaders {
   getLoaders (): {
     lastResult: DataLoader<string, ResultRequestDbObjectNormalized, string>
     requests: DataLoader<string, ResultRequestDbObjectNormalized, string>
+    logos: DataLoader<string, string, string>
   } {
     return {
       lastResult: this.genericLoader<Promise<ResultRequestDbObjectNormalized>>(
@@ -72,6 +76,9 @@ export class Loaders {
           filter.feedFullName,
           timestamp
         )
+      }),
+      logos: new DataLoader(async (logos: Array<string>) => {
+        return Object.values(await this.svgCache.getMany(logos))
       })
     }
   }

packages/api/src/readDataFeeds.ts

@@ -0,0 +1,106 @@
+import axios from 'axios'
+import path from 'path'
+import fs from 'fs'
+import { RouterDataFeedsConfig, FeedInfo, FeedInfoConfig } from './types'
+import { normalizeConfig } from './utils'
+
+export async function fetchDataFeedsRouterConfig (): Promise<
+  RouterDataFeedsConfig | null
+> {
+  return await axios
+    .get(
+      'https://raw.github.com/witnet/data-feeds-explorer/main/packages/api/src/dataFeedsRouter.json'
+    )
+    .then(res => {
+      return res.data
+    })
+    .catch(err => {
+      console.log('There was an error fetching the config file', err)
+      return null
+    })
+}
+/**
+ * FIXME(#197): normalizeAndValidateDataFeedConfig could be refactored to include the ABI to avoid
+ * have multiple functions to build the object. So we should review how we are fetching,
+ * validating and normalizing the configuration file We can even review the normalized object
+ * structure to check if it has sense right now or only because it was the previous configuration
+ * format
+ */
+export function normalizeAndValidateDataFeedConfig (
+  config: RouterDataFeedsConfig
+): Array<FeedInfo> {
+  const dataFeeds: Array<Omit<
+    FeedInfoConfig,
+    'abi' | 'routerAbi'
+  >> = normalizeConfig(config)
+
+  // Throw and error if config file is not valid
+  validateDataFeeds(dataFeeds)
+
+  return dataFeeds.map(dataFeed => ({
+    ...dataFeed,
+    routerAbi: JSON.parse(
+      fs.readFileSync(
+        path.resolve(
+          process.env.DATA_FEED_ROUTER_ABI_PATH ||
+            './src/abi/PriceFeedRouter.json'
+        ),
+        'utf-8'
+      )
+    ),
+    abi: JSON.parse(
+      fs.readFileSync(
+        path.resolve(
+          process.env.DATA_FEED_ABI_PATH || './src/abi/PriceFeed.json'
+        ),
+        'utf-8'
+      )
+    )
+  }))
+}
+
+// Throw an error if a field is missing in the data feed config file
+function validateDataFeeds (
+  dataFeeds: Array<Omit<FeedInfoConfig, 'abi' | 'routerAbi'>>
+) {
+  const expectedFields = [
+    'feedFullName',
+    'id',
+    'address',
+    'contractId',
+    'routerAddress',
+    'network',
+    'networkName',
+    'chain',
+    'name',
+    'label',
+    'pollingPeriod',
+    'color',
+    'blockExplorer',
+    'deviation',
+    'heartbeat',
+    'finality'
+  ]
+
+  dataFeeds.forEach((feedInfoConfig, index) => {
+    expectedFields.forEach(field => {
+      // Validate nested keys in a field
+      field.split('.').reduce((acc, val) => {
+        // Throw error if the key is not found or has a falsy value
+        if (!(val in acc) || !acc[val]) {
+          throw new Error(
+            `Missing field ${field} in index ${index} in data feed config file`
+          )
+        } else {
+          // Throw error if not validated new fields are added in the config file
+          if (Object.keys(feedInfoConfig).length !== expectedFields.length) {
+            throw new Error(
+              `There are more fields in the feed config than expected`
+            )
+          }
+          return acc[val]
+        }
+      }, feedInfoConfig)
+    })
+  })
+}

packages/api/src/repository/Feed.ts

@@ -44,8 +44,8 @@ export class FeedRepository {
     }
 
     return {
-      feeds,
-      total: feeds.length
+      feeds: feeds || [],
+      total: feeds ? feeds.length : 0
     }
   }