Merge pull request #1124 from jetstreamapp/feat/add-ip-api-service

Add IP lookup service

apps/cron-tasks/project.json

@@ -16,8 +16,12 @@
             "entryPath": "apps/cron-tasks/src/save-analytics-summary.ts"
           },
           {
-            "entryName": "geo-ip-updater",
-            "entryPath": "apps/cron-tasks/src/geo-ip-updater.ts"
+            "entryName": "geo-ip-api-updater",
+            "entryPath": "apps/cron-tasks/src/geo-ip-api-updater.ts"
+          },
+          {
+            "entryName": "geo-ip-db-updater",
+            "entryPath": "apps/cron-tasks/src/geo-ip-db-updater.ts"
           }
         ],
         "tsConfig": "apps/cron-tasks/tsconfig.app.json",

apps/cron-tasks/src/config/env-config.ts

@@ -31,4 +31,8 @@ export const ENV = {
 
   MAX_MIND_ACCOUNT_ID: process.env.MAX_MIND_ACCOUNT_ID,
   MAX_MIND_LICENSE_KEY: process.env.MAX_MIND_LICENSE_KEY,
+
+  GEO_IP_API_HOSTNAME: process.env.GEO_IP_API_HOSTNAME,
+  GEO_IP_API_USERNAME: process.env.MAX_MIND_ACCOUNT_ID,
+  GEO_IP_API_PASSWORD: process.env.MAX_MIND_LICENSE_KEY,
 };

apps/cron-tasks/src/geo-ip-api-updater.ts

@@ -0,0 +1,46 @@
+/* eslint-disable @typescript-eslint/no-non-null-assertion */
+import { ENV } from './config/env-config';
+import { logger } from './config/logger.config';
+import { getExceptionLog } from './utils/utils';
+
+const GEO_IP_API_HOSTNAME = ENV.GEO_IP_API_HOSTNAME!;
+const GEO_IP_API_USERNAME = ENV.GEO_IP_API_USERNAME!;
+const GEO_IP_API_PASSWORD = ENV.GEO_IP_API_PASSWORD!;
+
+if (!GEO_IP_API_HOSTNAME) {
+  logger.error('GEO_IP_API_HOSTNAME environment variable is not set');
+  process.exit(1);
+}
+if (!GEO_IP_API_USERNAME) {
+  logger.error('GEO_IP_API_USERNAME environment variable is not set');
+  process.exit(1);
+}
+if (!GEO_IP_API_PASSWORD) {
+  logger.error('GEO_IP_API_PASSWORD environment variable is not set');
+  process.exit(1);
+}
+
+async function initiateDownload() {
+  const response = await fetch(GEO_IP_API_HOSTNAME, {
+    method: 'POST',
+    headers: {
+      Authorization: `Basic ${Buffer.from(`${GEO_IP_API_USERNAME}:${GEO_IP_API_PASSWORD}`).toString('base64')}`,
+    },
+  });
+
+  if (!response.ok) {
+    throw new Error(`Failed to download: ${response.statusText}`);
+  }
+
+  return response.json();
+}
+
+async function main() {
+  const results = await initiateDownload();
+  logger.info(results, 'Download completed successfully');
+}
+
+main().catch((error) => {
+  logger.error(getExceptionLog(error), 'Fatal error: %s', error.message);
+  process.exit(1);
+});

apps/cron-tasks/src/geo-ip-updater.ts → apps/cron-tasks/src/geo-ip-db-updater.ts

@@ -8,6 +8,9 @@ import { logger } from './config/logger.config';
 import { getExceptionLog } from './utils/utils';
 
 /**
+
+NOTICE: this is no longer used in production as put too much strain on the database server
+
 CREATE TABLE IF NOT EXISTS geo_ip.network (
 	network cidr NOT NULL,
 	geoname_id int,

apps/geo-ip-api/.eslintrc.json

@@ -0,0 +1,18 @@
+{
+  "extends": ["../../.eslintrc.json"],
+  "ignorePatterns": ["!**/*"],
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.ts", "*.tsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.js", "*.jsx"],
+      "rules": {}
+    }
+  ]
+}

apps/geo-ip-api/.gitignore

@@ -0,0 +1,2 @@
+src/downloads/*
+!src/downloads/.gitkeep

apps/geo-ip-api/jest.config.ts

@@ -0,0 +1,9 @@
+export default {
+  displayName: 'geo-ip-api',
+  preset: '../../jest.preset.js',
+  testEnvironment: 'node',
+  transform: {
+    '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '<rootDir>/tsconfig.spec.json' }],
+  },
+  moduleFileExtensions: ['ts', 'js', 'html'],
+};

apps/geo-ip-api/project.json

@@ -0,0 +1,72 @@
+{
+  "name": "geo-ip-api",
+  "$schema": "../../node_modules/nx/schemas/project-schema.json",
+  "sourceRoot": "apps/geo-ip-api/src",
+  "projectType": "application",
+  "tags": ["scope:server"],
+  "targets": {
+    "build": {
+      "executor": "@nx/esbuild:esbuild",
+      "outputs": ["{options.outputPath}"],
+      "defaultConfiguration": "production",
+      "options": {
+        "platform": "node",
+        "format": ["cjs"],
+        "bundle": true,
+        "outputPath": "dist/apps/geo-ip-api",
+        "main": "apps/geo-ip-api/src/main.ts",
+        "tsConfig": "apps/geo-ip-api/tsconfig.app.json",
+        "assets": [
+          {
+            "glob": "**/*",
+            "input": "apps/geo-ip-api/src/downloads",
+            "output": "downloads",
+            "ignore": [".gitkeep"]
+          }
+        ],
+        "generatePackageJson": true,
+        "sourcemap": true,
+        "esbuildOptions": {
+          "sourcemap": true,
+          "outExtension": {
+            ".js": ".js"
+          }
+        }
+      },
+      "configurations": {
+        "development": {
+          "inspect": true
+        },
+        "production": {}
+      }
+    },
+    "serve": {
+      "executor": "@nx/js:node",
+      "defaultConfiguration": "development",
+      "options": {
+        "buildTarget": "geo-ip-api:build",
+        "inspect": "inspect",
+        "port": 7778
+      },
+      "configurations": {
+        "development": {
+          "buildTarget": "geo-ip-api:build:development"
+        },
+        "production": {
+          "buildTarget": "geo-ip-api:build:production"
+        }
+      }
+    },
+    "lint": {
+      "executor": "@nx/eslint:lint",
+      "outputs": ["{options.outputFile}"]
+    },
+    "test": {
+      "executor": "@nx/jest:jest",
+      "outputs": ["{workspaceRoot}/coverage/{projectRoot}"],
+      "options": {
+        "jestConfig": "apps/geo-ip-api/jest.config.ts"
+      }
+    }
+  }
+}

apps/geo-ip-api/src/main.ts

@@ -0,0 +1,172 @@
+import { ENV, getExceptionLog, httpLogger, logger } from '@jetstream/api-config';
+import { json, urlencoded } from 'body-parser';
+import express from 'express';
+import { z, ZodError } from 'zod';
+import { downloadMaxMindDb, initMaxMind, lookupIpAddress, validateIpAddress } from './maxmind.service';
+import { createRoute } from './route.utils';
+
+const DISK_PATH = process.env.DISK_PATH ?? __dirname;
+
+if (ENV.ENVIRONMENT !== 'development' && (!ENV.GEO_IP_API_USERNAME || !ENV.GEO_IP_API_PASSWORD)) {
+  logger.error('GEO_IP_API_USERNAME/GEO_IP_API_PASSWORD environment variables are not set');
+  process.exit(1);
+}
+
+const app = express();
+
+app.use(json({ limit: '20mb' }));
+app.use(urlencoded({ extended: true }));
+
+app.use(httpLogger);
+
+app.use('/healthz', (req, res) => {
+  res.status(200).json({
+    error: false,
+    uptime: process.uptime(),
+    message: 'Healthy',
+  });
+});
+
+app.use((req, res, next) => {
+  try {
+    const authHeader = req.headers.authorization;
+    if (typeof authHeader !== 'string') {
+      throw new Error('Unauthorized');
+    }
+    const [type, token] = authHeader.split(' ');
+    if (type !== 'Basic') {
+      throw new Error('Unauthorized');
+    }
+    const [username, password] = Buffer.from(token, 'base64').toString().split(':');
+    if (username !== ENV.GEO_IP_API_USERNAME || password !== ENV.GEO_IP_API_PASSWORD) {
+      throw new Error('Unauthorized');
+    }
+    next();
+  } catch (ex) {
+    res.header('WWW-Authenticate', 'Basic realm="Geo IP API"');
+    res.status(401).json({
+      success: false,
+      message: 'Unauthorized',
+    });
+  }
+});
+
+app.post(
+  '/api/download',
+  createRoute({}, async (_, req, res, next) => {
+    try {
+      const startTime = Date.now();
+      await downloadMaxMindDb(DISK_PATH);
+      const timeTaken = Date.now() - startTime;
+      res.status(200).json({
+        success: true,
+        message: 'MaxMind database downloaded',
+        timeTaken,
+      });
+    } catch (ex) {
+      res.log.error(getExceptionLog(ex, true), 'Failed to download MaxMind database');
+      next(ex);
+    }
+  })
+);
+
+/**
+ * Lookup a single IP address
+ */
+app.get(
+  '/api/lookup',
+  createRoute({ query: z.object({ ip: z.string() }) }, async ({ query }, req, res, next) => {
+    try {
+      const ipAddress = query.ip;
+      await initMaxMind(DISK_PATH);
+      if (!validateIpAddress(ipAddress)) {
+        res.status(400).json({ success: false, message: 'IP address is invalid' });
+        return;
+      }
+      const results = lookupIpAddress(ipAddress);
+      res.status(200).json({ success: true, results });
+    } catch (ex) {
+      res.log.error(getExceptionLog(ex, true), 'Failed to lookup IP address');
+      next(ex);
+    }
+  })
+);
+
+/**
+ * Lookup multiple IP addresses
+ */
+app.post(
+  '/api/lookup',
+  createRoute({ body: z.object({ ips: z.string().array() }) }, async ({ body }, req, res, next) => {
+    try {
+      const ipAddresses = body.ips;
+      await initMaxMind(DISK_PATH);
+
+      const results = ipAddresses.map((ipAddress) => {
+        const isValid = validateIpAddress(ipAddress);
+        return {
+          ipAddress,
+          isValid,
+          ...(isValid ? lookupIpAddress(ipAddress) : null),
+        };
+      });
+
+      res.status(200).json({ success: true, results });
+    } catch (ex) {
+      res.log.error(getExceptionLog(ex, true), 'Failed to lookup IP address');
+      next(ex);
+    }
+  })
+);
+
+app.use('*', (req, res, next) => {
+  res.status(404).json({
+    success: false,
+    message: 'Not found',
+  });
+});
+
+app.use((err: Error | ZodError, req: express.Request, res: express.Response, next: express.NextFunction) => {
+  res.log.error('Unhandled error:', err);
+
+  if (!res.statusCode) {
+    res.status(500);
+  }
+
+  if (err instanceof ZodError) {
+    res.json({
+      success: false,
+      message: 'Validation error',
+      details: err.errors,
+    });
+    return;
+  }
+
+  res.json({
+    success: false,
+    message: err.message,
+  });
+});
+
+const port = Number(process.env.PORT || 3334);
+
+const server = app.listen(port, () => {
+  logger.info(`Listening at http://localhost:${port}/api`);
+});
+server.on('error', (error) => {
+  logger.error(getExceptionLog(error, true), 'Server error: %s', error.message);
+});
+
+process.on('SIGTERM', () => {
+  logger.info('SIGTERM received, shutting down gracefully');
+  server.close(() => {
+    logger.info('Server closed');
+    process.exit(0);
+  });
+
+  // Force close after 30s
+  setTimeout(() => {
+    logger.error('Could not close connections in time, forcefully shutting down');
+    process.exit(1);
+  }, 30_000);
+});