Firewall demo hidden release INTER-378

.github/workflows/ci.yml

@@ -5,8 +5,13 @@ on:
   pull_request:
     branches: [main]
 env:
-  # Playwright headless browsers running in CI get low confidence scores, causing flaky tests. Lower the confidence score threshold for CI testing.
+  # Playwright headless browsers running in CI get low confidence scores, causing flaky e2e tests. Lower the confidence score threshold for CI testing.
   MIN_CONFIDENCE_SCORE: 0
+  # Staging Cloudflare credentials and IDs for e2e tests
+  CLOUDFLARE_API_TOKEN: '${{ secrets.CLOUDFLARE_API_TOKEN }}'
+  CLOUDFLARE_ZONE_ID: '${{ secrets.CLOUDFLARE_ZONE_ID }}'
+  CLOUDFLARE_RULESET_ID: '${{ secrets.CLOUDFLARE_RULESET_ID }}'
+
 jobs:
   lint:
     name: Lint
@@ -111,7 +116,12 @@ jobs:
         run: yarn build
 
       - name: Run Playwright tests
-        run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+        run: npx playwright test --grep-invert CHROME_ONLY --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+
+      # Some tests are only run on Chrome, marked with CHROME_ONLY in their name
+      - name: Run Chrome-only Playwright tests
+        run: npx playwright test --grep CHROME_ONLY --project='chromium'
+        if: matrix.shardIndex == 1
 
       - name: Upload Playwright report
         uses: actions/upload-artifact@v3

.gitignore

@@ -56,3 +56,6 @@ tsconfig.tsbuildinfo
 
 # MacOS Finder
 .DS_Store
+
+# Local experiments
+.scratchpad/*

cron-jobs/delete_expired_ip_rules.ts

@@ -0,0 +1,44 @@
+import { BlockedIpDbModel } from '../src/server/botd-firewall/blockedIpsDatabase';
+import { Op } from 'sequelize';
+import { syncFirewallRuleset } from '../src/server/botd-firewall/cloudflareApiHelper';
+import { schedule } from 'node-cron';
+import { HOUR_MS } from '../src/shared/timeUtils';
+import 'dotenv/config';
+
+/**
+ * In production, run this file in conjunction with the production web server like:
+ * yarn start:with-cron-jobs
+ */
+
+// Every 5 minutes
+schedule('*/5 * * * *', () => {
+  deleteOldIpBlocks();
+});
+
+const IP_BLOCK_TIME_TO_LIVE_MS = HOUR_MS;
+
+async function deleteOldIpBlocks() {
+  try {
+    // Remove expired IP blocks
+    const deletedCount = await BlockedIpDbModel.destroy({
+      where: {
+        timestamp: {
+          [Op.lt]: new Date(Date.now() - IP_BLOCK_TIME_TO_LIVE_MS).toISOString(),
+        },
+      },
+    });
+
+    console.log(`Deleted ${deletedCount} expired blocked IPs from the database.`);
+
+    /**
+     * Construct updated firewall rules from the blocked IP database and apply them to the Cloudflare application.
+     * Note: We do this even if no IPs were deleted:
+     * A user might have blocked their IP but the database might have been cleared during site deployment right after,
+     * potentially leaving the IP blocked beyond the desired TTL. Safer to sync the firewall ruleset every time.
+     */
+    await syncFirewallRuleset();
+    console.log(`Updated Cloudflare firewall.`);
+  } catch (error) {
+    console.error(`Error deleting old blocked IPs: ${error}`);
+  }
+}

e2e/bot-firewall.spec.ts

@@ -0,0 +1,48 @@
+import { expect, test } from '@playwright/test';
+import { resetScenarios } from './resetHelper';
+import { TEST_IDS } from '../src/client/testIDs';
+import { BOT_FIREWALL_COPY } from '../src/client/bot-firewall/botFirewallCopy';
+
+/**
+ * CHROME_ONLY flag tells the GitHub action to run this test only using Chrome.
+ * This test relies on a single common Cloudflare ruleset, we we cannot run multiple instances of it at the same time.
+ */
+test.describe('Bot Firewall Demo CHROME_ONLY', () => {
+  test.beforeEach(async ({ page }) => {
+    await page.goto('/coupon-fraud');
+    await resetScenarios(page);
+  });
+
+  test('Should display bot visit and allow blocking/unblocking its IP address', async ({ page, context }) => {
+    // Record bot visit in web-scraping page
+    await page.goto('/web-scraping');
+    await expect(page.getByTestId(TEST_IDS.common.alert)).toContainText('Malicious bot detected');
+
+    // Check bot visit record and block IP
+    await page.goto('/bot-firewall');
+    await page.getByRole('button', { name: BOT_FIREWALL_COPY.blockIp }).first().click();
+    await page.getByText('was blocked in the application firewall').waitFor();
+    await page.waitForTimeout(3000);
+
+    /**
+     * Try to visit web-scraping page, should be blocked by Cloudflare
+     * Checking the response code here as parsing the actual page if flaky for some reason.
+     * Using a separate tab also seems to help with flakiness.
+     */
+    const secondTab = await context.newPage();
+    await secondTab.goto('https://staging.fingerprinthub.com/web-scraping');
+    await secondTab.reload();
+    await secondTab.getByRole('heading', { name: 'Sorry, you have been blocked' }).waitFor();
+
+    // Unblock IP
+    await page.goto('/bot-firewall');
+    await page.getByRole('button', { name: BOT_FIREWALL_COPY.unblockIp }).first().click();
+    await page.getByText('was unblocked in the application firewall').waitFor();
+    await page.waitForTimeout(3000);
+
+    // Try to visit web-scraping page, should be allowed again
+    await secondTab.goto('https://staging.fingerprinthub.com/web-scraping');
+    await secondTab.reload();
+    await expect(secondTab.getByTestId(TEST_IDS.common.alert)).toContainText('Malicious bot detected');
+  });
+});

e2e/scraping/protected.spec.ts

@@ -3,8 +3,7 @@ import { TEST_IDS } from '../../src/client/testIDs';
 
 test.describe('Scraping flights', () => {
   test('is not possible with Bot detection on', async ({ page }) => {
-    await page.goto('/web-scraping');
-    await page.waitForLoadState('networkidle');
+    await page.goto('/web-scraping', { waitUntil: 'networkidle' });
     await expect(page.getByTestId(TEST_IDS.common.alert)).toContainText('Malicious bot detected');
   });
 });

e2e/scraping/unprotected.spec.ts

@@ -11,13 +11,11 @@ const scrapeText = async (parent: Locator, testId: string) => {
 
 test.describe('Scraping flights', () => {
   test('is possible with Bot detection off', async ({ page }) => {
-    await page.goto('/web-scraping?disableBotDetection=1');
-    await page.waitForLoadState('networkidle');
+    await page.goto('/web-scraping?disableBotDetection=1', { waitUntil: 'networkidle' });
     // Artificial wait necessary to prevent flakiness
     await page.waitForTimeout(3000);
 
     const flightCards = await page.getByTestId(TEST_ID.card).all();
-    console.log('Found flight cards: ', flightCards.length);
     expect(flightCards.length > 0).toBe(true);
 
     const flightData = [];
@@ -34,6 +32,5 @@ test.describe('Scraping flights', () => {
 
     expect(flightData.length > 0).toBe(true);
     writeFileSync('./e2e/output/flightData.json', JSON.stringify(flightData, null, 2));
-    console.log("Scraped flight data saved to 'e2e/output/flightData.json'");
   });
 });

e2e/zodUtils.ts

@@ -36,7 +36,7 @@ export function isAgentResponse(obj: unknown): boolean {
     agentResponseSchema.parse(obj);
     return true;
   } catch (error) {
-    console.log(error);
+    console.error(error);
     return false;
   }
 }
@@ -102,7 +102,7 @@ export function isServerResponse(obj: unknown): boolean {
     serverResponseSchema.parse(obj);
     return true;
   } catch (error) {
-    console.log(error);
+    console.error(error);
     return false;
   }
 }

next.config.js

@@ -6,7 +6,6 @@ const path = require('path');
  **/
 module.exports = {
   images: {
-    domains: ['images.unsplash.com', 'localhost'],
     formats: ['image/webp'],
   },
   sassOptions: {

package.json

@@ -6,6 +6,8 @@
     "dev": "next dev",
     "build": "next build",
     "start": "next start",
+    "start:with-cron-jobs": "run-p start cron",
+    "cron": "tsx cron-jobs/delete_expired_ip_rules.ts",
     "lint": "next lint",
     "lint:fix": "yarn lint --fix",
     "prettier": "prettier src --check",
@@ -36,6 +38,7 @@
     "classnames": "^2.3.2",
     "framer-motion": "^10.13.2",
     "include-media": "^2.0.0",
+    "is-ip": "^5.0.1",
     "leaflet": "^1.9.4",
     "next": "^14.0.3",
     "next-usequerystate": "^1.9.1",
@@ -46,7 +49,8 @@
     "react-query": "^3.39.1",
     "react-syntax-highlighter": "^15.5.0",
     "react-use": "^17.4.0",
-    "sequelize": "^6.19.0",
+    "sequelize": "^6.35.2",
+    "sharp": "^0.33.1",
     "socket.io": "^4.5.4",
     "socket.io-client": "^4.5.4",
     "sqlite3": "^5.0.8",
@@ -57,18 +61,23 @@
     "@playwright/test": "^1.40.1",
     "@types/leaflet": "^1.9.3",
     "@types/node": "^18.11.18",
+    "@types/node-cron": "^3.0.11",
     "@types/react": "^18.0.27",
     "@typescript-eslint/eslint-plugin": "^6.13.2",
     "@vitejs/plugin-react": "^3.0.1",
     "dotenv": "^16.3.1",
-    "eslint": "^8.55.0",
+    "eslint": "^8.56.0",
     "eslint-config-next": "^14.0.4",
     "eslint-config-prettier": "^9.1.0",
+    "eslint-plugin-jsx-a11y": "^6.8.0",
     "eslint-plugin-prettier": "^5.0.1",
     "eslint-plugin-react-hooks": "^4.6.0",
     "jsdom": "^21.1.0",
+    "node-cron": "^3.0.3",
+    "npm-run-all": "^4.1.5",
     "prettier": "^3.1.0",
     "sass": "^1.64.1",
+    "tsx": "^4.7.0",
     "typescript": "^4.9.5",
     "vitest": "^0.28.3"
   }

src/client/bot-firewall/botFirewallCopy.ts

@@ -0,0 +1,4 @@
+export const BOT_FIREWALL_COPY = {
+  blockIp: 'Block this IP',
+  unblockIp: 'Unblock',
+} as const;

src/client/components/common/Header/Header.tsx

@@ -80,11 +80,11 @@ export default function Header({ notificationBar, darkMode }: HeaderProps) {
                     darkMode,
                     leftColumns: [
                       {
-                        list: USE_CASES_NAVIGATION.slice(0, 3),
+                        list: USE_CASES_NAVIGATION.slice(0, 4),
                         cardBackground: true,
                       },
                       {
-                        list: USE_CASES_NAVIGATION.slice(3),
+                        list: USE_CASES_NAVIGATION.slice(4),
                         cardBackground: true,
                       },
                     ],

src/client/components/common/content.tsx

@@ -28,6 +28,7 @@ export type UseCase = {
     title: string;
     url: string;
   }[];
+  hiddenInNavigation?: boolean;
 };
 
 export const USE_CASES = {
@@ -227,6 +228,7 @@ export const USE_CASES = {
     title: 'Paywall',
     titleMeta: 'Fingerprint Use Cases | Content Paywall Live Demo',
     url: '/paywall',
+    articleUrl: 'https://fingerprint.com/blog/how-paywalls-work-paywall-protection-tutorial/',
     iconSvg: PaywallIcon,
     descriptionHomepage: [
       <p key="1">
@@ -371,6 +373,80 @@ export const USE_CASES = {
       },
     ],
   },
+  botFirewall: {
+    title: 'Bot-Detection-powered Firewall',
+    titleMeta: 'Fingerprint Use Cases | Bot-Detection-powered Firewall',
+    url: '/bot-firewall',
+    articleUrl: 'https://fingerprint.com/blog/bot-detection-powered-firewall/',
+    iconSvg: ScrapingIcon,
+    descriptionHomepage: [
+      <p key="1">
+        Integrate Fingerprint Bot Detection with your Web Application Firewall and dynamically block IP addresses linked
+        to past bot visits.
+      </p>,
+      <p key="2">
+        Block previously recognized bots on their next visit completely — before they even reach your web page.
+      </p>,
+    ],
+    description: (
+      <>
+        <p>
+          Integrate Fingerprint Bot Detection with your Web Application Firewall and dynamically block IP addresses
+          linked to past bot visits.
+        </p>
+        <p>
+          Fingerprint Bot Detection allows you to identify sophisticated bots and headless browsers by collecting and
+          analyzing browser signals. See our{' '}
+          <Link href={'/web-scraping'} target="_blank">
+            Web scraping demo
+          </Link>{' '}
+          for an example of protecting client-site content from bots. This demo goes a step further and uses Bot
+          detection results to block previously recognized bots on their next visit completely — before they even reach
+          your web page.
+        </p>
+      </>
+    ),
+    descriptionMeta:
+      'Integrate Fingerprint Bot Detection with your Web Application Firewall and dynamically block IP addresses linked to past bot visits.',
+    doNotMentionResetButton: true,
+    instructions: [
+      <>
+        Use a locally running instance of Playwright, Cypress, or another headless browser tool to visit the{' '}
+        <Link href={'/web-scraping'} target="_blank">
+          web scraping demo
+        </Link>
+        .
+      </>,
+      <>
+        Your headless browser will be recognized as a bot, and your IP address will be saved to the bot visit database
+        displayed below.
+      </>,
+      <>
+        Click <b>Block this IP</b> to prevent the bot from loading the page at all going forward. For demo purposes, you
+        are only allowed to block your own IP.
+      </>,
+      <>
+        Try visiting the{' '}
+        <Link href={'/web-scraping'} target="_blank">
+          web scraping demo
+        </Link>{' '}
+        again (either as a bot or using your regular browser).
+      </>,
+      <>Your IP address is blocked from the page completely.</>,
+    ],
+    moreResources: [
+      {
+        url: 'https://fingerprint.com/blog/preventing-content-scraping/',
+        type: 'Use case tutorial',
+        title: 'Web Scraping Prevention',
+      },
+      {
+        url: 'https://fingerprint.com/blog/betting-bots/',
+        type: 'Article',
+        title: 'Betting Bots',
+      },
+    ],
+  },
 } as const satisfies Record<string, UseCase>;
 
 export const PLAYGROUND_METADATA: Pick<
@@ -391,9 +467,14 @@ export const PLAYGROUND_METADATA: Pick<
   descriptionMeta: 'Analyze your browser with Fingerprint Pro and see all the available signals.',
 };
 
-export const USE_CASES_ARRAY = Object.values(USE_CASES);
+export const USE_CASES_ARRAY = Object.values(USE_CASES)
+  // TODO: Remove this when ready the final of bot firewall demo is ready
+  .filter((useCase) => useCase.url !== USE_CASES.botFirewall.url);
 
-export const USE_CASES_NAVIGATION = USE_CASES_ARRAY.map((useCase) => ({ title: useCase.title, url: useCase.url }));
+export const USE_CASES_NAVIGATION = USE_CASES_ARRAY.map((useCase) => ({
+  title: useCase.title,
+  url: useCase.url,
+}));
 export const PLATFORM_NAVIGATION = [PLAYGROUND_METADATA];
 
 type HomePageCard = Pick<UseCase, 'title' | 'url' | 'iconSvg' | 'descriptionHomepage'>;