Merge pull request #1 from stevan-borus/fix/use-local-referers

fix/use local referers
stevan-borus · Oct 11, 2024 · b6a63b2 · b6a63b2
2 parents 08980be + 189adcf
commit b6a63b2
Show file tree

Hide file tree

Showing 8 changed files with 3,532 additions and 71 deletions.
diff --git a/README.md b/README.md
@@ -2,7 +2,7 @@
 
 A React-compatible referer parser library inspired by [nodejs-referer-parser](https://github.com/snowplow-referer-parser/nodejs-referer-parser).
 
-The implementation uses the shared 'database' of known referers found in [file](https://s3-eu-west-1.amazonaws.com/snowplow-hosted-assets/third-party/referer-parser/referers-latest.yaml) as it says in the [snowplow-referer-parser/referer-parser](https://github.com/snowplow-referer-parser/referer-parser) README.
+The implementation uses the shared 'database' of known referers found in [file](https://s3-eu-west-1.amazonaws.com/snowplow-hosted-assets/third-party/referer-parser/referers-latest.json) as it says in the [snowplow-referer-parser/referer-parser](https://github.com/snowplow-referer-parser/referer-parser) README.
 
 ## Installation
 
@@ -45,7 +45,7 @@ async function example() {
   );
 
   console.log("Search engine referral:", result);
-  // Output: Search engine referral: { medium: 'search', referer: 'google', term: 'gateway oracle cards denise linn' }
+  // Output: Search engine referral: { medium: 'search', referer: 'Google', term: 'gateway oracle cards denise linn' }
 }
 
 example();

diff --git a/package.json b/package.json
@@ -22,9 +22,6 @@
     "type": "git",
     "url": "https://github.com/stevan-borus/react-referer-parser.git"
   },
-  "dependencies": {
-    "js-yaml": "^4.1.0"
-  },
   "devDependencies": {
     "@biomejs/biome": "1.9.3",
     "@types/js-yaml": "^4.0.9",

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/src/index.test.ts b/src/index.test.ts
@@ -79,7 +79,7 @@ describe('parse function', () => {
     const result = await parse('http://www.google.com/search?q=test');
     expect(result).toEqual({
       medium: 'search',
-      referer: 'google',
+      referer: 'Google',
       term: 'test',
     });
   });
@@ -88,7 +88,7 @@ describe('parse function', () => {
     const result = await parse('http://www.google.co.uk/search?q=test');
     expect(result).toEqual({
       medium: 'search',
-      referer: 'google',
+      referer: 'Google',
       term: 'test',
     });
   });
@@ -97,7 +97,7 @@ describe('parse function', () => {
     const result = await parse('http://www.bing.com/search?q=test');
     expect(result).toEqual({
       medium: 'search',
-      referer: 'bing',
+      referer: 'Bing',
       term: 'test',
     });
   });
@@ -106,7 +106,7 @@ describe('parse function', () => {
     const result = await parse('http://search.yahoo.com/search?p=test');
     expect(result).toEqual({
       medium: 'search',
-      referer: 'yahoo',
+      referer: 'Yahoo!',
       term: 'test',
     });
   });
@@ -115,7 +115,7 @@ describe('parse function', () => {
     const result = await parse('http://www.facebook.com/page');
     expect(result).toEqual({
       medium: 'social',
-      referer: 'facebook',
+      referer: 'Facebook',
       term: null,
     });
   });
@@ -124,7 +124,7 @@ describe('parse function', () => {
     const result = await parse('http://t.co/abcde');
     expect(result).toEqual({
       medium: 'social',
-      referer: 'twitter',
+      referer: 'Twitter',
       term: null,
     });
   });
@@ -133,7 +133,7 @@ describe('parse function', () => {
     const result = await parse('http://www.linkedin.com/share');
     expect(result).toEqual({
       medium: 'social',
-      referer: 'linkedin',
+      referer: 'LinkedIn',
       term: null,
     });
   });
@@ -142,7 +142,7 @@ describe('parse function', () => {
     const result = await parse('http://lnkd.in/abcde');
     expect(result).toEqual({
       medium: 'social',
-      referer: 'linkedin',
+      referer: 'LinkedIn',
       term: null,
     });
   });
@@ -151,7 +151,7 @@ describe('parse function', () => {
     const result = await parse('http://www.instagram.com/p/abcde');
     expect(result).toEqual({
       medium: 'social',
-      referer: 'instagram',
+      referer: 'Instagram',
       term: null,
     });
   });
@@ -160,7 +160,7 @@ describe('parse function', () => {
     const result = await parse('http://www.pinterest.com/pin/abcde');
     expect(result).toEqual({
       medium: 'social',
-      referer: 'pinterest',
+      referer: 'Pinterest',
       term: null,
     });
   });
@@ -169,7 +169,7 @@ describe('parse function', () => {
     const result = await parse('http://mail.google.com/mail/u/0');
     expect(result).toEqual({
       medium: 'email',
-      referer: 'gmail',
+      referer: 'Gmail',
       term: null,
     });
   });
@@ -187,7 +187,7 @@ describe('parse function', () => {
     const result = await parse('http://www.google.com/');
     expect(result).toEqual({
       medium: 'search',
-      referer: 'google',
+      referer: 'Google',
       term: null,
     });
   });
@@ -196,7 +196,7 @@ describe('parse function', () => {
     const result = await parse('http://www.google.com/search?q=');
     expect(result).toEqual({
       medium: 'search',
-      referer: 'google',
+      referer: 'Google',
       term: null,
     });
   });
@@ -205,7 +205,7 @@ describe('parse function', () => {
     const result = await parse('http://www.google.com/search?q=test&hl=en');
     expect(result).toEqual({
       medium: 'search',
-      referer: 'google',
+      referer: 'Google',
       term: 'test',
     });
   });
@@ -221,7 +221,7 @@ describe('parse function', () => {
     const result = await parse('http://WWW.GOOGLE.COM/search?q=test');
     expect(result).toEqual({
       medium: 'search',
-      referer: 'google',
+      referer: 'Google',
       term: 'test',
     });
   });
@@ -230,7 +230,7 @@ describe('parse function', () => {
     const result = await parse('http://www.google.com:8080/search?q=test');
     expect(result).toEqual({
       medium: 'search',
-      referer: 'google',
+      referer: 'Google',
       term: 'test',
     });
   });
@@ -239,7 +239,7 @@ describe('parse function', () => {
     const result = await parse('http://user:[email protected]/search?q=test');
     expect(result).toEqual({
       medium: 'search',
-      referer: 'google',
+      referer: 'Google',
       term: 'test',
     });
   });

diff --git a/src/index.ts b/src/index.ts
@@ -1,32 +1,11 @@
-import { load } from 'js-yaml';
-import type { Medium, Referer, RefererData } from './types';
-
-let cachedRefererData: RefererData | null = null;
-let lastFetchTime = 0;
-let CACHE_DURATION = 24 * 60 * 60 * 1000;
-
-async function fetchAndParseRefererData(): Promise<RefererData> {
-  let response = await fetch(
-    'https://s3-eu-west-1.amazonaws.com/snowplow-hosted-assets/third-party/referer-parser/referers-latest.yaml',
-  );
-
-  let yamlText = await response.text();
-
-  return load(yamlText) as RefererData;
-}
+import type { Referer, RefererData } from './types';
+import refererData from './utils/latest-referers.json';
 
 export async function parse(
-  refererUrl: string,
+  refererUrl: string | null,
   pageUrl?: string,
   internalDomains: string[] = [],
 ): Promise<Referer> {
-  let currentTime = Date.now();
-
-  if (!cachedRefererData || currentTime - lastFetchTime > CACHE_DURATION) {
-    cachedRefererData = await fetchAndParseRefererData();
-    lastFetchTime = currentTime;
-  }
-
   if (!refererUrl) {
     return { medium: 'direct', referer: null, term: null };
   }
@@ -41,7 +20,7 @@ export async function parse(
     return { medium: 'internal', referer: null, term: null };
   }
 
-  let bestMatch = findBestMatch(refererParsed, cachedRefererData);
+  let bestMatch = findBestMatch(refererParsed, refererData as RefererData);
 
   return (
     bestMatch || {
@@ -103,8 +82,8 @@ function findBestMatch(refererParsed: URL, cachedRefererData: RefererData): Refe
   return null;
 }
 
-function findTerm(refererParsed: URL, parameters: string[]): string | null {
-  if (parameters.length === 0) {
+function findTerm(refererParsed: URL, parameters?: string[]): string | null {
+  if (!parameters || parameters.length === 0) {
     return null;
   }
 

diff --git a/src/types.ts b/src/types.ts
@@ -10,17 +10,86 @@ export type Medium =
   | 'direct'
   | (string & {});
 
-export interface Referer {
+type RefererSource =
+  | 'Google'
+  | 'Yandex Maps'
+  | 'Yahoo!'
+  | 'TalkTalk'
+  | '1.cz'
+  | 'Softonic'
+  | 'GAIS'
+  | 'Freecause'
+  | '360.cn'
+  | 'RPMFind'
+  | 'Comcast'
+  | 'Voila'
+  | 'Nifty'
+  | 'Atlas'
+  | 'Lo.st'
+  | 'DasTelefonbuch'
+  | 'Fireball'
+  | '1und1'
+  | 'Virgilio'
+  | 'Telstra'
+  | 'Web.nl'
+  | 'Plazoo'
+  | 'Goyellow.de'
+  | 'AOL'
+  | 'Acoon'
+  | 'Free'
+  | 'Apollo Latvia'
+  | 'HighBeam'
+  | 'I-play'
+  | 'FriendFeed'
+  | 'Yasni'
+  | 'Gigablast'
+  | 'Arcor'
+  | 'arama'
+  | 'Fixsuche'
+  | 'Apontador'
+  | 'Search.com'
+  | 'Monstercrawler'
+  | 'Google Images'
+  | 'ABCsøk'
+  | 'Google Product Search'
+  | 'Blogpulse'
+  | 'Hooseek.com'
+  | 'Dalesearch'
+  | 'Alice Adsl'
+  | 'T-Online'
+  | 'Sogou'
+  | 'Hit-Parade'
+  | 'SearchCanvas'
+  | 'Jungle Key'
+  | 'Interia'
+  | 'Genieo'
+  | 'Tiscali'
+  | 'Gomeo'
+  | 'Facebook'
+  | 'Twitter'
+  | 'LinkedIn'
+  | 'Instagram'
+  | 'Pinterest'
+  | 'YouTube'
+  | 'Reddit'
+  | 'Tumblr'
+  | 'WhatsApp'
+  | 'Telegram'
+  | 'Snapchat'
+  | 'TikTok'
+  | (string & {});
+
+export type Referer = {
   medium: Medium;
-  referer: string | null;
+  referer: RefererSource | null;
   term: string | null;
-}
+};
 
 export interface RefererData {
   [medium: string]: {
     [source: string]: {
       domains: string[];
-      parameters: string[];
+      parameters?: string[];
     };
   };
 }