Update the structure to include more fingerprints patterns, which fixes https://github.com/projectdiscovery/wappalyzergo/issues/39

.github/workflows/build-test.yml

@@ -13,7 +13,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.18
+          go-version: 1.21.x
 
       - name: Check out code
         uses: actions/checkout@v3

.github/workflows/fingerprint-update.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Setup golang
         uses: actions/setup-go@v4
         with:
-          go-version: 1.18
+          go-version: 1.21.x
 
       - name: Installing Update binary
         run: |

.github/workflows/lint-test.yml

@@ -14,7 +14,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.18
+          go-version: 1.21.x
       - name: Run golangci-lint
         uses: golangci/[email protected]
         with:

README.md

@@ -26,7 +26,7 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
+	"io"
 	"log"
 	"net/http"
 
@@ -38,7 +38,7 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
-	data, _ := ioutil.ReadAll(resp.Body) // Ignoring error for example
+	data, _ := io.ReadAll(resp.Body) // Ignoring error for example
 
 	wappalyzerClient, err := wappalyzer.New()
 	fingerprints := wappalyzerClient.Fingerprint(resp.Header, data)

cmd/update-fingerprints/main.go

@@ -5,7 +5,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"log"
 	"net/http"
 	"os"
@@ -24,12 +24,14 @@ type Fingerprints struct {
 
 // Fingerprint is a single piece of information about a tech
 type Fingerprint struct {
+	Cats        []int                  `json:"cats"`
 	CSS         interface{}            `json:"css"`
 	Cookies     map[string]string      `json:"cookies"`
 	JS          map[string]string      `json:"js"`
 	Headers     map[string]string      `json:"headers"`
 	HTML        interface{}            `json:"html"`
-	Script      interface{}            `json:"script"`
+	Script      interface{}            `json:"scripts"`
+	ScriptSrc   interface{}            `json:"scriptSrc"`
 	Meta        map[string]interface{} `json:"meta"`
 	Implies     interface{}            `json:"implies"`
 	Description string                 `json:"description"`
@@ -45,12 +47,14 @@ type OutputFingerprints struct {
 
 // OutputFingerprint is a single piece of information about a tech validated and normalized
 type OutputFingerprint struct {
+	Cats        []int               `json:"cats,omitempty"`
+	CSS         []string            `json:"css,omitempty"`
 	Cookies     map[string]string   `json:"cookies,omitempty"`
 	JS          []string            `json:"js,omitempty"`
 	Headers     map[string]string   `json:"headers,omitempty"`
 	HTML        []string            `json:"html,omitempty"`
-	Script      []string            `json:"script,omitempty"`
-	CSS         []string            `json:"css,omitempty"`
+	Script      []string            `json:"scripts,omitempty"`
+	ScriptSrc   []string            `json:"scriptSrc,omitempty"`
 	Meta        map[string][]string `json:"meta,omitempty"`
 	Implies     []string            `json:"implies,omitempty"`
 	Description string              `json:"description,omitempty"`
@@ -123,7 +127,7 @@ func gatherFingerprintsFromURL(URL string, fingerprints *Fingerprints) error {
 	}
 	defer resp.Body.Close()
 
-	data, err := ioutil.ReadAll(resp.Body)
+	data, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
@@ -145,6 +149,7 @@ func normalizeFingerprints(fingerprints *Fingerprints) *OutputFingerprints {
 
 	for app, fingerprint := range fingerprints.Apps {
 		output := OutputFingerprint{
+			Cats:        fingerprint.Cats,
 			Cookies:     make(map[string]string),
 			Headers:     make(map[string]string),
 			Meta:        make(map[string][]string),
@@ -184,7 +189,7 @@ func normalizeFingerprints(fingerprints *Fingerprints) *OutputFingerprints {
 			sort.Strings(output.HTML)
 		}
 
-		// Use reflection type switch for determining Script tag type
+		// Use reflection type switch for determining Script type
 		if fingerprint.Script != nil {
 			v := reflect.ValueOf(fingerprint.Script)
 
@@ -203,6 +208,25 @@ func normalizeFingerprints(fingerprints *Fingerprints) *OutputFingerprints {
 			sort.Strings(output.Script)
 		}
 
+		// Use reflection type switch for determining ScriptSrc type
+		if fingerprint.ScriptSrc != nil {
+			v := reflect.ValueOf(fingerprint.ScriptSrc)
+
+			switch v.Kind() {
+			case reflect.String:
+				data := v.Interface().(string)
+				output.ScriptSrc = append(output.ScriptSrc, strings.ToLower(data))
+			case reflect.Slice:
+				data := v.Interface().([]interface{})
+				for _, pattern := range data {
+					pat := pattern.(string)
+					output.ScriptSrc = append(output.ScriptSrc, strings.ToLower(pat))
+				}
+			}
+
+			sort.Strings(output.ScriptSrc)
+		}
+
 		for header, pattern := range fingerprint.Meta {
 			v := reflect.ValueOf(pattern)
 

examples/main.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
+	"io"
 	"log"
 	"net/http"
 
@@ -14,11 +14,16 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
-	data, _ := ioutil.ReadAll(resp.Body) // Ignoring error for example
+	data, _ := io.ReadAll(resp.Body) // Ignoring error for example
 
 	wappalyzerClient, err := wappalyzer.New()
+	if err != nil {
+		log.Fatal(err)
+	}
 	fingerprints := wappalyzerClient.Fingerprint(resp.Header, data)
 	fmt.Printf("%v\n", fingerprints)
-
 	// Output: map[Acquia Cloud Platform:{} Amazon EC2:{} Apache:{} Cloudflare:{} Drupal:{} PHP:{} Percona:{} React:{} Varnish:{}]
+
+	fingerprintsWithCats := wappalyzerClient.FingerprintWithCats(resp.Header, data)
+	fmt.Printf("%v\n", fingerprintsWithCats)
 }

fingerprints.go

@@ -15,12 +15,14 @@ type Fingerprints struct {
 
 // Fingerprint is a single piece of information about a tech validated and normalized
 type Fingerprint struct {
+	Cats        []int               `json:"cats"`
+	CSS         []string            `json:"css"`
 	Cookies     map[string]string   `json:"cookies"`
 	JS          []string            `json:"js"`
 	Headers     map[string]string   `json:"headers"`
 	HTML        []string            `json:"html"`
-	CSS         []string            `json:"css"`
 	Script      []string            `json:"scripts"`
+	ScriptSrc   []string            `json:"scriptSrcs"`
 	Meta        map[string][]string `json:"meta"`
 	Implies     []string            `json:"implies"`
 	Description string              `json:"description"`
@@ -35,6 +37,8 @@ type CompiledFingerprints struct {
 
 // CompiledFingerprint contains the compiled fingerprints from the tech json
 type CompiledFingerprint struct {
+	// cats contain categories that are implicit with this tech
+	cats []int
 	// implies contains technologies that are implicit with this tech
 	implies []string
 	// description contains fingerprint description
@@ -49,8 +53,10 @@ type CompiledFingerprint struct {
 	headers map[string]*versionRegex
 	// html contains fingerprints for the target HTML
 	html []*versionRegex
-	// script contains fingerprints for script tags
+	// script contains fingerprints for scripts
 	script []*versionRegex
+	// scriptSrc contains fingerprints for script srcs
+	scriptSrc []*versionRegex
 	// meta contains fingerprints for meta tags
 	meta map[string][]*versionRegex
 }
@@ -61,6 +67,11 @@ type AppInfo struct {
 	Website     string
 }
 
+// CatsInfo contains basic information about an App.
+type CatsInfo struct {
+	Cats []int
+}
+
 type versionRegex struct {
 	regex     *regexp.Regexp
 	skipRegex bool
@@ -130,6 +141,7 @@ const (
 // loadPatterns loads the fingerprint patterns and compiles regexes
 func compileFingerprint(fingerprint *Fingerprint) *CompiledFingerprint {
 	compiled := &CompiledFingerprint{
+		cats:        fingerprint.Cats,
 		implies:     fingerprint.Implies,
 		description: fingerprint.Description,
 		website:     fingerprint.Website,
@@ -138,6 +150,7 @@ func compileFingerprint(fingerprint *Fingerprint) *CompiledFingerprint {
 		headers:     make(map[string]*versionRegex),
 		html:        make([]*versionRegex, 0, len(fingerprint.HTML)),
 		script:      make([]*versionRegex, 0, len(fingerprint.Script)),
+		scriptSrc:   make([]*versionRegex, 0, len(fingerprint.ScriptSrc)),
 		meta:        make(map[string][]*versionRegex),
 	}
 
@@ -181,6 +194,14 @@ func compileFingerprint(fingerprint *Fingerprint) *CompiledFingerprint {
 		compiled.script = append(compiled.script, fingerprint)
 	}
 
+	for _, pattern := range fingerprint.ScriptSrc {
+		fingerprint, err := newVersionRegex(pattern)
+		if err != nil {
+			continue
+		}
+		compiled.scriptSrc = append(compiled.scriptSrc, fingerprint)
+	}
+
 	for meta, patterns := range fingerprint.Meta {
 		var compiledList []*versionRegex
 
@@ -213,7 +234,7 @@ func (f *CompiledFingerprints) matchString(data string, part part) []string {
 				}
 			}
 		case scriptPart:
-			for _, pattern := range fingerprint.script {
+			for _, pattern := range fingerprint.scriptSrc {
 				if valid, versionString := pattern.MatchString(data); valid {
 					matched = true
 					version = versionString