From 94fdb5efa029f1f77c23044e09bf9b0db71dce21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Wo=C5=BAniak?= Date: Sat, 10 Dec 2022 13:18:19 +0100 Subject: [PATCH] First commit --- Makefile | 52 ++++++++++ README.md | 106 ++++++++++++++++++++ go.mod | 13 +++ go.sum | 14 +++ main.go | 287 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 472 insertions(+) create mode 100644 Makefile create mode 100644 README.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d3d459a --- /dev/null +++ b/Makefile @@ -0,0 +1,52 @@ +# Source from https://github.com/krisnova/go-nova/blob/main/Makefile + +all: compile + +version ?= @(git describe --tags --abbrev=0) +org ?= y0rune +target ?= fortios-release +authorname ?= Marcin Woźniak +authoremail ?= y0rune@aol.com +license ?= MIT +year ?= 2022 +copyright ?= Copyright (c) $(year) +gofile ?= main.go + +compile: + @echo "Compiling..." + go build -ldflags "\ + -X 'github.com/$(org)/$(target).Version=$(version)' \ + -X 'github.com/$(org)/$(target).AuthorName=$(authorname)' \ + -X 'github.com/$(org)/$(target).AuthorEmail=$(authoremail)' \ + -X 'github.com/$(org)/$(target).Copyright=$(copyright)' \ + -X 'github.com/$(org)/$(target).License=$(license)' \ + -X 'github.com/$(org)/$(target).Name=$(target)'" \ + -o $(target) $(gofile) + +# install: +# @echo "Installing..." +# sudo cp $(target) /usr/bin/$(target) + +build: clean compile + +clean: + @echo "Cleaning..." + rm -rvf release/* + rm -rvf $(target) + +# test: clean compile install +# test: clean compile +# @echo "Testing..." +# go test -v . + +.PHONY: release +release: + mkdir -p release + GOOS="linux" GOARCH="amd64" go build -ldflags "-X 'github.com/$(org)/$(target).Version=$(version)'" -o release/$(target)-linux-amd64 $(gofile) + GOOS="linux" GOARCH="arm" go build -ldflags "-X 'github.com/$(org)/$(target).Version=$(version)'" -o release/$(target)-linux-arm $(gofile) + GOOS="linux" GOARCH="arm64" go build -ldflags "-X 'github.com/$(org)/$(target).Version=$(version)'" -o release/$(target)-linux-arm64 $(gofile) + GOOS="darwin" GOARCH="amd64" go build -ldflags "-X 'github.com/$(org)/$(target).Version=$(version)'" -o release/$(target)-darwin-amd64 $(gofile) + +.PHONY: help +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[32m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/README.md b/README.md new file mode 100644 index 0000000..d3de199 --- /dev/null +++ b/README.md @@ -0,0 +1,106 @@ +# fortios-release + +The `fortios-release` is script for creating a better release notes. +You can give the version(s) and it will remove duplicates and change the status +of the bug. + +## Instalation from release + +- `MacOS` + +```bash +wget https://github.com/y0rune/fortios-release/blob/main/release/fortios-release-darwin-amd64 -O fortios-release +chmod +x fortios-release +./fortios-release +``` + +- `amd64` + +```bash +wget https://github.com/y0rune/fortios-release/blob/main/release/fortios-release-darwin-amd64 -O fortios-release +chmod +x fortios-release +./fortios-release +``` + +- `arm64` + +```bash +wget https://github.com/y0rune/fortios-release/blob/main/release/fortios-release-darwin-arm -O fortios-release +chmod +x fortios-release +./fortios-release +``` + +- `arm` + +```bash +wget https://github.com/y0rune/fortios-release/blob/main/release/fortios-release-darwin-amd64 -O fortios-release +chmod +x fortios-release +./fortios-release +``` + +## Instalation from source + +```bash +git clone https://github.com/y0rune/fortios-release.git +go get +make build +``` + +## Arguments + +``` + -recordsFile string + Name of the unsorted records from versions (default "records.csv") + -sorted + Get a sorted release notes + -sortedFile string + Name of the sorted output file (default "final.csv") + -version value + Version(s) of the FortiOS +``` + +## Example of usage + +Getting the known and resolved issues in version `7.0.0` and `7.0.1`. + +```bash +$ ./fortios-release -version 7.0.0 -version 7.0.1 -recordsFile issues-from-7.0.0-to-7.0.1.csv -sortedFile final.csv -sorted + +2022/12/10 13:09:19 Starting gathering links for version 7.0.0 +2022/12/10 13:09:21 The knownIssuesUrl is https://docs.fortinet.com/document/fortigate/7.0.0/fortios-release-notes/236526/known-issues +2022/12/10 13:09:21 The resolvedIssuesUrl is https://docs.fortinet.com/document/fortigate/7.0.0/fortios-release-notes/289806/resolved-issues +2022/12/10 13:09:21 Getting the resolved issue for version 7.0.0 +2022/12/10 13:09:21 Starting parsing https://docs.fortinet.com/document/fortigate/7.0.0/fortios-release-notes/289806/resolved-issues data to table +2022/12/10 13:09:22 Starting writing the to file issues-from-7.0.0-to-7.0.1.csv +2022/12/10 13:09:22 Getting the known issue for version 7.0.0 +2022/12/10 13:09:22 Starting parsing https://docs.fortinet.com/document/fortigate/7.0.0/fortios-release-notes/236526/known-issues data to table +2022/12/10 13:09:22 Starting writing the to file issues-from-7.0.0-to-7.0.1.csv +2022/12/10 13:09:22 Starting gathering links for version 7.0.1 +2022/12/10 13:09:23 The knownIssuesUrl is https://docs.fortinet.com/document/fortigate/7.0.1/fortios-release-notes/236526/known-issues +2022/12/10 13:09:23 The resolvedIssuesUrl is https://docs.fortinet.com/document/fortigate/7.0.1/fortios-release-notes/289806/resolved-issues +2022/12/10 13:09:23 Getting the resolved issue for version 7.0.1 +2022/12/10 13:09:23 Starting parsing https://docs.fortinet.com/document/fortigate/7.0.1/fortios-release-notes/289806/resolved-issues data to table +2022/12/10 13:09:24 Starting writing the to file issues-from-7.0.0-to-7.0.1.csv +2022/12/10 13:09:24 Getting the known issue for version 7.0.1 +2022/12/10 13:09:24 Starting parsing https://docs.fortinet.com/document/fortigate/7.0.1/fortios-release-notes/236526/known-issues data to table +2022/12/10 13:09:25 Starting writing the to file issues-from-7.0.0-to-7.0.1.csv +2022/12/10 13:09:25 Starting writing the to file final.csv +``` + +All known and resolved issues in version `7.0.0` and `7.0.1`. + +``` +$ cat issues-from-7.0.0-to-7.0.1.csv | head -n3 +BugID,Description,Status,Version +650160,"When using email filter profile, emails are being queued due to IMAP proxy being in stuck state.",resolved,7.0.0 +524571,Quarantined files cannot be fetched in the AV log page if the file was already quarantined under another protocol.,resolved,7.0.0 +``` + +Final output + +``` +$ cat final.csv | head -n3 +BugID,Description,Status,Version +650160,"When using email filter profile, emails are being queued due to IMAP proxy being in stuck state.",resolved,7.0.0 +524571,Quarantined files cannot be fetched in the AV log page if the file was already quarantined under another protocol.,resolved,7.0.0 +``` diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..94d33c2 --- /dev/null +++ b/go.mod @@ -0,0 +1,13 @@ +module fortiosrelease + +go 1.19 + +require ( + github.com/PuerkitoBio/goquery v1.8.0 + github.com/hashicorp/go-version v1.6.0 +) + +require ( + github.com/andybalholm/cascadia v1.3.1 // indirect + golang.org/x/net v0.0.0-20220909164309-bea034e7d591 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..49fa05f --- /dev/null +++ b/go.sum @@ -0,0 +1,14 @@ +github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/main.go b/main.go new file mode 100644 index 0000000..fcfdbc4 --- /dev/null +++ b/main.go @@ -0,0 +1,287 @@ +package main + +import ( + "encoding/csv" + "flag" + "io" + "log" + "net/http" + "os" + "regexp" + "sort" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" + "github.com/hashicorp/go-version" +) + +const staticVersion string = "1.0.0" +const fortiUrl string = "https://docs.fortinet.com" +const fortidocument string = fortiUrl + "/document/fortigate/" +const fileCSV string = "records.csv" +const fileUnDuplicated string = "final.csv" +const headerRow string = "BugID,Description,Status,Version\n" + +type fortiTable struct { + BugID string `header:"Bug ID"` + Description string `header:"Description"` + Status string + Version string +} +type arrayVersions []string + +var versions arrayVersions + +func (i *arrayVersions) String() string { + return "my string representation" +} + +func (i *arrayVersions) Set(value string) error { + *i = append(*i, value) + return nil +} + +func getUrlIssues(version string) (string, string) { + log.Printf("Starting gathering links for version %s\n", version) + + // Create HTTP client with timeout + client := &http.Client{ + Timeout: 30 * time.Second, + } + + // https://docs.fortinet.com/document/fortigate/6.4.0/fortios-release-notes + url := fortidocument + version + "/fortios-release-notes" + + // Make request + response, err := client.Get(url) + if err != nil { + log.Fatal(err) + } + defer response.Body.Close() + + bodyString := "" + + if response.StatusCode == http.StatusOK { + bodyBytes, err := io.ReadAll(response.Body) + if err != nil { + log.Fatal(err) + } + bodyString = string(bodyBytes) + } + + r, _ := regexp.Compile("href=.*known-issues") + knownIssues := r.FindString(bodyString) + knownIssues = strings.ReplaceAll(knownIssues, "href=\"", "") + + r, _ = regexp.Compile("href=.*resolved-issues") + resolvedIssues := r.FindString(bodyString) + resolvedIssues = strings.ReplaceAll(resolvedIssues, "href=\"", "") + + knownIssuesUrl := fortiUrl + knownIssues + resolvedIssuesUrl := fortiUrl + resolvedIssues + + log.Printf("The knownIssuesUrl is %s\n", knownIssuesUrl) + log.Printf("The resolvedIssuesUrl is %s\n", resolvedIssuesUrl) + + return knownIssuesUrl, resolvedIssuesUrl +} + +func returnTable(url string, version string, status string) []fortiTable { + log.Printf("Starting parsing %s data to table", url) + var table []fortiTable + + doc, err := goquery.NewDocument(url) + if err != nil { + log.Fatal(err) + } + + doc.Find("tbody tr").Each(func(_ int, tr *goquery.Selection) { + + e := fortiTable{} + + tr.Find("td").Each(func(ix int, td *goquery.Selection) { + switch ix { + case 0: + e.BugID = strings.TrimSpace(td.Text()) + case 1: + // Source + // https://stackoverflow.com/questions/37290693/how-to-remove-redundant-spaces-whitespace-from-a-string-in-golang + desc := strings.TrimSpace(td.Text()) + desc = strings.Join(strings.Fields(desc), " ") + e.Description = strings.TrimSpace(desc) + } + }) + + e.Version = version + e.Status = status + + table = append(table, e) + }) + return table +} + +func getResolvedIssues(resolvedIssuesUrl string, version string) []fortiTable { + log.Printf("Getting the resolved issue for version %s", version) + resolvedIssues := returnTable(resolvedIssuesUrl, version, "resolved") + return resolvedIssues +} + +func getKnownIssues(knownIssuesUrl string, version string) []fortiTable { + log.Printf("Getting the known issue for version %s", version) + knownIssues := returnTable(knownIssuesUrl, version, "unresolved") + return knownIssues +} + +func writeToCSV(table []fortiTable, fileName string) { + log.Printf("Starting writing the to file %s", fileName) + + var isCreated bool + + if _, err := os.Stat(fileName); err == nil { + isCreated = false + } else { + isCreated = true + } + + // Source + // https://articles.wesionary.team/read-and-write-csv-file-in-go-b445e34968e9 + + file, err := os.OpenFile(fileName, + os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Println(err) + } + + if isCreated { + file.WriteString(headerRow) + } + + defer file.Close() + + if err != nil { + log.Fatalln("Failed to open file", err) + } + w := csv.NewWriter(file) + defer w.Flush() + + for _, record := range table { + row := []string{record.BugID, record.Description, record.Status, record.Version} + if err := w.Write(row); err != nil { + log.Fatalln("Error writing record to file", err) + } + } +} + +func createFortiList(data [][]string) []fortiTable { + var table []fortiTable + for i, line := range data { + if i > 0 { // omit header line + var rec fortiTable + for j, field := range line { + switch j { + case 0: + rec.BugID = field + case 1: + rec.Description = field + case 2: + rec.Status = field + case 3: + rec.Version = field + } + } + table = append(table, rec) + } + } + return table +} + +func removeDuplicates(input string, output string) { + f, err := os.Open(input) + if err != nil { + log.Fatal(err) + } + + defer f.Close() + + csvReader := csv.NewReader(f) + data, err := csvReader.ReadAll() + if err != nil { + log.Fatal(err) + } + + listOrg := createFortiList(data) + + var listFinal []fortiTable + var listPreFinal []fortiTable + var listWithoutDuplicates []fortiTable + + visited := make(map[fortiTable]bool, 0) + visitedFinal := make(map[fortiTable]bool, 0) + + // Removing a duplicates + for _, item := range listOrg { + if _, value := visited[item]; !value { + visited[item] = true + listWithoutDuplicates = append(listWithoutDuplicates, item) + } + } + + // Logic for issues + versionList := []string{} + for _, itemOrg := range listOrg { + finalItem := fortiTable{} + for _, item := range listWithoutDuplicates { + if item.BugID == itemOrg.BugID { + versionList = append(versionList, item.Version) + } + } + sort.Slice(versionList, func(i, j int) bool { + v1, _ := version.NewVersion(versionList[i]) + v2, _ := version.NewVersion(versionList[j]) + return v1.GreaterThanOrEqual(v2) + }) + + for _, item := range listWithoutDuplicates { + if item.BugID == itemOrg.BugID && versionList[0] == item.Version { + finalItem.BugID = item.BugID + finalItem.Description = item.Description + finalItem.Version = item.Version + finalItem.Status = item.Status + } + } + listPreFinal = append(listPreFinal, finalItem) + versionList = nil + + } + + // Removing a duplicates + for _, item := range listPreFinal { + if _, value := visitedFinal[item]; !value { + visitedFinal[item] = true + listFinal = append(listFinal, item) + } + } + + writeToCSV(listFinal, output) +} + +func main() { + flag.Var(&versions, "version", "Version(s) of the FortiOS") + recordsFile := flag.String("recordsFile", fileCSV, "Name of the unsorted records from versions") + sorted := flag.Bool("sorted", false, "Get a sorted release notes") + sortedFile := flag.String("sortedFile", fileUnDuplicated, "Name of the sorted output file") + flag.Parse() + + for _, version := range versions { + knownIssuesUrl, resolvedIssuesUrl := getUrlIssues(version) + resolvedIssues := getResolvedIssues(resolvedIssuesUrl, version) + writeToCSV(resolvedIssues, *recordsFile) + knownIssuses := getKnownIssues(knownIssuesUrl, version) + writeToCSV(knownIssuses, *recordsFile) + } + + if *sorted { + removeDuplicates(*recordsFile, *sortedFile) + } +}