Add: smoketest for openvas-nasl-lint (#1125)

.github/workflows/smoketest.yml

@@ -0,0 +1,29 @@
+name: "SmokeTest"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  nasl-lint:
+    name: Smoketest openvas-nasl-lint
+    runs-on: ubuntu-latest
+    container: ${{ github.repository }}-build:unstable
+    steps:
+      - name: Check out openvas-scanner
+        uses: actions/checkout@v3
+      - name: Setup go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '>=1.16.0'
+      - name: build
+        run: |
+          cmake -Bbuild -DCMAKE_BUILD_TYPE=Release
+          cmake --build build
+      - name: Run Smoketest linter
+        run: |
+          make build
+          ./run -e ../../build/nasl/openvas-nasl-lint
+        working-directory: smoketest_lint

smoketest_lint/Makefile

@@ -0,0 +1,13 @@
+.PHONY: build run clean all
+
+build:
+	go build -o run cmd/main.go
+
+run: build
+	./run
+
+clean:
+	rm run
+
+all: build run clean
+

smoketest_lint/README.md

@@ -0,0 +1,19 @@
+# smoke-test linter
+
+Contains a bunch of predefined nasl finds which are used by a go program to test the expected functionality of openvas-nasl-lint.
+
+To build and run the tests a Makefile is provided:
+- make build - builds the file `run` in the root directory
+- make run - runs the program `run` builded with `make build`
+- make clean - removes the builded program `run`
+- make all - automatically builds, runs and cleans
+
+To verify in your local environment you need to have `go` installed:
+
+```
+make all
+```
+
+The current version supports two arguments:
+- openvasDir - Location of the openvas-nasl-lint executable, has to be absolute or relative to test files directory. If `openvas-nasl-lint` is located within `$PATH` It can be left empty
+- testFiles - Folder containing the nasl test files.

smoketest_lint/cmd/main.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/greenbone/openvas-scanner/smoketest_lint/test"
+)
+
+func main() {
+
+	openvasExe := flag.String("e", "openvas-nasl-lint", "openvas-nasl-lint executable, must be either in $PATH, an absolute path or relative to the data folder")
+	path := flag.String("d", "data", "data folder for the nasl test data")
+	flag.Parse()
+
+	tfs := test.TestFiles()
+	fmt.Println("Parsing Nasl Test Files")
+	err := filepath.Walk(*path, tfs.Parse)
+	if err != nil {
+		fmt.Printf("Unable to parse files in %s: %s\n", *path, err)
+	}
+
+	fmt.Println("Testing: Compare actual with expected output")
+	for _, tf := range tfs.Tfs {
+		errs := tf.Test(*openvasExe)
+		if len(errs) > 0 {
+			fmt.Printf("%d error(s) while processing %s:\n", len(errs), tf.Name)
+			for _, err := range errs {
+				fmt.Println(err)
+			}
+			os.Exit(1)
+		}
+	}
+	fmt.Println("No errors were found")
+}

smoketest_lint/data/undeclaredv11.nasl

@@ -0,0 +1,92 @@
+barlist2 = get_kb_list( "bar/foo" );
+foreach bar3( barlist2 ) {#!none
+  bar4 = bar3;
+  if( bar4 ) {}
+}
+
+
+foolist = get_kb_list( "foo/bar");
+foo = foolist[1];
+
+
+barlist = get_kb_list( "bar/foo" );
+foreach bar( barlist ) {
+  bar2 = bar;
+  if( bar2 ) {}
+}
+
+
+foonull = NULL;
+fooempty = "";
+if( foonull ){}
+if( fooempty ){}
+
+
+fooempty3 += "";
+foonull3 += NULL;
+if( fooempty3 ){}
+if( foonull3 ){}
+
+
+fooincrement++;
+if( fooincrement++ ){}
+
+
+foostring += string("foo");
+if( foostring ) {}
+
+
+if(!get_port_state(foostateport)){}
+#!undeclared:foostateport
+
+
+function foreachfunc(foreachport) {
+  local_var foreachport, foreachports;
+  foreachports = make_list(123,456);
+  foreach foreachport( foreachports ) {}
+}
+foreachprot = 123;
+foreachfunc(foreachport:foreachport);
+#!undeclared:foreachport
+
+
+fooarraythere['1'] = 2;
+if(fooarraynotthere[0]) {}
+#!undeclared:fooarraynotthere
+
+
+foo = toupper(fooarraynotthere2["foo"]);
+#!undeclared:fooarraynotthere2
+
+
+function fooFunc(foo) {
+  local_var foo;
+}
+if(!fooFunc(foo:foofile)){}
+#!undeclared:foofile
+
+
+function fooPort( _fooport ) {
+  local_var fooport;
+  fooport = _fooport;
+}
+fooPort();
+log_message(port:fooport);
+#!undeclared:fooport
+
+
+display(displaybar);
+#!undeclared:displaybar
+displaybar = 0;
+
+
+foreach bar3( barlist3 ) {}
+#!undeclared:barlist3
+
+
+foonull = NULL;
+fooempty = "";
+if( foonull2 ){}
+#!undeclared:foonull2
+if( fooempty2 ){}
+#!undeclared:fooempty2

smoketest_lint/go.mod

@@ -0,0 +1,3 @@
+module github.com/greenbone/openvas-scanner/smoketest_lint
+
+go 1.16

smoketest_lint/test/testcase.go

@@ -0,0 +1,35 @@
+package test
+
+import "fmt"
+
+type testCase struct {
+	msg    string
+	line   int
+	tested bool
+}
+
+type testError struct {
+	line     int
+	expected string
+	occurred string
+}
+
+func (t testError) Error() string {
+	return fmt.Sprintf("line %d:\nexpected: %s\noccurred: %s", t.line, t.expected, t.occurred)
+}
+
+func (t1 *testCase) test(t2 *testCase) error {
+	if t1.line != t2.line {
+		return nil
+	}
+	t1.tested = true
+	t2.tested = true
+	if t1.msg != t2.msg {
+		return testError{
+			t1.line,
+			t1.msg,
+			t2.msg,
+		}
+	}
+	return nil
+}

smoketest_lint/test/testfile.go

@@ -0,0 +1,147 @@
+package test
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+type testFile struct {
+	Name      string
+	dir       string
+	testCases []*testCase
+}
+
+// testFileFromPath Parses a Nasl file and creates testCases for a testFile
+// based on
+// specific comments within the file.
+func testFileFromPath(path string, info os.FileInfo) (testFile, error) {
+	// Error Handling
+	if info.IsDir() {
+		return testFile{}, fmt.Errorf("unable to parse %s: File is Directory", info.Name())
+	}
+	file, err := os.Open(path)
+	if err != nil {
+		return testFile{}, fmt.Errorf("unable to parse %s: %s", info.Name(), err)
+	}
+
+	// Prepare
+	tc := testFile{
+		info.Name(),
+		filepath.Dir(path),
+		make([]*testCase, 0),
+	}
+	i := 0
+	// Scan File Line by Line
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		i = i + 1
+		line := scanner.Text()
+
+		// Make testcase out of line
+		for _, tct := range testCaseTypes {
+			msg := tct(line)
+			if msg == "" {
+				continue
+			}
+			tc.testCases = append(tc.testCases, &testCase{
+				msg,
+				i - 1,
+				false,
+			})
+			break
+		}
+	}
+	return tc, nil
+}
+
+// test tests the openvas-nasl-lint with file t. Note that file t must be
+// parsed first. test return a list of unexpected events that occurred
+// as a list of errors
+func (t testFile) Test(openvasExe string) []error {
+	errs := make([]error, 0)
+	// run openvas-nasl-lint and collect output
+	cmd := exec.Command(openvasExe, t.Name)
+	cmd.Dir = t.dir
+	// Ignoring ExitErrors as openvas-nasl-lint does not exit with 0 when errors
+	// were found
+	out, err := cmd.CombinedOutput()
+	var target *exec.ExitError
+	if err != nil && !errors.As(err, &target) {
+		panic(fmt.Sprintf("Unable to run openvas-nasl-lint: %s", err))
+	}
+
+	// test output line by line
+	lines := strings.Split(string(out), "\n")
+	for _, line := range lines {
+		// parse line into testCase
+		if !strings.HasPrefix(line, "lib") {
+			continue
+		}
+		matches := regexp.MustCompile(fmt.Sprintf("lib  nasl-Message: \\d\\d:\\d\\d:\\d\\d\\.\\d\\d\\d: \\[\\d*\\]\\(%s:(\\d*)\\) (.*)", t.Name)).FindAllStringSubmatch(line, -1)
+		if matches == nil || len(matches[0]) != 3 {
+			continue
+		}
+		lineNumber, err := strconv.Atoi(matches[0][1])
+		if err != nil {
+			continue
+		}
+
+		// testing
+		tc := testCase{
+			matches[0][2],
+			lineNumber,
+			false,
+		}
+		for _, test := range t.testCases {
+			err := test.test(&tc)
+			if err != nil {
+				errs = append(errs, err)
+			}
+		}
+		if !tc.tested {
+			errs = append(errs, fmt.Errorf("on line %d the error '%s' occurred, but was not expected", tc.line, tc.msg))
+		}
+	}
+	for _, tc := range t.testCases {
+		if !tc.tested {
+			errs = append(errs, fmt.Errorf("on line %d the error '%s' was expected, but did not occurred", tc.line, tc.msg))
+		}
+	}
+	return errs
+}
+
+type testFiles struct {
+	Tfs []testFile
+}
+
+func TestFiles() testFiles {
+	return testFiles{
+		make([]testFile, 0),
+	}
+}
+
+// parse is a function designed to fit into a filepath.Walkfunc so a bunch of
+// testFiles is created automatically given a Folder with nasl files.
+func (t *testFiles) Parse(path string, info os.FileInfo, err error) error {
+	if err != nil {
+		fmt.Printf("Unable to parse %s: %s\n", path, err)
+		return nil
+	}
+	if info.IsDir() {
+		return nil
+	}
+
+	tf, err := testFileFromPath(path, info)
+	if err != nil {
+		return err
+	}
+	t.Tfs = append(t.Tfs, tf)
+	return nil
+}