Add smoke tests

.github/workflows/ci.yml

@@ -0,0 +1,35 @@
+name: CI
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    name: Tests
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1
+        with:
+          install: true
+
+      - name: Build Docker image and store in cache
+        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825
+        with:
+          context: .
+          push: false
+          load: true
+          tags: exercism/common-lisp-test-runner
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Run Tests in Docker
+        run: bin/run-tests-in-docker.sh

.gitignore

@@ -1,3 +1,4 @@
 *.json
 *.fasl
 *~
+!tests/*/expected_results.json

bin/run-in-docker.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env sh
+
+# Synopsis:
+# Run the test runner on a solution using the test runner Docker image.
+# The test runner Docker image is built automatically.
+
+# Arguments:
+# $1: exercise slug
+# $2: path to solution folder
+# $3: path to output directory
+
+# Output:
+# Writes the test results to a results.json file in the passed-in output directory.
+# The test results are formatted according to the specifications at https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md
+
+# Example:
+# ./bin/run-in-docker.sh two-fer path/to/solution/folder/ path/to/output/directory/
+
+# Stop executing when a command returns a non-zero return code
+set -e
+
+# If any required arguments is missing, print the usage and exit
+if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
+    echo "usage: ./bin/run-in-docker.sh exercise-slug path/to/solution/folder/ path/to/output/directory/"
+    exit 1
+fi
+
+slug="$1"
+solution_dir=$(realpath "${2%/}")
+output_dir=$(realpath "${3%/}")
+
+# Create the output directory if it doesn't exist
+mkdir -p "${output_dir}"
+
+# Build the Docker image
+docker build --rm -t exercism/common-lisp-test-runner .
+
+# Run the Docker image using the settings mimicking the production environment
+docker run \
+    --rm \
+    --network none \
+    --read-only \
+    --mount type=bind,src="${solution_dir}",dst=/solution \
+    --mount type=bind,src="${output_dir}",dst=/output \
+    --mount type=tmpfs,dst=/tmp \
+    exercism/common-lisp-test-runner "${slug}" /solution /output 

bin/run-tests-in-docker.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env sh
+
+# Synopsis:
+# Test the test runner Docker image by running it against a predefined set of 
+# solutions with an expected output.
+# The test runner Docker image is built automatically.
+
+# Output:
+# Outputs the diff of the expected test results against the actual test results
+# generated by the test runner Docker image.
+
+# Example:
+# ./bin/run-tests-in-docker.sh
+
+# Stop executing when a command returns a non-zero return code
+set -e
+
+# Build the Docker image
+docker build --rm -t exercism/common-lisp-test-runner .
+
+# Run the Docker image using the settings mimicking the production environment
+docker run \
+    --rm \
+    --network none \
+    --read-only \
+    --mount type=bind,src="${PWD}/tests",dst=/opt/test-runner/tests \
+    --mount type=tmpfs,dst=/tmp \
+    --volume "${PWD}/bin/run-tests.sh:/opt/test-runner/bin/run-tests.sh" \
+    --workdir /opt/test-runner \
+    --entrypoint /opt/test-runner/bin/run-tests.sh \
+    exercism/common-lisp-test-runner

bin/run-tests.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env sh
+
+# Synopsis:
+# Test the test runner by running it against a predefined set of solutions 
+# with an expected output.
+
+# Output:
+# Outputs the diff of the expected test results against the actual test results
+# generated by the test runner.
+
+# Example:
+# ./bin/run-tests.sh
+
+exit_code=0
+
+# Iterate over all test directories
+for test_dir in tests/*; do
+    test_dir_name=$(basename "${test_dir}")
+    test_dir_path=$(realpath "${test_dir}")
+    results_file_path="${test_dir_path}/results.json"
+    expected_results_file_path="${test_dir_path}/expected_results.json"
+
+    bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}"
+
+    # OPTIONAL: Normalize the results file
+    # If the results.json file contains information that changes between 
+    # different test runs (e.g. timing information or paths), you should normalize
+    # the results file to allow the diff comparison below to work as expected
+    # sed -i -E \
+    #   -e 's/Elapsed time: [0-9]+\.[0-9]+ seconds//g' \
+    #   -e "s~${test_dir_path}~/solution~g" \
+    #   "${results_file_path}"
+
+    echo "${test_dir_name}: comparing results.json to expected_results.json"
+    diff "${results_file_path}" "${expected_results_file_path}"
+
+    if [ $? -ne 0 ]; then
+        exit_code=1
+    fi
+done
+
+exit ${exit_code}

tests/all-fail/all-fail-test.lisp

@@ -0,0 +1,30 @@
+;; Ensures that all-fail.lisp and the testing library are always loaded
+(eval-when (:compile-toplevel :load-toplevel :execute)
+  (load "all-fail")
+  (quicklisp-client:quickload :fiveam))
+
+;; Defines the testing package with symbols from leap and FiveAM in scope
+;; The `run-tests` function is exported for use by both the user and test-runner
+(defpackage :all-fail-test
+  (:use :cl :fiveam)
+  (:export :run-tests))
+
+;; Enter the testing package
+(in-package :all-fail-test)
+
+;; Define and enter a new FiveAM test-suite
+(def-suite* all-fail-suite)
+
+(test vanilla-leap-year (is (all-fail:leap-year-p 1996)))
+
+(test any-old-year (is (not (all-fail:leap-year-p 1997))))
+
+(test non-leap-even-year (is (not (all-fail:leap-year-p 1998))))
+
+(test century (is (not (all-fail:leap-year-p 1900))))
+
+(test exceptional-century (is (all-fail:leap-year-p 2400)))
+
+(defun run-tests (&optional (test-or-suite 'leap-suite))
+  "Provides human readable results of test run. Default to entire suite."
+  (run! test-or-suite))

tests/all-fail/all-fail.lisp

@@ -0,0 +1,14 @@
+(cl:defpackage :all-fail
+  (:use :common-lisp)
+  (:export :leap-year-p))
+
+(in-package :all-fail)
+
+(defun divisible-by-p (n d) (= 0 (rem n d)))
+
+(defun leap-year-p (year)
+  (not
+    (and (divisible-by-p year 4)
+        (or (not (divisible-by-p year 100))
+      (divisible-by-p year 400)))))
+

tests/all-fail/expected_results.json

@@ -0,0 +1 @@
+{"version":2,"status":"fail","tests":[{"name":"VANILLA-LEAP-YEAR (1/1)","status":"fail","message":"\n1996\n\n evaluated to \n\n1996\n\n which does not satisfy \n\nALL-FAIL:LEAP-YEAR-P\n\n","output":"","test_code":"(is (all-fail:leap-year-p 1996))"},{"name":"ANY-OLD-YEAR (1/1)","status":"fail","message":"\n1997\n\n evaluated to \n\n1997\n\n which satisfies \n\nALL-FAIL:LEAP-YEAR-P\n\n (it should not)","output":"","test_code":"(is (not (all-fail:leap-year-p 1997)))"},{"name":"NON-LEAP-EVEN-YEAR (1/1)","status":"fail","message":"\n1998\n\n evaluated to \n\n1998\n\n which satisfies \n\nALL-FAIL:LEAP-YEAR-P\n\n (it should not)","output":"","test_code":"(is (not (all-fail:leap-year-p 1998)))"},{"name":"CENTURY (1/1)","status":"fail","message":"\n1900\n\n evaluated to \n\n1900\n\n which satisfies \n\nALL-FAIL:LEAP-YEAR-P\n\n (it should not)","output":"","test_code":"(is (not (all-fail:leap-year-p 1900)))"},{"name":"EXCEPTIONAL-CENTURY (1/1)","status":"fail","message":"\n2400\n\n evaluated to \n\n2400\n\n which does not satisfy \n\nALL-FAIL:LEAP-YEAR-P\n\n","output":"","test_code":"(is (all-fail:leap-year-p 2400))"}]}

tests/empty-file/empty-file-test.lisp

@@ -0,0 +1,30 @@
+;; Ensures that empty-file.lisp and the testing library are always loaded
+(eval-when (:compile-toplevel :load-toplevel :execute)
+  (load "empty-file")
+  (quicklisp-client:quickload :fiveam))
+
+;; Defines the testing package with symbols from leap and FiveAM in scope
+;; The `run-tests` function is exported for use by both the user and test-runner
+(defpackage :empty-file-test
+  (:use :cl :fiveam)
+  (:export :run-tests))
+
+;; Enter the testing package
+(in-package :empty-file-test)
+
+;; Define and enter a new FiveAM test-suite
+(def-suite* empty-file-suite)
+
+(test vanilla-leap-year (is (empty-file:leap-year-p 1996)))
+
+(test any-old-year (is (not (empty-file:leap-year-p 1997))))
+
+(test non-leap-even-year (is (not (empty-file:leap-year-p 1998))))
+
+(test century (is (not (empty-file:leap-year-p 1900))))
+
+(test exceptional-century (is (empty-file:leap-year-p 2400)))
+
+(defun run-tests (&optional (test-or-suite 'leap-suite))
+  "Provides human readable results of test run. Default to entire suite."
+  (run! test-or-suite))

tests/empty-file/expected_results.json

@@ -0,0 +1 @@
+{"version":2,"status":"error","message":"READ error during LOAD:\n\n  Package EMPTY-FILE does not exist.\n\n    Line: 18, Column: 51\n\n    Stream: #<SB-INT:FORM-TRACKING-STREAM for \"file /opt/test-runner/tests/empty-file/empty-file-test.lisp\" {100095BBA3}>"}

tests/partial-fail/expected_results.json

@@ -0,0 +1 @@
+{"version":2,"status":"fail","tests":[{"name":"VANILLA-LEAP-YEAR (1/1)","status":"pass","message":null,"output":"","test_code":"(is (partial-fail:leap-year-p 1996))"},{"name":"ANY-OLD-YEAR (1/1)","status":"pass","message":null,"output":"","test_code":"(is (not (partial-fail:leap-year-p 1997)))"},{"name":"NON-LEAP-EVEN-YEAR (1/1)","status":"pass","message":null,"output":"","test_code":"(is (not (partial-fail:leap-year-p 1998)))"},{"name":"CENTURY (1/1)","status":"pass","message":null,"output":"","test_code":"(is (not (partial-fail:leap-year-p 1900)))"},{"name":"EXCEPTIONAL-CENTURY (1/1)","status":"fail","message":"\n2400\n\n evaluated to \n\n2400\n\n which does not satisfy \n\nPARTIAL-FAIL:LEAP-YEAR-P\n\n","output":"","test_code":"(is (partial-fail:leap-year-p 2400))"}]}

tests/partial-fail/partial-fail-test.lisp

@@ -0,0 +1,30 @@
+;; Ensures that partial-fail.lisp and the testing library are always loaded
+(eval-when (:compile-toplevel :load-toplevel :execute)
+  (load "partial-fail")
+  (quicklisp-client:quickload :fiveam))
+
+;; Defines the testing package with symbols from leap and FiveAM in scope
+;; The `run-tests` function is exported for use by both the user and test-runner
+(defpackage :partial-fail-test
+  (:use :cl :fiveam)
+  (:export :run-tests))
+
+;; Enter the testing package
+(in-package :partial-fail-test)
+
+;; Define and enter a new FiveAM test-suite
+(def-suite* leap-suite)
+
+(test vanilla-leap-year (is (partial-fail:leap-year-p 1996)))
+
+(test any-old-year (is (not (partial-fail:leap-year-p 1997))))
+
+(test non-leap-even-year (is (not (partial-fail:leap-year-p 1998))))
+
+(test century (is (not (partial-fail:leap-year-p 1900))))
+
+(test exceptional-century (is (partial-fail:leap-year-p 2400)))
+
+(defun run-tests (&optional (test-or-suite 'leap-suite))
+  "Provides human readable results of test run. Default to entire suite."
+  (run! test-or-suite))

tests/partial-fail/partial-fail.lisp

@@ -0,0 +1,12 @@
+(cl:defpackage :partial-fail
+  (:use :common-lisp)
+  (:export :leap-year-p))
+
+(in-package :partial-fail)
+
+(defun divisible-by-p (n d) (= 0 (rem n d)))
+
+(defun leap-year-p (year)
+  (and (divisible-by-p year 4)
+       (or (not (divisible-by-p year 100))
+	   (divisible-by-p year 404))))

tests/success/expected_results.json

@@ -0,0 +1 @@
+{"version":2,"status":"pass","tests":[{"name":"VANILLA-LEAP-YEAR (1/1)","status":"pass","message":null,"output":"","test_code":"(is (success:leap-year-p 1996))"},{"name":"ANY-OLD-YEAR (1/1)","status":"pass","message":null,"output":"","test_code":"(is (not (success:leap-year-p 1997)))"},{"name":"NON-LEAP-EVEN-YEAR (1/1)","status":"pass","message":null,"output":"","test_code":"(is (not (success:leap-year-p 1998)))"},{"name":"CENTURY (1/1)","status":"pass","message":null,"output":"","test_code":"(is (not (success:leap-year-p 1900)))"},{"name":"EXCEPTIONAL-CENTURY (1/1)","status":"pass","message":null,"output":"","test_code":"(is (success:leap-year-p 2400))"}]}

tests/success/success-test.lisp

@@ -0,0 +1,31 @@
+
+;; Ensures that success.lisp and the testing library are always loaded
+(eval-when (:compile-toplevel :load-toplevel :execute)
+  (load "success")
+  (quicklisp-client:quickload :fiveam))
+
+;; Defines the testing package with symbols from leap and FiveAM in scope
+;; The `run-tests` function is exported for use by both the user and test-runner
+(defpackage :success-test
+  (:use :cl :fiveam)
+  (:export :run-tests))
+
+;; Enter the testing package
+(in-package :success-test)
+
+;; Define and enter a new FiveAM test-suite
+(def-suite* success-suite)
+
+(test vanilla-leap-year (is (success:leap-year-p 1996)))
+
+(test any-old-year (is (not (success:leap-year-p 1997))))
+
+(test non-leap-even-year (is (not (success:leap-year-p 1998))))
+
+(test century (is (not (success:leap-year-p 1900))))
+
+(test exceptional-century (is (success:leap-year-p 2400)))
+
+(defun run-tests (&optional (test-or-suite 'leap-suite))
+  "Provides human readable results of test run. Default to entire suite."
+  (run! test-or-suite))

tests/success/success.lisp

@@ -0,0 +1,12 @@
+(cl:defpackage :success
+  (:use :common-lisp)
+  (:export :leap-year-p))
+
+(in-package :success)
+
+(defun divisible-by-p (n d) (= 0 (rem n d)))
+
+(defun leap-year-p (year)
+  (and (divisible-by-p year 4)
+       (or (not (divisible-by-p year 100))
+	   (divisible-by-p year 400))))

tests/syntax-error/expected_results.json

@@ -0,0 +1 @@
+{"version":2,"status":"error","message":"READ error during LOAD:\n\n  end of file on #<SB-INT:FORM-TRACKING-STREAM for \"file /opt/test-runner/tests/syntax-error/syntax-error.lisp\" {1001A86233}>\n\n  (in form starting at line: 1, column: 0, position: 0)"}

tests/syntax-error/syntax-error-test.lisp

@@ -0,0 +1,30 @@
+;; Ensures that syntax-error.lisp and the testing library are always loaded
+(eval-when (:compile-toplevel :load-toplevel :execute)
+  (load "syntax-error")
+  (quicklisp-client:quickload :fiveam))
+
+;; Defines the testing package with symbols from leap and FiveAM in scope
+;; The `run-tests` function is exported for use by both the user and test-runner
+(defpackage :syntax-error-test
+  (:use :cl :fiveam)
+  (:export :run-tests))
+
+;; Enter the testing package
+(in-package :syntax-error-test)
+
+;; Define and enter a new FiveAM test-suite
+(def-suite* syntax-error-suite)
+
+(test vanilla-leap-year (is (syntax-error:leap-year-p 1996)))
+
+(test any-old-year (is (not (syntax-error:leap-year-p 1997))))
+
+(test non-leap-even-year (is (not (syntax-error:leap-year-p 1998))))
+
+(test century (is (not (syntax-error:leap-year-p 1900))))
+
+(test exceptional-century (is (syntax-error:leap-year-p 2400)))
+
+(defun run-tests (&optional (test-or-suite 'leap-suite))
+  "Provides human readable results of test run. Default to entire suite."
+  (run! test-or-suite))

tests/syntax-error/syntax-error.lisp

@@ -0,0 +1 @@
+(defpackage $&*@#$B