Merge pull request #361 from byte-physics/add-changelog-for-next-release

Add changelog for next release

License.txt

@@ -1,4 +1,4 @@
-Copyright 2013-2022 byte physics and contributors
+Copyright 2013-2023 byte physics and contributors
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:

create-public-release.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-newVersion=1.08
+newVersion=1.09
 revision=UnitTestingFramework-v$newVersion
 
 filesToWatch="procedures docu helper INSTALL.txt"

docu/doxygen/Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = "Unit Testing Framework"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 1.08
+PROJECT_NUMBER         = 1.09
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

docu/examples/CoverageDemoCode.ipf

@@ -1,4 +1,5 @@
 #pragma TextEncoding = "UTF-8"
+#pragma version=1.09
 #pragma rtGlobals=3 // Use modern global access method and strict wave access.
 #pragma rtFunctionErrors=1
 

docu/examples/Example6/example6-automatic-invocation.ipf

@@ -1,5 +1,6 @@
 #pragma rtGlobals=3
 #pragma TextEncoding="UTF-8"
+#pragma version=1.09
 #pragma ModuleName=Example6
 
 #include "unit-testing"

docu/examples/Example6/example6-runner.ipf

@@ -1,5 +1,6 @@
 #pragma rtGlobals=3
 #pragma TextEncoding="UTF-8"
+#pragma version=1.09
 
 #include "unit-testing"
 

docu/examples/TAP_Examples/tap_example1a-uncaught-aborts.ipf

@@ -1,5 +1,6 @@
 #pragma rtGlobals=3
 #pragma ModuleName=TAPExample1a
+#pragma version=1.09
 
 #include "unit-testing"
 

docu/examples/TAP_Examples/tap_example1b-warn_and_error.ipf

@@ -1,4 +1,5 @@
 #pragma rtGlobals=3
+#pragma version=1.09
 
 #include "unit-testing"
 

docu/examples/example1-plain.ipf

@@ -1,5 +1,6 @@
 #pragma rtGlobals=3
 #pragma TextEncoding="UTF-8"
+#pragma version=1.09
 
 #include "unit-testing"
 

docu/examples/example10-peakfind.ipf

@@ -1,4 +1,5 @@
 #pragma TextEncoding = "UTF-8"
+#pragma version=1.09
 #pragma rtGlobals=3		// Use modern global access method and strict wave access.
 
 #include "unit-testing"

docu/examples/example11-background.ipf

@@ -1,5 +1,6 @@
 #pragma rtGlobals=3
 #pragma TextEncoding="UTF-8"
+#pragma version=1.09
 #pragma ModuleName=Example11
 
 #include "unit-testing"

docu/examples/example12-background-using-hooks.ipf

@@ -1,5 +1,6 @@
 #pragma rtGlobals=3
 #pragma TextEncoding="UTF-8"
+#pragma version=1.09
 #pragma ModuleName=Example12
 
 #include "unit-testing"

docu/examples/example13-multi-test-data.ipf

@@ -1,5 +1,6 @@
 #pragma rtGlobals=3
 #pragma ModuleName=Example13
+#pragma version=1.09
 
 #include "unit-testing"
 

docu/examples/example14-info.ipf

@@ -1,4 +1,5 @@
 #pragma TextEncoding = "UTF-8"
+#pragma version=1.09
 #pragma rtGlobals=3				// Use modern global access method and strict wave access
 #pragma DefaultTab={3,20,4}		// Set default tab width in Igor Pro 9 and later
 #pragma ModuleName=InfoTest

docu/examples/example2-plain.ipf

@@ -1,5 +1,6 @@
 #pragma rtGlobals=3
 #pragma TextEncoding="UTF-8"
+#pragma version=1.09
 #pragma ModuleName=Example2
 
 #include "unit-testing"

docu/examples/example3-plain.ipf

@@ -1,5 +1,6 @@
 #pragma rtGlobals=3
 #pragma TextEncoding="UTF-8"
+#pragma version=1.09
 #pragma ModuleName=Example3
 
 #include "unit-testing"

docu/examples/example4-wavechecking.ipf

@@ -1,5 +1,6 @@
 #pragma rtGlobals=3
 #pragma TextEncoding="UTF-8"
+#pragma version=1.09
 #pragma ModuleName=Example4
 
 #include "unit-testing"

docu/examples/example5-extensionhooks-otherSuite.ipf

@@ -1,5 +1,6 @@
 #pragma rtGlobals=3
 #pragma TextEncoding="UTF-8"
+#pragma version=1.09
 
 #include "unit-testing"
 

docu/examples/example5-extensionhooks.ipf

@@ -1,5 +1,6 @@
 #pragma rtGlobals=3
 #pragma TextEncoding="UTF-8"
+#pragma version=1.09
 #pragma ModuleName=Example5
 
 #include "unit-testing"

docu/examples/example7-uncaught-aborts.ipf

@@ -1,5 +1,6 @@
 #pragma rtGlobals=3
 #pragma TextEncoding="UTF-8"
+#pragma version=1.09
 #pragma ModuleName=Example7
 
 #include "unit-testing"

docu/examples/example8-uncaught-runtime-errors.ipf

@@ -1,5 +1,6 @@
 #pragma rtGlobals=3
 #pragma TextEncoding="UTF-8"
+#pragma version=1.09
 #pragma ModuleName=Example8
 
 #include "unit-testing"

docu/examples/example9-IM.ipf

@@ -1,4 +1,5 @@
 #pragma TextEncoding = "UTF-8"
+#pragma version=1.09
 #pragma rtGlobals=3
 #pragma IndependentModule=Example9
 

docu/sphinx/source/advanced.rst

@@ -215,6 +215,8 @@ Abort button.
    terminate only the current test case and continue with the next queued test
    case.
 
+.. _tests_with_background_activity:
+
 Test Cases with Background Activity
 -----------------------------------
 
@@ -260,6 +262,8 @@ Test Cases with background activity are supported from multi data test cases, se
 
  See also :ref:`example12`.
 
+.. _multi_data_test_cases:
+
 Multi Data Test Cases
 ---------------------
 
@@ -390,6 +394,8 @@ test execution is aborted.
      PASS()
    End
 
+.. _multi_multi_data_test_cases:
+
 Multi-Multi Data Test Cases
 ---------------------------
 
@@ -478,6 +484,8 @@ The indices are changed for all setup variables. The first variables changes fas
 If Multi-Multi-Data test cases are combined with functions with background activity the reentry function must have the same
 signature.
 
+.. _code_coverage:
+
 Code Coverage Determination
 ---------------------------
 
@@ -534,6 +542,8 @@ At the end of a run with code coverage determination Igor Pro outputs the global
 The following regular expression can be used in CI services (e.g. in GitLab) to retrieve the number
 ``(?:^Coverage: )(\d+.\d+)(?:%$)``.
 
+.. _coverage_statistics:
+
 Statistics
 ^^^^^^^^^^
 

docu/sphinx/source/changelog.rst

@@ -7,6 +7,90 @@ All notable changes to this project will be documented in this file.
 
 The format is based on `Keep a Changelog <https://keepachangelog.com/en/1.0.0/>`_.
 
+1.09 (01/04/2023)
+-----------------
+
+General
+~~~~~~~
+
+   - Add Code Coverage determination, see :ref:`here <code_coverage>` (IP9 Build 38812 or higher)
+   - Allow analytics of code coverage tracing data, see :ref:`here <coverage_statistics>`
+   - Add support for :ref:`multi data <multi_data_test_cases>` and :ref:`multi-multi data
+     <multi_multi_data_test_cases>` test cases
+   - Add :cpp:func:`INFO` function to output more information on failed test assertions, see :ref:`here <example14>`
+   - Add support for test code running in background functions, see also :ref:`here <tests_with_background_activity>`.
+   - Add support for checking for free/local wave leaks (IP9 Build 39622 or higher)
+   - Add ``UTF_SKIP`` tag
+   - Mark test cases with zero sized data generator waves as skipped
+   - Call data generator only once for MD/MMD test cases
+   - Enforce that we have at least one assertion in each test case
+   - Fix hitting the sprintf limit (IP 8 or lower)
+   - Abort flag does no longer cover runtime errors
+   - Execute the test cases from top to bottom in each test suite
+   - Test the basic parts of our testing framework using the very tiny test environment ``VTTE``
+   - Add generic function to report wrapper results
+   - Reorganize code and split it into more files
+   - TestCaseEnd: Silently ignore non-killable working folder
+   - Output state messages to stdout (IP 8 or higher)
+   - Enhance output on failed test assertion in test cases
+   - Tighten the check for test case signatures
+   - AfterFileOpenHook: Make it more robust
+   - Execute the builtin hooks also for failing user test hooks
+   - Always clear runtime errors before ``AbortOnRTE``
+   - .gitlab.ci.yml: Add CI
+   - Moved ``NULL_WAVE`` flag to major flags for wave comparison in documentation
+   - Output failure summary at the end
+   - Allow unsaved experiments in some cases
+   - Allow fixed log file naming
+   - New option ``debugMode`` for more fine-grained debug control
+
+Test assertions
+~~~~~~~~~~~~~~~
+
+   - :cpp:func:`*_CLOSE_VAR <CHECK_CLOSE_VAR>`: Prevent singularity
+   - :cpp:func:`*_SMALL_VAR <CHECK_SMALL_VAR>`: Change tolerance so that ``0`` is considered small with zero tolerance
+   - Add assertions <, <=, >, >= for double arguments, see :cpp:func:`*_LT_VAR <CHECK_LT_VAR>`,
+     :cpp:func:`*_LE_VAR <CHECK_LE_VAR>`, :cpp:func:`*_GT_VAR <CHECK_GT_VAR>`, and :cpp:func:`*_GE_VAR <CHECK_GE_VAR>`
+   - :cpp:func:`*_EQUAL_STR <CHECK_EQUAL_STR>`: Make case sensitive comparison the default
+   - Added assertions for Int64 and UInt64 variables, see :cpp:func:`*_EQUAL_INT64 <CHECK_EQUAL_INT64>` and
+     :cpp:func:`*_EQUAL_UINT64 <CHECK_EQUAL_UINT64>`
+
+:cpp:func:`*_WAVE <CHECK_WAVE>`
+"""""""""""""""""""""""""""""""
+
+   - Require valid types as arguments
+   - Make the output more human readable
+
+:cpp:func:`*_EQUAL_WAVES <CHECK_EQUAL_WAVES>` and :cpp:func:`*_EQUAL_TEXTWAVES <CHECK_EQUAL_TEXTWAVES>`
+"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+
+   - Allow to check matching sizes of waves of different types
+   - Allow identical wave references
+   - Compare zero sized waves properly with set mode
+   - Make error reporting for custom mode patterns better
+   - Output descriptive error messages in ``WAVE_DATA`` mode
+   - Handle invalid mode correctly
+   - Work around EqualWaves dimension labels bug with certain IP versions
+   - Complain for unknown modes
+   - Make modes wave signed thus allowing -1 to be passed in to check all modes
+
+JUNIT output
+~~~~~~~~~~~~
+
+   - Improve accuracy of test case/suite durations
+   - Remove optional TestSuite attribute ``disabled``
+   - Add a ``<failure>`` tag for each failed assertion
+   - Handle expected failure test cases as skipped
+   - Add JUNIT reference and updated JUNIT section in documentation
+   - Drop timezone information as required by the "standard"
+   - Nicify properties output
+
+TAP output
+~~~~~~~~~~
+
+   - Handle ``TODO`` gracefully
+   - Now also holds skipped testcases
+
 1.08 (02/15/2019)
 -----------------
 

docu/sphinx/source/conf.py

@@ -38,7 +38,7 @@ def getDocRoot():
 # -- Project information -----------------------------------------------------
 
 project = u'Igor Unit Testing Framework'
-copyright = u'2012-2022 All Contributors'
+copyright = u'2012-2023 All Contributors'
 author = u'All Contributors'
 
 version = getVersion()

docu/sphinx/source/examples.rst

@@ -473,6 +473,8 @@ case is called for each element of that wave.
 
    * :cpp:func:`CHECK`
 
+.. _example14:
+
 Example14
 ---------
 

internal_dev/DataGenFunctionResolution/IM_test_example.ipf

@@ -1,6 +1,7 @@
 #pragma rtGlobals=3
 #pragma TextEncoding="UTF-8"
 #pragma rtFunctionErrors=1
+#pragma version=1.09
 #pragma ModuleName=TestExampleMain
 #pragma IndependentModule=TestExample_IM
 

internal_dev/DataGenFunctionResolution/IM_test_example_sub1.ipf

@@ -1,6 +1,7 @@
 #pragma rtGlobals=3
 #pragma TextEncoding="UTF-8"
 #pragma rtFunctionErrors=1
+#pragma version=1.09
 #pragma ModuleName=TestExampleSub1
 #pragma IndependentModule=TestExample_IM
 

internal_dev/DataGenFunctionResolution/IM_test_example_sub2.ipf

@@ -1,6 +1,7 @@
 #pragma rtGlobals=3
 #pragma TextEncoding="UTF-8"
 #pragma rtFunctionErrors=1
+#pragma version=1.09
 #pragma IndependentModule=TestExample_IM
 
 Function/WAVE dataGenGlobal()

internal_dev/DataGenFunctionResolution/test_example.ipf

@@ -1,6 +1,7 @@
 #pragma rtGlobals=3
 #pragma TextEncoding="UTF-8"
 #pragma rtFunctionErrors=1
+#pragma version=1.09
 #pragma ModuleName=TestExampleMain
 
 #include "unit-testing"

internal_dev/DataGenFunctionResolution/test_example_sub1.ipf

@@ -1,6 +1,7 @@
 #pragma rtGlobals=3
 #pragma TextEncoding="UTF-8"
 #pragma rtFunctionErrors=1
+#pragma version=1.09
 #pragma ModuleName=TestExampleSub1
 
 #include "unit-testing"

internal_dev/DataGenFunctionResolution/test_example_sub2.ipf

@@ -1,6 +1,7 @@
 #pragma rtGlobals=3
 #pragma TextEncoding="UTF-8"
 #pragma rtFunctionErrors=1
+#pragma version=1.09
 
 Function/WAVE dataGenGlobal()
 	Make/FREE/T data = {"ProcGlobal"}

internal_dev/DataGenSpecificWaveTypes.ipf

@@ -1,5 +1,6 @@
 #pragma rtGlobals=3
 #pragma rtFunctionErrors=1
+#pragma version=1.09
 #pragma TextEncoding="UTF-8"
 
 #include "unit-testing"

internal_dev/EqualWaves_Examples.ipf

@@ -1,6 +1,7 @@
 #pragma rtGlobals=3
 #pragma TextEncoding = "UTF-8"
 #pragma rtFunctionErrors=1
+#pragma version=1.09
 
 #include "unit-testing"
 

internal_dev/FreeWaveLeaks.ipf

@@ -1,5 +1,6 @@
 #pragma TextEncoding = "UTF-8"
 #pragma rtGlobals=3				// Use modern global access method and strict wave access
+#pragma version=1.09
 #pragma DefaultTab={3,20,4}		// Set default tab width in Igor Pro 9 and later
 #pragma ModuleName=FreeWaveLeaks
 

internal_dev/IM_Test2.ipf

@@ -1,5 +1,6 @@
 #pragma rtGlobals=3
 #pragma rtFunctionErrors=1
+#pragma version=1.09
 #pragma IndependentModule=ABCD
 
 #include "unit-testing"

internal_dev/IM_Test3.ipf

@@ -1,5 +1,6 @@
 #pragma rtGlobals=3
 #pragma rtFunctionErrors=1
+#pragma version=1.09
 #pragma IndependentModule=ABCD
 
 #include "unit-testing"

internal_dev/IM_Test4.ipf

@@ -1,5 +1,6 @@
 #pragma rtGlobals=3
 #pragma rtFunctionErrors=1
+#pragma version=1.09
 #pragma IndependentModule=ABCD
 
 #include "unit-testing"