From 40e9724544a7161d334905f45c7da47d7349acfe Mon Sep 17 00:00:00 2001 From: Patrick Eads Date: Wed, 17 Jul 2024 00:16:19 -0400 Subject: [PATCH] Ci stuff (#35) * init * added a bunch of tests, and made minor code mods to increase (mostly) code coverage * maybe 3.11 does not like tuples in foreaches? --- .github/workflows/python-app.yml | 4 +- example.sh | 2 +- example_simo.sh | 2 +- src/dsp/dsp_processor.py | 30 ++++---- src/dsp/vfo_processor.py | 11 ++- src/misc/io_args.py | 84 ++++++++++++++--------- src/misc/keyboard_interruptable_thread.py | 4 +- src/misc/read_file.py | 1 - src/sdrterm.py | 26 +++---- test/dsp/dsp_processor_test.py | 25 ++++--- test/dsp/iq_correction_test.py | 5 +- test/misc/file_util_test.py | 7 ++ test/misc/general_util_test.py | 56 +++++++++++++++ test/misc/io_args_test.py | 41 +++++++++++ test/misc/read_file_test.py | 11 +++ 15 files changed, 221 insertions(+), 88 deletions(-) create mode 100644 test/misc/general_util_test.py create mode 100644 test/misc/io_args_test.py create mode 100644 test/misc/read_file_test.py diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index fc224bf..8b6529d 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -82,7 +82,7 @@ jobs: echo "checksum matched: ${i}" cnt=$(( cnt + 1 )); else - printf "${setBlue}FAILED: ${i}\n\tEXPECTED: ${sums["$i"]}\n\tRECEIVED: ${z["$i"]}\n${setNormal}" 1>&2; + printf "\033[31mFAILED: ${i}\n\tEXPECTED: ${sums["$i"]}\n\tRECEIVED: ${z["$i"]}\n\033[31m" 1>&2; fi done - (( cnt == total )) \ No newline at end of file + (( cnt == total )) diff --git a/example.sh b/example.sh index 064fa5d..223e9ca 100644 --- a/example.sh +++ b/example.sh @@ -30,7 +30,7 @@ if [[ -z ${DSD_CMD} ]]; then fi if [[ -z ${OUT_PATH} ]]; then - OUT_PATH=/mnt/d/simo; + OUT_PATH=/tmp; fi echo $SDRTERM_EXEC; diff --git a/example_simo.sh b/example_simo.sh index 6df58ad..34fabd8 100644 --- a/example_simo.sh +++ b/example_simo.sh @@ -39,7 +39,7 @@ if [[ -z ${DSD_CMD} ]]; then fi if [[ -z ${OUT_PATH} ]]; then - OUT_PATH=/mnt/d/simo; + OUT_PATH=/tmp; fi declare -A pids; diff --git a/src/dsp/dsp_processor.py b/src/dsp/dsp_processor.py index f54302e..366a466 100644 --- a/src/dsp/dsp_processor.py +++ b/src/dsp/dsp_processor.py @@ -25,7 +25,7 @@ from scipy.signal import decimate, dlti, savgol_filter, sosfilt, ellip from dsp.data_processor import DataProcessor -from dsp.demodulation import amDemod, fmDemod +from dsp.demodulation import amDemod, fmDemod, realOutput, imagOutput from misc.general_util import vprint @@ -56,6 +56,7 @@ def __init__(self, fileInfo: dict = None, **kwargs): + self._demod = None self._pool = None self._shift = None self.bandwidth \ @@ -131,36 +132,39 @@ def demod(x): setattr(self, 'demod', demod) return ret - def _demod(self, y: ndarray[any, dtype[complex64 | complex128]]) -> ndarray[any, dtype[float32 | float64]]: - pass - def _setDemod(self, fun: Callable[[ndarray[any, dtype[complex64 | complex128]]], ndarray[any, dtype[float32 | float64]]], *filters) \ -> Callable[[ndarray[any, dtype[complex64 | complex128]]], ndarray[any, dtype[float32 | float64]]]: - if fun is not None and len(filters): + if fun is not None: self._outputFilters.clear() - self._outputFilters.extend(*filters) + if len(filters): + self._outputFilters.extend(*filters) setattr(self, '_demod', fun) return self.demod raise ValueError("Demodulation function, or filters not defined") - def selectOuputFm(self): + def selectOutputFm(self): vprint('NFM Selected') self.bandwidth = 12500 self._setDemod(fmDemod, generateEllipFilter(self.__decimatedFs, self._FILTER_DEGREE, self.omegaOut, 'lowpass')) - def selectOuputWfm(self): - vprint('WFM Selected') - self.bandwidth = 25000 - self._setDemod(fmDemod, generateEllipFilter(self.__decimatedFs, self._FILTER_DEGREE, self.omegaOut, 'lowpass')) - - def selectOuputAm(self): + def selectOutputAm(self): vprint('AM Selected') self.bandwidth = 10000 self._setDemod(amDemod, generateEllipFilter(self.__decimatedFs, self._FILTER_DEGREE, self.omegaOut, 'lowpass')) + def selectOutputReal(self): + vprint('I output Selected') + self.bandwidth = self.decimatedFs + self._setDemod(realOutput) + + def selectOutputImag(self): + vprint('Q output Selected') + self.bandwidth = self.decimatedFs + self._setDemod(imagOutput) + def _processChunk(self, y: ndarray) -> ndarray[any, dtype[float32 | float64]]: if self._shift is not None: ''' diff --git a/src/dsp/vfo_processor.py b/src/dsp/vfo_processor.py index 08b666d..5e7c00b 100644 --- a/src/dsp/vfo_processor.py +++ b/src/dsp/vfo_processor.py @@ -46,8 +46,12 @@ def __init__(self, fs, vfoHost: str = 'localhost', vfos: str = None, **kwargs): self.vfos = array(self.vfos) self._nFreq = len(self.vfos) self.__omega = -2j * pi * (self.vfos / self.fs) - self.host = vfoHost - + if ':' in vfoHost: + self.host, self.port = vfoHost.split(':') + self.port = int(self.port) + else: + self.host = vfoHost + self.port = findPort(self.host) self.__queue: Queue[str] | None = None self.__clients: dict[str, RawIOBase] | None = None self.__event: Event | None = None @@ -98,7 +102,7 @@ def handle(self): self.outer_self.queue.task_done() self.outer_self.event.wait() - with ThreadedTCPServer((self.host, findPort(self.host)), ThreadedTCPRequestHandler) as server: + with ThreadedTCPServer((self.host, self.port), ThreadedTCPRequestHandler) as server: st = Thread(target=server.serve_forever) try: @@ -120,5 +124,6 @@ def handle(self): with self.__queue.all_tasks_done: self.__queue.all_tasks_done.notify_all() st.join() + self._pool.shutdown(wait=False, cancel_futures=True) vprint('Multi-VFO writer halted') return diff --git a/src/misc/io_args.py b/src/misc/io_args.py index 14c45b0..3bae4c8 100644 --- a/src/misc/io_args.py +++ b/src/misc/io_args.py @@ -17,17 +17,34 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +from enum import Enum from multiprocessing import Queue, Process from typing import Callable +from misc.general_util import tprint -def selectDemodulation(demodType: str, processor) -> Callable: - if demodType == 'fm' or demodType == 'nfm': - return processor.selectOuputFm - elif demodType == 'am': - return processor.selectOuputAm - else: - raise ValueError(f'Invalid demod type {demodType}') + +class DemodulationChoices(str, Enum): + FM = "fm" + AM = "am" + REAL = "re" + IMAG = "im" + + def __str__(self): + return self.value + + +def selectDemodulation(demodType: DemodulationChoices, processor) -> Callable: + tprint(f'{demodType} requested') + if 'fm' == demodType or 'nfm' == demodType: + return processor.selectOutputFm + elif 'am' == demodType: + return processor.selectOutputAm + elif 're' == demodType: + return processor.selectOutputReal + elif 'im' == demodType: + return processor.selectOutputImag + raise ValueError(f'Invalid demod type {demodType}') def selectPlotType(plotType: str): @@ -65,12 +82,12 @@ def __init__(self, verbose: int = 0, **kwargs): kwargs['fileInfo'] = checkWavHeader(kwargs['inFile'], kwargs['fs'], kwargs['enc']) kwargs['fs'] = kwargs['fileInfo']['sampRate'] - IOArgs.__initializeOutputHandlers(**kwargs) + IOArgs._initializeOutputHandlers(**kwargs) kwargs['isDead'].value = 0 @classmethod - def __initializeProcess(cls, isDead: Value, processor, *args, - name: str = 'Process', **kwargs) -> tuple[Queue, Process]: + def _initializeProcess(cls, isDead: Value, processor, *args, + name: str = 'Process', **kwargs) -> tuple[Queue, Process]: if processor is None: raise ValueError('Processor must be provided') buffer = Queue() @@ -79,16 +96,16 @@ def __initializeProcess(cls, isDead: Value, processor, *args, return buffer, proc @classmethod - def __initializeOutputHandlers(cls, - isDead: Value = None, - fs: int = 0, - dm: str = None, - outFile: str = None, - simo: bool = False, - pl: str = None, - processes: list[Process] = None, - buffers: list[Queue] = None, - **kwargs) -> None: + def _initializeOutputHandlers(cls, + isDead: Value = None, + fs: int = 0, + dm: DemodulationChoices | str = None, + outFile: str = None, + simo: bool = False, + pl: str = None, + processes: list[Process] = None, + buffers: list[Queue] = None, + **kwargs) -> None: import os from misc.general_util import eprint @@ -107,19 +124,18 @@ def __initializeOutputHandlers(cls, else: for p in pl.split(','): psplot = selectPlotType(p) - if psplot is not None: - kwargs['bandwidth'] = cls.strct['processor'].bandwidth - buffer, proc = cls.__initializeProcess(isDead, - psplot, - fs, name="Plotter-", - **kwargs) - processes.append(proc) - buffers.append(buffer) - - buffer, proc = cls.__initializeProcess(isDead, - cls.strct['processor'], - outFile, - name="File writer-", - **kwargs) + kwargs['bandwidth'] = cls.strct['processor'].bandwidth + buffer, proc = cls._initializeProcess(isDead, + psplot, + fs, name="Plotter-", + **kwargs) + processes.append(proc) + buffers.append(buffer) + + buffer, proc = cls._initializeProcess(isDead, + cls.strct['processor'], + outFile, + name="File writer-", + **kwargs) processes.append(proc) buffers.append(buffer) diff --git a/src/misc/keyboard_interruptable_thread.py b/src/misc/keyboard_interruptable_thread.py index 3376128..13af782 100644 --- a/src/misc/keyboard_interruptable_thread.py +++ b/src/misc/keyboard_interruptable_thread.py @@ -23,6 +23,7 @@ class KeyboardInterruptableThread(Thread): def __init__(self, func: Callable[[], None], target: Callable, group=None, name=None, args=(), daemon=None): + self._handleException = None if func is None: raise ValueError("func cannot be None") super().__init__(group=group, target=target, name=name, args=args, daemon=daemon) @@ -30,9 +31,6 @@ def __init__(self, func: Callable[[], None], target: Callable, group=None, name= import threading threading.excepthook = self.handleException - def _handleException(self): - pass - def handleException(self, e): from misc.general_util import tprint from sys import __excepthook__ diff --git a/src/misc/read_file.py b/src/misc/read_file.py index f0809d2..5b1c6da 100644 --- a/src/misc/read_file.py +++ b/src/misc/read_file.py @@ -35,7 +35,6 @@ def readFile(bitsPerSample: dtype = None, correctIq: bool = False, normalize: bool = False, **_) -> None: - from array import array from io import BufferedReader from sys import stdin diff --git a/src/sdrterm.py b/src/sdrterm.py index aba8493..7800372 100644 --- a/src/sdrterm.py +++ b/src/sdrterm.py @@ -18,7 +18,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -from enum import Enum from multiprocessing import Value from os import getpid from typing import Annotated @@ -27,16 +26,7 @@ from typer import run as typerRun, Option from misc.file_util import DataType - - -class DemodulationChoices(str, Enum): - FM = "fm" - AM = "am" - REAL = "re" - IMAG = "im" - - def __str__(self): - return self.value +from misc.io_args import DemodulationChoices def parseStrDataType(value: str) -> str: @@ -46,9 +36,11 @@ def parseStrDataType(value: str) -> str: raise BadParameter(str(ex)) -def parseIntString(value: str) -> int: +def parseIntString(value: str | int) -> int: if value is None: raise BadParameter('Value cannot be None') + elif isinstance(value, int): + return value elif 'k' in value: return int(float(value.replace('k', '')) * 10E+2) elif 'M' in value: @@ -67,7 +59,7 @@ def main(fs: Annotated[int, Option('--fs', '-r', center: Annotated[int, Option('--center-frequency', '-c', metavar='NUMBER', parser=parseIntString, - help='Offset from tuned frequency in k/M/Hz')] = '0', + help='Offset from tuned frequency in k/M/Hz')] = 0, inFile: Annotated[str, Option('--input', '-i', show_default='stdin', help='Input device')] = None, @@ -93,7 +85,7 @@ def main(fs: Annotated[int, Option('--fs', '-r', omegaOut: Annotated[int, Option('--omega-out', '-w', metavar='NUMBER', parser=parseIntString, - help='Output cutoff frequency in k/M/Hz')] = '12500', + help='Output cutoff frequency in k/M/Hz')] = 12500, correct_iq: Annotated[bool, Option(help='Toggle iq correction')] = False, simo: Annotated[bool, Option(help=''' Enable using sockets to output data processed from multiple channels specified by the vfos option. @@ -111,7 +103,6 @@ def main(fs: Annotated[int, Option('--fs', '-r', help='Swap input endianness', show_default='False => system-default, or as defined in RIFF header')] = False, normalize_input: Annotated[bool, Option(help='Normalize input data.')] = False, ): - from misc.io_args import IOArgs from misc.read_file import readFile from multiprocessing import Process, Queue @@ -181,13 +172,12 @@ def __setStartMethod(): from misc.general_util import printException if 'spawn' in get_all_start_methods(): + from multiprocessing import set_start_method try: - from multiprocessing import set_start_method - set_start_method('spawn') except Exception as e: printException(e) - raise NotImplementedError('Setting start method to spawn failed') + set_start_method('spawn', True) def __generatePidFile(pid): diff --git a/test/dsp/dsp_processor_test.py b/test/dsp/dsp_processor_test.py index 4952174..13653c7 100644 --- a/test/dsp/dsp_processor_test.py +++ b/test/dsp/dsp_processor_test.py @@ -5,7 +5,7 @@ import dsp.demodulation as dem import dsp.dsp_processor as dsp -from misc.general_util import eprint +from dsp.data_processor import DataProcessor DEFAULT_FS = 48000 DEFAULT_CENTER = -1000 @@ -22,28 +22,26 @@ def test_init(processor): assert processor.fs == DEFAULT_FS assert processor.decimation == 2 assert processor.decimatedFs == DEFAULT_FS >> 1 - processor.selectOuputFm() + processor.selectOutputFm() assert processor._demod == dem.fmDemod - processor.selectOuputWfm() - assert processor._demod == dem.fmDemod - processor.selectOuputAm() + processor.selectOutputAm() assert processor._demod == dem.amDemod with pytest.raises(ValueError) as e: processor.decimation = 1 - eprint(f'\n{e.type.__name__}: {e.value}') + print(f'\n{e.type.__name__}: {e.value}') with pytest.raises(AttributeError) as e: processor.decimatedFs = 1 - eprint(f'\n{e.type.__name__}: {e.value}') + print(f'\n{e.type.__name__}: {e.value}') with pytest.raises(ValueError) as e: processor._setDemod(None, ()) - eprint(f'\n{e.type.__name__}: {e.value}') + print(f'\n{e.type.__name__}: {e.value}') with pytest.raises(TypeError) as e: processor._setDemod("asdf", None) - eprint(f'\n{e.type.__name__}: {e.value}') + print(f'\n{e.type.__name__}: {e.value}') processor.decimation = DEFAULT_DECIMATION_FACTOR assert processor.decimation == DEFAULT_DECIMATION_FACTOR @@ -61,12 +59,17 @@ def test_init(processor): with pytest.raises(FileNotFoundError) as e: processor.processData(None, None, '') - eprint(f'\n{e.type.__name__}: {e.value}') + print(f'\n{e.type.__name__}: {e.value}') with pytest.raises(AttributeError) as e: processor.processData(None, None, None) - eprint(f'\n{e.type.__name__}: {e.value}') + print(f'\n{e.type.__name__}: {e.value}') assert locals().get('processor') is not None del processor assert locals().get('processor') is None + + with pytest.raises(TypeError) as e: + DataProcessor() + print(f'\n{e.type.__name__}: {e.value}') + DataProcessor.processData(None) diff --git a/test/dsp/iq_correction_test.py b/test/dsp/iq_correction_test.py index 0b9843f..62e0b0e 100644 --- a/test/dsp/iq_correction_test.py +++ b/test/dsp/iq_correction_test.py @@ -2,15 +2,18 @@ import pytest from dsp.iq_correction import IQCorrection + DEFAULT_SAMPLE_RATE = 2000 DEFAULT_IMPEDANCE = 50 NEXT_SAMPLE_RATE = 3750 NEXT_IMPEDANCE = 75 + @pytest.fixture def corrector(): return IQCorrection(DEFAULT_SAMPLE_RATE) + def test_iqCorrection(corrector): assert corrector.sampleRate == DEFAULT_SAMPLE_RATE assert corrector.inductance == DEFAULT_IMPEDANCE / DEFAULT_SAMPLE_RATE @@ -33,7 +36,7 @@ def test_iqCorrection(corrector): corrector.correctIq(someData) assert someData.size == size assert someCorrectedData.size == size - for x,y in zip(someData, someCorrectedData): + for x, y in zip(someData, someCorrectedData): assert x == y assert locals().get('corrector') is not None diff --git a/test/misc/file_util_test.py b/test/misc/file_util_test.py index 25aa5a5..8b16668 100644 --- a/test/misc/file_util_test.py +++ b/test/misc/file_util_test.py @@ -300,3 +300,10 @@ def test_checkAlawHeader(alawHeader): with pytest.raises(ValueError) as e: checkWavHeader(alawHeader, 8000, 'B') print(f'\n{e.value}') + + +def test_enum(): + for x, y in zip(DataType, DataType.tuples()): + k, v = y + assert x.name == k + assert x.value == v diff --git a/test/misc/general_util_test.py b/test/misc/general_util_test.py new file mode 100644 index 0000000..386ab59 --- /dev/null +++ b/test/misc/general_util_test.py @@ -0,0 +1,56 @@ +import os +import signal +from multiprocessing import get_context, Value, Event + +import pytest + +from misc.general_util import setSignalHandlers, traceOn + + +def fail(value: Value): + value.value = 1 + + +def target(pid: Value, event: Event, value: Value): + setSignalHandlers(os.getpid(), lambda: fail(value)) + traceOn() + pid.value = os.getpid() + event.set() + while not value.value: + pass + + +@pytest.fixture(scope='function') +def context(): + ctx = get_context('spawn') + return ctx, ctx.Value('B', 0), ctx.Event(), ctx.Value('L', 0) + + +def test_general_util(context): + print('\n') + ctx, value, event, pid = context + for x in [signal.SIGINT, signal.SIGQUIT, signal.SIGHUP, signal.SIGTERM, signal.SIGXCPU, signal.SIGABRT, ]: + thread = ctx.Process(target=target, args=(pid, event, value,)) + thread.start() + event.wait() + print(f'Sent {signal.Signals(x).name} to {pid.value}') + [os.kill(pid.value, x) for x in (signal.SIGTSTP, signal.SIGTTIN, signal.SIGTTOU)] + os.kill(pid.value, x) + [os.kill(pid.value, x) for x in (signal.SIGTSTP, signal.SIGTTIN, signal.SIGTTOU)] + thread.join() + assert value.value + del thread + event.clear() + value.value = 0 + + +def test_general_util2(context): + print('\n') + value = context[1] + setSignalHandlers(os.getpid(), lambda: fail(value)) + for x in [signal.SIGINT, signal.SIGQUIT, signal.SIGHUP, signal.SIGTERM, signal.SIGXCPU, signal.SIGABRT, ]: + [os.kill(os.getpid(), x) for x in (signal.SIGTSTP, signal.SIGTTIN, signal.SIGTTOU)] + os.kill(os.getpid(), x) + [os.kill(os.getpid(), x) for x in (signal.SIGTSTP, signal.SIGTTIN, signal.SIGTTOU)] + assert value.value + value.value = 0 diff --git a/test/misc/io_args_test.py b/test/misc/io_args_test.py new file mode 100644 index 0000000..d8f1dd1 --- /dev/null +++ b/test/misc/io_args_test.py @@ -0,0 +1,41 @@ +import os + +import pytest + +from misc.io_args import IOArgs, DemodulationChoices + + +@pytest.fixture(scope='function') +def osEnv(): + ret = os.environ.pop('DISPLAY') if 'DISPLAY' in os.environ else None + yield ret + + +def test_ioargs(osEnv): + with pytest.raises(ValueError): + IOArgs._initializeProcess(None, None, name='p008') + + IOArgs.strct = kwargs = {'simo': True, + 'vfo_host': 'localhost:1234', + 'inFile': '/mnt/d/uint8.wav', + 'outFile': "/dev/null", + 'omegaOut': 5000, + 'tuned': 155685000, + 'center': -350000, + 'vfos': "0", + } + IOArgs._initializeOutputHandlers(fs=1024000, + dm=DemodulationChoices.FM, + pl='ps', + processes=[], + buffers=[], + **kwargs) + if osEnv is not None: + os.environ['DISPLAY'] = osEnv + # TODO more tests + # os.environ.pop('DISPLAY') + # IOArgs._initializeOutputHandlers(fs=1024000, + # dm=DemodulationChoices.FM, + # processes=[], + # buffers=[], + # **kwargs) diff --git a/test/misc/read_file_test.py b/test/misc/read_file_test.py new file mode 100644 index 0000000..789b98d --- /dev/null +++ b/test/misc/read_file_test.py @@ -0,0 +1,11 @@ +import pytest + +from misc.read_file import readFile + + +def test_read_file(): + with pytest.raises(ValueError) as e: + readFile() + print(f'\n{e.value}') + +# TODO more tests \ No newline at end of file