Setup py fixes and atexit hang/not halting fixes and plot sutff

.gitignore

@@ -11,7 +11,7 @@ CMakeFiles
 junk/
 *.pyc
 *.bin
-.venv*
-venv*
+*venv*
 build
 *egg-info*
+*wenv*

setup.py

@@ -17,33 +17,33 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from setuptools import setup
+from setuptools import setup, find_namespace_packages
 
 setup(
     name='sdrterm',
-    version='0.0.01',    
+    version='0.0.01',
     description='Provides simple terminal-based user-interface for processing/analysis of SDR data',
     url='https://github.com/peads/sdrterm',
     author='Patrick Eads',
     author_email='[email protected]',
     license='GPLv3',
-    package_dir = {"": "src"},
-#    packages=['sdrterm'],
+    package_dir={"": "src"},
+    packages=find_namespace_packages(where='src'),
     install_requires=['scipy',
                       'numpy',
                       'matplotlib',
-                      'typer',                     
+                      'typer',
                       ],
-    license_files = ('LICENSE',),
+    license_files=('LICENSE',),
     classifiers=[
-        'Development Status :: 2 - Pre-Alpha'
+        'Development Status :: 2 - Pre-Alpha',
         'Intended Audience :: Developers',
         'Intended Audience :: Science/Research',
         'Intended Audience :: Telecommunications Industry',
         'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
         'Environment :: Console',
         'Natural Language :: English',
-        'Operating System :: POSIX :: Linux',        
+        'Operating System :: POSIX :: Linux',
         'Operating System :: Microsoft :: Windows :: Windows 7',
         'Operating System :: Microsoft :: Windows :: Windows 8',
         'Operating System :: Microsoft :: Windows :: Windows 8.1',

src/dsp/dsp_processor.py

@@ -30,29 +30,24 @@
 
 
 class DspProcessor(DataProcessor):
+    _FILTER_DEGREE = 4
+
     def __init__(self,
-                 fs: str,
-                 decimation: str,
-                 centerFreq: str,
+                 fs: int,
+                 decimation: int,
+                 centerFreq: float,
                  omegaOut: int,
                  demod: str = None,
-                 tunedFreq: str = None,
+                 tunedFreq: int = None,
                  vfos: str = None,
                  normalize: bool = False):
-        try:
-            fs = int(fs)
-            centerFreq = float(centerFreq)
-            tunedFreq = float(tunedFreq) if tunedFreq is not None else None
-            decimation = int(decimation) if decimation is not None else 1
-        except (ValueError, TypeError) as e:
-            raise ValueError(e)
 
-        self._FILTER_DEGREE = 8
+        decimation = decimation if decimation is not None else 1
         self.outputFilters = []
         self.sosIn = None
         self.fs = fs
         self.decimationFactor = decimation if decimation is not None and decimation > 0 else (
-                    np.floor(np.log2(fs / 1000)) - 8)
+                np.floor(np.log2(fs / 1000)) - 8)
         self.logDecimationFactor = self.decimationFactor
         self.decimatedFs = fs >> int(
             np.round(self.decimationFactor)) if self.decimationFactor > 0 else fs
@@ -62,8 +57,7 @@ def __init__(self,
         self.demod = demod if demod is not None else realDemod
         self.bandwidth = None
         self.tunedFreq = tunedFreq
-        self.vfos = [float(x) for x in vfos.split(',')] if not (
-                    vfos is None or len(vfos) < 1) else None
+        self.vfos = vfos
         self.normalize = normalize
         self.omegaOut = omegaOut
         eprint(
@@ -92,6 +86,7 @@ def selectOuputFm(self):
         eprint('NFM Selected')
         self.bandwidth = 12500
         self.sosIn = generateFmInputFilters(self.decimatedFs, self._FILTER_DEGREE, self.bandwidth)
+        # self.outputFilters = [signal.ellip(self._FILTER_DEGREE, 1, 30, self.omegaOut,
         self.outputFilters = [signal.butter(self._FILTER_DEGREE, self.omegaOut,
                                             btype='lowpass',
                                             analog=False,
@@ -115,7 +110,8 @@ def selectOuputAm(self):
         self.setDemod(amDemod)
 
     def processData(self, isDead, pipe, f) -> None:
-        if f is None or (isinstance(f, str)) and len(f) < 1:
+        if f is None or (isinstance(f, str)) and len(f) < 1 \
+                or self.demod is None:
             raise ValueError('f is not defined')
         reader, writer = pipe
         normalize = cnormalize if self.normalize else lambda x: x
@@ -124,6 +120,8 @@ def processData(self, isDead, pipe, f) -> None:
                 while not isDead.value:
                     writer.close()
                     y = reader.recv()
+                    if y is None or len(y) < 1:
+                        break
                     y = deinterleave(y)
                     y = convertDeinterlRealToComplex(y)
                     y = normalize(y)
@@ -134,8 +132,9 @@ def processData(self, isDead, pipe, f) -> None:
                     y = applyFilters(y, self.outputFilters)
                     file.write(struct.pack(len(y) * 'd', *y))
             except (EOFError, KeyboardInterrupt):
-                pass
+                isDead.value = 1
             except Exception as e:
+                isDead.value = 1
                 printException(e)
             finally:
                 file.write(b'')

src/dsp/util.py

@@ -53,6 +53,7 @@ def generateDeemphFilter(fs: Real, f=7.5e-5) -> np.ndarray[any, np.number]:
 
 
 def generateInputFilters(fs: float, deg: int, omega: Number | np.number):
+    # return signal.ellip(deg, 1, 30, [1, omega],
     return signal.butter(deg, [1, omega],
                          btype='bandpass',
                          analog=False,
@@ -69,6 +70,7 @@ def generateAmInputFilters(fs: float, deg: int, omega: Number | np.number = 1000
 
 
 def generateBroadcastOutputFilter(fs: float, deg: int, omega=18000):
+    # return signal.ellip(deg, 1, 30, omega,
     return signal.butter(deg, omega,
                          btype='lowpass',
                          analog=False,
@@ -98,3 +100,35 @@ def cnormalize(Z: np.ndarray[any, np.complex_]) -> np.ndarray[any, np.complex_]:
     ix = np.isnan(ret[:, ])
     ret[ix] = ret[np.ix_(ix)].all(0)
     return ret
+
+
+def rms(inp):
+    def func(a):
+        return np.sqrt(-np.square(np.sum(a)) + len(a) * np.sum(a * a)) / len(a)
+
+    ret = np.apply_along_axis(func, -1, inp)
+    return ret
+
+
+def generateDomain(dataType: str):
+    xmin, xmax = None, None
+    match dataType:
+        case 'B':
+            xmin, xmax = 0, 255
+        case 'b':
+            xmin, xmax = -128, 127
+        case 'H':
+            xmin, xmax = 0, 65536
+        case 'h':
+            xmin, xmax = -32768, 32767
+        case 'I':
+            xmin, xmax = 0, 4294967295
+        case 'i':
+            xmin, xmax = -2147483648, 2147483647
+        case 'L':
+            xmin, xmax = 0, 18446744073709551615
+        case 'l':
+            xmin, xmax = -9223372036854775808, 9223372036854775807
+        case _:
+            pass
+    return xmin, xmax

src/dsp/vfo_processor.py

@@ -0,0 +1,64 @@
+import os
+import struct
+from multiprocessing import Pool
+
+from scipy import signal
+
+from dsp.dsp_processor import DspProcessor
+from dsp.util import applyFilters, cnormalize, convertDeinterlRealToComplex, shiftFreq
+from misc.general_util import deinterleave, eprint, printException
+
+
+class VfoProcessor(DspProcessor):
+
+    def handleOutput(self, file, freq, y):
+        # with open(name, 'wb') as file:
+        y = shiftFreq(y, freq, self.decimatedFs)
+        y = signal.sosfilt(self.sosIn, y)
+        y = self.demod(y)
+        y = applyFilters(y, self.outputFilters)
+        return os.write(file, struct.pack(len(y) * 'd', *y))
+
+    def processData(self, isDead, pipe, f) -> None:
+        if f is None or (isinstance(f, str)) and len(f) < 1 \
+                or self.demod is None:
+            raise ValueError('f is not defined')
+        reader, writer = pipe
+        normalize = cnormalize if self.normalize else lambda x: x
+
+        namedPipes = []
+        for i in range(len(self.vfos)):
+            # uid = uuid.uuid4()
+            name = "/tmp/pipe-" + str(i)  # str(uid)
+            os.mkfifo(name)
+            namedPipes.append((name, open(name, 'wb', os.O_WRONLY | os.O_NONBLOCK)))
+
+        with Pool(processes=len(self.vfos) - 1) as pool:
+            try:
+                while not isDead.value:
+                    writer.close()
+                    y = reader.recv()
+                    if y is None or len(y) < 1:
+                        break
+                    y = deinterleave(y)
+                    y = convertDeinterlRealToComplex(y)
+                    y = normalize(y)
+                    y = shiftFreq(y, self.centerFreq, self.fs)
+                    y = signal.decimate(y, self.decimationFactor, ftype='fir')
+                    results = [pool.apply_async(self.handleOutput, (file.fileno(), freq, y)) for (name, file), freq in zip(namedPipes, self.vfos)]
+                    # y = signal.sosfilt(self.sosIn, y)
+                    # y = self.demod(y)
+                    # y = applyFilters(y, self.outputFilters)
+                    # file.write(struct.pack(len(y) * 'd', *y))
+                    results = [r.get() for r in results]
+
+            except (EOFError, KeyboardInterrupt):
+                isDead.value = 1
+            except Exception as e:
+                printException(e)
+            finally:
+                for n, fd in namedPipes:
+                    os.close(fd.fileno())
+                    os.unlink(n)
+                reader.close()
+                eprint(f'File writer halted')

src/misc/file_util.py

@@ -45,6 +45,9 @@
 
 
 def parseRawType(bits, enc, fs):
+    if fs is None or fs < 1:
+        raise ValueError('fs is required for raw input')
+    fs = int(fs)
     if bits is None and enc is None:
         result = zipRet((0, 0, 0, fs, 0, 0, (3, 'd')))
         result['dataOffset'] = 0

src/misc/read_file.py

@@ -1,44 +1,35 @@
+import mmap
+import os
 import struct
+import sys
+from functools import partial
 from multiprocessing import Pipe, Process, Value
 from uuid import UUID
 
-from misc.general_util import eprint, printException
+from misc.general_util import applyIgnoreException, eprint, printException
 
 
-# def generateDomain(dataType: str):
-#     xmin, xmax = None, None
-#     match dataType:
-#         case 'B':
-#             xmin, xmax = 0, 255
-#         case 'b':
-#             xmin, xmax = -128, 127
-#         case 'H':
-#             xmin, xmax = 0, 65536
-#         case 'h':
-#             xmin, xmax = -32768, 32767
-#         case 'I':
-#             xmin, xmax = 0, 4294967295
-#         case 'i':
-#             xmin, xmax = -2147483648, 2147483647
-#         case 'L':
-#             xmin, xmax = 0, 18446744073709551615
-#         case 'l':
-#             xmin, xmax = -9223372036854775808, 9223372036854775807
-#         case _:
-#             pass
-#     return xmin, xmax
-
-
-def readFile(wordtype, isDead: Value, pipes: dict[UUID, Pipe],
+def readFile(wordtype, fullFileRead: bool, isDead: Value, pipes: dict[UUID, Pipe],
              processes: dict[UUID, Process], f: str, offset=0, readSize=8192):
     MIN_BUF_SIZE = 16
     bitdepth, structtype = wordtype
     readSize <<= bitdepth
-    with open(f, 'rb') as file:
+    with open(f, 'rb') as ff:
+        if sys.stdin.fileno() == ff.fileno() or not fullFileRead:
+            file = ff
+        else:
+            eprint('Reading full file to memory')
+            if os.name != 'POSIX':
+                file = mmap.mmap(ff.fileno(), 0, access=mmap.ACCESS_READ)
+            else:
+                file = mmap.mmap(ff.fileno(), 0, prot=mmap.PROT_READ)
+                file.madvise(mmap.MADV_SEQUENTIAL)
+            eprint(f'Read: {file.size()} bytes')
+
         if offset:
             file.seek(offset)  # skip the wav header(s)
+
         try:
-            # data = []
             data = bytearray()
             while not isDead.value:
                 for _ in range(MIN_BUF_SIZE):
@@ -67,5 +58,9 @@ def readFile(wordtype, isDead: Value, pipes: dict[UUID, Pipe],
         except Exception as e:
             printException(e)
         finally:
+            file.close()
+            for (uuid, (r, w)) in list(pipes.items()):
+                applyIgnoreException(partial(w.send, b''))
+                w.close()
             isDead.value = 1
             eprint(f'Reader halted')

src/plots/abstract_plot.py

@@ -49,10 +49,13 @@ def __init__(self, **kwargs):
             'centerFreq'] is not None else 0
         self.bandwidth = (kwargs['bandwidth'] if 'bandwidth' in kwargs.keys() and kwargs[
             'bandwidth'] is not None else self.fs) / 2
-        self.tunedFreq = kwargs['tunedFreq'] if 'tunedFreq' in kwargs.keys() and kwargs['tunedFreq'] is not None else 0
+        self.tunedFreq = kwargs['tunedFreq'] if 'tunedFreq' in kwargs.keys() and kwargs[
+            'tunedFreq'] is not None else 0
         self.uuid = uuid4()
+        self.vfos = kwargs['vfos'] if 'vfos' in kwargs.keys() and len(kwargs['vfos']) > 1 else None
         self.close = None
-        self.iq = IQCorrection(self.fs)
+        self.correctIq = kwargs['iq']
+        self.iqCorrector = IQCorrection(self.fs) if self.correctIq else None
 
     @abstractmethod
     def animate(self, y: list | np.ndarray):
@@ -85,18 +88,25 @@ def processData(self, isDead, pipe, _=None) -> None:
         try:
             while not (self._isDead or isDead.value):
                 writer.close()
-                y = np.array(reader.recv())
+                y = reader.recv()
+                if y is None or len(y) < 1:
+                    break
+                y = np.array(y)
                 y = y[0::2] + 1j * y[1::2]
-                y = self.iq.correctIq(y)
+                if self.correctIq:
+                    y = self.iqCorrector.correctIq(y)
                 y = shiftFreq(y, self.centerFreq, self.fs)
                 self.animate(y)
-        except (EOFError, KeyboardInterrupt):
+        except EOFError:
             pass
+        except KeyboardInterrupt:
+            isDead.value = 1
         except Exception as e:
+            isDead.value = 1
             printException(e)
         finally:
-            self._isDead = True
-            reader.close()
             if 'spawn' not in multiprocessing.get_start_method():
                 isDead.value = 1
+            self._isDead = True
+            reader.close()
             eprint(f'Figure {type(self).__name__}-{self.uuid} halted')