Update sort_packets to actually sort packets where a PSC rollover occ…

…urs.

emit_pcap_to_hosc_raw.py

@@ -66,9 +66,12 @@ def main():
                 # Our ethernet frames only contain a single CCSDS Packet ...
                 pkt = dp.CCSDSPacket(stream=in_bytes)
 
+                # Get course time and convert to UTC
+                # TODO: Use leapseconds.dat or other common config file to determine leap seconds
                 course_time = int.from_bytes(pkt.body[:4], "big")
                 utc_time = time.gmtime(course_time + GPS_UTC_DELTA - LEAP_SECONDS)
 
+                # Write out HOSC files separately for engineering stream (APID 1674) and science stream (APID 1675)
                 if pkt.apid == 1674:
                     if pkt_cnt_1674 == 0:
                         start_time_1674 = utc_time
@@ -88,6 +91,7 @@ def main():
                     outfile_1675.write(pkt.body)
                     pkt_cnt_1675 += 1
 
+    # Rename output files using format "emit_<APID>_<START_TIME>_<STOP_TIME>_<CUR_TIME>_hsc.bin"
     current_utc_time = datetime.datetime.utcnow()
     renamed_1674 = out_file_1674.replace("hsc.bin", "_".join([time.strftime("%y%m%d%H%M%S", start_time_1674),
                                                               time.strftime("%y%m%d%H%M%S", stop_time_1674),

setup.py

@@ -0,0 +1,36 @@
+"""
+This is the setup file for managing the emit_sds_l0 package
+
+Author: Winston Olson-Duvall, [email protected]
+"""
+
+import setuptools
+
+with open("README.md", "r") as fh:
+    long_description = fh.read()
+
+setuptools.setup(
+    name="emit_sds_l0",
+    version="0.0.0",
+    author="Winston Olson-Duvall",
+    author_email="[email protected]",
+    description="""
+        L0 scripts and PGEs for EMIT data processing, including pcap->hosc, hosc->ccsds, and helper utilities.
+        """,
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.jpl.nasa.gov/emit-sds/emit-sds-l0",
+    packages=setuptools.find_packages(),
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: Apache Software License",
+        "Operating System :: OS Independent",
+    ],
+    python_requires=">=3",
+    install_requires=[
+        "sortedcontainers>=2.4.0",
+        "pytest>=6.2.1",
+        "pytest-cov>=2.10.1",
+        "pycodestyle>=2.6.0"
+    ]
+)

sort_packets.py

@@ -1,20 +1,29 @@
 #!/usr/bin/env python
 
 import argparse
-from collections import OrderedDict
 from sortedcontainers import SortedDict
 
 import emit.data_products as dp
 
 HOSC_HEADER_SIZE = 28
+HOSC_HEADER = bytes(28)
+MAX_PSC = 790
 
 
-def sort_pscs(psc_dict):
-    sorted = OrderedDict()
-    for k in sorted(psc_dict.keys()):
-        sorted[k] = psc_dict[k]
-    print(sorted)
-
+def sort_packets_by_psc(packets):
+    # This function sorts packets in a given course_time + fine_time group where PSCs have rolled over
+    # due to hitting the MAX_PSC limit.  This condition is met when the min and max PSCs in a particular
+    # group differ by more than 256 (an arbitrary number, but so far, I haven't seen more than 8 PSCs
+    # in a given group).
+    for psc, pkt in packets.items():
+        if psc < 256:
+            print(f"Removing psc {psc} and inserting as {psc + MAX_PSC}")
+            packet = packets.pop(psc)
+            # Reinsert the popped packet with a PSC that will allow it to be sorted correctly.  Note that this doesn't
+            # impact the underlying data, just the ordering for writing out the data. The end result should be
+            # properly sorted packets with the PSC rollover accounted for.
+            packets[psc + MAX_PSC] = packet
+    return packets
 
 
 def main():
@@ -31,33 +40,55 @@ def main():
     pkts = SortedDict()
     while True:
         try:
+            # TODO: Preserve hosc_hdr in output file
             hosc_hdr = in_file.read(HOSC_HEADER_SIZE)
             pkt = dp.CCSDSPacket(in_file)
-            print(pkt)
+            # print(pkt)
+
+            # Get course time, fine, time and psc to sort packets
             course_time = int.from_bytes(pkt.body[:4], "big")
             fine_time = pkt.body[4]
             psc = pkt.pkt_seq_cnt
             print(f"<course={course_time} fine={fine_time} psc={psc}>")
+
+            # Create a time_key index by combining padded course and fine times. This allows for sorting at the level
+            # of course and fine time.  PSCs will be further sorted after accounting for PSC rollover
             time_key = str(course_time).zfill(10) + str(fine_time).zfill(3)
             # print(f"time_key: {time_key}")
+
+            # Insert packets into SortedDict by time_key.  Use a sub-dictionary to track PSC max and min values within
+            # a given time_key index. Also use another SortedDict to store packets indexed by PSC.  The
+            # "sort_packets_by_psc" function sorts packets in cases where PSC rollovers occur for a given time_key index
             if time_key in pkts:
-                # print(psc)
-                pkts[time_key][psc] = pkt
-                # sort_pscs(pkts[time_key])
+                min_psc = min(psc, pkts[time_key]["min"])
+                max_psc = max(psc, pkts[time_key]["max"])
+                pkts[time_key]["min"] = min_psc
+                pkts[time_key]["max"] = max_psc
+                pkts[time_key]["pkts"][psc] = pkt
+                if max_psc - min_psc > 256:
+                    pkts[time_key]["pkts"] = sort_packets_by_psc(pkts[time_key]["pkts"])
             else:
-                pkts[time_key] = SortedDict()
-                pkts[time_key][psc] = pkt
+                pkts[time_key] = {
+                    "min": psc,
+                    "max": psc,
+                    "pkts": SortedDict()
+                }
+                pkts[time_key]["pkts"][psc] = pkt
 
             cnt += 1
         except EOFError:
             break
 
     print(f"Count: {cnt}")
 
-    # Iterate over pkts
-    for time, pscs in pkts.items():
-        for psc in pscs.keys():
-            print(f"{time}: {psc}")
+    # Iterate over pkts and its sub-dictionary and write out packets in sorted order
+    with open(args.input_file.replace(".bin", "_sorted.bin"), "wb") as outfile:
+        for time, packets in pkts.items():
+            for psc, pkt in packets["pkts"].items():
+                print(f"{time}: {psc}")
+                outfile.write(HOSC_HEADER)
+                outfile.write(pkt.hdr_data)
+                outfile.write(pkt.body)
 
 
 if __name__ == "__main__":

util/insert_psc_rollover.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+
+import argparse
+
+import emit.data_products as dp
+
+HOSC_HEADER_SIZE = 28
+MAX_PSC = 790
+
+
+def main():
+
+    parser = argparse.ArgumentParser(
+        description="Artificially insert a PSC rollover in a HOSC stream for testing purposes"
+    )
+    parser.add_argument('input_file')
+    args = parser.parse_args()
+
+    in_file = open(args.input_file, 'rb')
+    outfile = open(args.input_file.replace(".bin", "_rollover.bin"), "wb")
+
+    cnt = 0
+    while True:
+        try:
+            # Read past HOSC header
+            hosc_hdr = in_file.read(HOSC_HEADER_SIZE)
+
+            # Read in packet and copy header to update PSCs
+            pkt = dp.CCSDSPacket(in_file)
+            hdr = bytearray(pkt.hdr_data)
+
+            print(f"PSC before: {pkt.pkt_seq_cnt}")
+            pkt.pkt_seq_cnt = pkt.pkt_seq_cnt % MAX_PSC
+            print(f"PSC after: {pkt.pkt_seq_cnt}")
+
+            # Update PSC bytes in header
+            psc_bytes = pkt.pkt_seq_cnt.to_bytes(2, byteorder="big", signed=False)
+            print("psc: " + str([bin(psc_bytes[i])[2:].zfill(8) for i in range(2)]))
+            # Reset 14 bits of PSC to 0
+            hdr[2] = hdr[2] & 0xC0
+            hdr[3] = 0x00
+            # Add new PSC (preserving the leftmost 2 bits)
+            hdr[2] = hdr[2] | psc_bytes[0]
+            hdr[3] = hdr[3] | psc_bytes[1]
+
+            # Write out file
+            outfile.write(hosc_hdr)
+            outfile.write(hdr)
+            outfile.write(pkt.body)
+            cnt += 1
+        except EOFError:
+            break
+
+    print(f"Count: {cnt}")
+
+
+if __name__ == "__main__":
+    main()

util/reverse_packets.py

@@ -1,14 +1,11 @@
 #!/usr/bin/env python
 
 import argparse
-from collections import OrderedDict
-from sortedcontainers import SortedDict
 
 import emit.data_products as dp
 
-HOSC_HEADER = bytes(28)
 HOSC_HEADER_SIZE = 28
-
+HOSC_HEADER = bytes(HOSC_HEADER_SIZE)
 
 def main():