From b3c540d1089d62481d65af0844fe999a958f5c7f Mon Sep 17 00:00:00 2001 From: Winston Olson-Duvall Date: Thu, 22 Jul 2021 11:32:19 -0700 Subject: [PATCH] Update sort_packets to actually sort packets where a PSC rollover occurs. --- emit_pcap_to_hosc_raw.py | 4 +++ setup.py | 36 ++++++++++++++++++++ sort_packets.py | 65 +++++++++++++++++++++++++++---------- util/insert_psc_rollover.py | 58 +++++++++++++++++++++++++++++++++ util/reverse_packets.py | 5 +-- 5 files changed, 147 insertions(+), 21 deletions(-) create mode 100644 setup.py create mode 100644 util/insert_psc_rollover.py diff --git a/emit_pcap_to_hosc_raw.py b/emit_pcap_to_hosc_raw.py index 5eba730..6658319 100644 --- a/emit_pcap_to_hosc_raw.py +++ b/emit_pcap_to_hosc_raw.py @@ -66,9 +66,12 @@ def main(): # Our ethernet frames only contain a single CCSDS Packet ... pkt = dp.CCSDSPacket(stream=in_bytes) + # Get course time and convert to UTC + # TODO: Use leapseconds.dat or other common config file to determine leap seconds course_time = int.from_bytes(pkt.body[:4], "big") utc_time = time.gmtime(course_time + GPS_UTC_DELTA - LEAP_SECONDS) + # Write out HOSC files separately for engineering stream (APID 1674) and science stream (APID 1675) if pkt.apid == 1674: if pkt_cnt_1674 == 0: start_time_1674 = utc_time @@ -88,6 +91,7 @@ def main(): outfile_1675.write(pkt.body) pkt_cnt_1675 += 1 + # Rename output files using format "emit_____hsc.bin" current_utc_time = datetime.datetime.utcnow() renamed_1674 = out_file_1674.replace("hsc.bin", "_".join([time.strftime("%y%m%d%H%M%S", start_time_1674), time.strftime("%y%m%d%H%M%S", stop_time_1674), diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..1977273 --- /dev/null +++ b/setup.py @@ -0,0 +1,36 @@ +""" +This is the setup file for managing the emit_sds_l0 package + +Author: Winston Olson-Duvall, winston.olson-duvall@jpl.nasa.gov +""" + +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name="emit_sds_l0", + version="0.0.0", + author="Winston Olson-Duvall", + author_email="winston.olson-duvall@jpl.nasa.gov", + description=""" + L0 scripts and PGEs for EMIT data processing, including pcap->hosc, hosc->ccsds, and helper utilities. + """, + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.jpl.nasa.gov/emit-sds/emit-sds-l0", + packages=setuptools.find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + ], + python_requires=">=3", + install_requires=[ + "sortedcontainers>=2.4.0", + "pytest>=6.2.1", + "pytest-cov>=2.10.1", + "pycodestyle>=2.6.0" + ] +) diff --git a/sort_packets.py b/sort_packets.py index e4f06bb..e3709bf 100644 --- a/sort_packets.py +++ b/sort_packets.py @@ -1,20 +1,29 @@ #!/usr/bin/env python import argparse -from collections import OrderedDict from sortedcontainers import SortedDict import emit.data_products as dp HOSC_HEADER_SIZE = 28 +HOSC_HEADER = bytes(28) +MAX_PSC = 790 -def sort_pscs(psc_dict): - sorted = OrderedDict() - for k in sorted(psc_dict.keys()): - sorted[k] = psc_dict[k] - print(sorted) - +def sort_packets_by_psc(packets): + # This function sorts packets in a given course_time + fine_time group where PSCs have rolled over + # due to hitting the MAX_PSC limit. This condition is met when the min and max PSCs in a particular + # group differ by more than 256 (an arbitrary number, but so far, I haven't seen more than 8 PSCs + # in a given group). + for psc, pkt in packets.items(): + if psc < 256: + print(f"Removing psc {psc} and inserting as {psc + MAX_PSC}") + packet = packets.pop(psc) + # Reinsert the popped packet with a PSC that will allow it to be sorted correctly. Note that this doesn't + # impact the underlying data, just the ordering for writing out the data. The end result should be + # properly sorted packets with the PSC rollover accounted for. + packets[psc + MAX_PSC] = packet + return packets def main(): @@ -31,22 +40,40 @@ def main(): pkts = SortedDict() while True: try: + # TODO: Preserve hosc_hdr in output file hosc_hdr = in_file.read(HOSC_HEADER_SIZE) pkt = dp.CCSDSPacket(in_file) - print(pkt) + # print(pkt) + + # Get course time, fine, time and psc to sort packets course_time = int.from_bytes(pkt.body[:4], "big") fine_time = pkt.body[4] psc = pkt.pkt_seq_cnt print(f"") + + # Create a time_key index by combining padded course and fine times. This allows for sorting at the level + # of course and fine time. PSCs will be further sorted after accounting for PSC rollover time_key = str(course_time).zfill(10) + str(fine_time).zfill(3) # print(f"time_key: {time_key}") + + # Insert packets into SortedDict by time_key. Use a sub-dictionary to track PSC max and min values within + # a given time_key index. Also use another SortedDict to store packets indexed by PSC. The + # "sort_packets_by_psc" function sorts packets in cases where PSC rollovers occur for a given time_key index if time_key in pkts: - # print(psc) - pkts[time_key][psc] = pkt - # sort_pscs(pkts[time_key]) + min_psc = min(psc, pkts[time_key]["min"]) + max_psc = max(psc, pkts[time_key]["max"]) + pkts[time_key]["min"] = min_psc + pkts[time_key]["max"] = max_psc + pkts[time_key]["pkts"][psc] = pkt + if max_psc - min_psc > 256: + pkts[time_key]["pkts"] = sort_packets_by_psc(pkts[time_key]["pkts"]) else: - pkts[time_key] = SortedDict() - pkts[time_key][psc] = pkt + pkts[time_key] = { + "min": psc, + "max": psc, + "pkts": SortedDict() + } + pkts[time_key]["pkts"][psc] = pkt cnt += 1 except EOFError: @@ -54,10 +81,14 @@ def main(): print(f"Count: {cnt}") - # Iterate over pkts - for time, pscs in pkts.items(): - for psc in pscs.keys(): - print(f"{time}: {psc}") + # Iterate over pkts and its sub-dictionary and write out packets in sorted order + with open(args.input_file.replace(".bin", "_sorted.bin"), "wb") as outfile: + for time, packets in pkts.items(): + for psc, pkt in packets["pkts"].items(): + print(f"{time}: {psc}") + outfile.write(HOSC_HEADER) + outfile.write(pkt.hdr_data) + outfile.write(pkt.body) if __name__ == "__main__": diff --git a/util/insert_psc_rollover.py b/util/insert_psc_rollover.py new file mode 100644 index 0000000..258dfbd --- /dev/null +++ b/util/insert_psc_rollover.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +import argparse + +import emit.data_products as dp + +HOSC_HEADER_SIZE = 28 +MAX_PSC = 790 + + +def main(): + + parser = argparse.ArgumentParser( + description="Artificially insert a PSC rollover in a HOSC stream for testing purposes" + ) + parser.add_argument('input_file') + args = parser.parse_args() + + in_file = open(args.input_file, 'rb') + outfile = open(args.input_file.replace(".bin", "_rollover.bin"), "wb") + + cnt = 0 + while True: + try: + # Read past HOSC header + hosc_hdr = in_file.read(HOSC_HEADER_SIZE) + + # Read in packet and copy header to update PSCs + pkt = dp.CCSDSPacket(in_file) + hdr = bytearray(pkt.hdr_data) + + print(f"PSC before: {pkt.pkt_seq_cnt}") + pkt.pkt_seq_cnt = pkt.pkt_seq_cnt % MAX_PSC + print(f"PSC after: {pkt.pkt_seq_cnt}") + + # Update PSC bytes in header + psc_bytes = pkt.pkt_seq_cnt.to_bytes(2, byteorder="big", signed=False) + print("psc: " + str([bin(psc_bytes[i])[2:].zfill(8) for i in range(2)])) + # Reset 14 bits of PSC to 0 + hdr[2] = hdr[2] & 0xC0 + hdr[3] = 0x00 + # Add new PSC (preserving the leftmost 2 bits) + hdr[2] = hdr[2] | psc_bytes[0] + hdr[3] = hdr[3] | psc_bytes[1] + + # Write out file + outfile.write(hosc_hdr) + outfile.write(hdr) + outfile.write(pkt.body) + cnt += 1 + except EOFError: + break + + print(f"Count: {cnt}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/util/reverse_packets.py b/util/reverse_packets.py index 0b7eae0..b77f6b4 100644 --- a/util/reverse_packets.py +++ b/util/reverse_packets.py @@ -1,14 +1,11 @@ #!/usr/bin/env python import argparse -from collections import OrderedDict -from sortedcontainers import SortedDict import emit.data_products as dp -HOSC_HEADER = bytes(28) HOSC_HEADER_SIZE = 28 - +HOSC_HEADER = bytes(HOSC_HEADER_SIZE) def main():