From 02cb6f2595b03a46cd26e6fa9fd84c5dc4092942 Mon Sep 17 00:00:00 2001 From: Evan Goetz <32753745+eagoetz@users.noreply.github.com> Date: Thu, 25 Jan 2024 13:36:21 -0800 Subject: [PATCH] Fix for gaps in segments causing unfilled pie chart plots (#392) * Fix for gaps in segments causing unfilled pie chart plots * Updates to improve the PR and a bugfix - Better define whether running before, during, or after the span for SegmentPiePlot - suptitle of the plot is only over the full span if 'include_future' is provided, otherwise based on before, during, or after the span of interest - better handle undefined / unknown times - gwsumm.segments.get_segments() now has optional ignore_undefined argument by default False, but if True, as is used in NetworkDutyPiePlot, this prevents an undefined segment time from impacting the network segments - gwsumm.segments.get_segments() improved reading segments from global memory and taking intersection (this also fixes an incorrect code comment claiming incorrectly that the & operator is a union) - fix a bug that was using an iterating variable - some code cleanup, more documentation and comments * Minor comment update and fix percentage of no detector when not running future data --------- Co-authored-by: Evan Goetz --- gwsumm/plot/segments.py | 121 ++++++++++++++++++++++++++++++++++------ gwsumm/segments.py | 29 ++++++---- 2 files changed, 124 insertions(+), 26 deletions(-) diff --git a/gwsumm/plot/segments.py b/gwsumm/plot/segments.py index ba1172e1..81af207c 100644 --- a/gwsumm/plot/segments.py +++ b/gwsumm/plot/segments.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (C) Duncan Macleod (2013) +# Evan Goetz (2023) # # This file is part of GWSumm. # @@ -1006,15 +1007,54 @@ def draw(self, outputfile=None): wedgeargs = self.parse_wedge_kwargs() plotargs = self.parse_plot_kwargs() + # Boolean logic flags to determine if this code is currently running: + # - before the span of interest (current time the code is running < + # start of span) + # - after the span of interest (current time the code is running >= end + # of span) + # - during the span of interest (any other time) + # The flag is set to True for the appropriate noun (before, during, or + # after). + # These flags are then used to set plot titles and labels and determine + # if there is any missing data. + before = during = after = False + if globalv.NOW < int(self.span[0]): + before = True + elif globalv.NOW >= int(self.span[1]): + after = True + else: + during = True + # use state to generate suptitle with GPS span + # this will be different depending on if `include_future` is given + # as an option or whether running before, during, or after the time + # interval requested if self.state: - self.pargs.setdefault( - 'suptitle', - '[%s-%s, state: %s]' % (self.span[0], self.span[1], - texify(str(self.state)))) + if future or after: + self.pargs.setdefault( + 'suptitle', + (f'[{self.span[0]}-{self.span[1]}, ' + f'state: {texify(str(self.state))}]')) + elif before: + self.pargs.setdefault( + 'suptitle', + (f'[{self.span[0]}-{self.span[0]}, ' + f'state: {texify(str(self.state))}]')) + else: + self.pargs.setdefault( + 'suptitle', + (f'[{self.span[0]}-{globalv.NOW}, ' + f'state: {texify(str(self.state))}]')) else: - self.pargs.setdefault( - 'suptitle', '[%s-%s]' % (self.span[0], self.span[1])) + if future or after: + self.pargs.setdefault( + 'suptitle', f'[{self.span[0]}-{self.span[1]}]') + elif before: + self.pargs.setdefault( + 'suptitle', f'[{self.span[0]}-{self.span[0]}]') + else: + self.pargs.setdefault( + 'suptitle', f'[{self.span[0]}-{globalv.NOW}]') # get segments data = [] @@ -1028,12 +1068,39 @@ def draw(self, outputfile=None): padding=self.padding).coalesce() data.append(float(abs(segs.active))) - # handle missing or future data + # handle missing (undefined) segments + # if running before then all the time is future because segments + # haven't been generated + # if running after then some segments may not cover the whole time + # if during, it is somewhere in between total = float(sum(data)) - if future or (total < alltime): - data.append(alltime - total) + undefined = future_seg = 0 + if before: + future_seg = alltime + elif after: + undefined = alltime - total + elif during: + future_seg = int(self.span[1]) - globalv.NOW + undefined = alltime - future_seg - total + current_total = globalv.NOW - int(self.span[0]) + + # figure out the extra pieces to include in the pie chart and labels + # TODO: There is something messed up about "labels" and + # "label" that should be cleaned up + if undefined > 0: + data.append(undefined) + if 'labels' in plotargs: + plotargs['labels'] = list(plotargs['labels']) + ['Undefined'] + elif 'label' in plotargs: + plotargs['label'] = list(plotargs['label']) + ['Undefined'] + if 'colors' in plotargs: + plotargs['colors'] = list(plotargs['colors']) + ['black'] + if future or before: + data.append(future_seg) if 'labels' in plotargs: plotargs['labels'] = list(plotargs['labels']) + [' '] + elif 'label' in plotargs: + plotargs['label'] = list(plotargs['label']) + [' '] if 'colors' in plotargs: plotargs['colors'] = list(plotargs['colors']) + ['white'] @@ -1058,7 +1125,12 @@ def draw(self, outputfile=None): pclabels.append(label) else: try: - pc = d / (total if future else alltime) * 100 + if future or after: + pc = d / alltime * 100 + elif during: + pc = d / current_total * 100 + else: + pc = 0.0 except ZeroDivisionError: pc = 0.0 pclabels.append(texify( @@ -1150,28 +1222,45 @@ def draw(self): # construct compound flags for each network size flags = dict((f[:2], f) for f in self.flags) network = ''.join(sorted(set(flags))) - self.pargs.setdefault('title', '%s network duty factor' % network) + self.pargs.setdefault('title', f'{network} network duty factor') networkflags = [] colors = [] labels = [] + # define an exclude DQ flag so that each subsequent time through + # We exclude triple time from double time and double time from single + # time exclude = DataQualityFlag() for i in list(range(len(flags)+1))[::-1]: name = self.NETWORK_NAME[i] - flag = '%s:%s' % (network, name) + flag = f'{network}:{name}' networksegs = DataQualityFlag(flag, known=valid) + # loop over the possible combinations inserting the flag to the + # network segments dictionary for ifoset in combinations(flags, i): if not ifoset: - compound = '!%s' % '!'.join(list(flags.values())) + compound = f"!{'!'.join(list(flags.values()))}" else: compound = '&'.join(flags[ifo] for ifo in ifoset) - segs = get_segments(compound, validity=valid, query=False, - padding=self.padding).coalesce() + segs = get_segments( + compound, validity=valid, query=False, + padding=self.padding, ignore_undefined=True).coalesce() networksegs += segs + # Final step in the loop for no detectors: if not wanting to plot + # future times, then exclude the time from now to the end of the + # span from the no detector network + if (i == 0 and + not self.pargs.get('include_future', False) and + globalv.NOW < self.span[1]): + exclude.active += SegmentList( + [Segment(globalv.NOW, self.span[1])]) + # insert this flag into the segments global variable and exclude + # any of the previous network (more detectors) time from this time globalv.SEGMENTS[flag] = networksegs.copy() globalv.SEGMENTS[flag].active -= exclude.active + # update the segements of times to exclude exclude = networksegs networkflags.append(flag) - labels.append('%s interferometer' % name.title()) + labels.append(f'{name.title()} interferometer') colors.append(self.NETWORK_COLOR.get(name)) self.pargs.setdefault('colors', colors) diff --git a/gwsumm/segments.py b/gwsumm/segments.py index 6aa419a0..b9f66aa1 100644 --- a/gwsumm/segments.py +++ b/gwsumm/segments.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (C) Duncan Macleod (2013) +# Evan Goetz (2023) # # This file is part of GWSumm. # @@ -22,7 +23,6 @@ import sys import operator import warnings -from functools import reduce from collections import OrderedDict from configparser import ( DEFAULTSECT, @@ -34,7 +34,7 @@ from astropy.io.registry import IORegistryError from gwpy.segments import (DataQualityFlag, DataQualityDict, - SegmentList, Segment) + SegmentListDict, SegmentList, Segment) from . import globalv from .utils import ( @@ -56,7 +56,8 @@ def get_segments(flag, validity=None, config=ConfigParser(), cache=None, query=True, return_=True, coalesce=True, padding=None, - segdb_error='raise', url=None, **read_kw): + ignore_undefined=False, segdb_error='raise', url=None, + **read_kw): """Retrieve the segments for a given flag Segments will be loaded from global memory if already defined, @@ -97,6 +98,11 @@ def get_segments(flag, validity=None, config=ConfigParser(), cache=None, `(start, end)` padding with which to pad segments that are downloaded/read + ignore_undefined : `bool`, optional, default: `False` + Special case needed for network calculation compound flags so that + when this is True, DataQualityFlag.known values are set to the same + value as ``validity`` + segdb_error : `str`, optional, default: ``'raise'`` how to handle errors returned from the segment database, one of @@ -168,11 +174,12 @@ def get_segments(flag, validity=None, config=ConfigParser(), cache=None, for f in allflags: globalv.SEGMENTS.setdefault(f, DataQualityFlag(f)) - # read segments from global memory and get the union of needed times + # read segments from global memory and get the intersection of needed times try: - old = reduce( - operator.and_, - (globalv.SEGMENTS.get(f, DataQualityFlag(f)).known for f in flags)) + old = SegmentListDict() + for f in flags: + old[f] = globalv.SEGMENTS.get(f, DataQualityFlag(f)).known + old = SegmentList(old.intersection(flags)) except TypeError: old = SegmentList() newsegs = validity - old @@ -260,15 +267,17 @@ def get_segments(flag, validity=None, config=ConfigParser(), cache=None, for compound in flags: union, intersection, exclude, notequal = split_compound_flag( compound) - if len(union + intersection) == 1: - out[compound].description = globalv.SEGMENTS[f].description - out[compound].padding = padding.get(f, (0, 0)) + if len(f := (union + intersection)) == 1: + out[compound].description = globalv.SEGMENTS[f[0]].description + out[compound].padding = padding.get(f[0], (0, 0)) for flist, op in zip([exclude, intersection, union, notequal], [operator.sub, operator.and_, operator.or_, not_equal]): for f in flist: pad = padding.get(f, (0, 0)) segs = globalv.SEGMENTS[f].copy() + if ignore_undefined: + segs.known = validity if isinstance(pad, (float, int)): segs = segs.pad(pad, pad) elif pad is not None: