From 0c3d6d7bd56e1b0886297f2f61003ff05ea6f674 Mon Sep 17 00:00:00 2001 From: Evan Goetz Date: Wed, 24 Jan 2024 15:07:23 -0800 Subject: [PATCH] Updates to improve the PR and a bugfix - Better define whether running before, during, or after the span for SegmentPiePlot - suptitle of the plot is only over the full span if 'include_future' is provided, otherwise based on before, during, or after the span of interest - better handle undefined / unknown times - gwsumm.segments.get_segments() now has optional ignore_undefined argument by default False, but if True, as is used in NetworkDutyPiePlot, this prevents an undefined segment time from impacting the network segments - gwsumm.segments.get_segments() improved reading segments from global memory and taking intersection (this also fixes an incorrect code comment claiming incorrectly that the & operator is a union) - fix a bug that was using an iterating variable - some code cleanup, more documentation and comments --- gwsumm/plot/segments.py | 110 +++++++++++++++++++++++++++++----------- gwsumm/segments.py | 29 +++++++---- 2 files changed, 100 insertions(+), 39 deletions(-) diff --git a/gwsumm/plot/segments.py b/gwsumm/plot/segments.py index b5bcd14a..468c39ad 100644 --- a/gwsumm/plot/segments.py +++ b/gwsumm/plot/segments.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (C) Duncan Macleod (2013) +# Evan Goetz (2023) # # This file is part of GWSumm. # @@ -1006,15 +1007,45 @@ def draw(self, outputfile=None): wedgeargs = self.parse_wedge_kwargs() plotargs = self.parse_plot_kwargs() + # Are we currently running before, during, or after? + before = during = after = False + if globalv.NOW < int(self.span[0]): + before = True + elif globalv.NOW >= int(self.span[1]): + after = True + else: + during = True + # use state to generate suptitle with GPS span + # this will be different depending on if `include_future` is given + # as an option or whether running before, during, or after the time + # interval requested if self.state: - self.pargs.setdefault( - 'suptitle', - '[%s-%s, state: %s]' % (self.span[0], self.span[1], - texify(str(self.state)))) + if future or after: + self.pargs.setdefault( + 'suptitle', + (f'[{self.span[0]}-{self.span[1]}, ', + f'state: {texify(str(self.state))}]')) + elif before: + self.pargs.setdefault( + 'suptitle', + (f'[{self.span[0]}-{self.span[0]}, ', + f'state: {texify(str(self.state))}]')) + else: + self.pargs.setdefault( + 'suptitle', + (f'[{self.span[0]}-{globalv.NOW}, ', + f'state: {texify(str(self.state))}]')) else: - self.pargs.setdefault( - 'suptitle', '[%s-%s]' % (self.span[0], self.span[1])) + if future or after: + self.pargs.setdefault( + 'suptitle', f'[{self.span[0]}-{self.span[1]}]') + elif before: + self.pargs.setdefault( + 'suptitle', f'[{self.span[0]}-{self.span[0]}]') + else: + self.pargs.setdefault( + 'suptitle', f'[{self.span[0]}-{globalv.NOW}]') # get segments data = [] @@ -1028,28 +1059,35 @@ def draw(self, outputfile=None): padding=self.padding).coalesce() data.append(float(abs(segs.active))) - # handle missing or future data + # handle missing (undefined) segments + # if running before then all the time is future because segments + # haven't been generated + # if running after then some segments may not cover the whole time + # if during, it is somewhere in between + total = float(sum(data)) + undefined = future_seg = 0 + if before: + future_seg = alltime + elif after: + undefined = alltime - total + elif during: + future_seg = int(self.span[1]) - globalv.NOW + undefined = alltime - future_seg - total + current_total = globalv.NOW - int(self.span[0]) + + # figure out the extra pieces to include in the pie chart and labels # TODO: There is something messed up about "labels" and # "label" that should be cleaned up - total = float(sum(data)) - current_alltime = globalv.NOW - int(self.span[0]) - future_time = int(self.span[1]) - globalv.NOW - if ((future_time > 0 and total < current_alltime) or - (future_time <= 0 and total < alltime)): - if future_time > 0 and total < current_alltime: - data.append(current_alltime - total) - else: - data.append(alltime - total) + if undefined > 0: + data.append(undefined) if 'labels' in plotargs: - plotargs['labels'] = ( - list(plotargs['labels']) + ['Missing segments']) + plotargs['labels'] = list(plotargs['labels']) + ['Undefined'] elif 'label' in plotargs: - plotargs['label'] = ( - list(plotargs['label']) + ['Missing segments']) + plotargs['label'] = list(plotargs['label']) + ['Undefined'] if 'colors' in plotargs: - plotargs['colors'] = list(plotargs['colors']) + ['red'] + plotargs['colors'] = list(plotargs['colors']) + ['black'] if future: - data.append(future_time) + data.append(future_seg) if 'labels' in plotargs: plotargs['labels'] = list(plotargs['labels']) + [' '] elif 'label' in plotargs: @@ -1078,7 +1116,12 @@ def draw(self, outputfile=None): pclabels.append(label) else: try: - pc = d / (total if future else alltime) * 100 + if future or after: + pc = d / alltime * 100 + elif during: + pc = d / current_total * 100 + else: + pc = 0.0 except ZeroDivisionError: pc = 0.0 pclabels.append(texify( @@ -1170,28 +1213,37 @@ def draw(self): # construct compound flags for each network size flags = dict((f[:2], f) for f in self.flags) network = ''.join(sorted(set(flags))) - self.pargs.setdefault('title', '%s network duty factor' % network) + self.pargs.setdefault('title', f'{network} network duty factor') networkflags = [] colors = [] labels = [] + # define an exclude DQ flag so that each subsequent time through + # We exclude triple time from double time and double time from single + # time exclude = DataQualityFlag() for i in list(range(len(flags)+1))[::-1]: name = self.NETWORK_NAME[i] - flag = '%s:%s' % (network, name) + flag = f'{network}:{name}' networksegs = DataQualityFlag(flag, known=valid) + # loop over the possible combinations inserting the flag to the + # network segments dictionary for ifoset in combinations(flags, i): if not ifoset: - compound = '!%s' % '!'.join(list(flags.values())) + compound = f"!{'!'.join(list(flags.values()))}" else: compound = '&'.join(flags[ifo] for ifo in ifoset) - segs = get_segments(compound, validity=valid, query=False, - padding=self.padding).coalesce() + segs = get_segments( + compound, validity=valid, query=False, + padding=self.padding, ignore_undefined=True).coalesce() networksegs += segs + # insert this flag into the segments global variable and exclude + # any of the previous network (more detectors) time from this time globalv.SEGMENTS[flag] = networksegs.copy() globalv.SEGMENTS[flag].active -= exclude.active + # update the segements of times to exclude exclude = networksegs networkflags.append(flag) - labels.append('%s interferometer' % name.title()) + labels.append(f'{name.title()} interferometer') colors.append(self.NETWORK_COLOR.get(name)) self.pargs.setdefault('colors', colors) diff --git a/gwsumm/segments.py b/gwsumm/segments.py index 6aa419a0..b9f66aa1 100644 --- a/gwsumm/segments.py +++ b/gwsumm/segments.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (C) Duncan Macleod (2013) +# Evan Goetz (2023) # # This file is part of GWSumm. # @@ -22,7 +23,6 @@ import sys import operator import warnings -from functools import reduce from collections import OrderedDict from configparser import ( DEFAULTSECT, @@ -34,7 +34,7 @@ from astropy.io.registry import IORegistryError from gwpy.segments import (DataQualityFlag, DataQualityDict, - SegmentList, Segment) + SegmentListDict, SegmentList, Segment) from . import globalv from .utils import ( @@ -56,7 +56,8 @@ def get_segments(flag, validity=None, config=ConfigParser(), cache=None, query=True, return_=True, coalesce=True, padding=None, - segdb_error='raise', url=None, **read_kw): + ignore_undefined=False, segdb_error='raise', url=None, + **read_kw): """Retrieve the segments for a given flag Segments will be loaded from global memory if already defined, @@ -97,6 +98,11 @@ def get_segments(flag, validity=None, config=ConfigParser(), cache=None, `(start, end)` padding with which to pad segments that are downloaded/read + ignore_undefined : `bool`, optional, default: `False` + Special case needed for network calculation compound flags so that + when this is True, DataQualityFlag.known values are set to the same + value as ``validity`` + segdb_error : `str`, optional, default: ``'raise'`` how to handle errors returned from the segment database, one of @@ -168,11 +174,12 @@ def get_segments(flag, validity=None, config=ConfigParser(), cache=None, for f in allflags: globalv.SEGMENTS.setdefault(f, DataQualityFlag(f)) - # read segments from global memory and get the union of needed times + # read segments from global memory and get the intersection of needed times try: - old = reduce( - operator.and_, - (globalv.SEGMENTS.get(f, DataQualityFlag(f)).known for f in flags)) + old = SegmentListDict() + for f in flags: + old[f] = globalv.SEGMENTS.get(f, DataQualityFlag(f)).known + old = SegmentList(old.intersection(flags)) except TypeError: old = SegmentList() newsegs = validity - old @@ -260,15 +267,17 @@ def get_segments(flag, validity=None, config=ConfigParser(), cache=None, for compound in flags: union, intersection, exclude, notequal = split_compound_flag( compound) - if len(union + intersection) == 1: - out[compound].description = globalv.SEGMENTS[f].description - out[compound].padding = padding.get(f, (0, 0)) + if len(f := (union + intersection)) == 1: + out[compound].description = globalv.SEGMENTS[f[0]].description + out[compound].padding = padding.get(f[0], (0, 0)) for flist, op in zip([exclude, intersection, union, notequal], [operator.sub, operator.and_, operator.or_, not_equal]): for f in flist: pad = padding.get(f, (0, 0)) segs = globalv.SEGMENTS[f].copy() + if ignore_undefined: + segs.known = validity if isinstance(pad, (float, int)): segs = segs.pad(pad, pad) elif pad is not None: