Add DataStream

include/iact_data/acada_data_source.hpp

@@ -51,33 +51,36 @@ class ACADACameraEventDataSource:
   virtual void release_borrowed_event(const EventMessage* event) = 0;
 };
 
-template<typename EventMessage, typename HeaderMessage>
+template<typename EventMessage, typename HeaderMessage, typename DataStreamMessage = void>
 class ACADACameraEventDataSourceWithRunHeader:
   public virtual ACADACameraEventDataSource<EventMessage>
 {
 public:
   CALIN_TYPEALIAS(event_type, EventMessage);
   CALIN_TYPEALIAS(header_type, HeaderMessage);
+  CALIN_TYPEALIAS(data_stream_type, DataStreamMessage);
 
   ACADACameraEventDataSourceWithRunHeader(): 
       ACADACameraEventDataSource<EventMessage>() {
     /* nothing to see here */ }
   virtual ~ACADACameraEventDataSourceWithRunHeader();
   virtual header_type* get_run_header() = 0;
+  virtual data_stream_type* get_data_stream() = 0;
 };
 
-template<typename EventMessage, typename HeaderMessage>
+template<typename EventMessage, typename HeaderMessage, typename DataStreamMessage = void>
 class ACADACameraEventRandomAccessDataSourceWithRunHeader:
   public virtual calin::io::data_source::RandomAccessDataSource<EventMessage>,
-  public virtual ACADACameraEventDataSourceWithRunHeader<EventMessage,HeaderMessage>
+  public virtual ACADACameraEventDataSourceWithRunHeader<EventMessage,HeaderMessage,DataStreamMessage>
 {
 public:
   CALIN_TYPEALIAS(event_type, EventMessage);
   CALIN_TYPEALIAS(header_type, HeaderMessage);
+  CALIN_TYPEALIAS(data_stream_type, DataStreamMessage);
 
   ACADACameraEventRandomAccessDataSourceWithRunHeader(): 
       calin::io::data_source::RandomAccessDataSource<EventMessage>(), 
-      ACADACameraEventDataSourceWithRunHeader<EventMessage,HeaderMessage>() {
+      ACADACameraEventDataSourceWithRunHeader<EventMessage,HeaderMessage,DataStreamMessage>() {
     /* nothing to see here */ }
   virtual ~ACADACameraEventRandomAccessDataSourceWithRunHeader();
 };

include/iact_data/acada_event_decoder.hpp

@@ -44,12 +44,13 @@ calin::ix::iact_data::telescope_event::TriggerType determine_trigger_type(
     const calin::ix::iact_data::telescope_event::TIBData* calin_tib_data,
     const calin::ix::iact_data::telescope_event::CDTSData* calin_cdts_data);
 
-template<typename EventMessage, typename HeaderMessage>
+template<typename EventMessage, typename HeaderMessage, typename DataStreamMessage = void>
 class ACADACameraEventDecoder
 {
 public:
   CALIN_TYPEALIAS(event_type, EventMessage);
   CALIN_TYPEALIAS(header_type, HeaderMessage);
+  CALIN_TYPEALIAS(data_stream_type, DataStreamMessage);
 
   virtual ~ACADACameraEventDecoder();
   virtual bool decode(
@@ -59,7 +60,8 @@ class ACADACameraEventDecoder
     calin::ix::iact_data::telescope_run_configuration::
       TelescopeRunConfiguration* run_config,
     const HeaderMessage* cta_run_header,
-    const EventMessage* cta_event) = 0;
+    const EventMessage* cta_event,
+    const DataStreamMessage* cta_data_stream = nullptr) = 0;
   virtual ACADACameraEventDecoder* clone() const = 0;
 };
 

include/iact_data/nectarcam_acada_event_decoder.hpp

@@ -62,6 +62,7 @@ class NectarCam_ACADACameraEventDecoder_L0:
 
   using calin::iact_data::acada_event_decoder::ACADACameraEventDecoder_L0::event_type;
   using calin::iact_data::acada_event_decoder::ACADACameraEventDecoder_L0::header_type;
+  using calin::iact_data::acada_event_decoder::ACADACameraEventDecoder_L0::data_stream_type;
 
   NectarCam_ACADACameraEventDecoder_L0(const std::string& filename, unsigned run_number = 0,
     const calin::ix::iact_data::nectarcam_data_source::NectarCamCameraEventDecoderConfig& config = default_config());
@@ -81,9 +82,9 @@ class NectarCam_ACADACameraEventDecoder_L0:
     const event_type* cta_event) override;
 
   bool decode_run_config(
-    calin::ix::iact_data::telescope_run_configuration::
-      TelescopeRunConfiguration* run_config,
-    const header_type* cta_run_header, const event_type* cta_event) override;
+    calin::ix::iact_data::telescope_run_configuration:: TelescopeRunConfiguration* run_config,
+    const header_type* cta_run_header, const event_type* cta_event, 
+    const data_stream_type* cta_data_stream = nullptr) override;
 
   NectarCam_ACADACameraEventDecoder_L0* clone() const override;
 
@@ -141,6 +142,7 @@ class NectarCam_ACADACameraEventDecoder_R1v0:
 
   using calin::iact_data::acada_event_decoder::ACADACameraEventDecoder_R1v0::event_type;
   using calin::iact_data::acada_event_decoder::ACADACameraEventDecoder_R1v0::header_type;
+  using calin::iact_data::acada_event_decoder::ACADACameraEventDecoder_R1v0::data_stream_type;
 
   NectarCam_ACADACameraEventDecoder_R1v0(const std::string& filename, unsigned run_number = 0,
     const config_type& config = default_config());
@@ -151,10 +153,10 @@ class NectarCam_ACADACameraEventDecoder_R1v0:
     calin::ix::iact_data::telescope_event::TelescopeEvent* event,
     const event_type* cta_event) override;
 
-  virtual bool decode_run_config(
-    calin::ix::iact_data::telescope_run_configuration::
-      TelescopeRunConfiguration* run_config,
-    const header_type* cta_run_header, const event_type* cta_event) override;
+  bool decode_run_config(
+    calin::ix::iact_data::telescope_run_configuration:: TelescopeRunConfiguration* run_config,
+    const header_type* cta_run_header, const event_type* cta_event, 
+    const data_stream_type* cta_data_stream = nullptr) override;
 
   NectarCam_ACADACameraEventDecoder_R1v0* clone() const override;
 

include/iact_data/zfits_acada_data_source.hpp

@@ -43,13 +43,19 @@ class ZFITSSingleFileSingleMessageDataSource:
 public:
   CALIN_TYPEALIAS(message_type, Message);
 
-  ZFITSSingleFileSingleMessageDataSource(const std::string& filename, const std::string& tablename = {});
+  ZFITSSingleFileSingleMessageDataSource(const std::string& filename, const std::string& tablename = {},
+    bool suppress_file_record = false);
   virtual ~ZFITSSingleFileSingleMessageDataSource();
 
+  const message_type* borrow_next_message(uint64_t& seq_index_out);
+  void release_borrowed_message(const message_type* message);
+
   message_type* get_next(uint64_t& seq_index_out,
     google::protobuf::Arena** arena = nullptr) override;
+
   uint64_t size() override;
   void set_next_index(uint64_t next_index) override;
+  uint64_t get_next_index() const { return next_message_index_; };
 
 private:
   std::string filename_;
@@ -59,16 +65,17 @@ class ZFITSSingleFileSingleMessageDataSource:
   uint64_t next_message_index_ = 0;
 };
 
-template<typename EventMessage, typename HeaderMessage>
+template<typename EventMessage, typename HeaderMessage, typename DataStreamMessage = void>
 class ZFITSSingleFileACADACameraEventDataSource:
   public calin::iact_data::acada_data_source::
-    ACADACameraEventRandomAccessDataSourceWithRunHeader<EventMessage,HeaderMessage>
+    ACADACameraEventRandomAccessDataSourceWithRunHeader<EventMessage,HeaderMessage,DataStreamMessage>
 {
 public:
   CALIN_TYPEALIAS(config_type,
     calin::ix::iact_data::zfits_data_source::ZFITSDataSourceConfig);
   CALIN_TYPEALIAS(event_type, EventMessage);
   CALIN_TYPEALIAS(header_type, HeaderMessage);
+  CALIN_TYPEALIAS(data_stream_type, DataStreamMessage);
 
   ZFITSSingleFileACADACameraEventDataSource(const std::string& filename,
     const config_type& config = default_config());
@@ -83,39 +90,41 @@ class ZFITSSingleFileACADACameraEventDataSource:
   void set_next_index(uint64_t next_index) override;
 
   header_type* get_run_header() override;
+  data_stream_type* get_data_stream() override;
 
   static config_type default_config();
   const config_type& config() const { return config_; }
 
 private:
   std::string filename_;
-  ADH::IO::ProtobufIFits* zfits_ = nullptr;
-  calin::ix::provenance::chronicle::FileIORecord* file_record_ = nullptr;
-  uint64_t next_event_index_ = 0;
+  ZFITSSingleFileSingleMessageDataSource<event_type>* zfits_;
   header_type* run_header_ = nullptr;
+  data_stream_type* data_stream_ = nullptr;
   config_type config_;
 };
 
-template<typename EventMessage, typename HeaderMessage>
+template<typename EventMessage, typename HeaderMessage, typename DataStreamMessage = void>
 class ZFITSACADACameraEventDataSource:
   public calin::io::data_source::BasicChainedRandomAccessDataSource<
     calin::iact_data::acada_data_source::
-      ACADACameraEventRandomAccessDataSourceWithRunHeader<EventMessage,HeaderMessage> >
+      ACADACameraEventRandomAccessDataSourceWithRunHeader<EventMessage,HeaderMessage,DataStreamMessage> >
 {
 public:
   CALIN_TYPEALIAS(config_type,
     calin::ix::iact_data::zfits_data_source::ZFITSDataSourceConfig);
   CALIN_TYPEALIAS(event_type, EventMessage);
   CALIN_TYPEALIAS(header_type, HeaderMessage);
+  CALIN_TYPEALIAS(data_stream_type, DataStreamMessage);
   CALIN_TYPEALIAS(BaseDataSource, calin::io::data_source::BasicChainedRandomAccessDataSource<
     calin::iact_data::acada_data_source::
-      ACADACameraEventRandomAccessDataSourceWithRunHeader<EventMessage,HeaderMessage> >);
+      ACADACameraEventRandomAccessDataSourceWithRunHeader<EventMessage,HeaderMessage,DataStreamMessage> >);
 
   ZFITSACADACameraEventDataSource(const std::string& filename,
     const config_type& config = default_config());
   virtual ~ZFITSACADACameraEventDataSource();
 
   header_type* get_run_header() override;
+  data_stream_type* get_data_stream() override;
 
   const event_type* borrow_next_event(uint64_t& seq_index_out) override;
   void release_borrowed_event(const event_type* event) override;
@@ -138,26 +147,28 @@ class ZFITSACADACameraEventDataSource:
 private:
   config_type config_;
   header_type* run_header_ = nullptr;
+  data_stream_type* data_stream_ = nullptr;
 };
 
-template<typename EventMessage, typename HeaderMessage>
+template<typename EventMessage, typename HeaderMessage, typename DataStreamMessage = void>
 class ZFITSACADACameraEventDataSourceOpener:
   public calin::io::data_source::DataSourceOpener<
     calin::iact_data::acada_data_source::
-      ACADACameraEventRandomAccessDataSourceWithRunHeader<EventMessage,HeaderMessage> >
+      ACADACameraEventRandomAccessDataSourceWithRunHeader<EventMessage,HeaderMessage,DataStreamMessage> >
 {
 public:
   CALIN_TYPEALIAS(config_type,
     calin::ix::iact_data::zfits_data_source::ZFITSDataSourceConfig);
   CALIN_TYPEALIAS(event_type, EventMessage);
   CALIN_TYPEALIAS(header_type, HeaderMessage);
+  CALIN_TYPEALIAS(data_stream_type, DataStreamMessage);
 
   ZFITSACADACameraEventDataSourceOpener(std::string filename,
     const config_type& config = default_config());
   virtual ~ZFITSACADACameraEventDataSourceOpener();
   unsigned num_sources() const override;
   std::string source_name(unsigned isource) const override;
-  ZFITSSingleFileACADACameraEventDataSource<EventMessage,HeaderMessage>* open(unsigned isource) override;
+  ZFITSSingleFileACADACameraEventDataSource<EventMessage,HeaderMessage,DataStreamMessage>* open(unsigned isource) override;
   bool has_opened_file() { return has_opened_file_; }
 
   static config_type default_config();

include/iact_data/zfits_data_source.hpp

@@ -39,6 +39,7 @@
 
 namespace calin { namespace iact_data { namespace zfits_data_source {
 
+#if 0 // UNUSED
 template<typename EventMessage, typename HeaderMessage>
 class ZFITSSingleFileDataSource:
   public calin::iact_data::telescope_data_source::
@@ -85,8 +86,10 @@ class ZFITSSingleFileDataSource:
   calin::ix::iact_data::telescope_run_configuration::
     TelescopeRunConfiguration* run_config_ = nullptr;
 };
+#endif // UNUSED
 
-template<typename EventMessage, typename HeaderMessage>
+
+template<typename EventMessage, typename HeaderMessage, typename DataStreamMessage = void>
 class ZFITSDataSource:
   public calin::iact_data::telescope_data_source::
     TelescopeRandomAccessDataSourceWithRunConfig,
@@ -98,12 +101,12 @@ class ZFITSDataSource:
 
   ZFITSDataSource(const std::string& filename,
     calin::iact_data::acada_event_decoder::
-      ACADACameraEventDecoder<EventMessage,HeaderMessage>* decoder,
+      ACADACameraEventDecoder<EventMessage,HeaderMessage,DataStreamMessage>* decoder,
     bool adopt_decoder = false,
     const config_type& config = default_config());
 
   ZFITSDataSource(const std::string& filename,
-    ZFITSDataSource<EventMessage,HeaderMessage>* base_datasource, 
+    ZFITSDataSource<EventMessage,HeaderMessage,DataStreamMessage>* base_datasource, 
     const config_type& config = default_config());
 
   virtual ~ZFITSDataSource();
@@ -126,10 +129,10 @@ class ZFITSDataSource:
 protected:
 
   calin::iact_data::acada_event_decoder::
-    ACADACameraEventDecoder<EventMessage,HeaderMessage>* decoder_;
+    ACADACameraEventDecoder<EventMessage,HeaderMessage,DataStreamMessage>* decoder_;
   bool adopt_decoder_ = false;
   calin::iact_data::zfits_acada_data_source::
-    ZFITSACADACameraEventDataSource<EventMessage,HeaderMessage>* acada_zfits_ = nullptr;
+    ZFITSACADACameraEventDataSource<EventMessage,HeaderMessage,DataStreamMessage>* acada_zfits_ = nullptr;
   bool adopt_acada_zfits_ = false;
   calin::ix::iact_data::telescope_run_configuration::
     TelescopeRunConfiguration* run_config_ = nullptr;

proto/iact_data/zfits_data_source.proto

@@ -54,18 +54,13 @@ message ZFITSDataSourceConfig {
     (CFO).desc = "Maximum event sequence number. This effectively sets the "
       "number of events that can be read. Zero means there is no limit." ];
   bool dont_read_run_header                                = 11 [
-    (CFO).desc = "Don't read the run header." ];
+    (CFO).desc = "Don't read the run header or data stream." ];
   bool ignore_run_header_errors                            = 12 [
-    (CFO).desc = "Don't log missing run header." ];
-  bool verify_file_after_open                              = 20 [
-    (CFO).desc = "Verify ZFits file consistency after opening it. This may "
-      "be slow." ];
-  bool repair_broken_file                                  = 21 [
-    (CFO).desc = "Attempt to repair any files that fail the verification step. "
-      "If this is selected it implicitly means \"verify_file_after_open\" is "
-      "also." ];
-  string run_header_table_name                             = 22 [
+    (CFO).desc = "Don't log missing run header or data stream." ];
+  string data_stream_table_name                            = 20 [
+    (CFO).desc = "Name of the telescope data stream table in the ZFits file" ];
+  string run_header_table_name                             = 21 [
     (CFO).desc = "Name of the run header table in the ZFits file" ];
-  string events_table_name                                 = 23 [
+  string events_table_name                                 = 22 [
     (CFO).desc = "Name of the events table in the ZFits file" ];
 };

src/iact_data/acada_data_source.cpp

@@ -31,15 +31,15 @@ ACADACameraEventDataSource<EventMessage>::
   // nothing to see here
 }
 
-template<typename EventMessage, typename HeaderMessage>
-ACADACameraEventDataSourceWithRunHeader<EventMessage,HeaderMessage>::
+template<typename EventMessage, typename HeaderMessage, typename DataStreamMessage>
+ACADACameraEventDataSourceWithRunHeader<EventMessage,HeaderMessage,DataStreamMessage>::
 ~ACADACameraEventDataSourceWithRunHeader()
 {
   // nothing to see here
 }
 
-template<typename EventMessage, typename HeaderMessage>
-ACADACameraEventRandomAccessDataSourceWithRunHeader<EventMessage,HeaderMessage>::
+template<typename EventMessage, typename HeaderMessage, typename DataStreamMessage>
+ACADACameraEventRandomAccessDataSourceWithRunHeader<EventMessage,HeaderMessage,DataStreamMessage>::
 ~ACADACameraEventRandomAccessDataSourceWithRunHeader()
 {
   // nothing to see here
@@ -56,7 +56,9 @@ template class ACADACameraEventDataSourceWithRunHeader<ACADA_EventMessage_R1v0,
 template class ACADACameraEventRandomAccessDataSourceWithRunHeader<ACADA_EventMessage_R1v0, ACADA_HeaderMessage_R1v0>;
 
 template class ACADACameraEventDataSource<ACADA_EventMessage_R1v1>;
-template class ACADACameraEventDataSourceWithRunHeader<ACADA_EventMessage_R1v1, ACADA_HeaderMessage_R1v1>;
-template class ACADACameraEventRandomAccessDataSourceWithRunHeader<ACADA_EventMessage_R1v1, ACADA_HeaderMessage_R1v1>;
+template class ACADACameraEventDataSourceWithRunHeader<
+  ACADA_EventMessage_R1v1, ACADA_HeaderMessage_R1v1, ACADA_DataStreamMessage_R1v1>;
+template class ACADACameraEventRandomAccessDataSourceWithRunHeader<
+  ACADA_EventMessage_R1v1, ACADA_HeaderMessage_R1v1, ACADA_DataStreamMessage_R1v1>;
 
 } } } // namespace calin::iact_data::acada_data_source

src/iact_data/acada_event_decoder.cpp

@@ -37,8 +37,9 @@ using calin::util::file::is_file;
 using calin::util::file::is_readable;
 using calin::util::file::expand_filename;
 
-template<typename EventMessage, typename HeaderMessage>
-ACADACameraEventDecoder<EventMessage, HeaderMessage>::~ACADACameraEventDecoder()
+template<typename EventMessage, typename HeaderMessage, typename DataStreamMessage>
+ACADACameraEventDecoder<EventMessage, HeaderMessage, DataStreamMessage>::
+~ACADACameraEventDecoder()
 {
   // nothing to see here
 }

src/iact_data/nectarcam_acada_event_decoder_l0.cpp

@@ -439,10 +439,10 @@ bool NectarCam_ACADACameraEventDecoder_L0::decode(
 }
 
 bool NectarCam_ACADACameraEventDecoder_L0::decode_run_config(
-  calin::ix::iact_data::telescope_run_configuration::
-    TelescopeRunConfiguration* calin_run_config,
+  calin::ix::iact_data::telescope_run_configuration::TelescopeRunConfiguration* calin_run_config,
   const header_type* cta_run_header,
-  const event_type* cta_event)
+  const event_type* cta_event,
+  const data_stream_type* cta_data_stream)
 {
   calin_run_config->set_data_transcoder(
     "calin::iact_data::nectarcam_acada_event_decoder::NectarCam_ACADACameraEventDecoder_L0");

src/iact_data/nectarcam_acada_event_decoder_r1v0.cpp

@@ -549,9 +549,10 @@ bool NectarCam_ACADACameraEventDecoder_R1v0::decode(
 }
 
 bool NectarCam_ACADACameraEventDecoder_R1v0::decode_run_config(
-  calin::ix::iact_data::telescope_run_configuration::
-    TelescopeRunConfiguration* calin_run_config,
-  const header_type* cta_run_header, const event_type* cta_event)
+  calin::ix::iact_data::telescope_run_configuration::TelescopeRunConfiguration* calin_run_config,
+  const header_type* cta_run_header, 
+  const event_type* cta_event, 
+  const data_stream_type* cta_data_stream)
 {
   calin_run_config->set_data_transcoder(
     "calin::iact_data::nectarcam_acada_event_decoder::NectarCam_ACADACameraEventDecoder_R1v0");