Unify loading of Standard Morphology Files. (#2218)

All morphology loaders now return a `loaded_morphology` object
comprising
- `morphology`: the morphology corresponding to the data on disk
- `segment_tree`: the raw segment tree
- `label_dict`: all labels defined in the file
- `asc`/`swc`: `axon`, `soma`, `dend`, and `apic` based on tag 1, 2, 3,
4.
- `nml`: the union of the label dicts contained in the metadata,
prefixed by `grp:`, `nmd:`, and `seg:`.
- `metadata`: a set of items given by the fileformat
  - `swc`: nothing
  - `asc`: a list of spines and marker sets
  - `nml`: 
    - `named_segments` a label dict
    - `segment_groups` a label dict `name` -> segment ids
    - `group_segments` a map from segment id to group name(s)
    - `segments` a label dict, one entry per id
    - `cell_id` id of the cell defining this morphology, if any
    - `id` id of the morphology

Note that loading neuroml still goes via the `neuroml` object and the
`cell_morphology`/`morphology`
accessors return an optional `loaded_morphology`.

## ⚠️ Breaking ⚠️ 

- Removed the `load_*_raw` functions, as `loaded_morphology` bundles the
segment tree.
- The interface of `load_*` now has an extra indirection.
- The interface of `neuroml` changed.
- `label_dict::import` is now called `extend` for future safety (=C++
modules) and Python; calling things `import` (=a keyword) is not well
received by `black` et al.

## Issues

Closes #1981

arbor/include/arbor/morph/label_dict.hpp

@@ -24,7 +24,7 @@ class ARB_ARBOR_API label_dict {
     // construct a label dict with SWC tags predefined
     label_dict& add_swc_tags();
 
-    void import(const label_dict& other, const std::string& prefix = "");
+    label_dict& extend(const label_dict& other, const std::string& prefix = "");
 
     label_dict& set(const std::string& name, locset ls);
     label_dict& set(const std::string& name, region reg);

arbor/morph/label_dict.cpp

@@ -45,7 +45,7 @@ label_dict& label_dict::set(const std::string& name, arb::iexpr e) {
     return *this;
 }
 
-void label_dict::import(const label_dict& other, const std::string& prefix) {
+label_dict& label_dict::extend(const label_dict& other, const std::string& prefix) {
     for (const auto& entry: other.locsets()) {
         set(prefix+entry.first, entry.second);
     }
@@ -55,6 +55,7 @@ void label_dict::import(const label_dict& other, const std::string& prefix) {
     for (const auto& entry: other.iexpressions()) {
         set(prefix+entry.first, entry.second);
     }
+    return *this;
 }
 
 std::optional<region> label_dict::region(const std::string& name) const {

arborio/include/arborio/loaded_morphology.hpp

@@ -0,0 +1,76 @@
+#pragma once
+
+#include <variant>
+
+#include <arborio/export.hpp>
+
+#include <arbor/morph/label_dict.hpp>
+#include <arbor/morph/morphology.hpp>
+#include <arbor/morph/segment_tree.hpp>
+
+namespace arborio {
+
+struct ARB_ARBORIO_API swc_metadata {};
+
+struct ARB_ARBORIO_API asc_color {
+    uint8_t r = 0;
+    uint8_t g = 0;
+    uint8_t b = 0;
+};
+
+struct ARB_ARBORIO_API asc_spine {
+    std::string name;
+    arb::mpoint location;
+};
+
+enum ARB_ARBORIO_API asc_marker { dot, circle, cross, none };
+
+struct ARB_ARBORIO_API asc_marker_set {
+    asc_color color;
+    asc_marker marker = asc_marker::none;
+    std::string name;
+    std::vector<arb::mpoint> locations;
+};
+
+struct ARB_ARBORIO_API asc_metadata {
+    std::vector<asc_marker_set> markers;
+    std::vector<asc_spine> spines;
+};
+
+// Bundle some detailed metadata for neuroml ingestion.
+struct ARB_SYMBOL_VISIBLE nml_metadata {
+    // Cell id, or empty if morphology was taken from a top-level <morphology> element.
+    std::optional<std::string> cell_id;
+
+    // Morphology id.
+    std::string id;
+
+    // One region expression for each segment id.
+    arb::label_dict segments;
+
+    // One region expression for each name applied to one or more segments.
+    arb::label_dict named_segments;
+
+    // One region expression for each segmentGroup id.
+    arb::label_dict groups;
+
+    // Map from segmentGroup ids to their corresponding segment ids.
+    std::unordered_map<std::string, std::vector<unsigned long long>> group_segments;
+};
+
+// Interface for ingesting morphology data
+struct ARB_ARBORIO_API loaded_morphology {
+    // Raw segment tree, identical to morphology.
+    arb::segment_tree segment_tree;
+
+    // Morphology constructed from description.
+    arb::morphology morphology;
+
+    // Regions and locsets defined in the description.
+    arb::label_dict labels;
+
+    // Loader specific metadata
+    std::variant<swc_metadata, asc_metadata, nml_metadata> metadata;
+};
+
+}

arborio/include/arborio/neurolucida.hpp

@@ -6,8 +6,8 @@
 #include <filesystem>
 
 #include <arbor/arbexcept.hpp>
-#include <arbor/morph/label_dict.hpp>
-#include <arbor/morph/morphology.hpp>
+
+#include <arborio/loaded_morphology.hpp>
 #include <arborio/export.hpp>
 
 namespace arborio {
@@ -33,23 +33,10 @@ struct ARB_SYMBOL_VISIBLE asc_unsupported: asc_exception {
     std::string message;
 };
 
-struct asc_morphology {
-    // Raw segment tree from ASC, identical to morphology.
-    arb::segment_tree segment_tree;
-
-    // Morphology constructed from asc description.
-    arb::morphology morphology;
-
-    // Regions and locsets defined in the asc description.
-    arb::label_dict labels;
-};
-
 // Perform the parsing of the input as a string.
-ARB_ARBORIO_API asc_morphology parse_asc_string(const char* input);
-ARB_ARBORIO_API arb::segment_tree parse_asc_string_raw(const char* input);
+ARB_ARBORIO_API loaded_morphology parse_asc_string(const char* input);
 
 // Load asc morphology from file with name filename.
-ARB_ARBORIO_API asc_morphology load_asc(const std::filesystem::path& filename);
-ARB_ARBORIO_API arb::segment_tree load_asc_raw(const std::filesystem::path&filename);
+ARB_ARBORIO_API loaded_morphology load_asc(const std::filesystem::path& filename);
 
 } // namespace arborio

arborio/include/arborio/neuroml.hpp

@@ -8,6 +8,7 @@
 #include <unordered_map>
 #include <vector>
 
+#include <arborio/loaded_morphology.hpp>
 #include <arbor/morph/label_dict.hpp>
 #include <arbor/morph/morphology.hpp>
 #include <arborio/export.hpp>
@@ -69,29 +70,6 @@ struct ARB_SYMBOL_VISIBLE nml_cyclic_dependency: neuroml_exception {
 // Note: segment id values are interpreted as unsigned long long values;
 // parsing larger segment ids will throw an exception.
 
-struct nml_morphology_data {
-    // Cell id, or empty if morphology was taken from a top-level <morphology> element.
-    std::optional<std::string> cell_id;
-
-    // Morphology id.
-    std::string id;
-
-    // Morphology constructed from a single NeuroML <morphology> element.
-    arb::morphology morphology;
-
-    // One region expression for each segment id.
-    arb::label_dict segments;
-
-    // One region expression for each name applied to one or more segments.
-    arb::label_dict named_segments;
-
-    // One region expression for each segmentGroup id.
-    arb::label_dict groups;
-
-    // Map from segmentGroup ids to their corresponding segment ids.
-    std::unordered_map<std::string, std::vector<unsigned long long>> group_segments;
-};
-
 // Represent NeuroML data determined by provided string.
 
 struct ARB_ARBORIO_API neuroml_impl;
@@ -126,8 +104,8 @@ struct ARB_ARBORIO_API neuroml {
     // Parse and retrieve top-level morphology or morphology associated with a cell.
     // Return nullopt if not found.
 
-    std::optional<nml_morphology_data> morphology(const std::string& morph_id, enum neuroml_options::values = neuroml_options::none) const;
-    std::optional<nml_morphology_data> cell_morphology(const std::string& cell_id, enum neuroml_options::values = neuroml_options::none) const;
+    std::optional<loaded_morphology> morphology(const std::string& morph_id, enum neuroml_options::values = neuroml_options::none) const;
+    std::optional<loaded_morphology> cell_morphology(const std::string& cell_id, enum neuroml_options::values = neuroml_options::none) const;
 
     ~neuroml();
 

arborio/include/arborio/swcio.hpp

@@ -3,9 +3,11 @@
 #include <iostream>
 #include <string>
 #include <vector>
+#include <filesystem>
 
 #include <arbor/arbexcept.hpp>
-#include <arbor/morph/morphology.hpp>
+
+#include <arborio/loaded_morphology.hpp>
 #include <arborio/export.hpp>
 
 namespace arborio {
@@ -114,7 +116,6 @@ struct ARB_ARBORIO_API swc_data {
 // conditions above are encountered.
 //
 // SWC records are returned in id order.
-
 ARB_ARBORIO_API swc_data parse_swc(std::istream&);
 ARB_ARBORIO_API swc_data parse_swc(const std::string&);
 
@@ -126,17 +127,15 @@ ARB_ARBORIO_API swc_data parse_swc(const std::string&);
 // one segment for each SWC record after the first: this record defines the tag
 // and distal point of the segment, while the proximal point is taken from the
 // parent record.
-
-ARB_ARBORIO_API arb::morphology load_swc_arbor(const swc_data& data);
-ARB_ARBORIO_API arb::segment_tree load_swc_arbor_raw(const swc_data& data);
+ARB_ARBORIO_API loaded_morphology load_swc_arbor(const swc_data& data);
+ARB_ARBORIO_API loaded_morphology load_swc_arbor(const std::filesystem::path& fn);
 
 // As above, will convert a valid, ordered sequence of SWC records into a morphology
 //
 // Note that 'one-point soma' SWC files are supported here
 //
 // Complies inferred SWC rules from NEURON, explicitly listed in the docs.
-
-ARB_ARBORIO_API arb::morphology load_swc_neuron(const swc_data& data);
-ARB_ARBORIO_API arb::segment_tree load_swc_neuron_raw(const swc_data& data);
+ARB_ARBORIO_API loaded_morphology load_swc_neuron(const swc_data& data);
+ARB_ARBORIO_API loaded_morphology load_swc_neuron(const std::filesystem::path& fn);
 
 } // namespace arborio