User documentation: Generate docs from macros, makeDocumentationBuilder::build infallable

Cargo.toml

@@ -52,6 +52,8 @@ members = [
     "datafusion-examples/examples/ffi/ffi_module_loader",
     "test-utils",
     "benchmarks",
+    "datafusion/macros",
+    "datafusion/doc",
 ]
 resolver = "2"
 
@@ -100,6 +102,7 @@ datafusion = { path = "datafusion/core", version = "43.0.0", default-features =
 datafusion-catalog = { path = "datafusion/catalog", version = "43.0.0" }
 datafusion-common = { path = "datafusion/common", version = "43.0.0", default-features = false }
 datafusion-common-runtime = { path = "datafusion/common-runtime", version = "43.0.0" }
+datafusion-doc = { path = "datafusion/doc", version = "43.0.0" }
 datafusion-execution = { path = "datafusion/execution", version = "43.0.0" }
 datafusion-expr = { path = "datafusion/expr", version = "43.0.0" }
 datafusion-expr-common = { path = "datafusion/expr-common", version = "43.0.0" }
@@ -110,6 +113,7 @@ datafusion-functions-aggregate-common = { path = "datafusion/functions-aggregate
 datafusion-functions-nested = { path = "datafusion/functions-nested", version = "43.0.0" }
 datafusion-functions-window = { path = "datafusion/functions-window", version = "43.0.0" }
 datafusion-functions-window-common = { path = "datafusion/functions-window-common", version = "43.0.0" }
+datafusion-macros = { path = "datafusion/macros", version = "43.0.0" }
 datafusion-optimizer = { path = "datafusion/optimizer", version = "43.0.0", default-features = false }
 datafusion-physical-expr = { path = "datafusion/physical-expr", version = "43.0.0", default-features = false }
 datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "43.0.0", default-features = false }

datafusion/doc/Cargo.toml

@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "datafusion-doc"
+description = "Documentation module for DataFusion query engine"
+keywords = ["datafusion", "query", "sql"]
+version = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
+authors = { workspace = true }
+rust-version = { workspace = true }
+
+[lints]
+workspace = true
+
+[lib]
+name = "datafusion_doc"
+path = "src/lib.rs"

datafusion/expr/src/udf_docs.rs → datafusion/doc/src/lib.rs

@@ -15,15 +15,13 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use datafusion_common::exec_err;
-use datafusion_common::Result;
-
-/// Documentation for use by [`ScalarUDFImpl`](crate::ScalarUDFImpl),
-/// [`AggregateUDFImpl`](crate::AggregateUDFImpl) and [`WindowUDFImpl`](crate::WindowUDFImpl) functions
+#[allow(rustdoc::broken_intra_doc_links)]
+/// Documentation for use by [`ScalarUDFImpl`](ScalarUDFImpl),
+/// [`AggregateUDFImpl`](AggregateUDFImpl) and [`WindowUDFImpl`](WindowUDFImpl) functions
 /// that will be used to generate public documentation.
 ///
-/// The name of the udf will be pulled from the [`ScalarUDFImpl::name`](crate::ScalarUDFImpl::name),
-/// [`AggregateUDFImpl::name`](crate::AggregateUDFImpl::name) or [`WindowUDFImpl::name`](crate::WindowUDFImpl::name)
+/// The name of the udf will be pulled from the [`ScalarUDFImpl::name`](ScalarUDFImpl::name),
+/// [`AggregateUDFImpl::name`](AggregateUDFImpl::name) or [`WindowUDFImpl::name`](WindowUDFImpl::name)
 /// function as appropriate.
 ///
 /// All strings in the documentation are required to be
@@ -79,18 +77,21 @@ pub struct DocSection {
 /// Example:
 ///
 /// ```rust
-/// # use datafusion_expr::Documentation;
-/// # use datafusion_expr::scalar_doc_sections::DOC_SECTION_MATH;
-/// # use datafusion_common::Result;
-/// #
-/// # fn main() -> Result<()> {
-///       let documentation = Documentation::builder()
-///           .with_doc_section(DOC_SECTION_MATH)
+///
+/// # fn main() {
+///     use datafusion_doc::{DocSection, Documentation};
+///     let doc_section = DocSection {
+///         include: true,
+///         label: "Display Label",
+///         description: None,
+///     };
+///
+///     let documentation = Documentation::builder()
+///           .with_doc_section(doc_section)
 ///           .with_description("Add one to an int32")
 ///           .with_syntax_example("add_one(2)")
 ///           .with_argument("arg_1", "The int32 number to add one to")
-///           .build()?;
-///       Ok(())  
+///           .build();
 /// # }
 pub struct DocumentationBuilder {
     pub doc_section: Option<DocSection>,
@@ -190,7 +191,10 @@ impl DocumentationBuilder {
         self
     }
 
-    pub fn build(self) -> Result<Documentation> {
+    /// Build the documentation from provided components
+    ///
+    /// Panics if `doc_section`, `description` or `syntax_example` is not set
+    pub fn build(self) -> Documentation {
         let Self {
             doc_section,
             description,
@@ -202,24 +206,24 @@ impl DocumentationBuilder {
         } = self;
 
         if doc_section.is_none() {
-            return exec_err!("Documentation must have a doc section");
+            panic!("Documentation must have a doc section");
         }
         if description.is_none() {
-            return exec_err!("Documentation must have a description");
+            panic!("Documentation must have a description");
         }
         if syntax_example.is_none() {
-            return exec_err!("Documentation must have a syntax_example");
+            panic!("Documentation must have a syntax_example");
         }
 
-        Ok(Documentation {
+        Documentation {
             doc_section: doc_section.unwrap(),
             description: description.unwrap(),
             syntax_example: syntax_example.unwrap(),
             sql_example,
             arguments,
             alternative_syntax,
             related_udfs,
-        })
+        }
     }
 }
 

datafusion/expr/Cargo.toml

@@ -44,6 +44,7 @@ arrow-array = { workspace = true }
 arrow-buffer = { workspace = true }
 chrono = { workspace = true }
 datafusion-common = { workspace = true }
+datafusion-doc = { workspace = true }
 datafusion-expr-common = { workspace = true }
 datafusion-functions-aggregate-common = { workspace = true }
 datafusion-functions-window-common = { workspace = true }

datafusion/expr/src/lib.rs

@@ -34,7 +34,6 @@ mod partition_evaluator;
 mod table_source;
 mod udaf;
 mod udf;
-mod udf_docs;
 mod udwf;
 
 pub mod conditional_expressions;
@@ -66,6 +65,7 @@ pub mod var_provider;
 pub mod window_frame;
 pub mod window_state;
 
+pub use datafusion_doc::{DocSection, Documentation, DocumentationBuilder};
 pub use datafusion_expr_common::accumulator::Accumulator;
 pub use datafusion_expr_common::columnar_value::ColumnarValue;
 pub use datafusion_expr_common::groups_accumulator::{EmitTo, GroupsAccumulator};
@@ -93,7 +93,6 @@ pub use udaf::{
     aggregate_doc_sections, AggregateUDF, AggregateUDFImpl, ReversedUDAF, StatisticsArgs,
 };
 pub use udf::{scalar_doc_sections, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl};
-pub use udf_docs::{DocSection, Documentation, DocumentationBuilder};
 pub use udwf::{window_doc_sections, ReversedUDWF, WindowUDF, WindowUDFImpl};
 pub use window_frame::{WindowFrame, WindowFrameBound, WindowFrameUnits};
 

datafusion/expr/src/udaf.rs

@@ -339,7 +339,6 @@ where
 ///             .with_syntax_example("geo_mean(2.0)")
 ///             .with_argument("arg1", "The Float64 number for the geometric mean")
 ///             .build()
-///             .unwrap()
 ///     })
 /// }
 ///    

datafusion/expr/src/udf.rs

@@ -382,7 +382,6 @@ pub struct ScalarFunctionArgs<'a> {
 ///             .with_syntax_example("add_one(2)")
 ///             .with_argument("arg1", "The int32 number to add one to")
 ///             .build()
-///             .unwrap()
 ///     })
 /// }
 ///

datafusion/expr/src/udwf.rs

@@ -30,10 +30,10 @@ use arrow::datatypes::{DataType, Field};
 
 use crate::expr::WindowFunction;
 use crate::{
-    function::WindowFunctionSimplification, Documentation, Expr, PartitionEvaluator,
-    Signature,
+    function::WindowFunctionSimplification, Expr, PartitionEvaluator, Signature,
 };
 use datafusion_common::{not_impl_err, Result};
+use datafusion_doc::Documentation;
 use datafusion_functions_window_common::expr::ExpressionArgs;
 use datafusion_functions_window_common::field::WindowUDFFieldArgs;
 use datafusion_functions_window_common::partition::PartitionEvaluatorArgs;
@@ -264,7 +264,6 @@ where
 ///             .with_syntax_example("smooth_it(2)")
 ///             .with_argument("arg1", "The int32 number to smooth by")
 ///             .build()
-///             .unwrap()
 ///     })
 /// }
 ///
@@ -557,7 +556,7 @@ impl WindowUDFImpl for AliasedWindowUDFImpl {
 
 // Window UDF doc sections for use in public documentation
 pub mod window_doc_sections {
-    use crate::DocSection;
+    use datafusion_doc::DocSection;
 
     pub fn doc_sections() -> Vec<DocSection> {
         vec![

datafusion/functions-aggregate/src/approx_distinct.rs

@@ -334,6 +334,5 @@ fn get_approx_distinct_doc() -> &'static Documentation {
             )
             .with_standard_argument("expression", None)
             .build()
-            .unwrap()
     })
 }

datafusion/functions-aggregate/src/approx_median.rs

@@ -147,6 +147,5 @@ fn get_approx_median_doc() -> &'static Documentation {
             )
             .with_standard_argument("expression", None)
             .build()
-            .unwrap()
     })
 }

datafusion/functions-aggregate/src/approx_percentile_cont.rs

@@ -298,7 +298,6 @@ fn get_approx_percentile_cont_doc() -> &'static Documentation {
             .with_argument("percentile", "Percentile to compute. Must be a float value between 0 and 1 (inclusive).")
             .with_argument("centroids", "Number of centroids to use in the t-digest algorithm. _Default is 100_. A higher number results in more accurate approximation but requires more memory.")
             .build()
-            .unwrap()
     })
 }
 

datafusion/functions-aggregate/src/approx_percentile_cont_with_weight.rs

@@ -184,7 +184,6 @@ fn get_approx_percentile_cont_with_weight_doc() -> &'static Documentation {
             .with_argument("weight", "Expression to use as weight. Can be a constant, column, or function, and any combination of arithmetic operators.")
             .with_argument("percentile", "Percentile to compute. Must be a float value between 0 and 1 (inclusive).")
             .build()
-            .unwrap()
     })
 }
 

datafusion/functions-aggregate/src/array_agg.rs

@@ -171,7 +171,6 @@ fn get_array_agg_doc() -> &'static Documentation {
             )
             .with_standard_argument("expression", None)
             .build()
-            .unwrap()
     })
 }
 

datafusion/functions-aggregate/src/average.rs

@@ -266,7 +266,6 @@ fn get_avg_doc() -> &'static Documentation {
             )
             .with_standard_argument("expression", None)
             .build()
-            .unwrap()
     })
 }
 

datafusion/functions-aggregate/src/bit_and_or_xor.rs

@@ -145,7 +145,6 @@ fn get_bit_and_doc() -> &'static Documentation {
             .with_syntax_example("bit_and(expression)")
             .with_standard_argument("expression", Some("Integer"))
             .build()
-            .unwrap()
     })
 }
 
@@ -159,7 +158,6 @@ fn get_bit_or_doc() -> &'static Documentation {
             .with_syntax_example("bit_or(expression)")
             .with_standard_argument("expression", Some("Integer"))
             .build()
-            .unwrap()
     })
 }
 
@@ -175,7 +173,6 @@ fn get_bit_xor_doc() -> &'static Documentation {
             .with_syntax_example("bit_xor(expression)")
             .with_standard_argument("expression", Some("Integer"))
             .build()
-            .unwrap()
     })
 }
 

datafusion/functions-aggregate/src/bool_and_or.rs

@@ -204,7 +204,6 @@ fn get_bool_and_doc() -> &'static Documentation {
             )
             .with_standard_argument("expression", None)
             .build()
-            .unwrap()
     })
 }
 
@@ -353,7 +352,6 @@ fn get_bool_or_doc() -> &'static Documentation {
             )
             .with_standard_argument("expression", None)
             .build()
-            .unwrap()
     })
 }
 

datafusion/functions-aggregate/src/correlation.rs

@@ -138,7 +138,6 @@ fn get_corr_doc() -> &'static Documentation {
             .with_standard_argument("expression1", Some("First"))
             .with_standard_argument("expression2", Some("Second"))
             .build()
-            .unwrap()
     })
 }
 

datafusion/functions-aggregate/src/count.rs

@@ -359,7 +359,6 @@ fn get_count_doc() -> &'static Documentation {
 ```"#)
             .with_standard_argument("expression", None)
             .build()
-            .unwrap()
     })
 }
 

datafusion/functions-aggregate/src/covariance.rs

@@ -154,7 +154,6 @@ fn get_covar_samp_doc() -> &'static Documentation {
             .with_standard_argument("expression1", Some("First"))
             .with_standard_argument("expression2", Some("Second"))
             .build()
-            .unwrap()
     })
 }
 
@@ -252,7 +251,6 @@ fn get_covar_pop_doc() -> &'static Documentation {
             .with_standard_argument("expression1", Some("First"))
             .with_standard_argument("expression2", Some("Second"))
             .build()
-            .unwrap()
     })
 }