Skip to content

Commit

Permalink
Merge branch 'main' into feat/graph-primitives
Browse files Browse the repository at this point in the history
  • Loading branch information
@grtlr
grtlr authored Nov 25, 2024
2 parents e4a6596 + d1b80ae commit 44ff86c
Show file tree
Hide file tree
Showing 248 changed files with 2,288 additions and 1,643 deletions.
19 changes: 12 additions & 7 deletions Cargo.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5933,6 +5933,7 @@ dependencies = [
"libc",
"ndarray",
"ndarray-rand",
"nohash-hasher",
"once_cell",
"parking_lot",
"rand",
Expand Down Expand Up @@ -6015,6 +6016,7 @@ name = "re_space_view"
version = "0.21.0-alpha.1+dev"
dependencies = [
"ahash",
"arrow",
"bytemuck",
"egui",
"glam",
Expand Down Expand Up @@ -6127,6 +6129,7 @@ version = "0.21.0-alpha.1+dev"
dependencies = [
"ahash",
"anyhow",
"arrow",
"bitflags 2.6.0",
"bytemuck",
"criterion",
Expand Down Expand Up @@ -6311,6 +6314,7 @@ version = "0.21.0-alpha.1+dev"
dependencies = [
"anyhow",
"array-init",
"arrow",
"bytemuck",
"document-features",
"ecolor",
Expand Down Expand Up @@ -6349,6 +6353,7 @@ dependencies = [
name = "re_types_blueprint"
version = "0.21.0-alpha.1+dev"
dependencies = [
"arrow",
"once_cell",
"re_arrow2",
"re_tracing",
Expand Down Expand Up @@ -9268,9 +9273,9 @@ checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082"

[[package]]
name = "wgpu"
version = "23.0.0"
version = "23.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76ab52f2d3d18b70d5ab8dd270a1cff3ebe6dbe4a7d13c1cc2557138a9777fdc"
checksum = "80f70000db37c469ea9d67defdc13024ddf9a5f1b89cb2941b812ad7cde1735a"
dependencies = [
"arrayvec",
"cfg_aliases 0.1.1",
Expand All @@ -9293,9 +9298,9 @@ dependencies = [

[[package]]
name = "wgpu-core"
version = "23.0.0"
version = "23.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e0c68e7b6322a03ee5b83fcd92caeac5c2a932f6457818179f4652ad2a9c065"
checksum = "d63c3c478de8e7e01786479919c8769f62a22eec16788d8c2ac77ce2c132778a"
dependencies = [
"arrayvec",
"bit-vec",
Expand All @@ -9318,9 +9323,9 @@ dependencies = [

[[package]]
name = "wgpu-hal"
version = "23.0.0"
version = "23.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de6e7266b869de56c7e3ed72a954899f71d14fec6cc81c102b7530b92947601b"
checksum = "89364b8a0b211adc7b16aeaf1bd5ad4a919c1154b44c9ce27838213ba05fd821"
dependencies = [
"android_system_properties",
"arrayvec",
Expand Down Expand Up @@ -9390,7 +9395,7 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.48.0",
]

[[package]]
Expand Down
16 changes: 8 additions & 8 deletions crates/build/re_types_builder/src/codegen/cpp/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use std::collections::HashSet;

use camino::{Utf8Path, Utf8PathBuf};
use itertools::Itertools;
use proc_macro2::{Ident, TokenStream};
use proc_macro2::{Ident, Literal, TokenStream};
use quote::{format_ident, quote};
use rayon::prelude::*;

Expand Down Expand Up @@ -1241,7 +1241,7 @@ impl QuotedObject {
let field_name = field_name_identifier(obj_field);

// We assign the arrow type index to the enum fields to make encoding simpler and faster:
let arrow_type_index = proc_macro2::Literal::usize_unsuffixed(enum_value as _);
let arrow_type_index = Literal::usize_unsuffixed(enum_value as _);

quote! {
#NEWLINE_TOKEN
Expand Down Expand Up @@ -1417,7 +1417,7 @@ fn arrow_data_type_method(
cpp_includes.insert_system("arrow/type_fwd.h");
hpp_declarations.insert("arrow", ForwardDecl::Class(format_ident!("DataType")));

let quoted_datatype = quote_arrow_data_type(
let quoted_datatype = quote_arrow_datatype(
&Type::Object(obj.fqname.clone()),
objects,
cpp_includes,
Expand Down Expand Up @@ -2234,7 +2234,7 @@ fn quote_field_type(includes: &mut Includes, obj_field: &ObjectField) -> TokenSt
Type::Array { elem_type, length } => {
includes.insert_system("array");
let elem_type = quote_element_type(includes, elem_type);
let length = proc_macro2::Literal::usize_unsuffixed(*length);
let length = Literal::usize_unsuffixed(*length);
quote! { std::array<#elem_type, #length> }
}
Type::Vector { elem_type } => {
Expand Down Expand Up @@ -2401,7 +2401,7 @@ fn quote_integer<T: std::fmt::Display>(t: T) -> TokenStream {
quote!(#t)
}

fn quote_arrow_data_type(
fn quote_arrow_datatype(
typ: &Type,
objects: &Objects,
includes: &mut Includes,
Expand Down Expand Up @@ -2444,7 +2444,7 @@ fn quote_arrow_data_type(
let quoted_fqname = quote_fqname_as_type_path(includes, fqname);
quote!(Loggable<#quoted_fqname>::arrow_datatype())
} else if obj.is_arrow_transparent() {
quote_arrow_data_type(&obj.fields[0].typ, objects, includes, false)
quote_arrow_datatype(&obj.fields[0].typ, objects, includes, false)
} else {
let quoted_fields = obj
.fields
Expand Down Expand Up @@ -2480,7 +2480,7 @@ fn quote_arrow_field_type(
includes: &mut Includes,
) -> TokenStream {
let name = &field.name;
let datatype = quote_arrow_data_type(&field.typ, objects, includes, false);
let datatype = quote_arrow_datatype(&field.typ, objects, includes, false);
let is_nullable = field.is_nullable || field.typ == Type::Unit; // null type is always nullable

quote! {
Expand All @@ -2494,7 +2494,7 @@ fn quote_arrow_elem_type(
includes: &mut Includes,
) -> TokenStream {
let typ: Type = elem_type.clone().into();
let datatype = quote_arrow_data_type(&typ, objects, includes, false);
let datatype = quote_arrow_datatype(&typ, objects, includes, false);
let is_nullable = typ == Type::Unit; // null type must be nullable
quote! {
arrow::field("item", #datatype, #is_nullable)
Expand Down
17 changes: 10 additions & 7 deletions crates/build/re_types_builder/src/codegen/rust/api.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -864,17 +864,17 @@ fn quote_trait_impls_for_datatype_or_component(
let quoted_arrow_datatype = if let Some(forwarded_type) = forwarded_type.as_ref() {
quote! {
#[inline]
fn arrow2_datatype() -> arrow2::datatypes::DataType {
#forwarded_type::arrow2_datatype()
fn arrow_datatype() -> arrow::datatypes::DataType {
#forwarded_type::arrow_datatype()
}
}
} else {
let datatype = ArrowDataTypeTokenizer(&datatype, false);
quote! {
#[inline]
fn arrow2_datatype() -> arrow2::datatypes::DataType {
fn arrow_datatype() -> arrow::datatypes::DataType {
#![allow(clippy::wildcard_imports)]
use arrow2::datatypes::*;
use arrow::datatypes::*;
#datatype
}
}
Expand Down Expand Up @@ -906,7 +906,8 @@ fn quote_trait_impls_for_datatype_or_component(
// re_tracing::profile_function!();

#![allow(clippy::wildcard_imports)]
use arrow2::{datatypes::*, array::*, buffer::*};
use arrow::datatypes::*;
use arrow2::{ array::*, buffer::*};
use ::re_types_core::{Loggable as _, ResultExt as _};

// This code-path cannot have null fields. If it does have a validity mask
Expand Down Expand Up @@ -948,7 +949,8 @@ fn quote_trait_impls_for_datatype_or_component(
// re_tracing::profile_function!();

#![allow(clippy::wildcard_imports)]
use arrow2::{datatypes::*, array::*, buffer::*};
use arrow::datatypes::*;
use arrow2::{ array::*, buffer::*};
use ::re_types_core::{Loggable as _, ResultExt as _};
Ok(#quoted_deserializer)
}
Expand Down Expand Up @@ -987,7 +989,8 @@ fn quote_trait_impls_for_datatype_or_component(

#![allow(clippy::wildcard_imports)]
#![allow(clippy::manual_is_variant_and)]
use arrow2::{datatypes::*, array::*};
use arrow::datatypes::*;
use arrow2::array::*;
use ::re_types_core::{Loggable as _, ResultExt as _};

Ok(#quoted_serializer)
Expand Down
20 changes: 14 additions & 6 deletions crates/build/re_types_builder/src/codegen/rust/arrow.rs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use arrow2::datatypes::DataType;
use proc_macro2::TokenStream;
use proc_macro2::{Literal, TokenStream};
use quote::quote;

// ---
Expand Down Expand Up @@ -37,6 +37,7 @@ impl quote::ToTokens for ArrowDataTypeTokenizer<'_> {

DataType::FixedSizeList(field, length) => {
let field = ArrowFieldTokenizer(field);
let length = Literal::usize_unsuffixed(*length);
quote!(DataType::FixedSizeList(std::sync::Arc::new(#field), #length))
}

Expand All @@ -47,25 +48,32 @@ impl quote::ToTokens for ArrowDataTypeTokenizer<'_> {
UnionMode::Sparse => quote!(UnionMode::Sparse),
};
if let Some(types) = types {
let types = types.iter().map(|&t| {
Literal::i8_unsuffixed(i8::try_from(t).unwrap_or_else(|_| {
panic!("Expect union type tag to be in 0-127; got {t}")
}))
});
quote!(DataType::Union(
std::sync::Arc::new(vec![ #(#fields,)* ]),
Some(std::sync::Arc::new(vec![ #(#types,)* ])),
UnionFields::new(
vec![ #(#types,)* ],
vec![ #(#fields,)* ],
),
#mode,
))
} else {
quote!(DataType::Union(std::sync::Arc::new(vec![ #(#fields,)* ]), None, #mode))
quote!(DataType::Union(UnionFields::from(vec![ #(#fields,)* ]), #mode))
}
}

DataType::Struct(fields) => {
let fields = fields.iter().map(ArrowFieldTokenizer);
quote!(DataType::Struct(std::sync::Arc::new(vec![ #(#fields,)* ])))
quote!(DataType::Struct(Fields::from(vec![ #(#fields,)* ])))
}

DataType::Extension(fqname, datatype, _metadata) => {
if *recursive {
let fqname_use = quote_fqname_as_type_path(fqname);
quote!(<#fqname_use>::arrow2_datatype())
quote!(<#fqname_use>::arrow_datatype())
} else {
let datatype = ArrowDataTypeTokenizer(datatype.to_logical_type(), false);
quote!(#datatype)
Expand Down
2 changes: 1 addition & 1 deletion crates/build/re_types_builder/src/codegen/rust/deserializer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ pub fn quote_arrow_deserializer(
let data_src = format_ident!("arrow_data");

let datatype = &arrow_registry.get(&obj.fqname);
let quoted_self_datatype = quote! { Self::arrow2_datatype() };
let quoted_self_datatype = quote! { Self::arrow_datatype() };

let obj_fqname = obj.fqname.as_str();
let is_enum = obj.is_enum();
Expand Down
26 changes: 13 additions & 13 deletions crates/build/re_types_builder/src/codegen/rust/serializer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ pub fn quote_arrow_serializer(
) -> TokenStream {
let datatype = &arrow_registry.get(&obj.fqname);

let quoted_datatype = quote! { Self::arrow2_datatype() };
let quoted_datatype = quote! { Self::arrow_datatype() };

let is_enum = obj.is_enum();
let is_arrow_transparent = obj.datatype.is_none();
Expand Down Expand Up @@ -211,7 +211,7 @@ pub fn quote_arrow_serializer(
#quoted_bitmap;

StructArray::new(
#quoted_datatype,
#quoted_datatype.into(),
vec![#(#quoted_field_serializers,)*],
bitmap,
).boxed()
Expand Down Expand Up @@ -257,14 +257,14 @@ pub fn quote_arrow_serializer(

let fields: Vec<_> = std::iter::repeat(
NullArray::new(
DataType::Null,
arrow2::datatypes::DataType::Null,
#data_src.len(),
).boxed()
).take(1 + num_variants) // +1 for the virtual `nulls` arm
.collect();

UnionArray::new(
#quoted_datatype,
#quoted_datatype.into(),
types,
fields,
None,
Expand All @@ -290,7 +290,7 @@ pub fn quote_arrow_serializer(
if obj_field.typ == crate::Type::Unit {
return quote! {
NullArray::new(
DataType::Null,
arrow2::datatypes::DataType::Null,
#data_src
.iter()
.filter(|datum| matches!(datum.as_deref(), Some(Self::#quoted_obj_field_name)))
Expand Down Expand Up @@ -336,7 +336,7 @@ pub fn quote_arrow_serializer(
let quoted_fields = quote! {
vec![
NullArray::new(
DataType::Null,
arrow2::datatypes::DataType::Null,
#data_src.iter().filter(|v| v.is_none()).count(),
).boxed(),
#(#quoted_field_serializers,)*
Expand Down Expand Up @@ -426,7 +426,7 @@ pub fn quote_arrow_serializer(
let offsets = Some(#quoted_offsets);

UnionArray::new(
#quoted_datatype,
#quoted_datatype.into(),
types,
fields,
offsets,
Expand Down Expand Up @@ -547,7 +547,7 @@ fn quote_arrow_field_serializer(
if datatype.to_logical_type() == &DataType::Boolean {
quote! {
BooleanArray::new(
#quoted_datatype,
#quoted_datatype.into(),
#data_src.into_iter() #quoted_transparent_mapping .collect(),
#bitmap_src,
).boxed()
Expand All @@ -558,14 +558,14 @@ fn quote_arrow_field_serializer(
// to a buffer type.
InnerRepr::ArrowBuffer => quote! {
PrimitiveArray::new(
#quoted_datatype,
#quoted_datatype.into(),
#data_src,
#bitmap_src,
).boxed()
},
InnerRepr::NativeIterable => quote! {
PrimitiveArray::new(
#quoted_datatype,
#quoted_datatype.into(),
#data_src.into_iter() #quoted_transparent_mapping .collect(),
#bitmap_src,
).boxed()
Expand Down Expand Up @@ -654,7 +654,7 @@ fn quote_arrow_field_serializer(
// It would be nice to use quote_comment here and put this safety notice in the generated code,
// but that seems to push us over some complexity limit causing rustfmt to fail.
#[allow(unsafe_code, clippy::undocumented_unsafe_blocks)]
unsafe { Utf8Array::<i32>::new_unchecked(#quoted_datatype, offsets, inner_data, #bitmap_src) }.boxed()
unsafe { Utf8Array::<i32>::new_unchecked(#quoted_datatype.into(), offsets, inner_data, #bitmap_src) }.boxed()
}}
}

Expand Down Expand Up @@ -809,7 +809,7 @@ fn quote_arrow_field_serializer(
let quoted_create = if let DataType::List(_) = datatype {
quote! {
ListArray::try_new(
#quoted_datatype,
#quoted_datatype.into(),
offsets,
#quoted_inner,
#bitmap_src,
Expand All @@ -818,7 +818,7 @@ fn quote_arrow_field_serializer(
} else {
quote! {
FixedSizeListArray::new(
#quoted_datatype,
#quoted_datatype.into(),
#quoted_inner,
#bitmap_src,
).boxed()
Expand Down
Loading

0 comments on commit 44ff86c

Please sign in to comment.