Skip to content

Commit

Permalink
Prepare for 15.0.0 Release (apache#4470)
Browse files Browse the repository at this point in the history 
* update version

* update versions for test-utils and parquet-test-utils

* CHANGELOG

* revert changing test-utils versions

* update Cargo.lock
  • Loading branch information
@andygrove
andygrove authored and zhangli20 committed Jan 5, 2023
1 parent 5d69fa3 commit 455a392
Show file tree
Hide file tree
Showing 18 changed files with 381 additions and 54 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,3 +98,5 @@ dev/dist

# CI
arrow-ballista

datafusion/CHANGELOG.md.bak
6 changes: 3 additions & 3 deletions benchmarks/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
[package]
name = "datafusion-benchmarks"
description = "DataFusion Benchmarks"
version = "14.0.0"
version = "15.0.0"
edition = "2021"
authors = ["Apache Arrow <[email protected]>"]
homepage = "https://github.com/apache/arrow-datafusion"
Expand All @@ -34,7 +34,7 @@ snmalloc = ["snmalloc-rs"]

[dependencies]
arrow = "28.0.0"
datafusion = { path = "../datafusion/core", version = "14.0.0", features = ["scheduler"] }
datafusion = { path = "../datafusion/core", version = "15.0.0", features = ["scheduler"] }
env_logger = "0.10"
futures = "0.3"
mimalloc = { version = "0.1", optional = true, default-features = false }
Expand All @@ -51,4 +51,4 @@ test-utils = { path = "../test-utils/", version = "0.1.0" }
tokio = { version = "^1.0", features = ["macros", "rt", "rt-multi-thread", "parking_lot"] }

[dev-dependencies]
datafusion-proto = { path = "../datafusion/proto", version = "14.0.0" }
datafusion-proto = { path = "../datafusion/proto", version = "15.0.0" }
16 changes: 8 additions & 8 deletions datafusion-cli/Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions datafusion-cli/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
[package]
name = "datafusion-cli"
description = "Command Line Client for DataFusion query engine."
version = "14.0.0"
version = "15.0.0"
authors = ["Apache Arrow <[email protected]>"]
edition = "2021"
keywords = [ "arrow", "datafusion", "query", "sql" ]
Expand All @@ -31,7 +31,7 @@ readme = "README.md"
[dependencies]
arrow = "28.0.0"
clap = { version = "3", features = ["derive", "cargo"] }
datafusion = { path = "../datafusion/core", version = "14.0.0" }
datafusion = { path = "../datafusion/core", version = "15.0.0" }
dirs = "4.0.0"
env_logger = "0.9"
mimalloc = { version = "0.1", default-features = false }
Expand Down
2 changes: 1 addition & 1 deletion datafusion-examples/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
[package]
name = "datafusion-examples"
description = "DataFusion usage examples"
version = "14.0.0"
version = "15.0.0"
homepage = "https://github.com/apache/arrow-datafusion"
repository = "https://github.com/apache/arrow-datafusion"
authors = ["Apache Arrow <[email protected]>"]
Expand Down
317 changes: 317 additions & 0 deletions datafusion/CHANGELOG.md
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion datafusion/common/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
[package]
name = "datafusion-common"
description = "Common functionality for DataFusion query engine"
version = "14.0.0"
version = "15.0.0"
homepage = "https://github.com/apache/arrow-datafusion"
repository = "https://github.com/apache/arrow-datafusion"
readme = "README.md"
Expand Down
16 changes: 8 additions & 8 deletions datafusion/core/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
[package]
name = "datafusion"
description = "DataFusion is an in-memory query engine that uses Apache Arrow as the memory model"
version = "14.0.0"
version = "15.0.0"
homepage = "https://github.com/apache/arrow-datafusion"
repository = "https://github.com/apache/arrow-datafusion"
readme = "../../README.md"
Expand Down Expand Up @@ -64,13 +64,13 @@ bytes = "1.1"
bzip2 = { version = "0.4.3", optional = true }
chrono = { version = "0.4.23", default-features = false }
dashmap = "5.4.0"
datafusion-common = { path = "../common", version = "14.0.0", features = ["parquet", "object_store"] }
datafusion-expr = { path = "../expr", version = "14.0.0" }
datafusion-jit = { path = "../jit", version = "14.0.0", optional = true }
datafusion-optimizer = { path = "../optimizer", version = "14.0.0" }
datafusion-physical-expr = { path = "../physical-expr", version = "14.0.0" }
datafusion-row = { path = "../row", version = "14.0.0" }
datafusion-sql = { path = "../sql", version = "14.0.0" }
datafusion-common = { path = "../common", version = "15.0.0", features = ["parquet", "object_store"] }
datafusion-expr = { path = "../expr", version = "15.0.0" }
datafusion-jit = { path = "../jit", version = "15.0.0", optional = true }
datafusion-optimizer = { path = "../optimizer", version = "15.0.0" }
datafusion-physical-expr = { path = "../physical-expr", version = "15.0.0" }
datafusion-row = { path = "../row", version = "15.0.0" }
datafusion-sql = { path = "../sql", version = "15.0.0" }
flate2 = { version = "1.0.24", optional = true }
futures = "0.3"
glob = "0.3.0"
Expand Down
16 changes: 10 additions & 6 deletions datafusion/core/src/execution/memory_manager/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ pub trait MemoryConsumer: Send + Sync {
/// reached for this consumer.
async fn try_grow(&self, required: usize) -> Result<()> {
let current = self.mem_used();
debug!(
log::info!(
"trying to acquire {} whiling holding {} from consumer {}",
human_readable_size(required),
human_readable_size(current),
Expand All @@ -183,7 +183,7 @@ pub trait MemoryConsumer: Send + Sync {
let can_grow_directly =
self.memory_manager().can_grow_directly(required, current);
if !can_grow_directly {
debug!(
log::info!(
"Failed to grow memory of {} directly from consumer {}, spilling first ...",
human_readable_size(required),
self.id()
Expand Down Expand Up @@ -276,7 +276,7 @@ impl MemoryManager {
match config {
MemoryManagerConfig::Existing(manager) => manager,
MemoryManagerConfig::New { .. } => {
debug!(
log::info!(
"Creating memory manager with initial size {}",
human_readable_size(pool_size)
);
Expand All @@ -297,10 +297,12 @@ impl MemoryManager {
}

pub(crate) fn grow_tracker_usage(&self, delta: usize) {
log::info!("XXX grow tracker usage: {}", delta);
self.trackers_total.fetch_add(delta, Ordering::SeqCst);
}

pub(crate) fn shrink_tracker_usage(&self, delta: usize) {
log::info!("XXX shrink tracker usage: {}", delta);
let update =
self.trackers_total
.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |x| {
Expand All @@ -325,6 +327,7 @@ impl MemoryManager {

/// Register a new memory requester
pub(crate) fn register_requester(&self, requester_id: &MemoryConsumerId) {
log::info!("XXX register_requester: id={}", id);
self.requesters.lock().insert(requester_id.clone());
}

Expand Down Expand Up @@ -357,7 +360,7 @@ impl MemoryManager {
} else if current < min_per_rqt {
// if we cannot acquire at lease 1/2n memory, just wait for others
// to spill instead spill self frequently with limited total mem
debug!(
log::info!(
"Cannot acquire a minimum amount of {} memory from the manager of total {}, waiting for others to spill ...",
human_readable_size(min_per_rqt), human_readable_size(self.pool_size));
let now = Instant::now();
Expand All @@ -379,7 +382,7 @@ impl MemoryManager {

fn record_free_then_acquire(&self, freed: usize, acquired: usize) {
let mut requesters_total = self.requesters_total.lock();
debug!(
log::info!(
"free_then_acquire: total {}, freed {}, acquired {}",
human_readable_size(*requesters_total),
human_readable_size(freed),
Expand All @@ -393,7 +396,7 @@ impl MemoryManager {

fn record_free(&self, freed: usize) {
let mut requesters_total = self.requesters_total.lock();
debug!(
log::info!(
"free: total {}, freed {}",
human_readable_size(*requesters_total),
human_readable_size(freed)
Expand All @@ -405,6 +408,7 @@ impl MemoryManager {

/// Drop a memory consumer and reclaim the memory
pub(crate) fn drop_consumer(&self, id: &MemoryConsumerId, mem_used: usize) {
log::info!("XXX drop_consumer: id={}, mem_used={}", id, mem_used);
// find in requesters first
{
let mut requesters = self.requesters.lock();
Expand Down
4 changes: 4 additions & 0 deletions datafusion/core/src/execution/runtime_env.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,21 +76,25 @@ impl RuntimeEnv {

/// Register the consumer to get it tracked
pub fn register_requester(&self, id: &MemoryConsumerId) {
log::info!("XXX runtime.register_requester: id={}", id);
self.memory_manager.register_requester(id);
}

/// Drop the consumer from get tracked, reclaim memory
pub fn drop_consumer(&self, id: &MemoryConsumerId, mem_used: usize) {
log::info!("XXX runtime.drop_consumer: id={}, mem_used={}", id, mem_used);
self.memory_manager.drop_consumer(id, mem_used)
}

/// Grow tracker memory of `delta`
pub fn grow_tracker_usage(&self, delta: usize) {
log::info!("XXX runtime.grow_tracker_usage: {}", delta);
self.memory_manager.grow_tracker_usage(delta)
}

/// Shrink tracker memory of `delta`
pub fn shrink_tracker_usage(&self, delta: usize) {
log::info!("XXX runtime.shrink_tracker_usage: {}", delta);
self.memory_manager.shrink_tracker_usage(delta)
}

Expand Down
4 changes: 2 additions & 2 deletions datafusion/expr/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
[package]
name = "datafusion-expr"
description = "Logical plan and expression representation for DataFusion query engine"
version = "14.0.0"
version = "15.0.0"
homepage = "https://github.com/apache/arrow-datafusion"
repository = "https://github.com/apache/arrow-datafusion"
readme = "README.md"
Expand All @@ -37,6 +37,6 @@ path = "src/lib.rs"
[dependencies]
ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
arrow = { version = "28.0.0", default-features = false }
datafusion-common = { path = "../common", version = "14.0.0" }
datafusion-common = { path = "../common", version = "15.0.0" }
log = "^0.4"
sqlparser = "0.27"
6 changes: 3 additions & 3 deletions datafusion/jit/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
[package]
name = "datafusion-jit"
description = "Just In Time (JIT) compilation support for DataFusion query engine"
version = "14.0.0"
version = "15.0.0"
homepage = "https://github.com/apache/arrow-datafusion"
repository = "https://github.com/apache/arrow-datafusion"
readme = "README.md"
Expand All @@ -41,7 +41,7 @@ cranelift = "0.89.0"
cranelift-jit = "0.89.0"
cranelift-module = "0.89.0"
cranelift-native = "0.89.0"
datafusion-common = { path = "../common", version = "14.0.0", features = ["jit"] }
datafusion-expr = { path = "../expr", version = "14.0.0" }
datafusion-common = { path = "../common", version = "15.0.0", features = ["jit"] }
datafusion-expr = { path = "../expr", version = "15.0.0" }

parking_lot = "0.12"
10 changes: 5 additions & 5 deletions datafusion/optimizer/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
[package]
name = "datafusion-optimizer"
description = "DataFusion Query Optimizer"
version = "14.0.0"
version = "15.0.0"
homepage = "https://github.com/apache/arrow-datafusion"
repository = "https://github.com/apache/arrow-datafusion"
readme = "README.md"
Expand All @@ -40,13 +40,13 @@ unicode_expressions = []
arrow = { version = "28.0.0", features = ["prettyprint"] }
async-trait = "0.1.41"
chrono = { version = "0.4.23", default-features = false }
datafusion-common = { path = "../common", version = "14.0.0" }
datafusion-expr = { path = "../expr", version = "14.0.0" }
datafusion-physical-expr = { path = "../physical-expr", version = "14.0.0" }
datafusion-common = { path = "../common", version = "15.0.0" }
datafusion-expr = { path = "../expr", version = "15.0.0" }
datafusion-physical-expr = { path = "../physical-expr", version = "15.0.0" }
hashbrown = { version = "0.13", features = ["raw"] }
log = "^0.4"

[dev-dependencies]
ctor = "0.1.22"
datafusion-sql = { path = "../sql", version = "14.0.0" }
datafusion-sql = { path = "../sql", version = "15.0.0" }
env_logger = "0.10.0"
8 changes: 4 additions & 4 deletions datafusion/physical-expr/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
[package]
name = "datafusion-physical-expr"
description = "Physical expression implementation for DataFusion query engine"
version = "14.0.0"
version = "15.0.0"
homepage = "https://github.com/apache/arrow-datafusion"
repository = "https://github.com/apache/arrow-datafusion"
readme = "README.md"
Expand Down Expand Up @@ -46,9 +46,9 @@ arrow-schema = "28.0.0"
blake2 = { version = "^0.10.2", optional = true }
blake3 = { version = "1.0", optional = true }
chrono = { version = "0.4.23", default-features = false }
datafusion-common = { path = "../common", version = "14.0.0" }
datafusion-expr = { path = "../expr", version = "14.0.0" }
datafusion-row = { path = "../row", version = "14.0.0" }
datafusion-common = { path = "../common", version = "15.0.0" }
datafusion-expr = { path = "../expr", version = "15.0.0" }
datafusion-row = { path = "../row", version = "15.0.0" }
half = { version = "2.1", default-features = false }
hashbrown = { version = "0.13", features = ["raw"] }
itertools = { version = "0.10", features = ["use_std"] }
Expand Down
8 changes: 4 additions & 4 deletions datafusion/proto/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
[package]
name = "datafusion-proto"
description = "Protobuf serialization of DataFusion logical plan expressions"
version = "14.0.0"
version = "15.0.0"
homepage = "https://github.com/apache/arrow-datafusion"
repository = "https://github.com/apache/arrow-datafusion"
readme = "README.md"
Expand All @@ -42,9 +42,9 @@ json = ["pbjson", "serde", "serde_json"]
[dependencies]
arrow = "28.0.0"
chrono = { version = "0.4", default-features = false }
datafusion = { path = "../core", version = "14.0.0" }
datafusion-common = { path = "../common", version = "14.0.0" }
datafusion-expr = { path = "../expr", version = "14.0.0" }
datafusion = { path = "../core", version = "15.0.0" }
datafusion-common = { path = "../common", version = "15.0.0" }
datafusion-expr = { path = "../expr", version = "15.0.0" }
object_store = { version = "0.5.0" }
parking_lot = { version = "0.12" }
pbjson = { version = "0.5", optional = true }
Expand Down
Loading

0 comments on commit 455a392

Please sign in to comment.