From fcf028c711e7095b437e9e08536922496eef24aa Mon Sep 17 00:00:00 2001 From: dimitrisstaratzis Date: Sat, 7 Jan 2023 03:47:28 +0200 Subject: [PATCH] optimize memory allocation of validity buffers --- .../java/io/trino/plugin/tiledb/TileDBMetadata.java | 3 +-- .../java/io/trino/plugin/tiledb/TileDBPageSink.java | 13 ++++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/main/java/io/trino/plugin/tiledb/TileDBMetadata.java b/src/main/java/io/trino/plugin/tiledb/TileDBMetadata.java index 2b95de90b7..532c5d44dd 100644 --- a/src/main/java/io/trino/plugin/tiledb/TileDBMetadata.java +++ b/src/main/java/io/trino/plugin/tiledb/TileDBMetadata.java @@ -69,7 +69,6 @@ import static io.trino.plugin.tiledb.TileDBColumnProperties.getExtent; import static io.trino.plugin.tiledb.TileDBColumnProperties.getFilterList; import static io.trino.plugin.tiledb.TileDBColumnProperties.getLowerBound; -import static io.trino.plugin.tiledb.TileDBColumnProperties.getNullable; import static io.trino.plugin.tiledb.TileDBColumnProperties.getUpperBound; import static io.trino.plugin.tiledb.TileDBErrorCode.TILEDB_CREATE_TABLE_ERROR; import static io.trino.plugin.tiledb.TileDBModule.tileDBTypeFromTrinoType; @@ -438,7 +437,7 @@ public TileDBOutputTableHandle beginCreateArray(ConnectorSession session, Connec if (filterPairs.isPresent()) { attribute.setFilterList(Util.createTileDBFilterList(localCtx, filterPairs.get())); } - attribute.setNullable(getNullable(columnProperties)); + attribute.setNullable(column.isNullable()); arraySchema.addAttribute(attribute); } diff --git a/src/main/java/io/trino/plugin/tiledb/TileDBPageSink.java b/src/main/java/io/trino/plugin/tiledb/TileDBPageSink.java index 63e2548d0f..f811c65b75 100644 --- a/src/main/java/io/trino/plugin/tiledb/TileDBPageSink.java +++ b/src/main/java/io/trino/plugin/tiledb/TileDBPageSink.java @@ -176,7 +176,7 @@ private void resetBuffers(Map> buffers) t buffers.clear(); query.resetBuffers(); - validityMaps = new short[columnHandles.size()][maxBufferSize]; + validityMaps = new short[columnHandles.size()][]; // Loop through each column for (int channel = 0; channel < columnHandles.size(); channel++) { // Datatype @@ -195,9 +195,16 @@ private void resetBuffers(Map> buffers) t if (isVariableLength) { offsets = new NativeArray(ctx, maxBufferSize, Datatype.TILEDB_UINT64); } - if (!columnHandle.getIsDimension()) { - Arrays.fill(validityMaps[channel], (short) 1); // all valid + + // allocate validity buffers for nullable attributes only. + if (array.getSchema().hasAttribute(columnName)) { + Attribute attribute = array.getSchema().getAttribute(columnHandle.getColumnName()); + if (attribute.getNullable()) { + validityMaps[channel] = new short[maxBufferSize / type.getNativeSize()]; //allocate for the max amount of values + Arrays.fill(validityMaps[channel], (short) 1); // all valid + } } + buffers.put(columnName, new Pair<>(offsets, values)); } }