feat: [ODIN-895] LIST and STRUCT types

addon/result_iterator.cc

@@ -141,80 +141,103 @@ Napi::Value ResultIterator::getRowObject(Napi::Env env) {
 }
 
 Napi::Value ResultIterator::getCellValue(Napi::Env env, duckdb::idx_t col_idx) {
-  auto val = current_chunk->data[col_idx].GetValue(chunk_offset);
+  auto value = current_chunk->data[col_idx].GetValue(chunk_offset);
+  return getMappedValue(env, value);
+}
 
-  if (val.is_null) {
+Napi::Value ResultIterator::getMappedValue(Napi::Env env, duckdb::Value value) {
+  if (value.is_null) {
     return env.Null();
   }
 
-  switch (result->types[col_idx].id()) {
+  switch (value.type().id()) {
   case duckdb::LogicalTypeId::BOOLEAN:
-    return Napi::Boolean::New(env, val.GetValue<bool>());
+    return Napi::Boolean::New(env, value.GetValue<bool>());
   case duckdb::LogicalTypeId::TINYINT:
-    return Napi::Number::New(env, val.GetValue<int8_t>());
+    return Napi::Number::New(env, value.GetValue<int8_t>());
   case duckdb::LogicalTypeId::SMALLINT:
-    return Napi::Number::New(env, val.GetValue<int16_t>());
+    return Napi::Number::New(env, value.GetValue<int16_t>());
   case duckdb::LogicalTypeId::INTEGER:
-    return Napi::Number::New(env, val.GetValue<int32_t>());
+    return Napi::Number::New(env, value.GetValue<int32_t>());
   case duckdb::LogicalTypeId::BIGINT:
-    return Napi::BigInt::New(env, val.GetValue<int64_t>());
+    return Napi::BigInt::New(env, value.GetValue<int64_t>());
   case duckdb::LogicalTypeId::HUGEINT: {
     // hugeint_t represents a signed 128 bit integer in two's complement
     // notation napi's BigInt is basically a regular signed integer (MSB) so we
     // want to make sure we pass the absolute value of the huge int into napi
     // plus the sign bit
-    auto huge_int = val.GetValue<duckdb::hugeint_t>();
+    auto huge_int = value.GetValue<duckdb::hugeint_t>();
     int is_negative = huge_int.upper < 0;
     duckdb::hugeint_t positive_huge_int =
         is_negative ? huge_int * duckdb::hugeint_t(-1) : huge_int;
     uint64_t arr[2]{positive_huge_int.lower, (uint64_t)positive_huge_int.upper};
     return Napi::BigInt::New(env, is_negative, 2, &arr[0]);
   }
   case duckdb::LogicalTypeId::FLOAT:
-    return Napi::Number::New(env, val.GetValue<float>());
+    return Napi::Number::New(env, value.GetValue<float>());
   case duckdb::LogicalTypeId::DOUBLE:
-    return Napi::Number::New(env, val.GetValue<double>());
+    return Napi::Number::New(env, value.GetValue<double>());
   case duckdb::LogicalTypeId::DECIMAL:
     return Napi::Number::New(
-        env, val.CastAs(duckdb::LogicalType::DOUBLE).GetValue<double>());
+        env, value.CastAs(duckdb::LogicalType::DOUBLE).GetValue<double>());
   case duckdb::LogicalTypeId::VARCHAR:
-    return Napi::String::New(env, val.GetValue<string>());
+    return Napi::String::New(env, value.GetValue<string>());
   case duckdb::LogicalTypeId::BLOB: {
-    int array_length = val.str_value.length();
+    int array_length = value.str_value.length();
     char char_array[array_length + 1];
     // TODO: multiple copies, improve
-    strcpy(char_array, val.str_value.c_str());
+    strcpy(char_array, value.str_value.c_str());
     return Napi::Buffer<char>::Copy(env, char_array, array_length);
   }
   case duckdb::LogicalTypeId::TIMESTAMP: {
-    if (result->types[col_idx].InternalType() != duckdb::PhysicalType::INT64) {
+    if (value.type().InternalType() != duckdb::PhysicalType::INT64) {
       throw runtime_error("expected int64 for timestamp");
     }
-    int64_t tval = val.GetValue<int64_t>();
+    int64_t tval = value.GetValue<int64_t>();
     return Napi::Number::New(env, tval / 1000);
   }
   case duckdb::LogicalTypeId::TIME: {
-    if (result->types[col_idx].InternalType() != duckdb::PhysicalType::INT64) {
+    if (value.type().InternalType() != duckdb::PhysicalType::INT64) {
       throw runtime_error("expected int64 for time");
     }
-    int64_t tval = val.GetValue<int64_t>();
+    int64_t tval = value.GetValue<int64_t>();
     return Napi::Number::New(env, GetTime(tval));
   }
   case duckdb::LogicalTypeId::INTERVAL: {
-    return Napi::String::New(env, val.ToString());
+    return Napi::String::New(env, value.ToString());
   }
   case duckdb::LogicalTypeId::UTINYINT:
-    return Napi::Number::New(env, val.GetValue<uint8_t>());
+    return Napi::Number::New(env, value.GetValue<uint8_t>());
   case duckdb::LogicalTypeId::USMALLINT:
-    return Napi::Number::New(env, val.GetValue<uint16_t>());
+    return Napi::Number::New(env, value.GetValue<uint16_t>());
   case duckdb::LogicalTypeId::UINTEGER:
     // GetValue is not supported for uint32_t, so using the wider type
-    return Napi::Number::New(env, val.GetValue<int64_t>());
+    return Napi::Number::New(env, value.GetValue<int64_t>());
+  case duckdb::LogicalTypeId::LIST: {
+    auto array = Napi::Array::New(env);
+    for (size_t i = 0; i < value.list_value.size(); i++) {
+      auto &element = value.list_value[i];
+      auto mapped_value = getMappedValue(env, element);
+      array.Set(i, mapped_value);
+    }
+    return array;
+  }
+  case duckdb::LogicalTypeId::STRUCT: {
+    auto object = Napi::Object::New(env);
+    for (size_t i = 0; i < value.struct_value.size(); i++) {
+      auto &key = value.type().child_types()[i].first;
+      auto &element = value.struct_value[i];
+      auto child_value = getMappedValue(env, element);
+      object.Set(key, child_value);
+    }
+    return object;
+  }
   default:
     // default to getting string representation
-    return Napi::String::New(env, val.ToString());
+    return Napi::String::New(env, value.ToString());
   }
 }
+
 Napi::Value ResultIterator::Close(const Napi::CallbackInfo &info) {
   result.reset();
   return info.Env().Undefined();

addon/result_iterator.h

@@ -26,6 +26,7 @@ class ResultIterator : public Napi::ObjectWrap<ResultIterator> {
   duckdb::unique_ptr<duckdb::DataChunk> current_chunk;
   uint64_t chunk_offset = 0;
   Napi::Value getCellValue(Napi::Env env, duckdb::idx_t col_idx);
+  Napi::Value getMappedValue(Napi::Env env, duckdb::Value duckdb_value);
   Napi::Value getRowArray(Napi::Env env);
   Napi::Value getRowObject(Napi::Env env);
 };

package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-duckdb",
-  "version": "0.0.70",
+  "version": "0.0.71",
   "private": false,
   "description": "DuckDB for Node.JS",
   "keywords": [

src/tests/data-types.test.ts

@@ -220,4 +220,69 @@ describe("Data type mapping", () => {
 
     expect(result.fetchRow()).toMatchObject([1]);
   });
+
+  it("supports LIST - integers", async () => {
+    const result = await connection.executeIterator<any[]>(`SELECT array_slice([1,2,3], 1, NULL)`);
+    expect(result.fetchRow()).toMatchObject({ "array_slice(list_value(1, 2, 3), 1, NULL)": [2, 3] });
+  });
+
+  it("supports LIST - strings", async () => {
+    const result = await connection.executeIterator<any[]>(`SELECT array_slice(['a','b','c'], 1, NULL)`);
+    expect(result.fetchRow()).toEqual({ "array_slice(list_value(a, b, c), 1, NULL)": ["b", "c"] });
+  });
+
+  it("supports LIST - STRUCTs", async () => {
+    const result = await connection.executeIterator<any[]>(
+      `SELECT raw_header from parquet_scan('src/tests/test-fixtures/crawl_urls.parquet')`,
+    );
+    expect(result.fetchRow()).toEqual({
+      raw_header: [
+        {
+          key: "Content-Encoding",
+          value: "gzip",
+        },
+        {
+          key: "X-Frame-Options",
+          value: "SAMEORIGIN",
+        },
+        {
+          key: "Connection",
+          value: "keep-alive",
+        },
+        {
+          key: "X-Xss-Protection",
+          value: "1; mode=block",
+        },
+        {
+          key: "Content-Type",
+          value: "text/html;charset=utf-8",
+        },
+        {
+          key: "Date",
+          value: "Tue, 18 Aug 2020 13:46:36 GMT",
+        },
+        {
+          key: "Vary",
+          value: "User-agent,Accept-Encoding",
+        },
+        {
+          key: "Server",
+          value: "nginx/1.10.3",
+        },
+        {
+          key: "X-Content-Type-Options",
+          value: "nosniff",
+        },
+        {
+          key: "Content-Length",
+          value: "1180",
+        },
+      ],
+    });
+  });
+
+  it("supports STRUCT", async () => {
+    const result = await connection.executeIterator<any[]>(`SELECT struct_pack(i := 4, s := 'string')`);
+    expect(result.fetchRow()).toEqual({ "struct_pack(4, string)": { i: 4, s: "string" } });
+  });
 });

yarn.lock

@@ -612,9 +612,9 @@
     fastq "^1.6.0"
 
 "@npmcli/arborist@^2.3.0", "@npmcli/arborist@^2.5.0":
-  version "2.5.0"
-  resolved "https://registry.yarnpkg.com/@npmcli/arborist/-/arborist-2.5.0.tgz#3a2bf74e47b4cbe20e0983291f291a2ec2c3d06f"
-  integrity sha512-YPSkV/8vofpbAJyeu52J12YnC5VTkYIcfcNkRoSW6qjfQG+QybgbJtCbcdx+M0YxfdzDKS6iDTjpNMoETZ8HOA==
+  version "2.6.1"
+  resolved "https://registry.yarnpkg.com/@npmcli/arborist/-/arborist-2.6.1.tgz#bdbd1311cc857583ffc85f4d3f24a50683303dda"
+  integrity "sha1-vb0TEcyFdYP/yF9NPySlBoMwPdo= sha512-OOlntFIOAo7RplEQaYXlA5U5NXE+EwZtnTCsit4Wtme5+llGiea6GBytuV8dOzdPMPlNx3fQQjBUE9E8k76yjQ=="
   dependencies:
     "@npmcli/installed-package-contents" "^1.0.7"
     "@npmcli/map-workspaces" "^1.0.2"
@@ -1815,9 +1815,9 @@ byte-size@^7.0.1:
   integrity sha512-crQdqyCwhokxwV1UyDzLZanhkugAgft7vt0qbbdt60C6Zf3CAiGmtUCylbtYwrU6loOUw3euGrNtW1J651ot1A==
 
 cacache@^15.0.3, cacache@^15.0.5, cacache@^15.0.6:
-  version "15.0.6"
-  resolved "https://registry.yarnpkg.com/cacache/-/cacache-15.0.6.tgz#65a8c580fda15b59150fb76bf3f3a8e45d583099"
-  integrity sha512-g1WYDMct/jzW+JdWEyjaX2zoBkZ6ZT9VpOyp2I/VMtDsNLffNat3kqPFfi1eDRSK9/SuKGyORDHcQMcPF8sQ/w==
+  version "15.2.0"
+  resolved "https://registry.yarnpkg.com/cacache/-/cacache-15.2.0.tgz#73af75f77c58e72d8c630a7a2858cb18ef523389"
+  integrity "sha1-c69193xY5y2MYwp6KFjLGO9SM4k= sha512-uKoJSHmnrqXgthDFx/IU6ED/5xd+NNGe+Bb+kLZy7Ku4P+BaiWEUflAKPZ7eAzsYGcsAGASJZsybXp+quEcHTw=="
   dependencies:
     "@npmcli/move-file" "^1.0.1"
     chownr "^2.0.0"
@@ -3602,9 +3602,9 @@ fs.realpath@^1.0.0:
   integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8=
 
 fsevents@^2.1.2:
-  version "2.3.1"
-  resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.1.tgz#b209ab14c61012636c8863507edf7fb68cc54e9f"
-  integrity sha512-YR47Eg4hChJGAB1O3yEAOkGO+rlzutoICGqGo9EZ4lKWokzZRSyIW1QmTzqjtw8MJdj9srP869CuWw/hyzSiBw==
+  version "2.3.2"
+  resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
+  integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
 
 fsevents@~2.1.2:
   version "2.1.3"
@@ -4109,9 +4109,9 @@ [email protected]:
     safer-buffer ">= 2.1.2 < 3"
 
 iconv-lite@^0.6.2:
-  version "0.6.2"
-  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.2.tgz#ce13d1875b0c3a674bd6a04b7f76b01b1b6ded01"
-  integrity sha512-2y91h5OpQlolefMPmUlivelittSWy0rP+oYVpn6A7GwVHNE8AWzoYOBNmlwks3LobaJxgHCYZAnyNo2GgpNRNQ==
+  version "0.6.3"
+  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501"
+  integrity "sha1-pS+AvzjaGVLrXGgXkHGYcaGnJQE= sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="
   dependencies:
     safer-buffer ">= 2.1.2 < 3.0.0"
 
@@ -5360,9 +5360,9 @@ libnpmdiff@^2.0.4:
     tar "^6.1.0"
 
 libnpmexec@^1.1.1:
-  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/libnpmexec/-/libnpmexec-1.1.1.tgz#aa510cc74612cac3945f7e2fd2ac4c650fa7ef66"
-  integrity sha512-uw6H2dzC6F6fdq7lAxfzXPimHCJ3/g6ycFKcv2Q2QXuNZ94EDmNPpMO6f4mwiC5F6Lyy/WK0IL7AZwRhmSvUdQ==
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/libnpmexec/-/libnpmexec-1.2.0.tgz#ab0294ab63bb599b3ac6921129b7a706d4d56da2"
+  integrity "sha1-qwKUq2O7WZs6xpIRKbenBtTVbaI= sha512-LkxnH2wsMUI4thsgUK0r+EFZ5iCjKlp21J68dFY7AzD5uaaIPqO3lqVvYbyl1Umz1R4rY9t3vFa1fF3hzo6Y2Q=="
   dependencies:
     "@npmcli/arborist" "^2.3.0"
     "@npmcli/ci-detect" "^1.3.0"
@@ -6183,9 +6183,9 @@ normalize-url@^4.1.0:
   integrity sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==
 
 npm-audit-report@^2.1.4:
-  version "2.1.4"
-  resolved "https://registry.yarnpkg.com/npm-audit-report/-/npm-audit-report-2.1.4.tgz#b14c4625131fb7bcacc4b1c83842af1f58c92c98"
-  integrity sha512-Tz7rnfskSdZ0msTzt2mENC/B+H2QI8u0jN0ck7o3zDsQYIQrek/l3MjEc+CARer+64LsVTU6ZIqNuh0X55QPhw==
+  version "2.1.5"
+  resolved "https://registry.yarnpkg.com/npm-audit-report/-/npm-audit-report-2.1.5.tgz#a5b8850abe2e8452fce976c8960dd432981737b5"
+  integrity "sha1-pbiFCr4uhFL86XbIlg3UMpgXN7U= sha512-YB8qOoEmBhUH1UJgh1xFAv7Jg1d+xoNhsDYiFQlEFThEBui0W1vIz2ZK6FVg4WZjwEdl7uBQlm1jy3MUfyHeEw=="
   dependencies:
     chalk "^4.0.0"
 
@@ -6252,9 +6252,9 @@ npm-profile@^5.0.3:
     npm-registry-fetch "^10.0.0"
 
 npm-registry-fetch@^10.0.0, npm-registry-fetch@^10.1.1:
-  version "10.1.1"
-  resolved "https://registry.yarnpkg.com/npm-registry-fetch/-/npm-registry-fetch-10.1.1.tgz#97bc7a0fca5e8f76cc5162185b8de8caa8bea639"
-  integrity sha512-F6a3l+ffCQ7hvvN16YG5bpm1rPZntCg66PLHDQ1apWJPOCUVHoKnL2w5fqEaTVhp42dmossTyXeR7hTGirfXrg==
+  version "10.1.2"
+  resolved "https://registry.yarnpkg.com/npm-registry-fetch/-/npm-registry-fetch-10.1.2.tgz#11ffe03d813c653e768bdf762cfc5f1afe91b8bd"
+  integrity "sha1-Ef/gPYE8ZT52i992LPxfGv6RuL0= sha512-KsM/TdPmntqgBFlfsbkOLkkE9ovZo7VpVcd+/eTdYszCrgy5zFl5JzWm+OxavFaEWlbkirpkou+ZYI00RmOBFA=="
   dependencies:
     lru-cache "^6.0.0"
     make-fetch-happen "^8.0.9"
@@ -8962,15 +8962,10 @@ [email protected]:
   dependencies:
     mkdirp "^0.5.1"
 
-"ws@>= 2.2.3":
-  version "7.4.4"
-  resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.4.tgz#383bc9742cb202292c9077ceab6f6047b17f2d59"
-  integrity sha512-Qm8k8ojNQIMx7S+Zp8u/uHOx7Qazv3Yv4q68MiWWWOJhiwG5W3x7iqmRtJo8xxrciZUY4vRxUTJCKuRnF28ZZw==
-
-ws@^7.2.3:
-  version "7.4.2"
-  resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.2.tgz#782100048e54eb36fe9843363ab1c68672b261dd"
-  integrity sha512-T4tewALS3+qsrpGI/8dqNMLIVdq/g/85U98HPMa6F0m6xTbvhXU6RCQLqPH3+SlomNV/LdY6RXEbBpMH6EOJnA==
+"ws@>= 2.2.3", ws@^7.2.3:
+  version "7.4.6"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.6.tgz#5654ca8ecdeee47c33a9a4bf6d28e2be2980377c"
+  integrity sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==
 
 xdg-basedir@^4.0.0:
   version "4.0.0"