From afd8428e5218535a01332b5e1ed1f3a09c65a5dd Mon Sep 17 00:00:00 2001 From: Michael Sumner Date: Wed, 19 Jun 2024 06:33:52 +0000 Subject: [PATCH] restore internal arrow read func --- DESCRIPTION | 6 +- R/RcppExports.R | 4 + R/read_stream_internal.R | 109 ++- .../gdalarrowstream/gdalvectorstream.h | 652 ++++++++---------- src/000-stream.cpp | 32 +- src/RcppExports.cpp | 22 + src/warpscratch.cpp | 49 -- 7 files changed, 398 insertions(+), 476 deletions(-) delete mode 100644 src/warpscratch.cpp diff --git a/DESCRIPTION b/DESCRIPTION index d87e3cff..86d44a1c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: vapour Title: Access to the 'Geospatial Data Abstraction Library' ('GDAL') -Version: 0.10.0 +Version: 0.10.9001 Authors@R: c(person("Michael", "Sumner", email = "mdsumner@gmail.com", role = c("aut", "cre"), comment = c(ORCID = "0000-0002-2471-7511")), person("Simon", "Wotherspoon", role = "ctb", comment = "RasterIO configuration for resampling options"), person("Mark", "Padgham", role = "ctb", comment = "helped get started :)"), @@ -27,8 +27,10 @@ LazyData: true LinkingTo: Rcpp Imports: jsonlite, + nanoarrow, Rcpp, - utils + utils, + wk RoxygenNote: 7.3.1 Roxygen: list(markdown = TRUE) Suggests: diff --git a/R/RcppExports.R b/R/RcppExports.R index e3855068..c974c8a6 100644 --- a/R/RcppExports.R +++ b/R/RcppExports.R @@ -1,6 +1,10 @@ # Generated by using Rcpp::compileAttributes() -> do not edit by hand # Generator token: 10BE3573-1514-4C36-9D1C-5A225CD40393 +gdal_dsn_read_vector_stream <- function(stream_xptr, dsn, layer, sql, options, quiet, drivers, extent, dsn_exists, fid_column_name, width) { + .Call('_vapour_gdal_dsn_read_vector_stream', PACKAGE = 'vapour', stream_xptr, dsn, layer, sql, options, quiet, drivers, extent, dsn_exists, fid_column_name, width) +} + warp_general_cpp <- function(dsn, target_crs, target_extent, target_dim, target_res, bands, resample, silent, band_output_type, options, dsn_outname, include_meta, nara) { .Call('_vapour_warp_general_cpp', PACKAGE = 'vapour', dsn, target_crs, target_extent, target_dim, target_res, bands, resample, silent, band_output_type, options, dsn_outname, include_meta, nara) } diff --git a/R/read_stream_internal.R b/R/read_stream_internal.R index dbc416ad..f47ceb61 100644 --- a/R/read_stream_internal.R +++ b/R/read_stream_internal.R @@ -1,57 +1,52 @@ -# gdal_ptrs <- function(dsource, layer) { -# layer <- if (missing(layer)) { -# character() -# } else { -# enc2utf8(layer) -# } -# #gdal_ptrs_cpp(dsource, layer) -# } -# -# gdal_vector_data <- function(dsource, layer, ..., sql = NA, options = NULL, quiet = FALSE, -# fid_column_name = character(0), -# if_drivers = character(0), -# extent = NA, -# optional = FALSE, return_stream = FALSE) { -# -# ## vapourize this -# layer <- if (missing(layer)) { -# character() -# } else { -# enc2utf8(layer) -# } -# if (nchar(dsource) < 1L) { -# stop("`dsour` must describe a valid source description for GDAL (input was an empty string).", call. = FALSE) -# } -# dsn_exists <- file.exists(dsource) -# -# if (length(dsource) == 1 && dsn_exists) { -# dsource <- enc2utf8(normalizePath(dsource)) -# } -# -# -# stream <- nanoarrow::nanoarrow_allocate_array_stream() -# -# # -# # info <- gdal_dsn_read_vector_stream(stream, -# # dsource, layer, sql, as.character(options), quiet, -# # if_drivers, extent, dsn_exists, fid_column_name, 80L) -# # -# # -# -# # geometry_column <- unlist(lapply( -# # stream$get_schema()$children, function(s) identical(s$metadata[["ARROW:extension:name"]], "ogc.wkb") -# # )) -# crs <- info[[1L]] -# if (info[[2L]] == -1) { -# num_features = NULL -# } -# -# # df = suppressWarnings(nanoarrow::convert_array_stream(x$stream, x$num_features)) -# #list(stream = stream, crs = crs, num_features = num_features) -# d <- nanoarrow::convert_array_stream(stream, num_features) -# #d$wkb_geometry <- wk::wkb(d$wkb_geometry, crs = crs) -# d -# } -# -# -# +gdal_ptrs <- function(dsource, layer) { + layer <- if (missing(layer)) { + character() + } else { + enc2utf8(layer) + } + #gdal_ptrs_cpp(dsource, layer) +} + +gdal_vector_data <- function(dsource, layer, ..., sql = NA, options = NULL, quiet = FALSE, + fid_column_name = character(0), + if_drivers = character(0), + extent = NA, + optional = FALSE, return_stream = FALSE) { + + ## vapourize this + layer <- if (missing(layer)) { + character() + } else { + enc2utf8(layer) + } + if (nchar(dsource) < 1L) { + stop("`dsour` must describe a valid source description for GDAL (input was an empty string).", call. = FALSE) + } + dsn_exists <- file.exists(dsource) + + if (length(dsource) == 1 && dsn_exists) { + dsource <- enc2utf8(normalizePath(dsource)) + } + + + stream <- nanoarrow::nanoarrow_allocate_array_stream() + + info <- gdal_dsn_read_vector_stream(stream, + dsource, layer, sql, as.character(options), quiet, + if_drivers, extent, dsn_exists, fid_column_name, 80L) + + if (return_stream) return(structure(list(stream), crs = info[[1]], num_features = info[[2]])) + # + crs <- info[[1L]] + if (info[[2L]] == -1) { + num_features = NULL + } + + + d <- nanoarrow::convert_array_stream(stream, num_features) + d$wkb_geometry <- wk::wkb(d$wkb_geometry, crs = crs) + d +} + + + diff --git a/inst/include/gdalarrowstream/gdalvectorstream.h b/inst/include/gdalarrowstream/gdalvectorstream.h index f9f293e2..6286d518 100644 --- a/inst/include/gdalarrowstream/gdalvectorstream.h +++ b/inst/include/gdalarrowstream/gdalvectorstream.h @@ -1,352 +1,300 @@ -// #ifndef GDALVECTORSTREAM_H -// #define GDALVECTORSTREAM_H -// -// // WIP https://gdal.org/development/rfc/rfc86_column_oriented_api.html -// // written by Dewey Dunnington and the GDAL project -// // https://github.com/paleolimbot/sf/tree/stream-reading -// -// #include -// #include -// #include "common/common_vapour.h" -// using namespace Rcpp; -// -// namespace gdalvectorstream { -// -// #if GDAL_VERSION_NUM >= GDAL_COMPUTE_VERSION(3,6,0) -// -// #include -// -// class GDALStreamWrapper { -// public: -// static void Make(struct ArrowArrayStream* stream, Rcpp::List shelter, -// struct ArrowArrayStream* stream_out) { -// stream_out->get_schema = &get_schema_wrap; -// stream_out->get_next = &get_next_wrap; -// stream_out->get_last_error = &get_last_error_wrap; -// stream_out->release = &release_wrap; -// stream_out->private_data = new GDALStreamWrapper(stream, shelter); -// } -// -// ~GDALStreamWrapper() { -// stream_.release(&stream_); -// GDALDataset* poDS = (GDALDataset*)R_ExternalPtrAddr(shelter_[0]); -// GDALClose(poDS); -// R_SetExternalPtrAddr(shelter_[0], nullptr); -// } -// -// private: -// // The parent stream as returned from GDAL -// struct ArrowArrayStream stream_; -// Rcpp::List shelter_; -// -// GDALStreamWrapper(struct ArrowArrayStream* stream, Rcpp::List shelter): -// shelter_(shelter) { -// memcpy(&stream_, stream, sizeof(struct ArrowArrayStream)); -// stream->release = nullptr; -// } -// -// int get_schema(struct ArrowSchema* out) { -// return stream_.get_schema(&stream_, out); -// } -// -// int get_next(struct ArrowArray* out) { -// return stream_.get_next(&stream_, out); -// } -// -// const char* get_last_error() { -// return stream_.get_last_error(&stream_); -// } -// -// static int get_schema_wrap(struct ArrowArrayStream* stream, struct ArrowSchema* out) { -// return reinterpret_cast(stream->private_data)->get_schema(out); -// } -// -// static int get_next_wrap(struct ArrowArrayStream* stream, struct ArrowArray* out) { -// return reinterpret_cast(stream->private_data)->get_next(out); -// } -// -// static const char* get_last_error_wrap(struct ArrowArrayStream* stream) { -// return reinterpret_cast(stream->private_data)->get_last_error(); -// } -// -// static void release_wrap(struct ArrowArrayStream* stream) { -// delete reinterpret_cast(stream->private_data); -// stream->release = nullptr; -// } -// }; -// -// -// static void finalize_dataset_xptr(SEXP dataset_xptr) { -// GDALDataset *poDS = (GDALDataset*)R_ExternalPtrAddr(dataset_xptr); -// if (poDS != nullptr) { -// GDALClose(poDS); -// } -// } -// -// inline Rcpp::List ogr_ptrs(Rcpp::CharacterVector datasource, -// Rcpp::CharacterVector layer) { -// -// GDALDataset *poDS; -// -// poDS = (GDALDataset *) GDALOpenEx( datasource[0], GDAL_OF_VECTOR | GDAL_OF_READONLY, -// NULL, NULL,NULL ); -// -// if( poDS == NULL ) { -// -// Rcpp::stop("Cannot open %s; ", datasource); -// } -// -// -// // Will close the dataset if some early return/exception prevents GDALClose() from being -// // called/allows the result to be accessed by the caller. -// Rcpp::RObject dataset_xptr = R_MakeExternalPtr(poDS, R_NilValue, R_NilValue); -// R_RegisterCFinalizer(dataset_xptr, &finalize_dataset_xptr); -// -// -// OGRLayer *poLayer; -// if (layer.size() == 0 ) { // no layer specified -// switch (poDS->GetLayerCount()) { -// case 0: { // error: -// Rcpp::stop("No layers in datasource."); -// } -// case 1: { // silent: -// poLayer = poDS->GetLayer(0); -// layer = Rcpp::CharacterVector::create(poLayer->GetName()); -// break; -// } -// default: { // select first layer: message + warning: -// poLayer = poDS->GetLayer(0); -// layer = Rcpp::CharacterVector::create(poLayer->GetName()); -// } -// } -// } -// -// poLayer = poDS->GetLayerByName(layer[0]); -// if (poLayer == NULL) { -// Rcpp::Rcout << "Cannot open layer " << layer[0] << std::endl; -// Rcpp::stop("Opening layer failed.\n"); -// } -// -// -// // Keeps the dataset external pointer alive as long as the layer external pointer is alive -// Rcpp::RObject layer_xptr = R_MakeExternalPtr(poLayer, R_NilValue, dataset_xptr); -// -// // This lets us pass it around in R -// R_SetExternalPtrAddr(dataset_xptr, poDS); -// R_SetExternalPtrAddr(layer_xptr, poLayer); -// -// return Rcpp::List::create(dataset_xptr, layer_xptr); -// } -// -// inline Rcpp::List ogr_layer_setup(Rcpp::CharacterVector datasource, -// Rcpp::CharacterVector layer, -// Rcpp::CharacterVector query, -// std::vector options, -// bool quiet, -// std::vector drivers, -// Rcpp::NumericVector ex, -// -// int width) { -// // adapted from the OGR tutorial @ www.gdal.org -// std::vector open_options; -// if (options.size() > 0) { -// open_options = string_to_charptr(options); -// } -// -// std::vector drivers_v = string_to_charptr(drivers); -// GDALDataset *poDS; -// -// poDS = (GDALDataset *) GDALOpenEx( datasource[0], GDAL_OF_VECTOR | GDAL_OF_READONLY, -// drivers.size() ? drivers_v.data() : NULL, -// open_options.size() ? open_options.data() : NULL, -// NULL ); -// -// if( poDS == NULL ) { -// -// Rcpp::stop("Cannot open %s; ", datasource); -// } -// -// // Will close the dataset if some early return/exception prevents GDALClose() from being -// // called/allows the result to be accessed by the caller. -// Rcpp::RObject dataset_xptr = R_MakeExternalPtr(poDS, R_NilValue, R_NilValue); -// R_RegisterCFinalizer(dataset_xptr, &finalize_dataset_xptr); -// -// -// if (layer.size() == 0 && Rcpp::CharacterVector::is_na(query[0])) { // no layer specified -// switch (poDS->GetLayerCount()) { -// case 0: { // error: -// Rcpp::stop("No layers in datasource."); -// } -// case 1: { // silent: -// OGRLayer *poLayer = poDS->GetLayer(0); -// layer = Rcpp::CharacterVector::create(poLayer->GetName()); -// break; -// } -// default: { // select first layer: message + warning: -// OGRLayer *poLayer = poDS->GetLayer(0); -// layer = Rcpp::CharacterVector::create(poLayer->GetName()); -// if (! quiet) { // #nocov start -// Rcpp::Rcout << "Multiple layers are present in data source " << datasource[0] << ", "; -// Rcpp::Rcout << "reading layer `" << layer[0] << "'." << std::endl; -// Rcpp::Rcout << "Use `st_layers' to list all layer names and their type in a data source." << std::endl; -// Rcpp::Rcout << "Set the `layer' argument in `st_read' to read a particular layer." << std::endl; -// } // #nocov end -// Rcpp::Function warning("warning"); -// warning("automatically selected the first layer in a data source containing more than one."); -// } -// } -// } -// -// -// OGRPolygon poly; -// OGRLinearRing ring; -// -// // set spatial filter? -// bool use_extent_filter = false; -// if (ex.length() == 4) { -// if (ex[1] <= ex[0] || ex[3] <= ex[2]) { -// if (ex[1] <= ex[0]) { -// Rcpp::warning("extent filter invalid (xmax <= xmin), ignoring"); -// } -// if (ex[3] <= ex[2]) { -// Rcpp::warning("extent filter invalid (ymax <= ymin), ignoring"); -// } -// } else { -// use_extent_filter = true; -// ring.addPoint(ex[0], ex[2]); //xmin, ymin -// ring.addPoint(ex[0], ex[3]); //xmin, ymax -// ring.addPoint(ex[1], ex[3]); //xmax, ymax -// ring.addPoint(ex[1], ex[2]); //xmax, ymin -// ring.closeRings(); -// poly.addRing(&ring); -// if (Rcpp::CharacterVector::is_na(query[0]) ) { -// query[0] = CPLSPrintf("SELECT * FROM %s", (char *) layer[0]); -// } -// } -// } -// -// OGRLayer *poLayer; -// if (! Rcpp::CharacterVector::is_na(query[0])) {// [[Rcpp::export]] -// // FIXME we need dialect here -// poLayer = poDS->ExecuteSQL(query[0], &poly, NULL); -// if (poLayer == NULL) -// Rcpp::stop("Query execution failed, cannot open layer.\n"); // #nocov -// if (layer.size()) -// Rcpp::warning("argument layer is ignored when query is specified\n"); // #nocov -// } else -// poLayer = poDS->GetLayerByName(layer[0]); -// if (poLayer == NULL) { -// Rcpp::Rcout << "Cannot open layer " << layer[0] << std::endl; -// Rcpp::stop("Opening layer failed.\n"); -// } -// -// if (! quiet) { -// if (! Rcpp::CharacterVector::is_na(query[0])) -// Rcpp::Rcout << "Reading query `" << query[0] << "'" << std::endl << "from data source "; -// else -// Rcpp::Rcout << "Reading layer `" << layer[0] << "' from data source "; -// // if (LENGTH(datasource[0]) > (width - (34 + LENGTH(layer[0])))) -// Rcpp::String ds(datasource(0)); -// if (layer.size()) { -// Rcpp::String la(layer(0)); -// if (strlen(ds.get_cstring()) > (width - (34 + strlen(la.get_cstring())))) -// Rcpp::Rcout << std::endl << " "; -// } -// Rcpp::Rcout << "`" << datasource[0] << "' "; -// if (((int) strlen(ds.get_cstring())) > (width - 25)) -// Rcpp::Rcout << std::endl << " "; -// Rcpp::Rcout << "using driver `" << poDS->GetDriverName() << "'" << std::endl; // #nocov -// } -// -// // Keeps the dataset external pointer alive as long as the layer external pointer is alive -// Rcpp::RObject layer_xptr = R_MakeExternalPtr(poLayer, R_NilValue, dataset_xptr); -// Rprintf("%s", "ogr_layer_setup:\n"); -// return Rcpp::List::create(dataset_xptr, layer_xptr); -// } -// -// inline Rcpp::List read_gdal_stream( -// Rcpp::RObject stream_xptr, -// Rcpp::CharacterVector datasource, -// Rcpp::CharacterVector layer, -// Rcpp::CharacterVector query, -// std::vector options, -// bool quiet, -// std::vector drivers, -// Rcpp::NumericVector extent, -// bool dsn_exists, -// Rcpp::CharacterVector fid_column, -// int width) { -// -// const char* array_stream_options[] = {"INCLUDE_FID=NO", nullptr}; -// if (fid_column.size() == 1) { -// array_stream_options[0] = "INCLUDE_FID=YES"; -// } -// -// Rcpp::List prep = ogr_layer_setup(datasource, layer, query, -// options, -// quiet, -// drivers, -// extent, -// width); -// OGRDataSource* poDS = (OGRDataSource*)(R_ExternalPtrAddr(prep[0])); -// OGRLayer* poLayer = (OGRLayer*)R_ExternalPtrAddr(prep[1]); -// auto stream_out = reinterpret_cast( -// R_ExternalPtrAddr(stream_xptr)); -// -// OGRSpatialReference* crs = poLayer->GetSpatialRef(); -// char* wkt_out; -// std::string wkt_str; -// if (crs) { -// crs->exportToWkt(&wkt_out); -// wkt_str = wkt_out; -// } else { -// wkt_str = ""; -// } -// CPLFree(wkt_out); -// -// struct ArrowArrayStream stream_temp; -// if (!poLayer->GetArrowStream(&stream_temp, array_stream_options)) { -// Rcpp::stop("Failed to open ArrayStream from Layer"); -// } -// -// GDALStreamWrapper::Make(&stream_temp, prep, stream_out); -// -// -// // The reported feature count is incorrect if there is a query -// double num_features; -// if (query.size() == 0) { -// num_features = (double) poLayer->GetFeatureCount(false); -// } else { -// num_features = -1; -// } -// -// if (! Rcpp::CharacterVector::is_na(query[0])) { -// //poDS->ReleaseResultSet(poLayer); -// } -// -// return Rcpp::List::create(wkt_str, Rcpp::NumericVector::create(num_features)); -// } -// -// #else -// -// inline Rcpp::List read_gdal_stream( -// Rcpp::RObject stream_xptr, -// Rcpp::CharacterVector datasource, -// Rcpp::CharacterVector layer, -// Rcpp::CharacterVector query, -// std::vector options, -// bool quiet, -// std::vector drivers, -// Rcpp::NumericVector extent, -// bool dsn_exists, -// Rcpp::CharacterVector fid_column, -// int width) { -// Rcpp::stop("read_stream() requires GDAL >= 3.6"); -// } -// -// #endif -// -// } // gdalvectorstream -// -// -// #endif +#ifndef GDALVECTORSTREAM_H +#define GDALVECTORSTREAM_H + +// WIP https://gdal.org/development/rfc/rfc86_column_oriented_api.html +// written by Dewey Dunnington and the GDAL project +// https://github.com/paleolimbot/sf/tree/stream-reading + +#include +#include +#include "common/common_vapour.h" +#if GDAL_VERSION_NUM >= GDAL_COMPUTE_VERSION(3,6,0) +#include +#endif + +using namespace Rcpp; + +namespace gdalvectorstream { + +#if GDAL_VERSION_NUM >= GDAL_COMPUTE_VERSION(3,6,0) + + +class GDALStreamWrapper { +public: + static void Make(struct ArrowArrayStream* stream, Rcpp::List shelter, + struct ArrowArrayStream* stream_out) { + stream_out->get_schema = &get_schema_wrap; + stream_out->get_next = &get_next_wrap; + stream_out->get_last_error = &get_last_error_wrap; + stream_out->release = &release_wrap; + stream_out->private_data = new GDALStreamWrapper(stream, shelter); + } + + ~GDALStreamWrapper() { + stream_.release(&stream_); + GDALDataset* poDS = (GDALDataset*)R_ExternalPtrAddr(shelter_[0]); + GDALClose(poDS); + R_SetExternalPtrAddr(shelter_[0], nullptr); + } + +private: + // The parent stream as returned from GDAL + struct ArrowArrayStream stream_; + Rcpp::List shelter_; + + GDALStreamWrapper(struct ArrowArrayStream* stream, Rcpp::List shelter): + shelter_(shelter) { + memcpy(&stream_, stream, sizeof(struct ArrowArrayStream)); + stream->release = nullptr; + } + + int get_schema(struct ArrowSchema* out) { + return stream_.get_schema(&stream_, out); + } + + int get_next(struct ArrowArray* out) { + return stream_.get_next(&stream_, out); + } + + const char* get_last_error() { + return stream_.get_last_error(&stream_); + } + + static int get_schema_wrap(struct ArrowArrayStream* stream, struct ArrowSchema* out) { + return reinterpret_cast(stream->private_data)->get_schema(out); + } + + static int get_next_wrap(struct ArrowArrayStream* stream, struct ArrowArray* out) { + return reinterpret_cast(stream->private_data)->get_next(out); + } + + static const char* get_last_error_wrap(struct ArrowArrayStream* stream) { + return reinterpret_cast(stream->private_data)->get_last_error(); + } + + static void release_wrap(struct ArrowArrayStream* stream) { + delete reinterpret_cast(stream->private_data); + stream->release = nullptr; + } +}; + + +static void finalize_dataset_xptr(SEXP dataset_xptr) { + GDALDataset *poDS = (GDALDataset*)R_ExternalPtrAddr(dataset_xptr); + if (poDS != nullptr) { + GDALClose(poDS); + } +} + +inline Rcpp::List ogr_layer_setup(Rcpp::CharacterVector datasource, + Rcpp::CharacterVector layer, + Rcpp::CharacterVector query, + std::vector options, + bool quiet, + std::vector drivers, + Rcpp::NumericVector ex, + + int width) { + // adapted from the OGR tutorial @ www.gdal.org + std::vector open_options; + if (options.size() > 0) { + open_options = string_to_charptr(options); + } + + std::vector drivers_v = string_to_charptr(drivers); + GDALDataset *poDS; + + poDS = (GDALDataset *) GDALOpenEx( datasource[0], GDAL_OF_VECTOR | GDAL_OF_READONLY, + drivers.size() ? drivers_v.data() : NULL, + open_options.size() ? open_options.data() : NULL, + NULL ); + + if( poDS == NULL ) { + + Rcpp::stop("Cannot open %s; ", datasource); + } + + // Will close the dataset if some early return/exception prevents GDALClose() from being + // called/allows the result to be accessed by the caller. + Rcpp::RObject dataset_xptr = R_MakeExternalPtr(poDS, R_NilValue, R_NilValue); + R_RegisterCFinalizer(dataset_xptr, &finalize_dataset_xptr); + + + if (layer.size() == 0 && Rcpp::CharacterVector::is_na(query[0])) { // no layer specified + switch (poDS->GetLayerCount()) { + case 0: { // error: + Rcpp::stop("No layers in datasource."); + } + case 1: { // silent: + OGRLayer *poLayer = poDS->GetLayer(0); + layer = Rcpp::CharacterVector::create(poLayer->GetName()); + break; + } + default: { // select first layer: message + warning: + OGRLayer *poLayer = poDS->GetLayer(0); + layer = Rcpp::CharacterVector::create(poLayer->GetName()); + if (! quiet) { // #nocov start + Rcpp::Rcout << "Multiple layers are present in data source " << datasource[0] << ", "; + Rcpp::Rcout << "reading layer `" << layer[0] << "'." << std::endl; + Rcpp::Rcout << "Use `st_layers' to list all layer names and their type in a data source." << std::endl; + Rcpp::Rcout << "Set the `layer' argument in `st_read' to read a particular layer." << std::endl; + } // #nocov end + Rcpp::Function warning("warning"); + warning("automatically selected the first layer in a data source containing more than one."); + } + } + } + + + OGRPolygon poly; + OGRLinearRing ring; + + // set spatial filter? + bool use_extent_filter = false; + if (ex.length() == 4) { + if (ex[1] <= ex[0] || ex[3] <= ex[2]) { + if (ex[1] <= ex[0]) { + Rcpp::warning("extent filter invalid (xmax <= xmin), ignoring"); + } + if (ex[3] <= ex[2]) { + Rcpp::warning("extent filter invalid (ymax <= ymin), ignoring"); + } + } else { + use_extent_filter = true; + ring.addPoint(ex[0], ex[2]); //xmin, ymin + ring.addPoint(ex[0], ex[3]); //xmin, ymax + ring.addPoint(ex[1], ex[3]); //xmax, ymax + ring.addPoint(ex[1], ex[2]); //xmax, ymin + ring.closeRings(); + poly.addRing(&ring); + if (Rcpp::CharacterVector::is_na(query[0]) ) { + query[0] = CPLSPrintf("SELECT * FROM %s", (char *) layer[0]); + } + } + } + + OGRLayer *poLayer; + if (! Rcpp::CharacterVector::is_na(query[0])) {// [[Rcpp::export]] + // FIXME we need dialect here + poLayer = poDS->ExecuteSQL(query[0], &poly, NULL); + if (poLayer == NULL) + Rcpp::stop("Query execution failed, cannot open layer.\n"); // #nocov + if (layer.size()) + Rcpp::warning("argument layer is ignored when query is specified\n"); // #nocov + } else + poLayer = poDS->GetLayerByName(layer[0]); + if (poLayer == NULL) { + Rcpp::Rcout << "Cannot open layer " << layer[0] << std::endl; + Rcpp::stop("Opening layer failed.\n"); + } + + if (! quiet) { + if (! Rcpp::CharacterVector::is_na(query[0])) + Rcpp::Rcout << "Reading query `" << query[0] << "'" << std::endl << "from data source "; + else + Rcpp::Rcout << "Reading layer `" << layer[0] << "' from data source "; + // if (LENGTH(datasource[0]) > (width - (34 + LENGTH(layer[0])))) + Rcpp::String ds(datasource(0)); + if (layer.size()) { + Rcpp::String la(layer(0)); + if (strlen(ds.get_cstring()) > (width - (34 + strlen(la.get_cstring())))) + Rcpp::Rcout << std::endl << " "; + } + Rcpp::Rcout << "`" << datasource[0] << "' "; + if (((int) strlen(ds.get_cstring())) > (width - 25)) + Rcpp::Rcout << std::endl << " "; + Rcpp::Rcout << "using driver `" << poDS->GetDriverName() << "'" << std::endl; // #nocov + } + + // Keeps the dataset external pointer alive as long as the layer external pointer is alive + Rcpp::RObject layer_xptr = R_MakeExternalPtr(poLayer, R_NilValue, dataset_xptr); + Rprintf("%s", "ogr_layer_setup:\n"); + return Rcpp::List::create(dataset_xptr, layer_xptr); +} + +inline Rcpp::List read_gdal_stream( + Rcpp::RObject stream_xptr, + Rcpp::CharacterVector datasource, + Rcpp::CharacterVector layer, + Rcpp::CharacterVector query, + std::vector options, + bool quiet, + std::vector drivers, + Rcpp::NumericVector extent, + bool dsn_exists, + Rcpp::CharacterVector fid_column, + int width) { + + const char* array_stream_options[] = {"INCLUDE_FID=NO", nullptr}; + if (fid_column.size() == 1) { + array_stream_options[0] = "INCLUDE_FID=YES"; + } + + Rcpp::List prep = ogr_layer_setup(datasource, layer, query, + options, + quiet, + drivers, + extent, + width); + OGRDataSource* poDS = (OGRDataSource*)(R_ExternalPtrAddr(prep[0])); + OGRLayer* poLayer = (OGRLayer*)R_ExternalPtrAddr(prep[1]); + auto stream_out = reinterpret_cast( + R_ExternalPtrAddr(stream_xptr)); + + OGRSpatialReference* crs = poLayer->GetSpatialRef(); + char* wkt_out; + std::string wkt_str; + if (crs) { + crs->exportToWkt(&wkt_out); + wkt_str = wkt_out; + } else { + wkt_str = ""; + } + CPLFree(wkt_out); + + struct ArrowArrayStream stream_temp; + if (!poLayer->GetArrowStream(&stream_temp, array_stream_options)) { + Rcpp::stop("Failed to open ArrayStream from Layer"); + } + + GDALStreamWrapper::Make(&stream_temp, prep, stream_out); + + + // The reported feature count is incorrect if there is a query + double num_features; + if (query.size() == 0) { + num_features = (double) poLayer->GetFeatureCount(false); + } else { + num_features = -1; + } + + if (! Rcpp::CharacterVector::is_na(query[0])) { + //poDS->ReleaseResultSet(poLayer); + } + + return Rcpp::List::create(wkt_str, Rcpp::NumericVector::create(num_features)); +} + +#else + +inline Rcpp::List read_gdal_stream( + Rcpp::RObject stream_xptr, + Rcpp::CharacterVector datasource, + Rcpp::CharacterVector layer, + Rcpp::CharacterVector query, + std::vector options, + bool quiet, + std::vector drivers, + Rcpp::NumericVector extent, + bool dsn_exists, + Rcpp::CharacterVector fid_column, + int width) { + Rcpp::stop("read_stream() requires GDAL >= 3.6"); +} + +#endif + +} // gdalvectorstream + + +#endif diff --git a/src/000-stream.cpp b/src/000-stream.cpp index 92b2d312..3f587739 100644 --- a/src/000-stream.cpp +++ b/src/000-stream.cpp @@ -5,21 +5,21 @@ using namespace Rcpp; // see process_cpl_read_ogr_stream -// // [[Rcpp::export]] -// List gdal_dsn_read_vector_stream(RObject stream_xptr, -// CharacterVector dsn, -// CharacterVector layer, -// CharacterVector sql, -// std::vector options, -// bool quiet, -// std::vector drivers, -// Rcpp::NumericVector extent, -// bool dsn_exists, -// Rcpp::CharacterVector fid_column_name, -// int width) { -// return gdalvectorstream::read_gdal_stream(stream_xptr, dsn, layer, sql, -// options, quiet, drivers, extent, dsn_exists, -// fid_column_name, width); -// } +// [[Rcpp::export]] +List gdal_dsn_read_vector_stream(RObject stream_xptr, + CharacterVector dsn, + CharacterVector layer, + CharacterVector sql, + std::vector options, + bool quiet, + std::vector drivers, + Rcpp::NumericVector extent, + bool dsn_exists, + Rcpp::CharacterVector fid_column_name, + int width) { + return gdalvectorstream::read_gdal_stream(stream_xptr, dsn, layer, sql, + options, quiet, drivers, extent, dsn_exists, + fid_column_name, width); +} diff --git a/src/RcppExports.cpp b/src/RcppExports.cpp index 51457e2d..15116d8e 100644 --- a/src/RcppExports.cpp +++ b/src/RcppExports.cpp @@ -10,6 +10,27 @@ Rcpp::Rostream& Rcpp::Rcout = Rcpp::Rcpp_cout_get(); Rcpp::Rostream& Rcpp::Rcerr = Rcpp::Rcpp_cerr_get(); #endif +// gdal_dsn_read_vector_stream +List gdal_dsn_read_vector_stream(RObject stream_xptr, CharacterVector dsn, CharacterVector layer, CharacterVector sql, std::vector options, bool quiet, std::vector drivers, Rcpp::NumericVector extent, bool dsn_exists, Rcpp::CharacterVector fid_column_name, int width); +RcppExport SEXP _vapour_gdal_dsn_read_vector_stream(SEXP stream_xptrSEXP, SEXP dsnSEXP, SEXP layerSEXP, SEXP sqlSEXP, SEXP optionsSEXP, SEXP quietSEXP, SEXP driversSEXP, SEXP extentSEXP, SEXP dsn_existsSEXP, SEXP fid_column_nameSEXP, SEXP widthSEXP) { +BEGIN_RCPP + Rcpp::RObject rcpp_result_gen; + Rcpp::RNGScope rcpp_rngScope_gen; + Rcpp::traits::input_parameter< RObject >::type stream_xptr(stream_xptrSEXP); + Rcpp::traits::input_parameter< CharacterVector >::type dsn(dsnSEXP); + Rcpp::traits::input_parameter< CharacterVector >::type layer(layerSEXP); + Rcpp::traits::input_parameter< CharacterVector >::type sql(sqlSEXP); + Rcpp::traits::input_parameter< std::vector >::type options(optionsSEXP); + Rcpp::traits::input_parameter< bool >::type quiet(quietSEXP); + Rcpp::traits::input_parameter< std::vector >::type drivers(driversSEXP); + Rcpp::traits::input_parameter< Rcpp::NumericVector >::type extent(extentSEXP); + Rcpp::traits::input_parameter< bool >::type dsn_exists(dsn_existsSEXP); + Rcpp::traits::input_parameter< Rcpp::CharacterVector >::type fid_column_name(fid_column_nameSEXP); + Rcpp::traits::input_parameter< int >::type width(widthSEXP); + rcpp_result_gen = Rcpp::wrap(gdal_dsn_read_vector_stream(stream_xptr, dsn, layer, sql, options, quiet, drivers, extent, dsn_exists, fid_column_name, width)); + return rcpp_result_gen; +END_RCPP +} // warp_general_cpp List warp_general_cpp(CharacterVector dsn, CharacterVector target_crs, NumericVector target_extent, IntegerVector target_dim, NumericVector target_res, IntegerVector bands, CharacterVector resample, LogicalVector silent, CharacterVector band_output_type, CharacterVector options, CharacterVector dsn_outname, LogicalVector include_meta, LogicalVector nara); RcppExport SEXP _vapour_warp_general_cpp(SEXP dsnSEXP, SEXP target_crsSEXP, SEXP target_extentSEXP, SEXP target_dimSEXP, SEXP target_resSEXP, SEXP bandsSEXP, SEXP resampleSEXP, SEXP silentSEXP, SEXP band_output_typeSEXP, SEXP optionsSEXP, SEXP dsn_outnameSEXP, SEXP include_metaSEXP, SEXP naraSEXP) { @@ -740,6 +761,7 @@ END_RCPP } static const R_CallMethodDef CallEntries[] = { + {"_vapour_gdal_dsn_read_vector_stream", (DL_FUNC) &_vapour_gdal_dsn_read_vector_stream, 11}, {"_vapour_warp_general_cpp", (DL_FUNC) &_vapour_warp_general_cpp, 13}, {"_vapour_warp_suggest_cpp", (DL_FUNC) &_vapour_warp_suggest_cpp, 2}, {"_vapour_set_gdal_config_cpp", (DL_FUNC) &_vapour_set_gdal_config_cpp, 2}, diff --git a/src/warpscratch.cpp b/src/warpscratch.cpp deleted file mode 100644 index 566a3e16..00000000 --- a/src/warpscratch.cpp +++ /dev/null @@ -1,49 +0,0 @@ -// #include -// using namespace Rcpp; -// -// -// #include "gdal_priv.h" -// #include "gdalwarper.h" -// #include "gdal_utils.h" // for GDALWarpAppOptions -// #include "gdalraster/gdalraster.h" -// #include "ogr_spatialref.h" // for OGRCreateCoordinateTransformation -// -// #include "gdalarrowstream/gdalvectorstream.h" -// -// -// -// // [[Rcpp::export]] -// List gdal_ptrs_cpp(CharacterVector dsn, CharacterVector layer) { -// return gdalvectorstream::ogr_ptrs(dsn, layer); -// } -// -// // [[Rcpp::export]] -// NumericVector timesTwo(std::vector x) { -// -// std::vector vops = string_to_charptr( x); -// auto psOptions = GDALWarpAppOptionsNew(vops.data(), nullptr); -// -// GDALWarpAppOptionsFree(psOptions); -// -// NumericVector num = NumericVector(1); -// num[0] = 2.0; -// return num; -// } -// -// -// // [[Rcpp::export]] -// IntegerVector n_layers(SEXP xp) { -// GDALDataset *poDS = (GDALDataset*)R_ExternalPtrAddr(xp); -// IntegerVector res(1); -// res[0] = poDS->GetLayerCount(); -// return res; -// } -// -// // [[Rcpp::export]] -// CharacterVector ptr_query(SEXP xp) { -// OGRLayer *poLayer = (OGRLayer*)R_ExternalPtrAddr(xp); -// CharacterVector res(1); -// res[0] = poLayer->GetDescription(); -// return res; -// } -//