ARROW-5683: [R] Add snappy to Rtools Windows builds

Following r-windows/rtools-backports#7 and r-windows/rtools-packages#19, R Windows builds can now support Snappy compression. This patch tweaks the necessary files and unskips tests, in addition to some other PKGBUILD script cleanup.

Passing build here: https://ci.appveyor.com/project/nealrichardson/arrow/builds/25507388

Author: Neal Richardson <[email protected]>

Closes apache#4681 from nealrichardson/r-snappy and squashes the following commits:

0996a30 <Neal Richardson> Add license info for rtools-backports
1af0841 <Neal Richardson> Revert "Only run mine for now"
a5d967f <Neal Richardson> Get snappy from backports after all
a1c6390 <Neal Richardson> -lsnappy
a415449 <Neal Richardson> Only run mine for now
b911568 <Neal Richardson> More comments; unskip tests
d7a7419 <Neal Richardson> Add snappy to PKGBUILD; prune some default cmake flags and add comment

LICENSE.txt

@@ -890,7 +890,7 @@ This project include code from Google's Asylo project.
 * cpp/src/arrow/result.h is based on status_or.h
 
 Copyright (c)  Copyright 2017 Asylo authors
-Homepage: https://asylo.dev/ 
+Homepage: https://asylo.dev/
 License: Apache 2.0
 
 ---------------------------------------------------------------------------------
@@ -900,8 +900,8 @@ This project includes code from Google's protobuf project
 * cpp/src/arrow/result.h ARROW_ASSIGN_OR_RAISE is based off ASSIGN_OR_RETURN
 
 Copyright 2008 Google Inc.  All rights reserved.
-Homepage: https://developers.google.com/protocol-buffers/ 
-License: 
+Homepage: https://developers.google.com/protocol-buffers/
+License:
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are
@@ -933,3 +933,15 @@ Code generated by the Protocol Buffer compiler is owned by the owner
 of the input file used when generating it.  This code is not
 standalone and requires a support library to be linked with it.  This
 support library is itself covered by the above license.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the rtools-backports project.
+
+* ci/PKGBUILD and ci/appveyor-build-r.sh are based on code
+  from the rtools-backports project.
+
+Copyright: Copyright (c) 2013 - 2019, Алексей and Jeroen Ooms.
+All rights reserved.
+Homepage: https://github.com/r-windows/rtools-backports
+License: 3-clause BSD

ci/PKGBUILD

@@ -27,11 +27,18 @@ license=("Apache-2.0")
 depends=("${MINGW_PACKAGE_PREFIX}-boost"
          "${MINGW_PACKAGE_PREFIX}-double-conversion"
          "${MINGW_PACKAGE_PREFIX}-thrift"
+         "${MINGW_PACKAGE_PREFIX}-snappy"
          "${MINGW_PACKAGE_PREFIX}-zlib")
 makedepends=("${MINGW_PACKAGE_PREFIX}-cmake"
              "${MINGW_PACKAGE_PREFIX}-gcc")
 options=("staticlibs" "strip" "!buildflags")
 source_dir=apache-${_realname}-${pkgver}
+
+# For released version:
+#source=("https://archive.apache.org/dist/arrow/arrow-${pkgver}/apache-arrow-${pkgver}.tar.gz")
+#sha256sums=("ac2a77dd9168e9892e432c474611e86ded0be6dfe15f689c948751d37f81391a")
+# For github dev version:
+# Append `#commit=54b1b2f688e5e84b4c664b1e12a95f93b94ab2f3` to the URL to select a revision
 source=("${source_dir}"::"git+https://github.com/apache/arrow")
 sha256sums=("SKIP")
 
@@ -58,6 +65,8 @@ build() {
   # Workaround to fix static libparquet
   export CXXFLAGS="-DARROW_STATIC"
 
+  # This is the difference between rtools-packages and rtools-backports
+  # Remove this when submitting to rtools-packages
   export CC="/C/Rtools${MINGW_PREFIX/mingw/mingw_}/bin/gcc"
   export CXX="/C/Rtools${MINGW_PREFIX/mingw/mingw_}/bin/g++"
   export PATH="/C/Rtools${MINGW_PREFIX/mingw/mingw_}/bin:$PATH"
@@ -72,13 +81,9 @@ build() {
     -DCMAKE_BUILD_TYPE=${cmake_build_type} \
     -DARROW_BUILD_STATIC=ON \
     -DARROW_BUILD_SHARED=OFF \
-    -DARROW_BUILD_TESTS=OFF \
     -DARROW_PARQUET=ON \
-    -DARROW_PLASMA=OFF \
     -DARROW_HDFS=OFF \
-    -DARROW_PYTHON=OFF \
     -DARROW_BOOST_USE_SHARED=OFF \
-    -DARROW_WITH_SNAPPY=OFF \
     -DARROW_WITH_ZSTD=OFF \
     -DARROW_WITH_LZ4=OFF \
     -DARROW_JEMALLOC=OFF \
@@ -90,6 +95,7 @@ build() {
     -Ddouble-conversion_ROOT="${MINGW_PREFIX}" \
     -DThrift_ROOT="${MINGW_PREFIX}"
 
+  # Workaround for a compile error that appears with GCC 4.9.3
   sed -i 's/-fPIC/ /g' flatbuffers_ep-prefix/src/flatbuffers_ep-stamp/flatbuffers_ep-configure-RELEASE.cmake
 
   make

ci/windows-pkg-arrow-for-r.sh

@@ -46,6 +46,11 @@ wget https://dl.bintray.com/rtools/backports/mingw-w64-x86_64-thrift-0.12.0-8000
 # wget https://dl.bintray.com/rtools/mingw32/mingw-w64-i686-thrift-0.12.0-1-any.pkg.tar.xz
 # wget https://dl.bintray.com/rtools/mingw64/mingw-w64-x86_64-thrift-0.12.0-1-any.pkg.tar.xz
 
+wget https://dl.bintray.com/rtools/backports/mingw-w64-i686-snappy-1.1.7-2-any.pkg.tar.xz
+wget https://dl.bintray.com/rtools/backports/mingw-w64-x86_64-snappy-1.1.7-2-any.pkg.tar.xz
+# wget https://dl.bintray.com/rtools/mingw32/mingw-w64-i686-snappy-1.1.7-2-any.pkg.tar.xz
+# wget https://dl.bintray.com/rtools/mingw64/mingw-w64-x86_64-snappy-1.1.7-2-any.pkg.tar.xz
+
 # double-conversion is only available in the Rtools4.0 builds, but apparently that's ok
 wget https://dl.bintray.com/rtools/mingw64/mingw-w64-x86_64-double-conversion-3.1.2-1-any.pkg.tar.xz
 wget https://dl.bintray.com/rtools/mingw32/mingw-w64-i686-double-conversion-3.1.2-1-any.pkg.tar.xz

r/configure.win

@@ -39,7 +39,7 @@ else
   # Set the right flags to point to and enable arrow/parquet
   RWINLIB="../windows/arrow-${VERSION}"
   PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_R_WITH_PARQUET -DARROW_R_WITH_ARROW"
-  PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow -lthrift -lboost_regex-mt-s -lboost_filesystem-mt-s -lboost_system-mt-s -ldouble-conversion -lz -lws2_32"
+  PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow -lthrift -lsnappy -lboost_regex-mt-s -lboost_filesystem-mt-s -lboost_system-mt-s -ldouble-conversion -lz -lws2_32"
 fi
 
 echo "*** Writing Makevars.win"

r/tests/testthat/test-json.R

@@ -41,7 +41,6 @@ test_that("Can read json file with scalars columns (ARROW-5503)", {
   tib <- as.data.frame(tab1)
   expect_equal(tib$hello, c(3.5, 3.25, 3.125, 0))
   expect_equal(tib$world, c(FALSE, NA, NA, TRUE))
-  skip_on_os("windows") # TODO: debug UTF-8 test
   expect_equal(tib$yo, c("thing", NA, "\u5fcd", NA))
 })
 
@@ -68,7 +67,6 @@ test_that("read_json_arrow() converts to tibble", {
 
   expect_equal(tab1$hello, c(3.5, 3.25, 3.125, 0))
   expect_equal(tab1$world, c(FALSE, NA, NA, TRUE))
-  skip_on_os("windows") # TODO: debug UTF-8 test
   expect_equal(tab1$yo, c("thing", NA, "\u5fcd", NA))
 })
 

r/tests/testthat/test-parquet.R

@@ -20,7 +20,6 @@ context("Parquet file reading/writing")
 pq_file <- system.file("v0.7.1.parquet", package="arrow")
 
 test_that("reading a known Parquet file to tibble", {
-  skip_on_os("windows") # TODO: enable snappy in windows build
   df <- read_parquet(pq_file)
   expect_true(tibble::is_tibble(df))
   expect_identical(dim(df), c(10L, 11L))