From bbd27a2321fc18e77766362fa902a7c51a45068f Mon Sep 17 00:00:00 2001 From: mgiessing Date: Thu, 11 Aug 2022 22:28:51 +0200 Subject: [PATCH] ARROW-17372: [Go][Parquet] Fix failures for ppc64le (#13840) This PR fixes failures regarding the apache arrow go&/parquet module as described here: https://issues.apache.org/jira/browse/ARROW-17372 Signed-off-by: Marvin Giessing Authored-by: Marvin Giessing Signed-off-by: Matt Topol --- go/arrow/bitutil/bitmap_ops_ppc64le.go | 25 +++++ go/arrow/math/Makefile | 6 +- go/arrow/math/float64_ppc64le.go | 25 +++++ go/arrow/math/int64_ppc64le.go | 25 +++++ go/arrow/math/math_ppc64le.go | 29 ++++++ go/arrow/math/type_ppc64le.go.tmpl | 25 +++++ go/arrow/math/uint64_ppc64le.go | 25 +++++ go/internal/utils/min_max_ppc64le.go | 31 ++++++ go/internal/utils/transpose_ints_ppc64le.go | 97 +++++++++++++++++++ .../internal/bmi/bitmap_bmi2_ppc64le.go | 24 +++++ .../internal/utils/bit_packing_ppc64le.go | 23 +++++ .../internal/utils/unpack_bool_ppc64le.go | 25 +++++ 12 files changed, 357 insertions(+), 3 deletions(-) create mode 100644 go/arrow/bitutil/bitmap_ops_ppc64le.go create mode 100644 go/arrow/math/float64_ppc64le.go create mode 100644 go/arrow/math/int64_ppc64le.go create mode 100644 go/arrow/math/math_ppc64le.go create mode 100644 go/arrow/math/type_ppc64le.go.tmpl create mode 100644 go/arrow/math/uint64_ppc64le.go create mode 100644 go/internal/utils/min_max_ppc64le.go create mode 100644 go/internal/utils/transpose_ints_ppc64le.go create mode 100644 go/parquet/internal/bmi/bitmap_bmi2_ppc64le.go create mode 100644 go/parquet/internal/utils/bit_packing_ppc64le.go create mode 100644 go/parquet/internal/utils/unpack_bool_ppc64le.go diff --git a/go/arrow/bitutil/bitmap_ops_ppc64le.go b/go/arrow/bitutil/bitmap_ops_ppc64le.go new file mode 100644 index 0000000000000..86c47639a9e80 --- /dev/null +++ b/go/arrow/bitutil/bitmap_ops_ppc64le.go @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !noasm +// +build !noasm + +package bitutil + +func init() { + bitAndOp.opAligned = alignedBitAndGo + bitOrOp.opAligned = alignedBitOrGo +} diff --git a/go/arrow/math/Makefile b/go/arrow/math/Makefile index 15255950d71cb..3cc392071cde1 100644 --- a/go/arrow/math/Makefile +++ b/go/arrow/math/Makefile @@ -50,15 +50,15 @@ INTEL_SOURCES := \ assembly: $(INTEL_SOURCES) generate: ../bin/tmpl - ../bin/tmpl -i -data=float64.tmpldata type.go.tmpl=float64.go type_amd64.go.tmpl=float64_amd64.go type_arm64.go.tmpl=float64_arm64.go type_s390x.go.tmpl=float64_s390x.go type_noasm.go.tmpl=float64_noasm.go type_test.go.tmpl=float64_test.go + ../bin/tmpl -i -data=float64.tmpldata type.go.tmpl=float64.go type_amd64.go.tmpl=float64_amd64.go type_arm64.go.tmpl=float64_arm64.go type_ppc64le.go.tmpl=float64_ppc64le.go type_s390x.go.tmpl=float64_s390x.go type_noasm.go.tmpl=float64_noasm.go type_test.go.tmpl=float64_test.go ../bin/tmpl -i -data=float64.tmpldata -d arch=avx2 type_simd_amd64.go.tmpl=float64_avx2_amd64.go ../bin/tmpl -i -data=float64.tmpldata -d arch=sse4 type_simd_amd64.go.tmpl=float64_sse4_amd64.go ../bin/tmpl -i -data=float64.tmpldata -d arch=neon type_simd_arm64.go.tmpl=float64_neon_arm64.go - ../bin/tmpl -i -data=int64.tmpldata type.go.tmpl=int64.go type_amd64.go.tmpl=int64_amd64.go type_arm64.go.tmpl=int64_arm64.go type_s390x.go.tmpl=int64_s390x.go type_noasm.go.tmpl=int64_noasm.go type_test.go.tmpl=int64_test.go + ../bin/tmpl -i -data=int64.tmpldata type.go.tmpl=int64.go type_amd64.go.tmpl=int64_amd64.go type_arm64.go.tmpl=int64_arm64.go type_ppc64le.go.tmpl=int64_ppc64le.go type_s390x.go.tmpl=int64_s390x.go type_noasm.go.tmpl=int64_noasm.go type_test.go.tmpl=int64_test.go ../bin/tmpl -i -data=int64.tmpldata -d arch=avx2 type_simd_amd64.go.tmpl=int64_avx2_amd64.go ../bin/tmpl -i -data=int64.tmpldata -d arch=sse4 type_simd_amd64.go.tmpl=int64_sse4_amd64.go ../bin/tmpl -i -data=int64.tmpldata -d arch=neon type_simd_arm64.go.tmpl=int64_neon_arm64.go - ../bin/tmpl -i -data=uint64.tmpldata type.go.tmpl=uint64.go type_amd64.go.tmpl=uint64_amd64.go type_arm64.go.tmpl=uint64_arm64.go type_s390x.go.tmpl=uint64_s390x.go type_noasm.go.tmpl=uint64_noasm.go type_test.go.tmpl=uint64_test.go + ../bin/tmpl -i -data=uint64.tmpldata type.go.tmpl=uint64.go type_amd64.go.tmpl=uint64_amd64.go type_arm64.go.tmpl=uint64_arm64.go type_ppc64le.go.tmpl=uint64_ppc64le.go type_s390x.go.tmpl=uint64_s390x.go type_noasm.go.tmpl=uint64_noasm.go type_test.go.tmpl=uint64_test.go ../bin/tmpl -i -data=uint64.tmpldata -d arch=avx2 type_simd_amd64.go.tmpl=uint64_avx2_amd64.go ../bin/tmpl -i -data=uint64.tmpldata -d arch=sse4 type_simd_amd64.go.tmpl=uint64_sse4_amd64.go ../bin/tmpl -i -data=uint64.tmpldata -d arch=neon type_simd_arm64.go.tmpl=uint64_neon_arm64.go diff --git a/go/arrow/math/float64_ppc64le.go b/go/arrow/math/float64_ppc64le.go new file mode 100644 index 0000000000000..f60be90721d3a --- /dev/null +++ b/go/arrow/math/float64_ppc64le.go @@ -0,0 +1,25 @@ +// Code generated by type_s390x.go.tmpl. DO NOT EDIT. + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !noasm + +package math + +func initFloat64Go() { + Float64.sum = sum_float64_go +} diff --git a/go/arrow/math/int64_ppc64le.go b/go/arrow/math/int64_ppc64le.go new file mode 100644 index 0000000000000..1a615a9b27dcf --- /dev/null +++ b/go/arrow/math/int64_ppc64le.go @@ -0,0 +1,25 @@ +// Code generated by type_s390x.go.tmpl. DO NOT EDIT. + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !noasm + +package math + +func initInt64Go() { + Int64.sum = sum_int64_go +} diff --git a/go/arrow/math/math_ppc64le.go b/go/arrow/math/math_ppc64le.go new file mode 100644 index 0000000000000..3daeac7efaff8 --- /dev/null +++ b/go/arrow/math/math_ppc64le.go @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !noasm + +package math + +func init() { + initGo() +} + +func initGo() { + initFloat64Go() + initInt64Go() + initUint64Go() +} diff --git a/go/arrow/math/type_ppc64le.go.tmpl b/go/arrow/math/type_ppc64le.go.tmpl new file mode 100644 index 0000000000000..49a7fc634fd67 --- /dev/null +++ b/go/arrow/math/type_ppc64le.go.tmpl @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !noasm + +package math + +{{with .In}} +func init{{.Name}}Go() { + {{.Name}}.sum = sum_{{.Type}}_go +} +{{end}} diff --git a/go/arrow/math/uint64_ppc64le.go b/go/arrow/math/uint64_ppc64le.go new file mode 100644 index 0000000000000..8f7419fd4846a --- /dev/null +++ b/go/arrow/math/uint64_ppc64le.go @@ -0,0 +1,25 @@ +// Code generated by type_s390x.go.tmpl. DO NOT EDIT. + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !noasm + +package math + +func initUint64Go() { + Uint64.sum = sum_uint64_go +} diff --git a/go/internal/utils/min_max_ppc64le.go b/go/internal/utils/min_max_ppc64le.go new file mode 100644 index 0000000000000..ed1b0e69343a4 --- /dev/null +++ b/go/internal/utils/min_max_ppc64le.go @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !noasm +// +build !noasm + +package utils + +func init() { + minmaxFuncs.i8 = int8MinMax + minmaxFuncs.ui8 = uint8MinMax + minmaxFuncs.i16 = int16MinMax + minmaxFuncs.ui16 = uint16MinMax + minmaxFuncs.i32 = int32MinMax + minmaxFuncs.ui32 = uint32MinMax + minmaxFuncs.i64 = int64MinMax + minmaxFuncs.ui64 = uint64MinMax +} diff --git a/go/internal/utils/transpose_ints_ppc64le.go b/go/internal/utils/transpose_ints_ppc64le.go new file mode 100644 index 0000000000000..2650cebe75917 --- /dev/null +++ b/go/internal/utils/transpose_ints_ppc64le.go @@ -0,0 +1,97 @@ +// Code generated by transpose_ints_s390x.go.tmpl. DO NOT EDIT. + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !noasm +// +build !noasm + +package utils + +// if building with the 'noasm' tag, then point to the pure go implementations +var ( + TransposeInt8Int8 = transposeInt8Int8 + TransposeInt8Uint8 = transposeInt8Uint8 + TransposeInt8Int16 = transposeInt8Int16 + TransposeInt8Uint16 = transposeInt8Uint16 + TransposeInt8Int32 = transposeInt8Int32 + TransposeInt8Uint32 = transposeInt8Uint32 + TransposeInt8Int64 = transposeInt8Int64 + TransposeInt8Uint64 = transposeInt8Uint64 + + TransposeUint8Int8 = transposeUint8Int8 + TransposeUint8Uint8 = transposeUint8Uint8 + TransposeUint8Int16 = transposeUint8Int16 + TransposeUint8Uint16 = transposeUint8Uint16 + TransposeUint8Int32 = transposeUint8Int32 + TransposeUint8Uint32 = transposeUint8Uint32 + TransposeUint8Int64 = transposeUint8Int64 + TransposeUint8Uint64 = transposeUint8Uint64 + + TransposeInt16Int8 = transposeInt16Int8 + TransposeInt16Uint8 = transposeInt16Uint8 + TransposeInt16Int16 = transposeInt16Int16 + TransposeInt16Uint16 = transposeInt16Uint16 + TransposeInt16Int32 = transposeInt16Int32 + TransposeInt16Uint32 = transposeInt16Uint32 + TransposeInt16Int64 = transposeInt16Int64 + TransposeInt16Uint64 = transposeInt16Uint64 + + TransposeUint16Int8 = transposeUint16Int8 + TransposeUint16Uint8 = transposeUint16Uint8 + TransposeUint16Int16 = transposeUint16Int16 + TransposeUint16Uint16 = transposeUint16Uint16 + TransposeUint16Int32 = transposeUint16Int32 + TransposeUint16Uint32 = transposeUint16Uint32 + TransposeUint16Int64 = transposeUint16Int64 + TransposeUint16Uint64 = transposeUint16Uint64 + + TransposeInt32Int8 = transposeInt32Int8 + TransposeInt32Uint8 = transposeInt32Uint8 + TransposeInt32Int16 = transposeInt32Int16 + TransposeInt32Uint16 = transposeInt32Uint16 + TransposeInt32Int32 = transposeInt32Int32 + TransposeInt32Uint32 = transposeInt32Uint32 + TransposeInt32Int64 = transposeInt32Int64 + TransposeInt32Uint64 = transposeInt32Uint64 + + TransposeUint32Int8 = transposeUint32Int8 + TransposeUint32Uint8 = transposeUint32Uint8 + TransposeUint32Int16 = transposeUint32Int16 + TransposeUint32Uint16 = transposeUint32Uint16 + TransposeUint32Int32 = transposeUint32Int32 + TransposeUint32Uint32 = transposeUint32Uint32 + TransposeUint32Int64 = transposeUint32Int64 + TransposeUint32Uint64 = transposeUint32Uint64 + + TransposeInt64Int8 = transposeInt64Int8 + TransposeInt64Uint8 = transposeInt64Uint8 + TransposeInt64Int16 = transposeInt64Int16 + TransposeInt64Uint16 = transposeInt64Uint16 + TransposeInt64Int32 = transposeInt64Int32 + TransposeInt64Uint32 = transposeInt64Uint32 + TransposeInt64Int64 = transposeInt64Int64 + TransposeInt64Uint64 = transposeInt64Uint64 + + TransposeUint64Int8 = transposeUint64Int8 + TransposeUint64Uint8 = transposeUint64Uint8 + TransposeUint64Int16 = transposeUint64Int16 + TransposeUint64Uint16 = transposeUint64Uint16 + TransposeUint64Int32 = transposeUint64Int32 + TransposeUint64Uint32 = transposeUint64Uint32 + TransposeUint64Int64 = transposeUint64Int64 + TransposeUint64Uint64 = transposeUint64Uint64 +) diff --git a/go/parquet/internal/bmi/bitmap_bmi2_ppc64le.go b/go/parquet/internal/bmi/bitmap_bmi2_ppc64le.go new file mode 100644 index 0000000000000..498d5452e17ad --- /dev/null +++ b/go/parquet/internal/bmi/bitmap_bmi2_ppc64le.go @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !noasm + +package bmi + +func init() { + funclist.extractBits = extractBitsGo + funclist.gtbitmap = greaterThanBitmapGo +} diff --git a/go/parquet/internal/utils/bit_packing_ppc64le.go b/go/parquet/internal/utils/bit_packing_ppc64le.go new file mode 100644 index 0000000000000..58f869c3f5d97 --- /dev/null +++ b/go/parquet/internal/utils/bit_packing_ppc64le.go @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !noasm + +package utils + +import "io" + +var unpack32 func(io.Reader, []uint32, int) int = unpack32Default diff --git a/go/parquet/internal/utils/unpack_bool_ppc64le.go b/go/parquet/internal/utils/unpack_bool_ppc64le.go new file mode 100644 index 0000000000000..d833c2b9d6280 --- /dev/null +++ b/go/parquet/internal/utils/unpack_bool_ppc64le.go @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !noasm + +package utils + +// BytesToBools when built with the noasm tag will direct to the pure go implementation +// for converting a bitmap to a slice of bools +func BytesToBools(in []byte, out []bool) { + bytesToBoolsGo(in, out) +}