From 7883ec8b98c2f40649254883df14902f00587fed Mon Sep 17 00:00:00 2001 From: Shingo Otsuka Date: Fri, 21 Dec 2018 18:33:15 +0900 Subject: [PATCH 01/70] Add CUDA Stereo Semi Global Matching --- modules/cudastereo/CMakeLists.txt | 2 +- modules/cudastereo/doc/cudastereo.bib | 10 + .../cudastereo/include/opencv2/cudastereo.hpp | 47 + modules/cudastereo/perf/perf_stereo.cpp | 34 + modules/cudastereo/src/cuda/stereosgm.cu | 2081 +++++++++++++++++ modules/cudastereo/src/cuda/stereosgm.hpp | 61 + modules/cudastereo/src/stereosgm.cpp | 153 ++ modules/cudastereo/test/test_sgm_funcs.cpp | 444 ++++ modules/cudastereo/test/test_stereo.cpp | 755 ++++++ 9 files changed, 3586 insertions(+), 1 deletion(-) create mode 100644 modules/cudastereo/doc/cudastereo.bib create mode 100644 modules/cudastereo/src/cuda/stereosgm.cu create mode 100644 modules/cudastereo/src/cuda/stereosgm.hpp create mode 100644 modules/cudastereo/src/stereosgm.cpp create mode 100644 modules/cudastereo/test/test_sgm_funcs.cpp diff --git a/modules/cudastereo/CMakeLists.txt b/modules/cudastereo/CMakeLists.txt index c02086913cf..feb4d3e8fd0 100644 --- a/modules/cudastereo/CMakeLists.txt +++ b/modules/cudastereo/CMakeLists.txt @@ -6,4 +6,4 @@ set(the_description "CUDA-accelerated Stereo Correspondence") ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4127 /wd4324 /wd4512 -Wundef -Wmissing-declarations -Wshadow) -ocv_define_module(cudastereo opencv_calib3d WRAP python) +ocv_define_module(cudastereo opencv_calib3d OPTIONAL opencv_cudev WRAP python) diff --git a/modules/cudastereo/doc/cudastereo.bib b/modules/cudastereo/doc/cudastereo.bib new file mode 100644 index 00000000000..f37177fd2ab --- /dev/null +++ b/modules/cudastereo/doc/cudastereo.bib @@ -0,0 +1,10 @@ +@InProceedings{Spangenberg2013, + author = {Spangenberg, Robert and Langner, Tobias and Rojas, Ra{\'u}l}, + title = {Weighted Semi-Global Matching and Center-Symmetric Census Transform for Robust Driver Assistance}, + booktitle = {Computer Analysis of Images and Patterns}, + year = {2013}, + pages = {34--41}, + publisher = {Springer Berlin Heidelberg}, + abstract = {Automotive applications based on stereo vision require robust and fast matching algorithms, which makes semi-global matching (SGM) a popular method in this field. Typically the Census transform is used as a cost function, since it is advantageous for outdoor scenes. We propose an extension based on center-symmetric local binary patterns, which allows better efficiency and higher matching quality. Our second contribution exploits knowledge about the three-dimensional structure of the scene to selectively enforce the smoothness constraints of SGM. It is shown that information about surface normals can be easily integrated by weighing the paths according to the gradient of the disparity. The different approaches are evaluated on the KITTI benchmark, which provides real imagery with LIDAR ground truth. The results indicate improved performance compared to state-of-the-art SGM based algorithms.}, + url = {https://www.mi.fu-berlin.de/inf/groups/ag-ki/publications/Semi-Global_Matching/caip2013rsp_fu.pdf} +} diff --git a/modules/cudastereo/include/opencv2/cudastereo.hpp b/modules/cudastereo/include/opencv2/cudastereo.hpp index 0c312054d7c..9cadd123b53 100644 --- a/modules/cudastereo/include/opencv2/cudastereo.hpp +++ b/modules/cudastereo/include/opencv2/cudastereo.hpp @@ -241,6 +241,53 @@ class CV_EXPORTS_W StereoConstantSpaceBP : public cuda::StereoBeliefPropagation CV_EXPORTS_W Ptr createStereoConstantSpaceBP(int ndisp = 128, int iters = 8, int levels = 4, int nr_plane = 4, int msg_type = CV_32F); +///////////////////////////////////////// +// StereoSGM + +/** @brief The class implements the modified H. Hirschmuller algorithm @cite HH08. +Limitation and difference are as follows: + +- By default, the algorithm uses only 4 directions which are horizontal and vertical path instead of 8. +Set mode=StereoSGM::MODE_HH in createStereoSGM to run the full variant of the algorithm. +- Mutual Information cost function is not implemented. +Instead, Center-Symmetric Census Transform with \f$9 \times 7\f$ window size from @cite Spangenberg2013 +is used for robustness. + +@sa cv::StereoSGBM +*/ +class CV_EXPORTS_W StereoSGM : public cv::StereoSGBM +{ +public: + /** @brief Computes disparity map for the specified stereo pair + + @param left Left 8-bit or 16-bit unsigned single-channel image. + @param right Right image of the same size and the same type as the left one. + @param disparity Output disparity map. It has the same size as the input images. + StereoSGM computes 16-bit fixed-point disparity map (where each disparity value has 4 fractional bits). + */ + CV_WRAP virtual void compute(InputArray left, InputArray right, OutputArray disparity) CV_OVERRIDE = 0; + + /** @brief Computes disparity map with specified CUDA Stream + + @sa compute + */ + CV_WRAP_AS(compute_with_stream) virtual void compute(InputArray left, InputArray right, OutputArray disparity, Stream& stream) = 0; +}; + +/** @brief Creates StereoSGM object. + +@param minDisparity Minimum possible disparity value. Normally, it is zero but sometimes rectification algorithms can shift images, so this parameter needs to be adjusted accordingly. +@param numDisparities Maximum disparity minus minimum disparity. The value must be 64, 128 or 256. +@param P1 The first parameter controlling the disparity smoothness.This parameter is used for the case of slanted surfaces (not fronto parallel). +@param P2 The second parameter controlling the disparity smoothness.This parameter is used for "solving" the depth discontinuities problem. +@param uniquenessRatio Margin in percentage by which the best (minimum) computed cost function +value should "win" the second best value to consider the found match correct. Normally, a value +within the 5-15 range is good enough. +@param mode Set it to StereoSGM::MODE_HH to run the full-scale two-pass dynamic programming algorithm. +It will consume O(W\*H\*numDisparities) bytes. By default, it is set to StereoSGM::MODE_HH4. +*/ +CV_EXPORTS_W Ptr createStereoSGM(int minDisparity = 0, int numDisparities = 128, int P1 = 10, int P2 = 120, int uniquenessRatio = 5, int mode = cv::cuda::StereoSGM::MODE_HH4); + ///////////////////////////////////////// // DisparityBilateralFilter diff --git a/modules/cudastereo/perf/perf_stereo.cpp b/modules/cudastereo/perf/perf_stereo.cpp index 50529c2fe09..2b999d9d120 100644 --- a/modules/cudastereo/perf/perf_stereo.cpp +++ b/modules/cudastereo/perf/perf_stereo.cpp @@ -252,4 +252,38 @@ PERF_TEST_P(Sz_Depth, DrawColorDisp, } } +////////////////////////////////////////////////////////////////////// +// StereoSGM + +PERF_TEST_P(ImagePair, StereoSGM, + Values(pair_string("gpu/perf/aloe.png", "gpu/perf/aloeR.png"))) +{ + declare.time(300.0); + + const cv::Mat imgLeft = readImage(GET_PARAM(0), cv::IMREAD_GRAYSCALE); + ASSERT_FALSE(imgLeft.empty()); + + const cv::Mat imgRight = readImage(GET_PARAM(1), cv::IMREAD_GRAYSCALE); + ASSERT_FALSE(imgRight.empty()); + + const int ndisp = 128; + + if (PERF_RUN_CUDA()) + { + cv::Ptr d_sgm = cv::cuda::createStereoSGM(0, ndisp); + + const cv::cuda::GpuMat d_imgLeft(imgLeft); + const cv::cuda::GpuMat d_imgRight(imgRight); + cv::cuda::GpuMat dst; + + TEST_CYCLE() d_sgm->compute(d_imgLeft, d_imgRight, dst); + + CUDA_SANITY_CHECK(dst); + } + else + { + FAIL_NO_CPU(); + } +} + }} // namespace diff --git a/modules/cudastereo/src/cuda/stereosgm.cu b/modules/cudastereo/src/cuda/stereosgm.cu new file mode 100644 index 00000000000..153bda3977a --- /dev/null +++ b/modules/cudastereo/src/cuda/stereosgm.cu @@ -0,0 +1,2081 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Author: The "adaskit Team" at Fixstars Corporation + +#include "opencv2/opencv_modules.hpp" + +#ifndef HAVE_OPENCV_CUDEV + +#error "opencv_cudev is required" + +#else + +#include +#include "stereosgm.hpp" +#include "opencv2/cudev/common.hpp" +#include "opencv2/cudev/warp/warp.hpp" +#include "opencv2/cudastereo.hpp" + +namespace cv { namespace cuda { namespace device { +namespace stereosgm +{ + +static constexpr uint16_t INVALID_DISP = static_cast(-1); + +namespace detail +{ + +template +__device__ __forceinline__ static T ldg(const T* const p) +{ +#if __CUDA_ARCH__ >= 350 + return __ldg(p); +#else + return *p; +#endif +} + +template +__device__ __forceinline__ static T shfl(T var, int srcLane, int width = cudev::WARP_SIZE, uint32_t mask = 0xFFFFFFFFU) +{ +#if __CUDA_ARCH__ >= 300 +#if CUDA_VERSION >= 9000 + return __shfl_sync(mask, var, srcLane, width); +#else + return __shfl(var, srcLane, width); +#endif // CUDA_VERSION +#else + static __shared__ T smem[WARPS_PER_BLOCK][cudev::WARP_SIZE]; + srcLane %= width; + smem[cudev::Warp::warpId()][cudev::Warp::laneId()] = var; + T ret = smem[cudev::Warp::warpId()][srcLane + (cudev::Warp::laneId() / width) * width]; + return ret; +#endif // __CUDA_ARCH__ +} + +template +__device__ __forceinline__ static T shfl_up(T var, unsigned int delta, int width = cudev::WARP_SIZE, uint32_t mask = 0xFFFFFFFFU) +{ +#if __CUDA_ARCH__ >= 300 +#if CUDA_VERSION >= 9000 + return __shfl_up_sync(mask, var, delta, width); +#else + return __shfl_up(var, delta, width); +#endif // CUDA_VERSION +#else + static __shared__ T smem[WARPS_PER_BLOCK][cudev::WARP_SIZE]; + smem[cudev::Warp::warpId()][cudev::Warp::laneId()] = var; + T ret = var; + if (cudev::Warp::laneId() % width >= delta) + { + ret = smem[cudev::Warp::warpId()][cudev::Warp::laneId() - delta]; + } + return ret; +#endif // __CUDA_ARCH__ +} + +template +__device__ __forceinline__ static T shfl_down(T var, unsigned int delta, int width = cudev::WARP_SIZE, uint32_t mask = 0xFFFFFFFFU) +{ +#if __CUDA_ARCH__ >= 300 +#if CUDA_VERSION >= 9000 + return __shfl_down_sync(mask, var, delta, width); +#else + return __shfl_down(var, delta, width); +#endif // CUDA_VERSION +#else + static __shared__ T smem[WARPS_PER_BLOCK][cudev::WARP_SIZE]; + smem[cudev::Warp::warpId()][cudev::Warp::laneId()] = var; + T ret = var; + if (cudev::Warp::laneId() % width + delta < width) + { + ret = smem[cudev::Warp::warpId()][cudev::Warp::laneId() + delta]; + } + return ret; +#endif // __CUDA_ARCH__ +} + +template +__device__ __forceinline__ static T shfl_xor(T var, int laneMask, int width = cudev::WARP_SIZE, uint32_t mask = 0xFFFFFFFFU) +{ +#if __CUDA_ARCH__ >= 300 +#if CUDA_VERSION >= 9000 + return __shfl_xor_sync(mask, var, laneMask, width); +#else + return __shfl_xor(var, laneMask, width); +#endif // CUDA_VERSION +#else + static __shared__ T smem[WARPS_PER_BLOCK][cudev::WARP_SIZE]; + smem[cudev::Warp::warpId()][cudev::Warp::laneId()] = var; + T ret = var; + if (((cudev::Warp::laneId() % width) ^ laneMask) < width) + { + ret = smem[cudev::Warp::warpId()][cudev::Warp::laneId() ^ laneMask]; + } + return ret; +#endif // __CUDA_ARCH__ +} + + +template +struct subgroup_min_impl +{ + static __device__ T call(T x, uint32_t mask) + { + x = ::min(x, shfl_xor(x, STEP / 2, GROUP_SIZE, mask)); + return subgroup_min_impl::call(x, mask); + } +}; +template +struct subgroup_min_impl +{ + static __device__ T call(T x, uint32_t) + { + return x; + } +}; + +template +struct subgroup_and_impl +{ + static __device__ bool call(bool x, uint32_t mask) + { + x &= shfl_xor(x, STEP / 2, GROUP_SIZE, mask); + return subgroup_and_impl::call(x, mask); + } +}; +template +struct subgroup_and_impl +{ + static __device__ bool call(bool x, uint32_t) + { + return x; + } +}; +} // namespace detail + + +template +__device__ inline T subgroup_min(T x, uint32_t mask) +{ + return detail::subgroup_min_impl::call(x, mask); +} + +template +__device__ inline bool subgroup_and(bool x, uint32_t mask) +{ + return detail::subgroup_and_impl::call(x, mask); +} + + +template +__device__ inline T load_as(const S *p) +{ + return *reinterpret_cast(p); +} + +template +__device__ inline void store_as(S *p, const T& x) +{ + *reinterpret_cast(p) = x; +} + + +template +__device__ inline uint32_t pack_uint8x4(T x, T y, T z, T w) +{ + uchar4 uint8x4; + uint8x4.x = static_cast(x); + uint8x4.y = static_cast(y); + uint8x4.z = static_cast(z); + uint8x4.w = static_cast(w); + return load_as(&uint8x4); +} + + +template +__device__ inline void load_uint8_vector(uint32_t *dest, const uint8_t *ptr); + +template <> +__device__ inline void load_uint8_vector<1u>(uint32_t *dest, const uint8_t *ptr) +{ + dest[0] = static_cast(ptr[0]); +} + +template <> +__device__ inline void load_uint8_vector<2u>(uint32_t *dest, const uint8_t *ptr) +{ + const auto uint8x2 = load_as(ptr); + dest[0] = uint8x2.x; dest[1] = uint8x2.y; +} + +template <> +__device__ inline void load_uint8_vector<4u>(uint32_t *dest, const uint8_t *ptr) +{ + const auto uint8x4 = load_as(ptr); + dest[0] = uint8x4.x; dest[1] = uint8x4.y; dest[2] = uint8x4.z; dest[3] = uint8x4.w; +} + +template <> +__device__ inline void load_uint8_vector<8u>(uint32_t *dest, const uint8_t *ptr) +{ + const auto uint32x2 = load_as(ptr); + load_uint8_vector<4u>(dest + 0, reinterpret_cast(&uint32x2.x)); + load_uint8_vector<4u>(dest + 4, reinterpret_cast(&uint32x2.y)); +} + +template <> +__device__ inline void load_uint8_vector<16u>(uint32_t *dest, const uint8_t *ptr) +{ + const auto uint32x4 = load_as(ptr); + load_uint8_vector<4u>(dest + 0, reinterpret_cast(&uint32x4.x)); + load_uint8_vector<4u>(dest + 4, reinterpret_cast(&uint32x4.y)); + load_uint8_vector<4u>(dest + 8, reinterpret_cast(&uint32x4.z)); + load_uint8_vector<4u>(dest + 12, reinterpret_cast(&uint32x4.w)); +} + + +template +__device__ inline void store_uint8_vector(uint8_t *dest, const uint32_t *ptr); + +template <> +__device__ inline void store_uint8_vector<1u>(uint8_t *dest, const uint32_t *ptr) +{ + dest[0] = static_cast(ptr[0]); +} + +template <> +__device__ inline void store_uint8_vector<2u>(uint8_t *dest, const uint32_t *ptr) +{ + uchar2 uint8x2; + uint8x2.x = static_cast(ptr[0]); + uint8x2.y = static_cast(ptr[0]); + store_as(dest, uint8x2); +} + +template <> +__device__ inline void store_uint8_vector<4u>(uint8_t *dest, const uint32_t *ptr) +{ + store_as(dest, pack_uint8x4(ptr[0], ptr[1], ptr[2], ptr[3])); +} + +template <> +__device__ inline void store_uint8_vector<8u>(uint8_t *dest, const uint32_t *ptr) +{ + uint2 uint32x2; + uint32x2.x = pack_uint8x4(ptr[0], ptr[1], ptr[2], ptr[3]); + uint32x2.y = pack_uint8x4(ptr[4], ptr[5], ptr[6], ptr[7]); + store_as(dest, uint32x2); +} + +template <> +__device__ inline void store_uint8_vector<16u>(uint8_t *dest, const uint32_t *ptr) +{ + uint4 uint32x4; + uint32x4.x = pack_uint8x4(ptr[0], ptr[1], ptr[2], ptr[3]); + uint32x4.y = pack_uint8x4(ptr[4], ptr[5], ptr[6], ptr[7]); + uint32x4.z = pack_uint8x4(ptr[8], ptr[9], ptr[10], ptr[11]); + uint32x4.w = pack_uint8x4(ptr[12], ptr[13], ptr[14], ptr[15]); + store_as(dest, uint32x4); +} + + +template +__device__ inline void load_uint16_vector(uint32_t *dest, const uint16_t *ptr); + +template <> +__device__ inline void load_uint16_vector<1u>(uint32_t *dest, const uint16_t *ptr) +{ + dest[0] = static_cast(ptr[0]); +} + +template <> +__device__ inline void load_uint16_vector<2u>(uint32_t *dest, const uint16_t *ptr) +{ + const auto uint16x2 = load_as(ptr); + dest[0] = uint16x2.x; dest[1] = uint16x2.y; +} + +template <> +__device__ inline void load_uint16_vector<4u>(uint32_t *dest, const uint16_t *ptr) +{ + const auto uint16x4 = load_as(ptr); + dest[0] = uint16x4.x; dest[1] = uint16x4.y; dest[2] = uint16x4.z; dest[3] = uint16x4.w; +} + +template <> +__device__ inline void load_uint16_vector<8u>(uint32_t *dest, const uint16_t *ptr) +{ + const auto uint32x4 = load_as(ptr); + load_uint16_vector<2u>(dest + 0, reinterpret_cast(&uint32x4.x)); + load_uint16_vector<2u>(dest + 2, reinterpret_cast(&uint32x4.y)); + load_uint16_vector<2u>(dest + 4, reinterpret_cast(&uint32x4.z)); + load_uint16_vector<2u>(dest + 6, reinterpret_cast(&uint32x4.w)); +} + + +template +__device__ inline void store_uint16_vector(uint16_t *dest, const uint32_t *ptr); + +template <> +__device__ inline void store_uint16_vector<1u>(uint16_t *dest, const uint32_t *ptr) +{ + dest[0] = static_cast(ptr[0]); +} + +template <> +__device__ inline void store_uint16_vector<2u>(uint16_t *dest, const uint32_t *ptr) +{ + ushort2 uint16x2; + uint16x2.x = static_cast(ptr[0]); + uint16x2.y = static_cast(ptr[1]); + store_as(dest, uint16x2); +} + +template <> +__device__ inline void store_uint16_vector<4u>(uint16_t *dest, const uint32_t *ptr) +{ + ushort4 uint16x4; + uint16x4.x = static_cast(ptr[0]); + uint16x4.y = static_cast(ptr[1]); + uint16x4.z = static_cast(ptr[2]); + uint16x4.w = static_cast(ptr[3]); + store_as(dest, uint16x4); +} + +template <> +__device__ inline void store_uint16_vector<8u>(uint16_t *dest, const uint32_t *ptr) +{ + uint4 uint32x4; + store_uint16_vector<2u>(reinterpret_cast(&uint32x4.x), &ptr[0]); + store_uint16_vector<2u>(reinterpret_cast(&uint32x4.y), &ptr[2]); + store_uint16_vector<2u>(reinterpret_cast(&uint32x4.z), &ptr[4]); + store_uint16_vector<2u>(reinterpret_cast(&uint32x4.w), &ptr[6]); + store_as(dest, uint32x4); +} + +template <> +__device__ inline void store_uint16_vector<16u>(uint16_t *dest, const uint32_t *ptr) +{ + store_uint16_vector<8u>(dest + 0, ptr + 0); + store_uint16_vector<8u>(dest + 8, ptr + 8); +} + +namespace census_transform +{ +namespace +{ +static constexpr int WINDOW_WIDTH = 9; +static constexpr int WINDOW_HEIGHT = 7; + +static constexpr int BLOCK_SIZE = 128; +static constexpr int LINES_PER_BLOCK = 16; + +template +__global__ void census_transform_kernel( + PtrStepSz src, + PtrStep dest) +{ + using pixel_type = T; + static const int SMEM_BUFFER_SIZE = WINDOW_HEIGHT + 1; + + const int half_kw = WINDOW_WIDTH / 2; + const int half_kh = WINDOW_HEIGHT / 2; + + __shared__ pixel_type smem_lines[SMEM_BUFFER_SIZE][BLOCK_SIZE]; + + const int tid = threadIdx.x; + const int x0 = blockIdx.x * (BLOCK_SIZE - WINDOW_WIDTH + 1) - half_kw; + const int y0 = blockIdx.y * LINES_PER_BLOCK; + + for (int i = 0; i < WINDOW_HEIGHT; ++i) + { + const int x = x0 + tid, y = y0 - half_kh + i; + pixel_type value = 0; + if (0 <= x && x < src.cols && 0 <= y && y < src.rows) + { + value = src(y, x); + } + smem_lines[i][tid] = value; + } + __syncthreads(); + +#pragma unroll + for (int i = 0; i < LINES_PER_BLOCK; ++i) + { + if (i + 1 < LINES_PER_BLOCK) + { + // Load to smem + const int x = x0 + tid, y = y0 + half_kh + i + 1; + pixel_type value = 0; + if (0 <= x && x < src.cols && 0 <= y && y < src.rows) + { + value = src(y, x); + } + const int smem_x = tid; + const int smem_y = (WINDOW_HEIGHT + i) % SMEM_BUFFER_SIZE; + smem_lines[smem_y][smem_x] = value; + } + + if (half_kw <= tid && tid < BLOCK_SIZE - half_kw) + { + // Compute and store + const int x = x0 + tid, y = y0 + i; + if (half_kw <= x && x < src.cols - half_kw && half_kh <= y && y < src.rows - half_kh) + { + const int smem_x = tid; + const int smem_y = (half_kh + i) % SMEM_BUFFER_SIZE; + int32_t f = 0; + for (int dy = -half_kh; dy < 0; ++dy) + { + const int smem_y1 = (smem_y + dy + SMEM_BUFFER_SIZE) % SMEM_BUFFER_SIZE; + const int smem_y2 = (smem_y - dy + SMEM_BUFFER_SIZE) % SMEM_BUFFER_SIZE; + for (int dx = -half_kw; dx <= half_kw; ++dx) + { + const int smem_x1 = smem_x + dx; + const int smem_x2 = smem_x - dx; + const auto a = smem_lines[smem_y1][smem_x1]; + const auto b = smem_lines[smem_y2][smem_x2]; + f = (f << 1) | (a > b); + } + } + for (int dx = -half_kw; dx < 0; ++dx) + { + const int smem_x1 = smem_x + dx; + const int smem_x2 = smem_x - dx; + const auto a = smem_lines[smem_y][smem_x1]; + const auto b = smem_lines[smem_y][smem_x2]; + f = (f << 1) | (a > b); + } + dest(y, x) = f; + } + } + __syncthreads(); + } +} +} // anonymous namespace + +void censusTransform(const GpuMat& src, GpuMat& dest, Stream& _stream) +{ + CV_Assert(src.size() == dest.size()); + CV_Assert(src.type() == CV_8UC1 || src.type() == CV_16UC1); + const int width_per_block = BLOCK_SIZE - WINDOW_WIDTH + 1; + const int height_per_block = LINES_PER_BLOCK; + const dim3 gdim( + cudev::divUp(src.cols, width_per_block), + cudev::divUp(src.rows, height_per_block)); + const dim3 bdim(BLOCK_SIZE); + cudaStream_t stream = StreamAccessor::getStream(_stream); + switch (src.type()) + { + case CV_8UC1: + census_transform_kernel<<>>(src, dest); + break; + case CV_16UC1: + census_transform_kernel<<>>(src, dest); + break; + } +} +} // namespace census_transform + +namespace path_aggregation +{ + +template < + unsigned int DP_BLOCK_SIZE, + unsigned int SUBGROUP_SIZE, + unsigned int WARPS_PER_BLOCK> + struct DynamicProgramming +{ + static_assert( + DP_BLOCK_SIZE >= 2, + "DP_BLOCK_SIZE must be greater than or equal to 2"); + static_assert( + (SUBGROUP_SIZE & (SUBGROUP_SIZE - 1)) == 0, + "SUBGROUP_SIZE must be a power of 2"); + + uint32_t last_min; + uint32_t dp[DP_BLOCK_SIZE]; + + __device__ DynamicProgramming() + : last_min(0) + { + for (unsigned int i = 0; i < DP_BLOCK_SIZE; ++i) + { + dp[i] = 0; + } + } + + __device__ void update( + uint32_t *local_costs, uint32_t p1, uint32_t p2, uint32_t mask) + { + const unsigned int lane_id = threadIdx.x % SUBGROUP_SIZE; + + const auto dp0 = dp[0]; + uint32_t lazy_out = 0, local_min = 0; + { + const unsigned int k = 0; + const uint32_t prev = detail::shfl_up(dp[DP_BLOCK_SIZE - 1], 1, cudev::WARP_SIZE, mask); + uint32_t out = ::min(dp[k] - last_min, p2); + if (lane_id != 0) + { + out = ::min(out, prev - last_min + p1); + } + out = ::min(out, dp[k + 1] - last_min + p1); + lazy_out = local_min = out + local_costs[k]; + } + for (unsigned int k = 1; k + 1 < DP_BLOCK_SIZE; ++k) + { + uint32_t out = ::min(dp[k] - last_min, p2); + out = ::min(out, dp[k - 1] - last_min + p1); + out = ::min(out, dp[k + 1] - last_min + p1); + dp[k - 1] = lazy_out; + lazy_out = out + local_costs[k]; + local_min = ::min(local_min, lazy_out); + } + { + const unsigned int k = DP_BLOCK_SIZE - 1; + const uint32_t next = detail::shfl_down(dp0, 1, cudev::WARP_SIZE, mask); + uint32_t out = ::min(dp[k] - last_min, p2); + out = ::min(out, dp[k - 1] - last_min + p1); + if (lane_id + 1 != SUBGROUP_SIZE) + { + out = ::min(out, next - last_min + p1); + } + dp[k - 1] = lazy_out; + dp[k] = out + local_costs[k]; + local_min = ::min(local_min, dp[k]); + } + last_min = subgroup_min(local_min, mask); + } +}; + +template +__device__ unsigned int generate_mask() +{ + static_assert(SIZE <= 32, "SIZE must be less than or equal to 32"); + return static_cast((1ull << SIZE) - 1u); +} + +namespace horizontal +{ +namespace +{ +static constexpr unsigned int DP_BLOCK_SIZE = 8u; +static constexpr unsigned int DP_BLOCKS_PER_THREAD = 1u; + +static constexpr unsigned int WARPS_PER_BLOCK = 4u; +static constexpr unsigned int BLOCK_SIZE = cudev::WARP_SIZE * WARPS_PER_BLOCK; + + +template +__global__ void aggregate_horizontal_path_kernel( + PtrStep left, + PtrStep right, + PtrStep dest, + int width, + int height, + unsigned int p1, + unsigned int p2, + int min_disp) +{ + static const unsigned int SUBGROUP_SIZE = MAX_DISPARITY / DP_BLOCK_SIZE; + static const unsigned int SUBGROUPS_PER_WARP = cudev::WARP_SIZE / SUBGROUP_SIZE; + static const unsigned int PATHS_PER_WARP = + cudev::WARP_SIZE * DP_BLOCKS_PER_THREAD / SUBGROUP_SIZE; + static const unsigned int PATHS_PER_BLOCK = + BLOCK_SIZE * DP_BLOCKS_PER_THREAD / SUBGROUP_SIZE; + + static_assert(DIRECTION == 1 || DIRECTION == -1, ""); + if (width == 0 || height == 0) + { + return; + } + + int32_t right_buffer[DP_BLOCKS_PER_THREAD][DP_BLOCK_SIZE]; + DynamicProgramming dp[DP_BLOCKS_PER_THREAD]; + + const unsigned int warp_id = cudev::Warp::warpId(); + const unsigned int group_id = cudev::Warp::laneId() / SUBGROUP_SIZE; + const unsigned int lane_id = threadIdx.x % SUBGROUP_SIZE; + const unsigned int shfl_mask = + generate_mask() << (group_id * SUBGROUP_SIZE); + + const unsigned int y0 = + PATHS_PER_BLOCK * blockIdx.x + + PATHS_PER_WARP * warp_id + + group_id; + const unsigned int feature_step = SUBGROUPS_PER_WARP; + const unsigned int dest_step = SUBGROUPS_PER_WARP * MAX_DISPARITY * width; + const unsigned int dp_offset = lane_id * DP_BLOCK_SIZE; + left = PtrStep(left.ptr(y0), left.step); + right = PtrStep(right.ptr(y0), right.step); + dest = PtrStep(&dest(0, y0 * width * MAX_DISPARITY), dest.step); + + if (y0 >= height) + { + return; + } + + if (DIRECTION > 0) + { + for (unsigned int i = 0; i < DP_BLOCKS_PER_THREAD; ++i) + { + for (unsigned int j = 0; j < DP_BLOCK_SIZE; ++j) + { + right_buffer[i][j] = 0; + } + } + } + else + { + for (unsigned int i = 0; i < DP_BLOCKS_PER_THREAD; ++i) + { + for (unsigned int j = 0; j < DP_BLOCK_SIZE; ++j) + { + const int x = static_cast(width - (min_disp + j + dp_offset)); + if (0 <= x && x < static_cast(width)) + { + right_buffer[i][j] = detail::ldg(&right(i * feature_step, x)); + } + else + { + right_buffer[i][j] = 0; + } + } + } + } + + int x0 = (DIRECTION > 0) ? 0 : static_cast((width - 1) & ~(DP_BLOCK_SIZE - 1)); + for (unsigned int iter = 0; iter < width; iter += DP_BLOCK_SIZE) + { + for (unsigned int i = 0; i < DP_BLOCK_SIZE; ++i) + { + const unsigned int x = x0 + (DIRECTION > 0 ? i : (DP_BLOCK_SIZE - 1 - i)); + if (x >= width) + { + continue; + } + for (unsigned int j = 0; j < DP_BLOCKS_PER_THREAD; ++j) + { + const unsigned int y = y0 + j * SUBGROUPS_PER_WARP; + if (y >= height) + { + continue; + } + const int32_t left_value = detail::ldg(&left(j * feature_step, x)); + if (DIRECTION > 0) + { + const int32_t t = right_buffer[j][DP_BLOCK_SIZE - 1]; + for (unsigned int k = DP_BLOCK_SIZE - 1; k > 0; --k) + { + right_buffer[j][k] = right_buffer[j][k - 1]; + } + right_buffer[j][0] = detail::shfl_up(t, 1, SUBGROUP_SIZE, shfl_mask); + if (lane_id == 0 && x >= min_disp) + { + right_buffer[j][0] = + detail::ldg(&right(j * feature_step, x - min_disp)); + } + } + else + { + const int32_t t = right_buffer[j][0]; + for (unsigned int k = 1; k < DP_BLOCK_SIZE; ++k) + { + right_buffer[j][k - 1] = right_buffer[j][k]; + } + right_buffer[j][DP_BLOCK_SIZE - 1] = detail::shfl_down(t, 1, SUBGROUP_SIZE, shfl_mask); + if (lane_id + 1 == SUBGROUP_SIZE) + { + if (x >= min_disp + dp_offset + DP_BLOCK_SIZE - 1) + { + right_buffer[j][DP_BLOCK_SIZE - 1] = + detail::ldg(&right(j * feature_step, x - (min_disp + dp_offset + DP_BLOCK_SIZE - 1))); + } + else + { + right_buffer[j][DP_BLOCK_SIZE - 1] = 0; + } + } + } + uint32_t local_costs[DP_BLOCK_SIZE]; + for (unsigned int k = 0; k < DP_BLOCK_SIZE; ++k) + { + local_costs[k] = __popc(left_value ^ right_buffer[j][k]); + } + dp[j].update(local_costs, p1, p2, shfl_mask); + store_uint8_vector( + &dest(0, j * dest_step + x * MAX_DISPARITY + dp_offset), + dp[j].dp); + } + } + x0 += static_cast(DP_BLOCK_SIZE) * DIRECTION; + } +} +} // anonymous namespace + +template +void aggregateLeft2RightPath( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& _stream) +{ + CV_Assert(left.size() == right.size()); + CV_Assert(left.type() == right.type()); + CV_Assert(left.type() == CV_32SC1); + static const unsigned int SUBGROUP_SIZE = MAX_DISPARITY / DP_BLOCK_SIZE; + static const unsigned int PATHS_PER_BLOCK = + BLOCK_SIZE * DP_BLOCKS_PER_THREAD / SUBGROUP_SIZE; + + const int gdim = cudev::divUp(left.rows, PATHS_PER_BLOCK); + const int bdim = BLOCK_SIZE; + cudaStream_t stream = cv::cuda::StreamAccessor::getStream(_stream); + aggregate_horizontal_path_kernel<1, MAX_DISPARITY><<>>( + left, right, dest, left.cols, left.rows, p1, p2, min_disp); +} + +template +void aggregateRight2LeftPath( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& _stream) +{ + CV_Assert(left.size() == right.size()); + CV_Assert(left.type() == right.type()); + CV_Assert(left.type() == CV_32SC1); + static const unsigned int SUBGROUP_SIZE = MAX_DISPARITY / DP_BLOCK_SIZE; + static const unsigned int PATHS_PER_BLOCK = + BLOCK_SIZE * DP_BLOCKS_PER_THREAD / SUBGROUP_SIZE; + + const int gdim = cudev::divUp(left.rows, PATHS_PER_BLOCK); + const int bdim = BLOCK_SIZE; + cudaStream_t stream = cv::cuda::StreamAccessor::getStream(_stream); + aggregate_horizontal_path_kernel<-1, MAX_DISPARITY><<>>( + left, right, dest, left.cols, left.rows, p1, p2, min_disp); +} + + +template CV_EXPORTS void aggregateLeft2RightPath<64u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& _stream); + +template CV_EXPORTS void aggregateLeft2RightPath<128u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& _stream); + +template CV_EXPORTS void aggregateLeft2RightPath<256u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& _stream); + +template CV_EXPORTS void aggregateRight2LeftPath<64u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& _stream); + +template CV_EXPORTS void aggregateRight2LeftPath<128u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& _stream); + +template CV_EXPORTS void aggregateRight2LeftPath<256u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& _stream); +} // namespace horizontal + +namespace vertical +{ +namespace +{ +static constexpr unsigned int DP_BLOCK_SIZE = 16u; +static constexpr unsigned int WARPS_PER_BLOCK = 8u; +static constexpr unsigned int BLOCK_SIZE = cudev::WARP_SIZE * WARPS_PER_BLOCK; + +template +__global__ void aggregate_vertical_path_kernel( + PtrStep left, + PtrStep right, + PtrStep dest, + int width, + int height, + unsigned int p1, + unsigned int p2, + int min_disp) +{ + static const unsigned int SUBGROUP_SIZE = MAX_DISPARITY / DP_BLOCK_SIZE; + static const unsigned int PATHS_PER_WARP = cudev::WARP_SIZE / SUBGROUP_SIZE; + static const unsigned int PATHS_PER_BLOCK = BLOCK_SIZE / SUBGROUP_SIZE; + + static const unsigned int RIGHT_BUFFER_SIZE = MAX_DISPARITY + PATHS_PER_BLOCK; + static const unsigned int RIGHT_BUFFER_ROWS = RIGHT_BUFFER_SIZE / DP_BLOCK_SIZE; + + static_assert(DIRECTION == 1 || DIRECTION == -1, ""); + if (width == 0 || height == 0) + { + return; + } + + __shared__ int32_t right_buffer[2 * DP_BLOCK_SIZE][RIGHT_BUFFER_ROWS + 1]; + DynamicProgramming dp; + + const unsigned int warp_id = cudev::Warp::warpId(); + const unsigned int group_id = cudev::Warp::laneId() / SUBGROUP_SIZE; + const unsigned int lane_id = threadIdx.x % SUBGROUP_SIZE; + const unsigned int shfl_mask = + generate_mask() << (group_id * SUBGROUP_SIZE); + + const unsigned int x = + blockIdx.x * PATHS_PER_BLOCK + + warp_id * PATHS_PER_WARP + + group_id; + const unsigned int right_x0 = blockIdx.x * PATHS_PER_BLOCK; + const unsigned int dp_offset = lane_id * DP_BLOCK_SIZE; + + const unsigned int right0_addr = + (right_x0 + PATHS_PER_BLOCK - 1) - x + dp_offset; + const unsigned int right0_addr_lo = right0_addr % DP_BLOCK_SIZE; + const unsigned int right0_addr_hi = right0_addr / DP_BLOCK_SIZE; + + for (unsigned int iter = 0; iter < height; ++iter) + { + const unsigned int y = (DIRECTION > 0 ? iter : height - 1 - iter); + // Load left to register + int32_t left_value; + if (x < width) + { + left_value = left(y, x); + } + // Load right to smem + for (unsigned int i0 = 0; i0 < RIGHT_BUFFER_SIZE; i0 += BLOCK_SIZE) + { + const unsigned int i = i0 + threadIdx.x; + if (i < RIGHT_BUFFER_SIZE) + { + const int x = static_cast(right_x0 + PATHS_PER_BLOCK - 1 - i - min_disp); + int32_t right_value = 0; + if (0 <= x && x < static_cast(width)) + { + right_value = right(y, x); + } + const unsigned int lo = i % DP_BLOCK_SIZE; + const unsigned int hi = i / DP_BLOCK_SIZE; + right_buffer[lo][hi] = right_value; + if (hi > 0) + { + right_buffer[lo + DP_BLOCK_SIZE][hi - 1] = right_value; + } + } + } + __syncthreads(); + // Compute + if (x < width) + { + int32_t right_values[DP_BLOCK_SIZE]; + for (unsigned int j = 0; j < DP_BLOCK_SIZE; ++j) + { + right_values[j] = right_buffer[right0_addr_lo + j][right0_addr_hi]; + } + uint32_t local_costs[DP_BLOCK_SIZE]; + for (unsigned int j = 0; j < DP_BLOCK_SIZE; ++j) + { + local_costs[j] = __popc(left_value ^ right_values[j]); + } + dp.update(local_costs, p1, p2, shfl_mask); + store_uint8_vector( + &dest(0, dp_offset + x * MAX_DISPARITY + y * MAX_DISPARITY * width), + dp.dp); + } + __syncthreads(); + } +} +} // anonymous namespace + +template +void aggregateUp2DownPath( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& _stream) +{ + static const unsigned int SUBGROUP_SIZE = MAX_DISPARITY / DP_BLOCK_SIZE; + static const unsigned int PATHS_PER_BLOCK = BLOCK_SIZE / SUBGROUP_SIZE; + + const Size size = left.size(); + const int gdim = cudev::divUp(size.width, PATHS_PER_BLOCK); + const int bdim = BLOCK_SIZE; + cudaStream_t stream = cv::cuda::StreamAccessor::getStream(_stream); + aggregate_vertical_path_kernel<1, MAX_DISPARITY><<>>( + left, right, dest, size.width, size.height, p1, p2, min_disp); +} + +template +void aggregateDown2UpPath( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& _stream) +{ + static const unsigned int SUBGROUP_SIZE = MAX_DISPARITY / DP_BLOCK_SIZE; + static const unsigned int PATHS_PER_BLOCK = BLOCK_SIZE / SUBGROUP_SIZE; + + const Size size = left.size(); + const int gdim = cudev::divUp(size.width, PATHS_PER_BLOCK); + const int bdim = BLOCK_SIZE; + cudaStream_t stream = cv::cuda::StreamAccessor::getStream(_stream); + aggregate_vertical_path_kernel<-1, MAX_DISPARITY><<>>( + left, right, dest, size.width, size.height, p1, p2, min_disp); +} + + +template CV_EXPORTS void aggregateUp2DownPath<64u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +template CV_EXPORTS void aggregateUp2DownPath<128u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +template CV_EXPORTS void aggregateUp2DownPath<256u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +template CV_EXPORTS void aggregateDown2UpPath<64u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +template CV_EXPORTS void aggregateDown2UpPath<128u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +template CV_EXPORTS void aggregateDown2UpPath<256u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +} // namespace vertical + +namespace oblique +{ +namespace +{ +static constexpr unsigned int DP_BLOCK_SIZE = 16u; +static constexpr unsigned int WARPS_PER_BLOCK = 8u; +static constexpr unsigned int BLOCK_SIZE = cudev::WARP_SIZE * WARPS_PER_BLOCK; + +template +__global__ void aggregate_oblique_path_kernel( + PtrStep left, + PtrStep right, + PtrStep dest, + int width, + int height, + unsigned int p1, + unsigned int p2, + int min_disp) +{ + static const unsigned int SUBGROUP_SIZE = MAX_DISPARITY / DP_BLOCK_SIZE; + static const unsigned int PATHS_PER_WARP = cudev::WARP_SIZE / SUBGROUP_SIZE; + static const unsigned int PATHS_PER_BLOCK = BLOCK_SIZE / SUBGROUP_SIZE; + + static const unsigned int RIGHT_BUFFER_SIZE = MAX_DISPARITY + PATHS_PER_BLOCK; + static const unsigned int RIGHT_BUFFER_ROWS = RIGHT_BUFFER_SIZE / DP_BLOCK_SIZE; + + static_assert(X_DIRECTION == 1 || X_DIRECTION == -1, ""); + static_assert(Y_DIRECTION == 1 || Y_DIRECTION == -1, ""); + if (width == 0 || height == 0) + { + return; + } + + __shared__ int32_t right_buffer[2 * DP_BLOCK_SIZE][RIGHT_BUFFER_ROWS]; + DynamicProgramming dp; + + const unsigned int warp_id = cudev::Warp::warpId(); + const unsigned int group_id = cudev::Warp::laneId() / SUBGROUP_SIZE; + const unsigned int lane_id = threadIdx.x % SUBGROUP_SIZE; + const unsigned int shfl_mask = + generate_mask() << (group_id * SUBGROUP_SIZE); + + const int x0 = + blockIdx.x * PATHS_PER_BLOCK + + warp_id * PATHS_PER_WARP + + group_id + + (X_DIRECTION > 0 ? -static_cast(height - 1) : 0); + const int right_x00 = + blockIdx.x * PATHS_PER_BLOCK + + (X_DIRECTION > 0 ? -static_cast(height - 1) : 0); + const unsigned int dp_offset = lane_id * DP_BLOCK_SIZE; + + const unsigned int right0_addr = + static_cast(right_x00 + PATHS_PER_BLOCK - 1 - x0) + dp_offset; + const unsigned int right0_addr_lo = right0_addr % DP_BLOCK_SIZE; + const unsigned int right0_addr_hi = right0_addr / DP_BLOCK_SIZE; + + for (unsigned int iter = 0; iter < height; ++iter) + { + const int y = static_cast(Y_DIRECTION > 0 ? iter : height - 1 - iter); + const int x = x0 + static_cast(iter) * X_DIRECTION; + const int right_x0 = right_x00 + static_cast(iter) * X_DIRECTION; + // Load right to smem + for (unsigned int i0 = 0; i0 < RIGHT_BUFFER_SIZE; i0 += BLOCK_SIZE) + { + const unsigned int i = i0 + threadIdx.x; + if (i < RIGHT_BUFFER_SIZE) + { + const int x = static_cast(right_x0 + PATHS_PER_BLOCK - 1 - i - min_disp); + int32_t right_value = 0; + if (0 <= x && x < static_cast(width)) + { + right_value = right(y, x); + } + const unsigned int lo = i % DP_BLOCK_SIZE; + const unsigned int hi = i / DP_BLOCK_SIZE; + right_buffer[lo][hi] = right_value; + if (hi > 0) + { + right_buffer[lo + DP_BLOCK_SIZE][hi - 1] = right_value; + } + } + } + __syncthreads(); + // Compute + if (0 <= x && x < static_cast(width)) + { + const int32_t left_value = detail::ldg(&left(y, x)); + int32_t right_values[DP_BLOCK_SIZE]; + for (unsigned int j = 0; j < DP_BLOCK_SIZE; ++j) + { + right_values[j] = right_buffer[right0_addr_lo + j][right0_addr_hi]; + } + uint32_t local_costs[DP_BLOCK_SIZE]; + for (unsigned int j = 0; j < DP_BLOCK_SIZE; ++j) + { + local_costs[j] = __popc(left_value ^ right_values[j]); + } + dp.update(local_costs, p1, p2, shfl_mask); + store_uint8_vector( + &dest(0, dp_offset + x * MAX_DISPARITY + y * MAX_DISPARITY * width), + dp.dp); + } + __syncthreads(); + } +} +} // anonymous namespace + +template +void aggregateUpleft2DownrightPath( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& _stream) +{ + static const unsigned int SUBGROUP_SIZE = MAX_DISPARITY / DP_BLOCK_SIZE; + static const unsigned int PATHS_PER_BLOCK = BLOCK_SIZE / SUBGROUP_SIZE; + + const Size size = left.size(); + const int gdim = cudev::divUp(size.width + size.height - 1, PATHS_PER_BLOCK); + const int bdim = BLOCK_SIZE; + cudaStream_t stream = StreamAccessor::getStream(_stream); + aggregate_oblique_path_kernel<1, 1, MAX_DISPARITY><<>>( + left, right, dest, size.width, size.height, p1, p2, min_disp); +} + +template +void aggregateUpright2DownleftPath( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& _stream) +{ + static const unsigned int SUBGROUP_SIZE = MAX_DISPARITY / DP_BLOCK_SIZE; + static const unsigned int PATHS_PER_BLOCK = BLOCK_SIZE / SUBGROUP_SIZE; + + const Size size = left.size(); + const int gdim = cudev::divUp(size.width + size.height - 1, PATHS_PER_BLOCK); + const int bdim = BLOCK_SIZE; + cudaStream_t stream = StreamAccessor::getStream(_stream); + aggregate_oblique_path_kernel<-1, 1, MAX_DISPARITY><<>>( + left, right, dest, size.width, size.height, p1, p2, min_disp); +} + +template +void aggregateDownright2UpleftPath( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& _stream) +{ + static const unsigned int SUBGROUP_SIZE = MAX_DISPARITY / DP_BLOCK_SIZE; + static const unsigned int PATHS_PER_BLOCK = BLOCK_SIZE / SUBGROUP_SIZE; + + const Size size = left.size(); + const int gdim = cudev::divUp(size.width + size.height - 1, PATHS_PER_BLOCK); + const int bdim = BLOCK_SIZE; + cudaStream_t stream = StreamAccessor::getStream(_stream); + aggregate_oblique_path_kernel<-1, -1, MAX_DISPARITY><<>>( + left, right, dest, size.width, size.height, p1, p2, min_disp); +} + +template +void aggregateDownleft2UprightPath( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& _stream) +{ + static const unsigned int SUBGROUP_SIZE = MAX_DISPARITY / DP_BLOCK_SIZE; + static const unsigned int PATHS_PER_BLOCK = BLOCK_SIZE / SUBGROUP_SIZE; + + const Size size = left.size(); + const int gdim = cudev::divUp(size.width + size.height - 1, PATHS_PER_BLOCK); + const int bdim = BLOCK_SIZE; + cudaStream_t stream = StreamAccessor::getStream(_stream); + aggregate_oblique_path_kernel<1, -1, MAX_DISPARITY><<>>( + left, right, dest, size.width, size.height, p1, p2, min_disp); +} + +template CV_EXPORTS void aggregateUpleft2DownrightPath<64u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +template CV_EXPORTS void aggregateUpleft2DownrightPath<128u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +template CV_EXPORTS void aggregateUpleft2DownrightPath<256u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +template CV_EXPORTS void aggregateUpright2DownleftPath<64u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +template CV_EXPORTS void aggregateUpright2DownleftPath<128u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +template CV_EXPORTS void aggregateUpright2DownleftPath<256u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +template CV_EXPORTS void aggregateDownright2UpleftPath<64u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +template CV_EXPORTS void aggregateDownright2UpleftPath<128u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +template CV_EXPORTS void aggregateDownright2UpleftPath<256u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +template CV_EXPORTS void aggregateDownleft2UprightPath<64u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +template CV_EXPORTS void aggregateDownleft2UprightPath<128u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +template CV_EXPORTS void aggregateDownleft2UprightPath<256u>( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +} // namespace oblique + +template +void PathAggregation::operator() (const GpuMat& left, const GpuMat& right, GpuMat& dest, int mode, int p1, int p2, int min_disp, Stream& stream) +{ + CV_Assert(left.size() == right.size()); + CV_Assert(left.type() == right.type()); + CV_Assert(left.type() == CV_32SC1); + + const int num_paths = mode == StereoSGBM::MODE_HH4 ? 4 : 8; + + stream.waitForCompletion(); + + const Size size = left.size(); + const int buffer_step = size.width * size.height * static_cast(MAX_DISPARITY); + CV_Assert(dest.rows == 1 && buffer_step * num_paths == dest.cols); + + for (int i = 0; i < num_paths; ++i) + { + subs[i] = dest.colRange(i * buffer_step, (i + 1) * buffer_step); + } + + vertical::aggregateUp2DownPath(left, right, subs[0], p1, p2, min_disp, streams[0]); + vertical::aggregateDown2UpPath(left, right, subs[1], p1, p2, min_disp, streams[1]); + horizontal::aggregateLeft2RightPath(left, right, subs[2], p1, p2, min_disp, streams[2]); + horizontal::aggregateRight2LeftPath(left, right, subs[3], p1, p2, min_disp, streams[3]); + + if (mode == StereoSGBM::MODE_HH) + { + oblique::aggregateUpleft2DownrightPath(left, right, subs[4], p1, p2, min_disp, streams[4]); + oblique::aggregateUpright2DownleftPath(left, right, subs[5], p1, p2, min_disp, streams[5]); + oblique::aggregateDownright2UpleftPath(left, right, subs[6], p1, p2, min_disp, streams[6]); + oblique::aggregateDownleft2UprightPath(left, right, subs[7], p1, p2, min_disp, streams[7]); + } + + // synchronization + for (int i = 0; i < num_paths; ++i) + { + events[i].record(streams[i]); + stream.waitEvent(events[i]); + streams[i].waitForCompletion(); + } +} + +template void PathAggregation::operator()< 64>(const GpuMat& left, const GpuMat& right, GpuMat& dest, int mode, int p1, int p2, int min_disp, Stream& stream); +template void PathAggregation::operator()<128>(const GpuMat& left, const GpuMat& right, GpuMat& dest, int mode, int p1, int p2, int min_disp, Stream& stream); +template void PathAggregation::operator()<256>(const GpuMat& left, const GpuMat& right, GpuMat& dest, int mode, int p1, int p2, int min_disp, Stream& stream); + +} // namespace path_aggregation + +namespace winner_takes_all +{ +namespace +{ +static constexpr unsigned int WARPS_PER_BLOCK = 8u; +static constexpr unsigned int BLOCK_SIZE = WARPS_PER_BLOCK * cudev::WARP_SIZE; + +__device__ inline uint32_t pack_cost_index(uint32_t cost, uint32_t index) +{ + union + { + uint32_t uint32; + ushort2 uint16x2; + } u; + u.uint16x2.x = static_cast(index); + u.uint16x2.y = static_cast(cost); + return u.uint32; +} + +__device__ uint32_t unpack_cost(uint32_t packed) +{ + return packed >> 16; +} + +__device__ int unpack_index(uint32_t packed) +{ + return packed & 0xffffu; +} + +using ComputeDisparity = uint32_t(*)(uint32_t, uint32_t, uint16_t*); + +__device__ inline uint32_t compute_disparity_normal(uint32_t disp, uint32_t cost = 0, uint16_t* smem = nullptr) +{ + return disp; +} + +template +__device__ inline uint32_t compute_disparity_subpixel(uint32_t disp, uint32_t cost, uint16_t* smem) +{ + uint32_t subp = disp; + subp <<= StereoSGBM::DISP_SHIFT; + if (disp > 0 && disp < MAX_DISPARITY - 1) + { + const int left = smem[disp - 1]; + const int right = smem[disp + 1]; + const int numer = left - right; + const int denom = left - 2 * cost + right; + subp += ((numer << StereoSGBM::DISP_SHIFT) + denom) / (2 * denom); + } + return subp; +} + + +template +__global__ void winner_takes_all_kernel( + const PtrStep _src, + PtrStep _left_dest, + PtrStep _right_dest, + int width, + int height, + float uniqueness) +{ + static const unsigned int ACCUMULATION_PER_THREAD = 16u; + static const unsigned int REDUCTION_PER_THREAD = MAX_DISPARITY / cudev::WARP_SIZE; + static const unsigned int ACCUMULATION_INTERVAL = ACCUMULATION_PER_THREAD / REDUCTION_PER_THREAD; + static const unsigned int UNROLL_DEPTH = + (REDUCTION_PER_THREAD > ACCUMULATION_INTERVAL) + ? REDUCTION_PER_THREAD + : ACCUMULATION_INTERVAL; + + const unsigned int cost_step = MAX_DISPARITY * width * height; + const unsigned int warp_id = cudev::Warp::warpId(); + const unsigned int lane_id = cudev::Warp::laneId(); + + const unsigned int y = blockIdx.x * WARPS_PER_BLOCK + warp_id; + const PtrStep src{ (uint8_t*)&_src(0, y * MAX_DISPARITY * width), height * width * MAX_DISPARITY * NUM_PATHS }; + PtrStep left_dest{ _left_dest.ptr(y), _left_dest.step }; + PtrStep right_dest{ _right_dest.ptr(y), _right_dest.step }; + + if (y >= height) + { + return; + } + + __shared__ uint16_t smem_cost_sum[WARPS_PER_BLOCK][ACCUMULATION_INTERVAL][MAX_DISPARITY]; + + uint32_t right_best[REDUCTION_PER_THREAD]; + for (unsigned int i = 0; i < REDUCTION_PER_THREAD; ++i) + { + right_best[i] = 0xffffffffu; + } + + for (unsigned int x0 = 0; x0 < width; x0 += UNROLL_DEPTH) + { +#pragma unroll + for (unsigned int x1 = 0; x1 < UNROLL_DEPTH; ++x1) + { + if (x1 % ACCUMULATION_INTERVAL == 0) + { + const unsigned int k = lane_id * ACCUMULATION_PER_THREAD; + const unsigned int k_hi = k / MAX_DISPARITY; + const unsigned int k_lo = k % MAX_DISPARITY; + const unsigned int x = x0 + x1 + k_hi; + if (x < width) + { + const unsigned int offset = x * MAX_DISPARITY + k_lo; + uint32_t sum[ACCUMULATION_PER_THREAD]; + for (unsigned int i = 0; i < ACCUMULATION_PER_THREAD; ++i) + { + sum[i] = 0; + } + for (unsigned int p = 0; p < NUM_PATHS; ++p) + { + uint32_t load_buffer[ACCUMULATION_PER_THREAD]; + load_uint8_vector( + load_buffer, &src(0, p * cost_step + offset)); + for (unsigned int i = 0; i < ACCUMULATION_PER_THREAD; ++i) + { + sum[i] += load_buffer[i]; + } + } + store_uint16_vector( + &smem_cost_sum[warp_id][k_hi][k_lo], sum); + } +#if CUDA_VERSION >= 9000 + __syncwarp(); +#else + __threadfence_block(); +#endif + } + const unsigned int x = x0 + x1; + if (x < width) + { + // Load sum of costs + const unsigned int smem_x = x1 % ACCUMULATION_INTERVAL; + const unsigned int k0 = lane_id * REDUCTION_PER_THREAD; + uint32_t local_cost_sum[REDUCTION_PER_THREAD]; + load_uint16_vector( + local_cost_sum, &smem_cost_sum[warp_id][smem_x][k0]); + // Pack sum of costs and dispairty + uint32_t local_packed_cost[REDUCTION_PER_THREAD]; + for (unsigned int i = 0; i < REDUCTION_PER_THREAD; ++i) + { + local_packed_cost[i] = pack_cost_index(local_cost_sum[i], k0 + i); + } + // Update left + uint32_t best = 0xffffffffu; + for (unsigned int i = 0; i < REDUCTION_PER_THREAD; ++i) + { + best = ::min(best, local_packed_cost[i]); + } + best = subgroup_min(best, 0xffffffffu); + // Update right +#pragma unroll + for (unsigned int i = 0; i < REDUCTION_PER_THREAD; ++i) + { + const unsigned int k = lane_id * REDUCTION_PER_THREAD + i; + const int p = static_cast(((x - k) & ~(MAX_DISPARITY - 1)) + k); + const unsigned int d = static_cast(x - p); + uint32_t recv = detail::shfl(local_packed_cost[(REDUCTION_PER_THREAD - i + x1) % REDUCTION_PER_THREAD], d / REDUCTION_PER_THREAD); + right_best[i] = ::min(right_best[i], recv); + if (d == MAX_DISPARITY - 1) + { + if (0 <= p) + { + right_dest(0, p) = compute_disparity_normal(unpack_index(right_best[i])); + } + right_best[i] = 0xffffffffu; + } + } + // Resume updating left to avoid execution dependency + const uint32_t bestCost = unpack_cost(best); + const int bestDisp = unpack_index(best); + bool uniq = true; + for (unsigned int i = 0; i < REDUCTION_PER_THREAD; ++i) + { + const uint32_t x = local_packed_cost[i]; + const bool uniq1 = unpack_cost(x) * uniqueness >= bestCost; + const bool uniq2 = ::abs(unpack_index(x) - bestDisp) <= 1; + uniq &= uniq1 || uniq2; + } + uniq = subgroup_and(uniq, 0xffffffffu); + if (lane_id == 0) + { + left_dest(0, x) = uniq ? compute_disparity(bestDisp, bestCost, smem_cost_sum[warp_id][smem_x]) : INVALID_DISP; + } + } + } + } + for (unsigned int i = 0; i < REDUCTION_PER_THREAD; ++i) + { + const unsigned int k = lane_id * REDUCTION_PER_THREAD + i; + const int p = static_cast(((width - k) & ~(MAX_DISPARITY - 1)) + k); + if (0 <= p && p < width) + { + right_dest(0, p) = compute_disparity_normal(unpack_index(right_best[i])); + } + } +} +} // anonymous namespace + +template +void winnerTakesAll(const GpuMat& src, GpuMat& left, GpuMat& right, float uniqueness, bool subpixel, int mode, cv::cuda::Stream& _stream) +{ + cv::Size size = left.size(); + int num_paths = mode == StereoSGBM::MODE_HH4 ? 4 : 8; + CV_Assert(src.rows == 1 && src.cols == size.width * size.height * static_cast(MAX_DISPARITY) * num_paths); + CV_Assert(size == right.size()); + CV_Assert(left.type() == right.type()); + CV_Assert(src.type() == CV_8UC1); + CV_Assert(mode == StereoSGBM::MODE_HH || mode == StereoSGBM::MODE_HH4); + const int gdim = cudev::divUp(size.height, WARPS_PER_BLOCK); + const int bdim = BLOCK_SIZE; + cudaStream_t stream = cv::cuda::StreamAccessor::getStream(_stream); + if (subpixel && mode == StereoSGBM::MODE_HH) + { + winner_takes_all_kernel><<>>( + src, left, right, size.width, size.height, uniqueness); + } + else if (subpixel && mode == StereoSGBM::MODE_HH4) + { + winner_takes_all_kernel><<>>( + src, left, right, size.width, size.height, uniqueness); + } + else if (!subpixel && mode == StereoSGBM::MODE_HH) + { + winner_takes_all_kernel<<>>( + src, left, right, size.width, size.height, uniqueness); + } + else /* if (!subpixel && mode == StereoSGBM::MODE_HH4) */ + { + winner_takes_all_kernel<<>>( + src, left, right, size.width, size.height, uniqueness); + } +} + +template CV_EXPORTS void winnerTakesAll< 64>(const GpuMat&, GpuMat&, GpuMat&, float, bool, int, cv::cuda::Stream&); +template CV_EXPORTS void winnerTakesAll<128>(const GpuMat&, GpuMat&, GpuMat&, float, bool, int, cv::cuda::Stream&); +template CV_EXPORTS void winnerTakesAll<256>(const GpuMat&, GpuMat&, GpuMat&, float, bool, int, cv::cuda::Stream&); +} // namespace winner_takes_all + +namespace median_filter +{ +namespace +{ +const int BLOCK_X = 16; +const int BLOCK_Y = 16; +const int KSIZE = 3; +const int RADIUS = KSIZE / 2; +const int KSIZE_SQ = KSIZE * KSIZE; + +template +__device__ inline void swap(T& x, T& y) +{ + T tmp(x); + x = y; + y = tmp; +} + +// sort, min, max of 1 element +template __device__ inline void dev_sort(T& x, T& y) +{ + if (x > y) swap(x, y); +} +template __device__ inline void dev_min(T& x, T& y) +{ + x = ::min(x, y); +} +template __device__ inline void dev_max(T& x, T& y) +{ + y = ::max(x, y); +} + +// sort, min, max of 2 elements +__device__ inline void dev_sort_2(uint32_t& x, uint32_t& y) +{ + const uint32_t mask = __vcmpgtu2(x, y); + const uint32_t tmp = (x ^ y) & mask; + x ^= tmp; + y ^= tmp; +} +__device__ inline void dev_min_2(uint32_t& x, uint32_t& y) +{ + x = __vminu2(x, y); +} +__device__ inline void dev_max_2(uint32_t& x, uint32_t& y) +{ + y = __vmaxu2(x, y); +} + +template <> __device__ inline void dev_sort(uint32_t& x, uint32_t& y) +{ + dev_sort_2(x, y); +} +template <> __device__ inline void dev_min(uint32_t& x, uint32_t& y) +{ + dev_min_2(x, y); +} +template <> __device__ inline void dev_max(uint32_t& x, uint32_t& y) +{ + dev_max_2(x, y); +} + +// sort, min, max of 4 elements +__device__ inline void dev_sort_4(uint32_t& x, uint32_t& y) +{ + const uint32_t mask = __vcmpgtu4(x, y); + const uint32_t tmp = (x ^ y) & mask; + x ^= tmp; + y ^= tmp; +} +__device__ inline void dev_min_4(uint32_t& x, uint32_t& y) +{ + x = __vminu4(x, y); +} +__device__ inline void dev_max_4(uint32_t& x, uint32_t& y) +{ + y = __vmaxu4(x, y); +} + +template <> __device__ inline void dev_sort(uint32_t& x, uint32_t& y) +{ + dev_sort_4(x, y); +} +template <> __device__ inline void dev_min(uint32_t& x, uint32_t& y) +{ + dev_min_4(x, y); +} +template <> __device__ inline void dev_max(uint32_t& x, uint32_t& y) +{ + dev_max_4(x, y); +} + +template +__device__ inline void median_selection_network_9(T* buf) +{ +#define SWAP_OP(i, j) dev_sort(buf[i], buf[j]) +#define MIN_OP(i, j) dev_min(buf[i], buf[j]) +#define MAX_OP(i, j) dev_max(buf[i], buf[j]) + + SWAP_OP(0, 1); SWAP_OP(3, 4); SWAP_OP(6, 7); + SWAP_OP(1, 2); SWAP_OP(4, 5); SWAP_OP(7, 8); + SWAP_OP(0, 1); SWAP_OP(3, 4); SWAP_OP(6, 7); + MAX_OP(0, 3); MAX_OP(3, 6); + SWAP_OP(1, 4); MIN_OP(4, 7); MAX_OP(1, 4); + MIN_OP(5, 8); MIN_OP(2, 5); + SWAP_OP(2, 4); MIN_OP(4, 6); MAX_OP(2, 4); + +#undef SWAP_OP +#undef MIN_OP +#undef MAX_OP +} + +__global__ void median_kernel_3x3_8u(const PtrStepSz src, PtrStep dst) +{ + const int x = blockIdx.x * blockDim.x + threadIdx.x; + const int y = blockIdx.y * blockDim.y + threadIdx.y; + if (x < RADIUS || x >= src.cols - RADIUS || y < RADIUS || y >= src.rows - RADIUS) + return; + + uint8_t buf[KSIZE_SQ]; + for (int i = 0; i < KSIZE_SQ; i++) + buf[i] = src(y - RADIUS + i / KSIZE, x - RADIUS + i % KSIZE); + + median_selection_network_9(buf); + + dst(y, x) = buf[KSIZE_SQ / 2]; +} + +__global__ void median_kernel_3x3_16u(const PtrStepSz src, PtrStep dst) +{ + const int x = blockIdx.x * blockDim.x + threadIdx.x; + const int y = blockIdx.y * blockDim.y + threadIdx.y; + if (x < RADIUS || x >= src.cols - RADIUS || y < RADIUS || y >= src.rows - RADIUS) + return; + + uint16_t buf[KSIZE_SQ]; + for (int i = 0; i < KSIZE_SQ; i++) + buf[i] = src(y - RADIUS + i / KSIZE, x - RADIUS + i % KSIZE); + + median_selection_network_9(buf); + + dst(y, x) = buf[KSIZE_SQ / 2]; +} + +__global__ void median_kernel_3x3_8u_v4(const PtrStepSz src, PtrStep dst) +{ + const int x_4 = 4 * (blockIdx.x * blockDim.x + threadIdx.x); + const int y = blockIdx.y * blockDim.y + threadIdx.y; + + if (y < RADIUS || y >= src.rows - RADIUS) + return; + + uint32_t buf[KSIZE_SQ]; + if (x_4 >= 4 && x_4 + 7 < src.cols) + { + buf[0] = *((const uint32_t*)&src(y - 1, x_4 - 4)); + buf[1] = *((const uint32_t*)&src(y - 1, x_4 - 0)); + buf[2] = *((const uint32_t*)&src(y - 1, x_4 + 4)); + + buf[3] = *((const uint32_t*)&src(y - 0, x_4 - 4)); + buf[4] = *((const uint32_t*)&src(y - 0, x_4 - 0)); + buf[5] = *((const uint32_t*)&src(y - 0, x_4 + 4)); + + buf[6] = *((const uint32_t*)&src(y + 1, x_4 - 4)); + buf[7] = *((const uint32_t*)&src(y + 1, x_4 - 0)); + buf[8] = *((const uint32_t*)&src(y + 1, x_4 + 4)); + + buf[0] = (buf[1] << 8) | (buf[0] >> 24); + buf[2] = (buf[1] >> 8) | (buf[2] << 24); + + buf[3] = (buf[4] << 8) | (buf[3] >> 24); + buf[5] = (buf[4] >> 8) | (buf[5] << 24); + + buf[6] = (buf[7] << 8) | (buf[6] >> 24); + buf[8] = (buf[7] >> 8) | (buf[8] << 24); + + median_selection_network_9(buf); + + *((uint32_t*)&dst(y, x_4)) = buf[KSIZE_SQ / 2]; + } + else if (x_4 == 0) + { + for (int x = RADIUS; x < 4; x++) + { + uint8_t* buf_u8 = (uint8_t*)buf; + for (int i = 0; i < KSIZE_SQ; i++) + buf_u8[i] = src(y - RADIUS + i / KSIZE, x - RADIUS + i % KSIZE); + + median_selection_network_9(buf_u8); + + dst(y, x) = buf_u8[KSIZE_SQ / 2]; + } + } + else if (x_4 < src.cols) + { + for (int x = x_4; x < ::min(x_4 + 4, src.cols - RADIUS); x++) + { + uint8_t* buf_u8 = (uint8_t*)buf; + for (int i = 0; i < KSIZE_SQ; i++) + buf_u8[i] = src(y - RADIUS + i / KSIZE, x - RADIUS + i % KSIZE); + + median_selection_network_9(buf_u8); + + dst(y, x) = buf_u8[KSIZE_SQ / 2]; + } + } +} + +__global__ void median_kernel_3x3_16u_v2(const PtrStepSz src, PtrStep dst) +{ + const int x_2 = 2 * (blockIdx.x * blockDim.x + threadIdx.x); + const int y = blockIdx.y * blockDim.y + threadIdx.y; + + if (y < RADIUS || y >= src.rows - RADIUS) + return; + + uint32_t buf[KSIZE_SQ]; + if (x_2 >= 2 && x_2 + 3 < src.cols) + { + buf[0] = *((const uint32_t*)&src(y - 1, x_2 - 2)); + buf[1] = *((const uint32_t*)&src(y - 1, x_2 - 0)); + buf[2] = *((const uint32_t*)&src(y - 1, x_2 + 2)); + + buf[3] = *((const uint32_t*)&src(y - 0, x_2 - 2)); + buf[4] = *((const uint32_t*)&src(y - 0, x_2 - 0)); + buf[5] = *((const uint32_t*)&src(y - 0, x_2 + 2)); + + buf[6] = *((const uint32_t*)&src(y + 1, x_2 - 2)); + buf[7] = *((const uint32_t*)&src(y + 1, x_2 - 0)); + buf[8] = *((const uint32_t*)&src(y + 1, x_2 + 2)); + + buf[0] = (buf[1] << 16) | (buf[0] >> 16); + buf[2] = (buf[1] >> 16) | (buf[2] << 16); + + buf[3] = (buf[4] << 16) | (buf[3] >> 16); + buf[5] = (buf[4] >> 16) | (buf[5] << 16); + + buf[6] = (buf[7] << 16) | (buf[6] >> 16); + buf[8] = (buf[7] >> 16) | (buf[8] << 16); + + median_selection_network_9(buf); + + *((uint32_t*)&dst(y, x_2)) = buf[KSIZE_SQ / 2]; + } + else if (x_2 == 0) + { + for (int x = RADIUS; x < 2; x++) + { + uint8_t* buf_u8 = (uint8_t*)buf; + for (int i = 0; i < KSIZE_SQ; i++) + buf_u8[i] = src(y - RADIUS + i / KSIZE, x - RADIUS + i % KSIZE); + + median_selection_network_9(buf_u8); + + dst(y, x) = buf_u8[KSIZE_SQ / 2]; + } + } + else if (x_2 < src.cols) + { + for (int x = x_2; x < ::min(x_2 + 2, src.cols - RADIUS); x++) + { + uint8_t* buf_u8 = (uint8_t*)buf; + for (int i = 0; i < KSIZE_SQ; i++) + buf_u8[i] = src(y - RADIUS + i / KSIZE, x - RADIUS + i % KSIZE); + + median_selection_network_9(buf_u8); + + dst(y, x) = buf_u8[KSIZE_SQ / 2]; + } + } +} + +template +void median_filter(const PtrStepSz d_src, PtrStep d_dst, Stream& _stream); + +template <> +void median_filter(const PtrStepSz d_src, PtrStep d_dst, Stream& _stream) +{ + cudaStream_t stream = cv::cuda::StreamAccessor::getStream(_stream); + + if ((d_src.step / sizeof(uint8_t)) % 4 == 0) + { + const dim3 block(BLOCK_X, BLOCK_Y); + const dim3 grid(cudev::divUp(d_src.cols / 4, block.x), cudev::divUp(d_src.rows, block.y)); + median_kernel_3x3_8u_v4<<>>(d_src, d_dst); + } + else + { + const dim3 block(BLOCK_X, BLOCK_Y); + const dim3 grid(cudev::divUp(d_src.cols, block.x), cudev::divUp(d_src.rows, block.y)); + median_kernel_3x3_8u<<>>(d_src, d_dst); + } + + CV_CUDEV_SAFE_CALL(cudaGetLastError()); +} + +template <> +void median_filter(const PtrStepSz d_src, PtrStep d_dst, Stream& _stream) +{ + cudaStream_t stream = cv::cuda::StreamAccessor::getStream(_stream); + + if ((d_src.step / sizeof(uint16_t)) % 2 == 0) + { + const dim3 block(BLOCK_X, BLOCK_Y); + const dim3 grid(cudev::divUp(d_src.cols / 2, block.x), cudev::divUp(d_src.rows, block.y)); + median_kernel_3x3_16u_v2<<>>(d_src, d_dst); + } + else + { + const dim3 block(BLOCK_X, BLOCK_Y); + const dim3 grid(cudev::divUp(d_src.cols, block.x), cudev::divUp(d_src.rows, block.y)); + median_kernel_3x3_16u<<>>(d_src, d_dst); + } + + CV_CUDEV_SAFE_CALL(cudaGetLastError()); +} +} // anonymous namespace + +void medianFilter(const GpuMat& src, GpuMat& dst, Stream& stream) +{ + CV_Assert(src.size() == dst.size()); + CV_Assert(src.type() == CV_16SC1); + CV_Assert(src.type() == dst.type()); + + switch (src.type()) + { + case CV_8UC1: + median_filter(src, dst, stream); + break; + case CV_16SC1: + case CV_16UC1: + median_filter(src, dst, stream); + break; + default: + CV_Error(cv::Error::BadDepth, "Unsupported depth"); + } +} +} // namespace median_filter + +namespace check_consistency +{ +namespace +{ +template +__global__ void check_consistency_kernel(PtrStep d_leftDisp, const PtrStep d_rightDisp, const PtrStep d_left, int width, int height, bool subpixel) +{ + + const int j = blockIdx.x * blockDim.x + threadIdx.x; + const int i = blockIdx.y * blockDim.y + threadIdx.y; + + // left-right consistency check, only on leftDisp, but could be done for rightDisp too + + SRC_T mask = d_left(i, j); + DST_T org = d_leftDisp(i, j); + int d = org; + if (subpixel) + { + d >>= StereoMatcher::DISP_SHIFT; + } + int k = j - d; + if (mask == 0 || org == INVALID_DISP || (k >= 0 && k < width && abs(d_rightDisp(i, k) - d) > 1)) + { + // masked or left-right inconsistent pixel -> invalid + d_leftDisp(i, j) = static_cast(INVALID_DISP); + } +} + +template +void check_consistency(PtrStep d_left_disp, const PtrStep d_right_disp, const PtrStep d_src_left, int width, int height, bool subpixel, Stream& _stream) +{ + const dim3 blocks(width / 16, height / 16); + const dim3 threads(16, 16); + cudaStream_t stream = cv::cuda::StreamAccessor::getStream(_stream); + + check_consistency_kernel<<>>(d_left_disp, d_right_disp, d_src_left, width, height, subpixel); + + CV_CUDEV_SAFE_CALL(cudaGetLastError()); +} +} // anonymous namespace + +void checkConsistency(GpuMat& left_disp, const GpuMat& right_disp, const GpuMat& src_left, bool subpixel, Stream& stream) +{ + Size size = left_disp.size(); + CV_Assert(size == right_disp.size()); + CV_Assert(size == src_left.size()); + CV_Assert(left_disp.type() == CV_16SC1); + CV_Assert(left_disp.type() == right_disp.type()); + CV_Assert(src_left.type() == CV_8UC1 || src_left.type() == CV_16UC1); + + switch (src_left.type()) + { + case CV_8UC1: + check_consistency(left_disp, right_disp, src_left, size.width, size.height, subpixel, stream); + break; + case CV_16SC1: + case CV_16UC1: + check_consistency(left_disp, right_disp, src_left, size.width, size.height, subpixel, stream); + break; + default: + CV_Error(cv::Error::BadDepth, "Unsupported depth"); + } +} +} // namespace check_consistency + +namespace correct_disparity_range +{ +namespace +{ +__global__ void correct_disparity_range_kernel( + PtrStepSz disp, + int min_disp_scaled, + int invalid_disp_scaled) +{ + const int x = blockIdx.x * blockDim.x + threadIdx.x; + const int y = blockIdx.y * blockDim.y + threadIdx.y; + + if (x >= disp.cols || y >= disp.rows) + { + return; + } + + uint16_t d = disp(y, x); + if (d == INVALID_DISP) + { + d = invalid_disp_scaled; + } + else + { + d += min_disp_scaled; + } + disp(y, x) = d; +} +} // anonymous namespace + +void correctDisparityRange( + GpuMat& disp, + bool subpixel, + int min_disp, + Stream& _stream) +{ + CV_Assert(disp.type() == CV_16SC1); + + static constexpr int SIZE = 16; + cv::Size size = disp.size(); + + const dim3 blocks(cudev::divUp(size.width, SIZE), cudev::divUp(size.height, SIZE)); + const dim3 threads(SIZE, SIZE); + cudaStream_t stream = cv::cuda::StreamAccessor::getStream(_stream); + + const int scale = subpixel ? StereoSGBM::DISP_SCALE : 1; + const int min_disp_scaled = min_disp * scale; + const int invalid_disp_scaled = (min_disp - 1) * scale; + + correct_disparity_range_kernel<<>>(disp, min_disp_scaled, invalid_disp_scaled); +} +} // namespace correct_disparity_range + +} // namespace stereosgm +}}} // namespace cv { namespace cuda { namespace device { + +#endif // HAVE_OPENCV_CUDEV diff --git a/modules/cudastereo/src/cuda/stereosgm.hpp b/modules/cudastereo/src/cuda/stereosgm.hpp new file mode 100644 index 00000000000..bc2cfc7a9c3 --- /dev/null +++ b/modules/cudastereo/src/cuda/stereosgm.hpp @@ -0,0 +1,61 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Author: The "adaskit Team" at Fixstars Corporation + +#ifndef OPENCV_CUDASTEREO_SGM_HPP +#define OPENCV_CUDASTEREO_SGM_HPP + +#include "opencv2/core/cuda.hpp" + +namespace cv { namespace cuda { namespace device { +namespace stereosgm +{ + +namespace census_transform +{ +CV_EXPORTS void censusTransform(const GpuMat& src, GpuMat& dest, cv::cuda::Stream& stream); +} + +namespace path_aggregation +{ +class PathAggregation +{ +private: + static constexpr unsigned int MAX_NUM_PATHS = 8; + + std::array streams; + std::array events; + std::array subs; +public: + template + void operator() (const GpuMat& left, const GpuMat& right, GpuMat& dest, int mode, int p1, int p2, int min_disp, Stream& stream); +}; +} + +namespace winner_takes_all +{ +template +void winnerTakesAll(const GpuMat& src, GpuMat& left, GpuMat& right, float uniqueness, bool subpixel, int mode, cv::cuda::Stream& stream); +} + +namespace median_filter +{ +void medianFilter(const GpuMat& src, GpuMat& dst, Stream& stream); +} + +namespace check_consistency +{ +void checkConsistency(GpuMat& left_disp, const GpuMat& right_disp, const GpuMat& src_left, bool subpixel, Stream& stream); +} + +namespace correct_disparity_range +{ +void correctDisparityRange(GpuMat& disp, bool subpixel, int min_disp, Stream& stream); +} + +} // namespace stereosgm +}}} // namespace cv { namespace cuda { namespace device { + +#endif /* OPENCV_CUDASTEREO_SGM_HPP */ diff --git a/modules/cudastereo/src/stereosgm.cpp b/modules/cudastereo/src/stereosgm.cpp new file mode 100644 index 00000000000..11f630a254f --- /dev/null +++ b/modules/cudastereo/src/stereosgm.cpp @@ -0,0 +1,153 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Author: The "adaskit Team" at Fixstars Corporation + +#include "precomp.hpp" + +using namespace cv; +using namespace cv::cuda; + +#if !defined (HAVE_CUDA) || defined (CUDA_DISABLER) + +Ptr cv::cuda::createStereoSGM(int, int, int, int, int, int) { throw_no_cuda(); return Ptr(); } + +#else /* !defined (HAVE_CUDA) */ + +#include "cuda/stereosgm.hpp" + +namespace +{ +struct StereoSGMParams +{ + int minDisparity; + int numDisparities; + int P1; + int P2; + int uniquenessRatio; + int mode; + StereoSGMParams(int minDisparity = 0, int numDisparities = 128, int P1 = 10, int P2 = 120, int uniquenessRatio = 5, int mode = StereoSGM::MODE_HH4) : minDisparity(minDisparity), numDisparities(numDisparities), P1(P1), P2(P2), uniquenessRatio(uniquenessRatio), mode(mode) {} +}; + +class StereoSGMImpl CV_FINAL : public StereoSGM +{ +public: + StereoSGMImpl(int minDisparity, int numDisparities, int P1, int P2, int uniquenessRatio, int mode); + + void compute(InputArray left, InputArray right, OutputArray disparity) CV_OVERRIDE; + void compute(InputArray left, InputArray right, OutputArray disparity, Stream& stream) CV_OVERRIDE; + + int getBlockSize() const CV_OVERRIDE { return -1; } + void setBlockSize(int /*blockSize*/) CV_OVERRIDE {} + + int getDisp12MaxDiff() const CV_OVERRIDE { return 1; } + void setDisp12MaxDiff(int /*disp12MaxDiff*/) CV_OVERRIDE {} + + int getMinDisparity() const CV_OVERRIDE { return params.minDisparity; } + void setMinDisparity(int minDisparity) CV_OVERRIDE { params.minDisparity = minDisparity; } + + int getNumDisparities() const CV_OVERRIDE { return params.numDisparities; } + void setNumDisparities(int numDisparities) CV_OVERRIDE { params.numDisparities = numDisparities; } + + int getSpeckleWindowSize() const CV_OVERRIDE { return 0; } + void setSpeckleWindowSize(int /*speckleWindowSize*/) CV_OVERRIDE {} + + int getSpeckleRange() const CV_OVERRIDE { return 0; } + void setSpeckleRange(int /*speckleRange*/) CV_OVERRIDE {} + + int getP1() const CV_OVERRIDE { return params.P1; } + void setP1(int P1) CV_OVERRIDE { params.P1 = P1; } + + int getP2() const CV_OVERRIDE { return params.P2; } + void setP2(int P2) CV_OVERRIDE { params.P2 = P2; } + + int getUniquenessRatio() const CV_OVERRIDE { return params.uniquenessRatio; } + void setUniquenessRatio(int uniquenessRatio) CV_OVERRIDE { params.uniquenessRatio = uniquenessRatio; } + + int getMode() const CV_OVERRIDE { return params.mode; } + void setMode(int mode) CV_OVERRIDE { params.mode = mode; } + + int getPreFilterCap() const CV_OVERRIDE { return -1; } + void setPreFilterCap(int /*preFilterCap*/) CV_OVERRIDE {} + +private: + StereoSGMParams params; + device::stereosgm::path_aggregation::PathAggregation pathAggregation; + GpuMat censused_left, censused_right; + GpuMat aggregated; + GpuMat left_disp_tmp, right_disp_tmp; + GpuMat right_disp; +}; + +StereoSGMImpl::StereoSGMImpl(int minDisparity, int numDisparities, int P1, int P2, int uniquenessRatio, int mode) + : params(minDisparity, numDisparities, P1, P2, uniquenessRatio, mode) +{ +} + +void StereoSGMImpl::compute(InputArray left, InputArray right, OutputArray disparity) +{ + compute(left, right, disparity, Stream::Null()); +} + +void StereoSGMImpl::compute(InputArray _left, InputArray _right, OutputArray _disparity, Stream& _stream) +{ + using namespace device::stereosgm; + + GpuMat left = _left.getGpuMat(); + GpuMat right = _right.getGpuMat(); + const Size size = left.size(); + + if (params.mode != MODE_HH && params.mode != MODE_HH4) + { + CV_Error(Error::StsBadArg, "Unsupported mode"); + } + const unsigned int num_paths = params.mode == MODE_HH4 ? 4 : 8; + + CV_Assert(left.type() == CV_8UC1 || left.type() == CV_16UC1); + CV_Assert(size == right.size() && left.type() == right.type()); + + _disparity.create(size, CV_16SC1); + ensureSizeIsEnough(size, CV_16SC1, right_disp); + GpuMat left_disp = _disparity.getGpuMat(); + + ensureSizeIsEnough(size, CV_32SC1, censused_left); + ensureSizeIsEnough(size, CV_32SC1, censused_right); + census_transform::censusTransform(left, censused_left, _stream); + census_transform::censusTransform(right, censused_right, _stream); + + ensureSizeIsEnough(1, size.width * size.height * params.numDisparities * num_paths, CV_8UC1, aggregated); + ensureSizeIsEnough(size, CV_16SC1, left_disp_tmp); + ensureSizeIsEnough(size, CV_16SC1, right_disp_tmp); + + switch (params.numDisparities) + { + case 64: + pathAggregation.operator()<64>(censused_left, censused_right, aggregated, params.mode, params.P1, params.P2, params.minDisparity, _stream); + winner_takes_all::winnerTakesAll<64>(aggregated, left_disp_tmp, right_disp_tmp, (float)(100 - params.uniquenessRatio) / 100, true, params.mode, _stream); + break; + case 128: + pathAggregation.operator()<128>(censused_left, censused_right, aggregated, params.mode, params.P1, params.P2, params.minDisparity, _stream); + winner_takes_all::winnerTakesAll<128>(aggregated, left_disp_tmp, right_disp_tmp, (float)(100 - params.uniquenessRatio) / 100, true, params.mode, _stream); + break; + case 256: + pathAggregation.operator()<256>(censused_left, censused_right, aggregated, params.mode, params.P1, params.P2, params.minDisparity, _stream); + winner_takes_all::winnerTakesAll<256>(aggregated, left_disp_tmp, right_disp_tmp, (float)(100 - params.uniquenessRatio) / 100, true, params.mode, _stream); + break; + default: + CV_Error(Error::StsBadArg, "Unsupported num of disparities"); + } + + median_filter::medianFilter(left_disp_tmp, left_disp, _stream); + median_filter::medianFilter(right_disp_tmp, right_disp, _stream); + check_consistency::checkConsistency(left_disp, right_disp, left, true, _stream); + correct_disparity_range::correctDisparityRange(left_disp, true, params.minDisparity, _stream); +} +} // anonymous namespace + +Ptr cv::cuda::createStereoSGM(int minDisparity, int numDisparities, int P1, int P2, int uniquenessRatio, int mode) +{ + return makePtr(minDisparity, numDisparities, P1, P2, uniquenessRatio, mode); +} + +#endif /* !defined (HAVE_CUDA) */ diff --git a/modules/cudastereo/test/test_sgm_funcs.cpp b/modules/cudastereo/test/test_sgm_funcs.cpp new file mode 100644 index 00000000000..3f185ee857c --- /dev/null +++ b/modules/cudastereo/test/test_sgm_funcs.cpp @@ -0,0 +1,444 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Author: The "adaskit Team" at Fixstars Corporation + +#include "test_precomp.hpp" + +#ifdef HAVE_CUDA + +#ifdef _WIN32 +#define popcnt64 __popcnt64 +#else +#define popcnt64 __builtin_popcountll +#endif + +#include "opencv2/core/cuda.hpp" + +namespace cv { namespace cuda { namespace device { +namespace stereosgm +{ + +namespace census_transform +{ +void censusTransform(const GpuMat& src, GpuMat& dest, cv::cuda::Stream& stream); +} + +namespace path_aggregation +{ +namespace horizontal +{ +template +void aggregateLeft2RightPath( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +template +void aggregateRight2LeftPath( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); +} + +namespace vertical +{ +template +void aggregateUp2DownPath( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +template +void aggregateDown2UpPath( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); +} + +namespace oblique +{ +template +void aggregateUpleft2DownrightPath( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +template +void aggregateUpright2DownleftPath( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +template +void aggregateDownright2UpleftPath( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); + +template +void aggregateDownleft2UprightPath( + const GpuMat& left, + const GpuMat& right, + GpuMat& dest, + unsigned int p1, + unsigned int p2, + int min_disp, + Stream& stream); +} +} // namespace path_aggregation + +namespace winner_takes_all +{ +template +void winnerTakesAll(const GpuMat& src, GpuMat& left, GpuMat& right, float uniqueness, bool subpixel, int mode, cv::cuda::Stream& stream); +} + +} // namespace stereosgm +}}} // namespace cv { namespace cuda { namespace device { + +namespace opencv_test { namespace { + + void census_transform(const cv::Mat& src, cv::Mat& dst) + { + const int hor = 9 / 2, ver = 7 / 2; + dst.create(src.size(), CV_32SC1); + dst = 0; + for (int y = ver; y < static_cast(src.rows) - ver; ++y) { + for (int x = hor; x < static_cast(src.cols) - hor; ++x) { + int32_t value = 0; + for (int dy = -ver; dy <= 0; ++dy) { + for (int dx = -hor; dx <= (dy == 0 ? -1 : hor); ++dx) { + const auto a = src.at(y + dy, x + dx); + const auto b = src.at(y - dy, x - dx); + value <<= 1; + if (a > b) { value |= 1; } + } + } + dst.at(y, x) = value; + } + } + } + + PARAM_TEST_CASE(StereoSGM_CensusTransformImage, cv::cuda::DeviceInfo, std::string, UseRoi) + { + cv::cuda::DeviceInfo devInfo; + std::string path; + bool useRoi; + + virtual void SetUp() + { + devInfo = GET_PARAM(0); + path = GET_PARAM(1); + useRoi = GET_PARAM(2); + + cv::cuda::setDevice(devInfo.deviceID()); + } + }; + + CUDA_TEST_P(StereoSGM_CensusTransformImage, Image) + { + cv::Mat image = readImage(path, cv::IMREAD_GRAYSCALE); + cv::Mat dst_gold; + census_transform(image, dst_gold); + + cv::cuda::GpuMat g_dst; + g_dst.create(image.size(), CV_32SC1); + cv::cuda::device::stereosgm::census_transform::censusTransform(loadMat(image, useRoi), g_dst, cv::cuda::Stream::Null()); + + cv::Mat dst; + g_dst.download(dst); + + EXPECT_MAT_NEAR(dst_gold, dst, 0); + } + + INSTANTIATE_TEST_CASE_P(CUDA_StereoSGM_funcs, StereoSGM_CensusTransformImage, testing::Combine( + ALL_DEVICES, + testing::Values("stereobm/aloe-L.png", "stereobm/aloe-R.png"), + WHOLE_SUBMAT)); + + PARAM_TEST_CASE(StereoSGM_CensusTransformRandom, cv::cuda::DeviceInfo, cv::Size, UseRoi) + { + cv::cuda::DeviceInfo devInfo; + cv::Size size; + bool useRoi; + + virtual void SetUp() + { + devInfo = GET_PARAM(0); + size = GET_PARAM(1); + useRoi = GET_PARAM(2); + + cv::cuda::setDevice(devInfo.deviceID()); + } + }; + + CUDA_TEST_P(StereoSGM_CensusTransformRandom, Random) + { + cv::Mat image = randomMat(size, CV_8UC1); + cv::Mat dst_gold; + census_transform(image, dst_gold); + + cv::cuda::GpuMat g_dst; + g_dst.create(image.size(), CV_32SC1); + cv::cuda::device::stereosgm::census_transform::censusTransform(loadMat(image, useRoi), g_dst, cv::cuda::Stream::Null()); + + cv::Mat dst; + g_dst.download(dst); + + EXPECT_MAT_NEAR(dst_gold, dst, 0); + } + + INSTANTIATE_TEST_CASE_P(CUDA_StereoSGM_funcs, StereoSGM_CensusTransformRandom, testing::Combine( + ALL_DEVICES, + DIFFERENT_SIZES, + WHOLE_SUBMAT)); + + static void path_aggregation( + const cv::Mat& left, + const cv::Mat& right, + cv::Mat& dst, + int max_disparity, int min_disparity, int p1, int p2, + int dx, int dy) + { + const int width = left.cols; + const int height = left.rows; + dst.create(cv::Size(width * height * max_disparity, 1), CV_8UC1); + std::vector before(max_disparity); + for (int i = (dy < 0 ? height - 1 : 0); 0 <= i && i < height; i += (dy < 0 ? -1 : 1)) { + for (int j = (dx < 0 ? width - 1 : 0); 0 <= j && j < width; j += (dx < 0 ? -1 : 1)) { + const int i2 = i - dy, j2 = j - dx; + const bool inside = (0 <= i2 && i2 < height && 0 <= j2 && j2 < width); + for (int k = 0; k < max_disparity; ++k) { + before[k] = inside ? dst.at(0, k + (j2 + i2 * width) * max_disparity) : 0; + } + const int min_cost = *min_element(before.begin(), before.end()); + for (int k = 0; k < max_disparity; ++k) { + const auto l = left.at(i, j); + const auto r = (k + min_disparity > j ? 0 : right.at(i, j - k - min_disparity)); + int cost = std::min(before[k] - min_cost, p2); + if (k > 0) { + cost = std::min(cost, before[k - 1] - min_cost + p1); + } + if (k + 1 < max_disparity) { + cost = std::min(cost, before[k + 1] - min_cost + p1); + } + cost += static_cast(popcnt64(l ^ r)); + dst.at(0, k + (j + i * width) * max_disparity) = static_cast(cost); + } + } + } + } + + static constexpr size_t DISPARITY = 128; + static constexpr int P1 = 10; + static constexpr int P2 = 120; + + PARAM_TEST_CASE(StereoSGM_PathAggregation, cv::cuda::DeviceInfo, cv::Size, UseRoi, int) + { + cv::cuda::DeviceInfo devInfo; + cv::Size size; + bool useRoi; + int minDisp; + + virtual void SetUp() + { + devInfo = GET_PARAM(0); + size = GET_PARAM(1); + useRoi = GET_PARAM(2); + minDisp = GET_PARAM(3); + + cv::cuda::setDevice(devInfo.deviceID()); + } + + template + void test_path_aggregation(T func, int dx, int dy) + { + cv::Mat left_image = randomMat(size, CV_32SC1, 0.0, static_cast(std::numeric_limits::max())); + cv::Mat right_image = randomMat(size, CV_32SC1, 0.0, static_cast(std::numeric_limits::max())); + cv::Mat dst_gold; + path_aggregation(left_image, right_image, dst_gold, DISPARITY, minDisp, P1, P2, dx, dy); + + cv::cuda::GpuMat g_dst; + g_dst.create(cv::Size(left_image.cols * left_image.rows * DISPARITY, 1), CV_8UC1); + func(loadMat(left_image, useRoi), loadMat(right_image, useRoi), g_dst, P1, P2, minDisp, cv::cuda::Stream::Null()); + + cv::Mat dst; + g_dst.download(dst); + + EXPECT_MAT_NEAR(dst_gold, dst, 0); + } + }; + + CUDA_TEST_P(StereoSGM_PathAggregation, RandomLeft2Right) + { + test_path_aggregation(cv::cuda::device::stereosgm::path_aggregation::horizontal::aggregateLeft2RightPath, 1, 0); + } + + CUDA_TEST_P(StereoSGM_PathAggregation, RandomRight2Left) + { + test_path_aggregation(cv::cuda::device::stereosgm::path_aggregation::horizontal::aggregateRight2LeftPath, -1, 0); + } + + CUDA_TEST_P(StereoSGM_PathAggregation, RandomUp2Down) + { + test_path_aggregation(cv::cuda::device::stereosgm::path_aggregation::vertical::aggregateUp2DownPath, 0, 1); + } + + CUDA_TEST_P(StereoSGM_PathAggregation, RandomDown2Up) + { + test_path_aggregation(cv::cuda::device::stereosgm::path_aggregation::vertical::aggregateDown2UpPath, 0, -1); + } + + CUDA_TEST_P(StereoSGM_PathAggregation, RandomUpLeft2DownRight) + { + test_path_aggregation(cv::cuda::device::stereosgm::path_aggregation::oblique::aggregateUpleft2DownrightPath, 1, 1); + } + + CUDA_TEST_P(StereoSGM_PathAggregation, RandomUpRight2DownLeft) + { + test_path_aggregation(cv::cuda::device::stereosgm::path_aggregation::oblique::aggregateUpright2DownleftPath, -1, 1); + } + + CUDA_TEST_P(StereoSGM_PathAggregation, RandomDownRight2UpLeft) + { + test_path_aggregation(cv::cuda::device::stereosgm::path_aggregation::oblique::aggregateDownright2UpleftPath, -1, -1); + } + + CUDA_TEST_P(StereoSGM_PathAggregation, RandomDownLeft2UpRight) + { + test_path_aggregation(cv::cuda::device::stereosgm::path_aggregation::oblique::aggregateDownleft2UprightPath, 1, -1); + } + + INSTANTIATE_TEST_CASE_P(CUDA_StereoSGM_funcs, StereoSGM_PathAggregation, testing::Combine( + ALL_DEVICES, + DIFFERENT_SIZES, + WHOLE_SUBMAT, + testing::Values(0, 1, 10))); + + + void winner_takes_all_left( + const cv::Mat& src, + cv::Mat& dst, + int width, int height, int disparity, int num_paths, + float uniqueness, bool subpixel) + { + dst.create(cv::Size(width, height), CV_16UC1); + for (int i = 0; i < height; ++i) { + for (int j = 0; j < width; ++j) { + std::vector> v; + for (int k = 0; k < disparity; ++k) { + int cost_sum = 0; + for (int p = 0; p < num_paths; ++p) { + cost_sum += static_cast(src.at(0, + p * disparity * width * height + + i * disparity * width + + j * disparity + + k)); + } + v.emplace_back(cost_sum, static_cast(k)); + } + const auto ite = std::min_element(v.begin(), v.end()); + assert(ite != v.end()); + const auto best = *ite; + const int best_cost = best.first; + int best_disp = best.second; + int ans = best_disp; + if (subpixel) { + ans <<= StereoMatcher::DISP_SHIFT; + if (0 < best_disp && best_disp < static_cast(disparity) - 1) { + const int left = v[best_disp - 1].first; + const int right = v[best_disp + 1].first; + const int numer = left - right; + const int denom = left - 2 * best_cost + right; + ans += ((numer << StereoMatcher::DISP_SHIFT) + denom) / (2 * denom); + } + } + for (const auto& p : v) { + const int cost = p.first; + const int disp = p.second; + if (cost * uniqueness < best_cost && abs(disp - best_disp) > 1) { + ans = -1; + break; + } + } + + dst.at(i, j) = static_cast(ans); + } + } + } + + PARAM_TEST_CASE(StereoSGM_WinnerTakesAll, cv::cuda::DeviceInfo, cv::Size, bool, int) + { + cv::cuda::DeviceInfo devInfo; + cv::Size size; + bool subpixel; + int mode; + + virtual void SetUp() + { + devInfo = GET_PARAM(0); + size = GET_PARAM(1); + subpixel = GET_PARAM(2); + mode = GET_PARAM(3); + + cv::cuda::setDevice(devInfo.deviceID()); + } + }; + + CUDA_TEST_P(StereoSGM_WinnerTakesAll, RandomLeft) + { + int num_paths = mode == cv::cuda::StereoSGM::MODE_HH4 ? 4 : 8; + cv::Mat aggregated = randomMat(cv::Size(size.width * size.height * DISPARITY * num_paths, 1), CV_8UC1, 0.0, 32.0); + cv::Mat dst_gold; + winner_takes_all_left(aggregated, dst_gold, size.width, size.height, DISPARITY, num_paths, 0.95f, subpixel); + + cv::cuda::GpuMat g_src, g_dst, g_dst_right; + g_src.upload(aggregated); + g_dst.create(size, CV_16UC1); + g_dst_right.create(size, CV_16UC1); + cv::cuda::device::stereosgm::winner_takes_all::winnerTakesAll(g_src, g_dst, g_dst_right, 0.95f, subpixel, mode, cv::cuda::Stream::Null()); + + cv::Mat dst; + g_dst.download(dst); + + EXPECT_MAT_NEAR(dst_gold, dst, 0); + } + + INSTANTIATE_TEST_CASE_P(CUDA_StereoSGM_funcs, StereoSGM_WinnerTakesAll, testing::Combine( + ALL_DEVICES, + DIFFERENT_SIZES, + testing::Values(false, true), + testing::Values(cv::cuda::StereoSGM::MODE_HH4, cv::cuda::StereoSGM::MODE_HH))); + +}} // namespace +#endif // HAVE_CUDA diff --git a/modules/cudastereo/test/test_stereo.cpp b/modules/cudastereo/test/test_stereo.cpp index ebcab085ceb..0d2a7c84e22 100644 --- a/modules/cudastereo/test/test_stereo.cpp +++ b/modules/cudastereo/test/test_stereo.cpp @@ -209,6 +209,761 @@ INSTANTIATE_TEST_CASE_P(CUDA_Stereo, ReprojectImageTo3D, testing::Combine( testing::Values(MatDepth(CV_8U), MatDepth(CV_16S)), WHOLE_SUBMAT)); +//////////////////////////////////////////////////////////////////////////////// +// StereoSGM + +/* +This is a regression test for stereo matching algorithms. This test gets some quality metrics +described in "A Taxonomy and Evaluation of Dense Two-Frame Stereo Correspondence Algorithms". +Daniel Scharstein, Richard Szeliski +*/ + +const float EVAL_BAD_THRESH = 1.f; +const int EVAL_TEXTURELESS_WIDTH = 3; +const float EVAL_TEXTURELESS_THRESH = 4.f; +const float EVAL_DISP_THRESH = 1.f; +const float EVAL_DISP_GAP = 2.f; +const int EVAL_DISCONT_WIDTH = 9; +const int EVAL_IGNORE_BORDER = 10; + +const int ERROR_KINDS_COUNT = 6; + +//============================== quality measuring functions ================================================= + +/* +Calculate textureless regions of image (regions where the squared horizontal intensity gradient averaged over +a square window of size=evalTexturelessWidth is below a threshold=evalTexturelessThresh) and textured regions. +*/ +void computeTextureBasedMasks(const Mat& _img, Mat* texturelessMask, Mat* texturedMask, + int texturelessWidth = EVAL_TEXTURELESS_WIDTH, float texturelessThresh = EVAL_TEXTURELESS_THRESH) +{ + if (!texturelessMask && !texturedMask) + return; + if (_img.empty()) + CV_Error(Error::StsBadArg, "img is empty"); + + Mat img = _img; + if (_img.channels() > 1) + { + Mat tmp; cvtColor(_img, tmp, COLOR_BGR2GRAY); img = tmp; + } + Mat dxI; Sobel(img, dxI, CV_32FC1, 1, 0, 3); + Mat dxI2; pow(dxI / 8.f/*normalize*/, 2, dxI2); + Mat avgDxI2; boxFilter(dxI2, avgDxI2, CV_32FC1, Size(texturelessWidth, texturelessWidth)); + + if (texturelessMask) + *texturelessMask = avgDxI2 < texturelessThresh; + if (texturedMask) + *texturedMask = avgDxI2 >= texturelessThresh; +} + +void checkTypeAndSizeOfDisp(const Mat& dispMap, const Size* sz) +{ + if (dispMap.empty()) + CV_Error(Error::StsBadArg, "dispMap is empty"); + if (dispMap.type() != CV_32FC1) + CV_Error(Error::StsBadArg, "dispMap must have CV_32FC1 type"); + if (sz && (dispMap.rows != sz->height || dispMap.cols != sz->width)) + CV_Error(Error::StsBadArg, "dispMap has incorrect size"); +} + +void checkTypeAndSizeOfMask(const Mat& mask, Size sz) +{ + if (mask.empty()) + CV_Error(Error::StsBadArg, "mask is empty"); + if (mask.type() != CV_8UC1) + CV_Error(Error::StsBadArg, "mask must have CV_8UC1 type"); + if (mask.rows != sz.height || mask.cols != sz.width) + CV_Error(Error::StsBadArg, "mask has incorrect size"); +} + +void checkDispMapsAndUnknDispMasks(const Mat& leftDispMap, const Mat& rightDispMap, + const Mat& leftUnknDispMask, const Mat& rightUnknDispMask) +{ + // check type and size of disparity maps + checkTypeAndSizeOfDisp(leftDispMap, 0); + if (!rightDispMap.empty()) + { + Size sz = leftDispMap.size(); + checkTypeAndSizeOfDisp(rightDispMap, &sz); + } + + // check size and type of unknown disparity maps + if (!leftUnknDispMask.empty()) + checkTypeAndSizeOfMask(leftUnknDispMask, leftDispMap.size()); + if (!rightUnknDispMask.empty()) + checkTypeAndSizeOfMask(rightUnknDispMask, rightDispMap.size()); + + // check values of disparity maps (known disparity values musy be positive) + double leftMinVal = 0, rightMinVal = 0; + if (leftUnknDispMask.empty()) + minMaxLoc(leftDispMap, &leftMinVal); + else + minMaxLoc(leftDispMap, &leftMinVal, 0, 0, 0, ~leftUnknDispMask); + if (!rightDispMap.empty()) + { + if (rightUnknDispMask.empty()) + minMaxLoc(rightDispMap, &rightMinVal); + else + minMaxLoc(rightDispMap, &rightMinVal, 0, 0, 0, ~rightUnknDispMask); + } + if (leftMinVal < 0 || rightMinVal < 0) + CV_Error(Error::StsBadArg, "known disparity values must be positive"); +} + +/* +Calculate occluded regions of reference image (left image) (regions that are occluded in the matching image (right image), +i.e., where the forward-mapped disparity lands at a location with a larger (nearer) disparity) and non occluded regions. +*/ +void computeOcclusionBasedMasks(const Mat& leftDisp, const Mat& _rightDisp, + Mat* occludedMask, Mat* nonOccludedMask, + const Mat& leftUnknDispMask = Mat(), const Mat& rightUnknDispMask = Mat(), + float dispThresh = EVAL_DISP_THRESH) +{ + if (!occludedMask && !nonOccludedMask) + return; + checkDispMapsAndUnknDispMasks(leftDisp, _rightDisp, leftUnknDispMask, rightUnknDispMask); + + Mat rightDisp; + if (_rightDisp.empty()) + { + if (!rightUnknDispMask.empty()) + CV_Error(Error::StsBadArg, "rightUnknDispMask must be empty if _rightDisp is empty"); + rightDisp.create(leftDisp.size(), CV_32FC1); + rightDisp.setTo(Scalar::all(0)); + for (int leftY = 0; leftY < leftDisp.rows; leftY++) + { + for (int leftX = 0; leftX < leftDisp.cols; leftX++) + { + if (!leftUnknDispMask.empty() && leftUnknDispMask.at(leftY, leftX)) + continue; + float leftDispVal = leftDisp.at(leftY, leftX); + int rightX = leftX - cvRound(leftDispVal), rightY = leftY; + if (rightX >= 0) + rightDisp.at(rightY, rightX) = max(rightDisp.at(rightY, rightX), leftDispVal); + } + } + } + else + _rightDisp.copyTo(rightDisp); + + if (occludedMask) + { + occludedMask->create(leftDisp.size(), CV_8UC1); + occludedMask->setTo(Scalar::all(0)); + } + if (nonOccludedMask) + { + nonOccludedMask->create(leftDisp.size(), CV_8UC1); + nonOccludedMask->setTo(Scalar::all(0)); + } + for (int leftY = 0; leftY < leftDisp.rows; leftY++) + { + for (int leftX = 0; leftX < leftDisp.cols; leftX++) + { + if (!leftUnknDispMask.empty() && leftUnknDispMask.at(leftY, leftX)) + continue; + float leftDispVal = leftDisp.at(leftY, leftX); + int rightX = leftX - cvRound(leftDispVal), rightY = leftY; + if (rightX < 0 && occludedMask) + occludedMask->at(leftY, leftX) = 255; + else + { + if (!rightUnknDispMask.empty() && rightUnknDispMask.at(rightY, rightX)) + continue; + float rightDispVal = rightDisp.at(rightY, rightX); + if (rightDispVal > leftDispVal + dispThresh) + { + if (occludedMask) + occludedMask->at(leftY, leftX) = 255; + } + else + { + if (nonOccludedMask) + nonOccludedMask->at(leftY, leftX) = 255; + } + } + } + } +} + +/* +Calculate depth discontinuty regions: pixels whose neiboring disparities differ by more than +dispGap, dilated by window of width discontWidth. +*/ +void computeDepthDiscontMask(const Mat& disp, Mat& depthDiscontMask, const Mat& unknDispMask = Mat(), + float dispGap = EVAL_DISP_GAP, int discontWidth = EVAL_DISCONT_WIDTH) +{ + if (disp.empty()) + CV_Error(Error::StsBadArg, "disp is empty"); + if (disp.type() != CV_32FC1) + CV_Error(Error::StsBadArg, "disp must have CV_32FC1 type"); + if (!unknDispMask.empty()) + checkTypeAndSizeOfMask(unknDispMask, disp.size()); + + Mat curDisp; disp.copyTo(curDisp); + if (!unknDispMask.empty()) + curDisp.setTo(Scalar(std::numeric_limits::min()), unknDispMask); + Mat maxNeighbDisp; dilate(curDisp, maxNeighbDisp, Mat(3, 3, CV_8UC1, Scalar(1))); + if (!unknDispMask.empty()) + curDisp.setTo(Scalar(std::numeric_limits::max()), unknDispMask); + Mat minNeighbDisp; erode(curDisp, minNeighbDisp, Mat(3, 3, CV_8UC1, Scalar(1))); + depthDiscontMask = max((Mat)(maxNeighbDisp - disp), (Mat)(disp - minNeighbDisp)) > dispGap; + if (!unknDispMask.empty()) + depthDiscontMask &= ~unknDispMask; + dilate(depthDiscontMask, depthDiscontMask, Mat(discontWidth, discontWidth, CV_8UC1, Scalar(1))); +} + +/* +Get evaluation masks excluding a border. +*/ +Mat getBorderedMask(Size maskSize, int border = EVAL_IGNORE_BORDER) +{ + CV_Assert(border >= 0); + Mat mask(maskSize, CV_8UC1, Scalar(0)); + int w = maskSize.width - 2 * border, h = maskSize.height - 2 * border; + if (w < 0 || h < 0) + mask.setTo(Scalar(0)); + else + mask(Rect(Point(border, border), Size(w, h))).setTo(Scalar(255)); + return mask; +} + +/* +Calculate root-mean-squared error between the computed disparity map (computedDisp) and ground truth map (groundTruthDisp). +*/ +float dispRMS(const Mat& computedDisp, const Mat& groundTruthDisp, const Mat& mask) +{ + checkTypeAndSizeOfDisp(groundTruthDisp, 0); + Size sz = groundTruthDisp.size(); + checkTypeAndSizeOfDisp(computedDisp, &sz); + + int pointsCount = sz.height*sz.width; + if (!mask.empty()) + { + checkTypeAndSizeOfMask(mask, sz); + pointsCount = countNonZero(mask); + } + return 1.f / sqrt((float)pointsCount) * (float)cvtest::norm(computedDisp, groundTruthDisp, NORM_L2, mask); +} + +/* +Calculate fraction of bad matching pixels. +*/ +float badMatchPxlsFraction(const Mat& computedDisp, const Mat& groundTruthDisp, const Mat& mask, + float _badThresh = EVAL_BAD_THRESH) +{ + int badThresh = cvRound(_badThresh); + checkTypeAndSizeOfDisp(groundTruthDisp, 0); + Size sz = groundTruthDisp.size(); + checkTypeAndSizeOfDisp(computedDisp, &sz); + + Mat badPxlsMap; + absdiff(computedDisp, groundTruthDisp, badPxlsMap); + badPxlsMap = badPxlsMap > badThresh; + int pointsCount = sz.height*sz.width; + if (!mask.empty()) + { + checkTypeAndSizeOfMask(mask, sz); + badPxlsMap = badPxlsMap & mask; + pointsCount = countNonZero(mask); + } + return 1.f / pointsCount * countNonZero(badPxlsMap); +} + +//===================== regression test for stereo matching algorithms ============================== + +const string ALGORITHMS_DIR = "stereomatching/algorithms/"; +const string DATASETS_DIR = "stereomatching/datasets/"; +const string DATASETS_FILE = "datasets.xml"; + +const string RUN_PARAMS_FILE = "_params.xml"; +const string RESULT_FILE = "_res.xml"; + +const string LEFT_IMG_NAME = "im2.png"; +const string RIGHT_IMG_NAME = "im6.png"; +const string TRUE_LEFT_DISP_NAME = "disp2.png"; +const string TRUE_RIGHT_DISP_NAME = "disp6.png"; + +string ERROR_PREFIXES[] = { "borderedAll", +"borderedNoOccl", +"borderedOccl", +"borderedTextured", +"borderedTextureless", +"borderedDepthDiscont" }; // size of ERROR_KINDS_COUNT + +string ROI_PREFIXES[] = { "roiX", +"roiY", +"roiWidth", +"roiHeight" }; + + +const string RMS_STR = "RMS"; +const string BAD_PXLS_FRACTION_STR = "BadPxlsFraction"; +const string ROI_STR = "ValidDisparityROI"; + +class QualityEvalParams +{ +public: + QualityEvalParams() + { + setDefaults(); + } + QualityEvalParams(int _ignoreBorder) + { + setDefaults(); + ignoreBorder = _ignoreBorder; + } + void setDefaults() + { + badThresh = EVAL_BAD_THRESH; + texturelessWidth = EVAL_TEXTURELESS_WIDTH; + texturelessThresh = EVAL_TEXTURELESS_THRESH; + dispThresh = EVAL_DISP_THRESH; + dispGap = EVAL_DISP_GAP; + discontWidth = EVAL_DISCONT_WIDTH; + ignoreBorder = EVAL_IGNORE_BORDER; + } + float badThresh; + int texturelessWidth; + float texturelessThresh; + float dispThresh; + float dispGap; + int discontWidth; + int ignoreBorder; +}; + +class CV_StereoMatchingTest : public cvtest::BaseTest +{ +public: + CV_StereoMatchingTest() + { + rmsEps.resize(ERROR_KINDS_COUNT, 0.01f); fracEps.resize(ERROR_KINDS_COUNT, 1.e-6f); + } +protected: + // assumed that left image is a reference image + virtual int runStereoMatchingAlgorithm(const Mat& leftImg, const Mat& rightImg, + Rect& calcROI, Mat& leftDisp, Mat& rightDisp, int caseIdx) = 0; // return ignored border width + + int readDatasetsParams(FileStorage& fs); + virtual int readRunParams(FileStorage& fs); + void writeErrors(const string& errName, const vector& errors, FileStorage* fs = 0); + void writeROI(const Rect& calcROI, FileStorage* fs = 0); + void readErrors(FileNode& fn, const string& errName, vector& errors); + void readROI(FileNode& fn, Rect& trueROI); + int compareErrors(const vector& calcErrors, const vector& validErrors, + const vector& eps, const string& errName); + int compareROI(const Rect& calcROI, const Rect& validROI); + int processStereoMatchingResults(FileStorage& fs, int caseIdx, bool isWrite, + const Mat& leftImg, const Mat& rightImg, + const Rect& calcROI, + const Mat& trueLeftDisp, const Mat& trueRightDisp, + const Mat& leftDisp, const Mat& rightDisp, + const QualityEvalParams& qualityEvalParams); + void run(int); + + vector rmsEps; + vector fracEps; + + struct DatasetParams + { + int dispScaleFactor; + int dispUnknVal; + }; + map datasetsParams; + + vector caseNames; + vector caseDatasets; +}; + +void CV_StereoMatchingTest::run(int) +{ + addDataSearchSubDirectory("cv"); + string algorithmName = name; + assert(!algorithmName.empty()); + + FileStorage datasetsFS(findDataFile(DATASETS_DIR + DATASETS_FILE), FileStorage::READ); + int code = readDatasetsParams(datasetsFS); + if (code != cvtest::TS::OK) + { + ts->set_failed_test_info(code); + return; + } + FileStorage runParamsFS(findDataFile(ALGORITHMS_DIR + algorithmName + RUN_PARAMS_FILE), FileStorage::READ); + code = readRunParams(runParamsFS); + if (code != cvtest::TS::OK) + { + ts->set_failed_test_info(code); + return; + } + + string fullResultFilename = findDataDirectory(ALGORITHMS_DIR) + algorithmName + RESULT_FILE; + FileStorage resFS(fullResultFilename, FileStorage::READ); + bool isWrite = true; // write or compare results + if (resFS.isOpened()) + isWrite = false; + else + { + resFS.open(fullResultFilename, FileStorage::WRITE); + if (!resFS.isOpened()) + { + ts->printf(cvtest::TS::LOG, "file %s can not be read or written\n", fullResultFilename.c_str()); + ts->set_failed_test_info(cvtest::TS::FAIL_BAD_ARG_CHECK); + return; + } + resFS << "stereo_matching" << "{"; + } + + int progress = 0, caseCount = (int)caseNames.size(); + for (int ci = 0; ci < caseCount; ci++) + { + progress = update_progress(progress, ci, caseCount, 0); + printf("progress: %d%%\n", progress); + fflush(stdout); + string datasetName = caseDatasets[ci]; + string datasetFullDirName = findDataDirectory(DATASETS_DIR) + datasetName + "/"; + Mat leftImg = imread(datasetFullDirName + LEFT_IMG_NAME); + Mat rightImg = imread(datasetFullDirName + RIGHT_IMG_NAME); + Mat trueLeftDisp = imread(datasetFullDirName + TRUE_LEFT_DISP_NAME, 0); + Mat trueRightDisp = imread(datasetFullDirName + TRUE_RIGHT_DISP_NAME, 0); + Rect calcROI; + + if (leftImg.empty() || rightImg.empty() || trueLeftDisp.empty()) + { + ts->printf(cvtest::TS::LOG, "images or left ground-truth disparities of dataset %s can not be read", datasetName.c_str()); + code = cvtest::TS::FAIL_INVALID_TEST_DATA; + continue; + } + int dispScaleFactor = datasetsParams[datasetName].dispScaleFactor; + Mat tmp; + + trueLeftDisp.convertTo(tmp, CV_32FC1, 1.f / dispScaleFactor); + trueLeftDisp = tmp; + tmp.release(); + + if (!trueRightDisp.empty()) + { + trueRightDisp.convertTo(tmp, CV_32FC1, 1.f / dispScaleFactor); + trueRightDisp = tmp; + tmp.release(); + } + + Mat leftDisp, rightDisp; + int ignBorder = max(runStereoMatchingAlgorithm(leftImg, rightImg, calcROI, leftDisp, rightDisp, ci), EVAL_IGNORE_BORDER); + + leftDisp.convertTo(tmp, CV_32FC1); + leftDisp = tmp; + tmp.release(); + + rightDisp.convertTo(tmp, CV_32FC1); + rightDisp = tmp; + tmp.release(); + + int tempCode = processStereoMatchingResults(resFS, ci, isWrite, + leftImg, rightImg, calcROI, trueLeftDisp, trueRightDisp, leftDisp, rightDisp, QualityEvalParams(ignBorder)); + code = tempCode == cvtest::TS::OK ? code : tempCode; + } + + if (isWrite) + resFS << "}"; // "stereo_matching" + + ts->set_failed_test_info(code); +} + +void calcErrors(const Mat& leftImg, const Mat& /*rightImg*/, + const Mat& trueLeftDisp, const Mat& trueRightDisp, + const Mat& trueLeftUnknDispMask, const Mat& trueRightUnknDispMask, + const Mat& calcLeftDisp, const Mat& /*calcRightDisp*/, + vector& rms, vector& badPxlsFractions, + const QualityEvalParams& qualityEvalParams) +{ + Mat texturelessMask, texturedMask; + computeTextureBasedMasks(leftImg, &texturelessMask, &texturedMask, + qualityEvalParams.texturelessWidth, qualityEvalParams.texturelessThresh); + Mat occludedMask, nonOccludedMask; + computeOcclusionBasedMasks(trueLeftDisp, trueRightDisp, &occludedMask, &nonOccludedMask, + trueLeftUnknDispMask, trueRightUnknDispMask, qualityEvalParams.dispThresh); + Mat depthDiscontMask; + computeDepthDiscontMask(trueLeftDisp, depthDiscontMask, trueLeftUnknDispMask, + qualityEvalParams.dispGap, qualityEvalParams.discontWidth); + + Mat borderedKnownMask = getBorderedMask(leftImg.size(), qualityEvalParams.ignoreBorder) & ~trueLeftUnknDispMask; + + nonOccludedMask &= borderedKnownMask; + occludedMask &= borderedKnownMask; + texturedMask &= nonOccludedMask; // & borderedKnownMask + texturelessMask &= nonOccludedMask; // & borderedKnownMask + depthDiscontMask &= nonOccludedMask; // & borderedKnownMask + + rms.resize(ERROR_KINDS_COUNT); + rms[0] = dispRMS(calcLeftDisp, trueLeftDisp, borderedKnownMask); + rms[1] = dispRMS(calcLeftDisp, trueLeftDisp, nonOccludedMask); + rms[2] = dispRMS(calcLeftDisp, trueLeftDisp, occludedMask); + rms[3] = dispRMS(calcLeftDisp, trueLeftDisp, texturedMask); + rms[4] = dispRMS(calcLeftDisp, trueLeftDisp, texturelessMask); + rms[5] = dispRMS(calcLeftDisp, trueLeftDisp, depthDiscontMask); + + badPxlsFractions.resize(ERROR_KINDS_COUNT); + badPxlsFractions[0] = badMatchPxlsFraction(calcLeftDisp, trueLeftDisp, borderedKnownMask, qualityEvalParams.badThresh); + badPxlsFractions[1] = badMatchPxlsFraction(calcLeftDisp, trueLeftDisp, nonOccludedMask, qualityEvalParams.badThresh); + badPxlsFractions[2] = badMatchPxlsFraction(calcLeftDisp, trueLeftDisp, occludedMask, qualityEvalParams.badThresh); + badPxlsFractions[3] = badMatchPxlsFraction(calcLeftDisp, trueLeftDisp, texturedMask, qualityEvalParams.badThresh); + badPxlsFractions[4] = badMatchPxlsFraction(calcLeftDisp, trueLeftDisp, texturelessMask, qualityEvalParams.badThresh); + badPxlsFractions[5] = badMatchPxlsFraction(calcLeftDisp, trueLeftDisp, depthDiscontMask, qualityEvalParams.badThresh); +} + +int CV_StereoMatchingTest::processStereoMatchingResults(FileStorage& fs, int caseIdx, bool isWrite, + const Mat& leftImg, const Mat& rightImg, + const Rect& calcROI, + const Mat& trueLeftDisp, const Mat& trueRightDisp, + const Mat& leftDisp, const Mat& rightDisp, + const QualityEvalParams& qualityEvalParams) +{ + // rightDisp is not used in current test virsion + int code = cvtest::TS::OK; + assert(fs.isOpened()); + assert(trueLeftDisp.type() == CV_32FC1); + assert(trueRightDisp.empty() || trueRightDisp.type() == CV_32FC1); + assert(leftDisp.type() == CV_32FC1 && (rightDisp.empty() || rightDisp.type() == CV_32FC1)); + + // get masks for unknown ground truth disparity values + Mat leftUnknMask, rightUnknMask; + DatasetParams params = datasetsParams[caseDatasets[caseIdx]]; + absdiff(trueLeftDisp, Scalar(params.dispUnknVal), leftUnknMask); + leftUnknMask = leftUnknMask < std::numeric_limits::epsilon(); + assert(leftUnknMask.type() == CV_8UC1); + if (!trueRightDisp.empty()) + { + absdiff(trueRightDisp, Scalar(params.dispUnknVal), rightUnknMask); + rightUnknMask = rightUnknMask < std::numeric_limits::epsilon(); + assert(rightUnknMask.type() == CV_8UC1); + } + + // calculate errors + vector rmss, badPxlsFractions; + calcErrors(leftImg, rightImg, trueLeftDisp, trueRightDisp, leftUnknMask, rightUnknMask, + leftDisp, rightDisp, rmss, badPxlsFractions, qualityEvalParams); + + if (isWrite) + { + fs << caseNames[caseIdx] << "{"; + fs.writeComment(RMS_STR, 0); + writeErrors(RMS_STR, rmss, &fs); + fs.writeComment(BAD_PXLS_FRACTION_STR, 0); + writeErrors(BAD_PXLS_FRACTION_STR, badPxlsFractions, &fs); + fs.writeComment(ROI_STR, 0); + writeROI(calcROI, &fs); + fs << "}"; // datasetName + } + else // compare + { + ts->printf(cvtest::TS::LOG, "\nquality of case named %s\n", caseNames[caseIdx].c_str()); + ts->printf(cvtest::TS::LOG, "%s\n", RMS_STR.c_str()); + writeErrors(RMS_STR, rmss); + ts->printf(cvtest::TS::LOG, "%s\n", BAD_PXLS_FRACTION_STR.c_str()); + writeErrors(BAD_PXLS_FRACTION_STR, badPxlsFractions); + ts->printf(cvtest::TS::LOG, "%s\n", ROI_STR.c_str()); + writeROI(calcROI); + + FileNode fn = fs.getFirstTopLevelNode()[caseNames[caseIdx]]; + vector validRmss, validBadPxlsFractions; + Rect validROI; + + readErrors(fn, RMS_STR, validRmss); + readErrors(fn, BAD_PXLS_FRACTION_STR, validBadPxlsFractions); + readROI(fn, validROI); + int tempCode = compareErrors(rmss, validRmss, rmsEps, RMS_STR); + code = tempCode == cvtest::TS::OK ? code : tempCode; + tempCode = compareErrors(badPxlsFractions, validBadPxlsFractions, fracEps, BAD_PXLS_FRACTION_STR); + code = tempCode == cvtest::TS::OK ? code : tempCode; + tempCode = compareROI(calcROI, validROI); + code = tempCode == cvtest::TS::OK ? code : tempCode; + } + return code; +} + +int CV_StereoMatchingTest::readDatasetsParams(FileStorage& fs) +{ + if (!fs.isOpened()) + { + ts->printf(cvtest::TS::LOG, "datasetsParams can not be read "); + return cvtest::TS::FAIL_INVALID_TEST_DATA; + } + datasetsParams.clear(); + FileNode fn = fs.getFirstTopLevelNode(); + assert(fn.isSeq()); + for (int i = 0; i < (int)fn.size(); i += 3) + { + String _name = fn[i]; + DatasetParams params; + String sf = fn[i + 1]; params.dispScaleFactor = atoi(sf.c_str()); + String uv = fn[i + 2]; params.dispUnknVal = atoi(uv.c_str()); + datasetsParams[_name] = params; + } + return cvtest::TS::OK; +} + +int CV_StereoMatchingTest::readRunParams(FileStorage& fs) +{ + if (!fs.isOpened()) + { + ts->printf(cvtest::TS::LOG, "runParams can not be read "); + return cvtest::TS::FAIL_INVALID_TEST_DATA; + } + caseNames.clear();; + caseDatasets.clear(); + return cvtest::TS::OK; +} + +void CV_StereoMatchingTest::writeErrors(const string& errName, const vector& errors, FileStorage* fs) +{ + assert((int)errors.size() == ERROR_KINDS_COUNT); + vector::const_iterator it = errors.begin(); + if (fs) + for (int i = 0; i < ERROR_KINDS_COUNT; i++, ++it) + *fs << ERROR_PREFIXES[i] + errName << *it; + else + for (int i = 0; i < ERROR_KINDS_COUNT; i++, ++it) + ts->printf(cvtest::TS::LOG, "%s = %f\n", string(ERROR_PREFIXES[i] + errName).c_str(), *it); +} + +void CV_StereoMatchingTest::writeROI(const Rect& calcROI, FileStorage* fs) +{ + if (fs) + { + *fs << ROI_PREFIXES[0] << calcROI.x; + *fs << ROI_PREFIXES[1] << calcROI.y; + *fs << ROI_PREFIXES[2] << calcROI.width; + *fs << ROI_PREFIXES[3] << calcROI.height; + } + else + { + ts->printf(cvtest::TS::LOG, "%s = %d\n", ROI_PREFIXES[0].c_str(), calcROI.x); + ts->printf(cvtest::TS::LOG, "%s = %d\n", ROI_PREFIXES[1].c_str(), calcROI.y); + ts->printf(cvtest::TS::LOG, "%s = %d\n", ROI_PREFIXES[2].c_str(), calcROI.width); + ts->printf(cvtest::TS::LOG, "%s = %d\n", ROI_PREFIXES[3].c_str(), calcROI.height); + } +} + +void CV_StereoMatchingTest::readErrors(FileNode& fn, const string& errName, vector& errors) +{ + errors.resize(ERROR_KINDS_COUNT); + vector::iterator it = errors.begin(); + for (int i = 0; i < ERROR_KINDS_COUNT; i++, ++it) + fn[ERROR_PREFIXES[i] + errName] >> *it; +} + +void CV_StereoMatchingTest::readROI(FileNode& fn, Rect& validROI) +{ + fn[ROI_PREFIXES[0]] >> validROI.x; + fn[ROI_PREFIXES[1]] >> validROI.y; + fn[ROI_PREFIXES[2]] >> validROI.width; + fn[ROI_PREFIXES[3]] >> validROI.height; +} + +int CV_StereoMatchingTest::compareErrors(const vector& calcErrors, const vector& validErrors, + const vector& eps, const string& errName) +{ + assert((int)calcErrors.size() == ERROR_KINDS_COUNT); + assert((int)validErrors.size() == ERROR_KINDS_COUNT); + assert((int)eps.size() == ERROR_KINDS_COUNT); + vector::const_iterator calcIt = calcErrors.begin(), + validIt = validErrors.begin(), + epsIt = eps.begin(); + bool ok = true; + for (int i = 0; i < ERROR_KINDS_COUNT; i++, ++calcIt, ++validIt, ++epsIt) + if (*calcIt - *validIt > *epsIt) + { + ts->printf(cvtest::TS::LOG, "bad accuracy of %s (valid=%f; calc=%f)\n", string(ERROR_PREFIXES[i] + errName).c_str(), *validIt, *calcIt); + ok = false; + } + return ok ? cvtest::TS::OK : cvtest::TS::FAIL_BAD_ACCURACY; +} + +int CV_StereoMatchingTest::compareROI(const Rect& calcROI, const Rect& validROI) +{ + int compare[4][2] = { + { calcROI.x, validROI.x }, + { calcROI.y, validROI.y }, + { calcROI.width, validROI.width }, + { calcROI.height, validROI.height }, + }; + bool ok = true; + for (int i = 0; i < 4; i++) + { + if (compare[i][0] != compare[i][1]) + { + ts->printf(cvtest::TS::LOG, "bad accuracy of %s (valid=%d; calc=%d)\n", ROI_PREFIXES[i].c_str(), compare[i][1], compare[i][0]); + ok = false; + } + } + return ok ? cvtest::TS::OK : cvtest::TS::FAIL_BAD_ACCURACY; +} + +//----------------------------------- StereoSGM test ----------------------------------------------------- + +class CV_Cuda_StereoSGMTest : public CV_StereoMatchingTest +{ +public: + CV_Cuda_StereoSGMTest() + { + name = "cuda_stereosgm"; + fill(rmsEps.begin(), rmsEps.end(), 0.25f); + fill(fracEps.begin(), fracEps.end(), 0.01f); + } + +protected: + struct RunParams + { + int ndisp; + int mode; + }; + vector caseRunParams; + + virtual int readRunParams(FileStorage& fs) + { + int code = CV_StereoMatchingTest::readRunParams(fs); + FileNode fn = fs.getFirstTopLevelNode(); + assert(fn.isSeq()); + for (int i = 0; i < (int)fn.size(); i += 4) + { + String caseName = fn[i], datasetName = fn[i + 1]; + RunParams params; + String ndisp = fn[i + 2]; params.ndisp = atoi(ndisp.c_str()); + String mode = fn[i + 3]; params.mode = atoi(mode.c_str()); + caseNames.push_back(caseName); + caseDatasets.push_back(datasetName); + caseRunParams.push_back(params); + } + return code; + } + + virtual int runStereoMatchingAlgorithm(const Mat& leftImg, const Mat& rightImg, + Rect& calcROI, Mat& leftDisp, Mat& /*rightDisp*/, int caseIdx) + { + RunParams params = caseRunParams[caseIdx]; + assert(params.ndisp % 16 == 0); + Ptr sgm = createStereoSGM(0, params.ndisp, 10, 120, 5, params.mode); + + cv::Mat G1, G2; + cv::cvtColor(leftImg, G1, cv::COLOR_RGB2GRAY); + cv::cvtColor(rightImg, G2, cv::COLOR_RGB2GRAY); + cv::cuda::GpuMat d_leftImg, d_rightImg, d_leftDisp; + d_leftImg.upload(G1); + d_rightImg.upload(G2); + sgm->compute(d_leftImg, d_rightImg, d_leftDisp); + d_leftDisp.download(leftDisp); + CV_Assert(leftDisp.type() == CV_16SC1); + leftDisp.convertTo(leftDisp, CV_32FC1, 1.0 / StereoMatcher::DISP_SCALE); + + calcROI.x = calcROI.y = 0; + calcROI.width = leftImg.cols; + calcROI.height = leftImg.rows; + return 0; + } +}; + +TEST(CudaStereo_StereoSGM, regression) { CV_Cuda_StereoSGMTest test; test.safe_run(); } }} // namespace #endif // HAVE_CUDA From 166e302a17d51381ef6bc1c7e4c214ef26051fa7 Mon Sep 17 00:00:00 2001 From: Steffen Urban Date: Tue, 1 Dec 2020 14:54:57 +0100 Subject: [PATCH 02/70] add js to aruco module --- modules/aruco/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/aruco/CMakeLists.txt b/modules/aruco/CMakeLists.txt index d3c7eaa66a9..e147a78376a 100644 --- a/modules/aruco/CMakeLists.txt +++ b/modules/aruco/CMakeLists.txt @@ -1,2 +1,2 @@ set(the_description "ArUco Marker Detection") -ocv_define_module(aruco opencv_core opencv_imgproc opencv_calib3d WRAP python java objc) +ocv_define_module(aruco opencv_core opencv_imgproc opencv_calib3d WRAP python java objc js) From ec294f62ce45609bf4b61a7f74228b1984815808 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sat, 5 Dec 2020 20:03:38 +0000 Subject: [PATCH 03/70] build warnings - GCC 4.8.5 / CentOS 7 --- modules/rgbd/src/pose_graph.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/rgbd/src/pose_graph.hpp b/modules/rgbd/src/pose_graph.hpp index bb0164723be..e8b7c34c53b 100644 --- a/modules/rgbd/src/pose_graph.hpp +++ b/modules/rgbd/src/pose_graph.hpp @@ -228,8 +228,8 @@ class PoseGraph virtual ~PoseGraph() = default; //! PoseGraph can be copied/cloned - PoseGraph(const PoseGraph& _poseGraph) = default; - PoseGraph& operator=(const PoseGraph& _poseGraph) = default; + PoseGraph(const PoseGraph&) = default; + PoseGraph& operator=(const PoseGraph&) = default; void addNode(const PoseGraphNode& node) { nodes.push_back(node); } void addEdge(const PoseGraphEdge& edge) { edges.push_back(edge); } From 5f99ff97b99de176e812feaba8f544750764c228 Mon Sep 17 00:00:00 2001 From: cudawarped Date: Mon, 7 Dec 2020 10:01:02 +0000 Subject: [PATCH 04/70] Update cudacodec to work with Nvidia Video Codec SDK 11.0 --- modules/cudacodec/include/opencv2/cudacodec.hpp | 1 + modules/cudacodec/src/video_decoder.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/modules/cudacodec/include/opencv2/cudacodec.hpp b/modules/cudacodec/include/opencv2/cudacodec.hpp index 3fad950e102..401054eefcd 100644 --- a/modules/cudacodec/include/opencv2/cudacodec.hpp +++ b/modules/cudacodec/include/opencv2/cudacodec.hpp @@ -255,6 +255,7 @@ enum Codec HEVC, VP8, VP9, + AV1, NumCodecs, Uncompressed_YUV420 = (('I'<<24)|('Y'<<16)|('U'<<8)|('V')), //!< Y,U,V (4:2:0) diff --git a/modules/cudacodec/src/video_decoder.cpp b/modules/cudacodec/src/video_decoder.cpp index a84f79c9b6f..1293055bff4 100644 --- a/modules/cudacodec/src/video_decoder.cpp +++ b/modules/cudacodec/src/video_decoder.cpp @@ -78,6 +78,7 @@ void cv::cudacodec::detail::VideoDecoder::create(const FormatInfo& videoFormat) #if ((CUDART_VERSION == 7500) || (CUDART_VERSION >= 9000)) codecSupported |= cudaVideoCodec_VP8 == _codec || cudaVideoCodec_VP9 == _codec || + cudaVideoCodec_AV1 == _codec || cudaVideoCodec_YUV420 == _codec; #endif #endif From cfcd5b6636a4f0b9ab0ce6595267179e1f13b1d9 Mon Sep 17 00:00:00 2001 From: Enrico Ronconi Date: Mon, 7 Dec 2020 17:43:18 +0100 Subject: [PATCH 05/70] surface_matching/PPF3DDetector::match : fix memory leak --- modules/surface_matching/src/ppf_match_3d.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/surface_matching/src/ppf_match_3d.cpp b/modules/surface_matching/src/ppf_match_3d.cpp index f8477a38f83..c5c25e27e88 100644 --- a/modules/surface_matching/src/ppf_match_3d.cpp +++ b/modules/surface_matching/src/ppf_match_3d.cpp @@ -583,9 +583,7 @@ void PPF3DDetector::match(const Mat& pc, std::vector& results, const poseList.push_back(pose); } -#if defined (_OPENMP) free(accumulator); -#endif } // TODO : Make the parameters relative if not arguments. From 9b5a80114807bd3d67bc109af1e0bebbb81dabd6 Mon Sep 17 00:00:00 2001 From: Kumataro Date: Tue, 8 Dec 2020 05:37:50 +0900 Subject: [PATCH 06/70] Merge pull request #2776 from Kumataro:master_freetype2_doc freetype2: Fix comments in freetype.hpp and README.md * freetype2: Update README.md * freetype2: Update freetype.hpp * freetype2: remove new blank line * Update README.md - Remove dots from the Headers - Align number of === / --- symbols on the next line * freetype: remove comment for color. --- modules/freetype/README.md | 46 +++++++++---------- modules/freetype/include/opencv2/freetype.hpp | 2 +- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/modules/freetype/README.md b/modules/freetype/README.md index 44ecb1648c3..cfea3e0e0dd 100644 --- a/modules/freetype/README.md +++ b/modules/freetype/README.md @@ -1,34 +1,34 @@ -FreeType Module -=========== +FreeType2 Wrapper Module +======================== -This FreeType module allows you to draw strings with outlines and bitmaps. +This FreeType2 wrapper module allows to draw strings with outlines and bitmaps. + +Requested external libraries +---------------------------- -Installation ------------ harfbuzz is requested to convert UTF8 to gid(GlyphID). + freetype library is requested to rasterize given gid. -harfbuzz https://www.freedesktop.org/wiki/Software/HarfBuzz/ -freetype https://www.freetype.org/ +- harfbuzz https://www.freedesktop.org/wiki/Software/HarfBuzz/ +- freetype https://www.freetype.org/ Usage ------------ -cv::freetype::FreeType2 ft2; -ft2.loadFontData("your-font.ttf", 0); -ft2.setSplitNumber( 4 ); // Bezier-line is splited by 4 segment. -ft2.putText(src, .... ) +----- + +``` +cv::Ptr ft2; +ft2 = cv::freetype::createFreeType2(); +ft2->loadFontData(ttf_pathname, 0); +ft2->putText(mat, "hello world", cv::Point(20, 200), + 30, CV_RGB(0, 0, 0), cv::FILLED, cv::LINE_AA, true); +``` Option ------------- +------ - 2nd argument of loadFontData is used if font file has many font data. - 3 drawing mode is available. --- outline mode is used if lineWidth is larger than 0. (like original putText) --- bitmap mode is used if lineWidth is less than 0. ---- 1bit bitmap mode is used if lineStyle is 4 or 8. ---- gray bitmap mode is used if lineStyle is 16. - -Future work ------------- -- test --- CJK and ... -- RTL,LTR,TTB,BTT... + - outline mode is used if lineWidth is larger than 0. (like original putText) + - bitmap mode is used if lineWidth is less than 0. + - 1bit bitmap mode is used if lineStyle is 4 or 8. + - gray bitmap mode is used if lineStyle is 16. diff --git a/modules/freetype/include/opencv2/freetype.hpp b/modules/freetype/include/opencv2/freetype.hpp index 6e41ec7ff4e..1c0a6e5e224 100644 --- a/modules/freetype/include/opencv2/freetype.hpp +++ b/modules/freetype/include/opencv2/freetype.hpp @@ -99,7 +99,7 @@ If you want to draw small glyph, small is better. The function putText renders the specified text string in the image. Symbols that cannot be rendered using the specified font are replaced by "Tofu" or non-drawn. -@param img Image. +@param img Image. (Only 8UC3 image is supported.) @param text Text string to be drawn. @param org Bottom-left/Top-left corner of the text string in the image. @param fontHeight Drawing font size by pixel unit. From bc1ea8f891fabb7bea9d1b9e95811ba737c94e5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iago=20Su=C3=A1rez?= Date: Wed, 9 Dec 2020 13:28:21 -0800 Subject: [PATCH 07/70] Merge pull request #2774 from iago-suarez:master * Added BEBLID local feature descriptor * Solved format problems, moved descriptor data to .hpp files and added BeblidSize enum * Deleting guards from BEBLID parameter files * Using int for the BeblidSize, using at<> to access matrix and releasing descriptors when no keypoints are detected. --- modules/xfeatures2d/README.md | 2 +- modules/xfeatures2d/doc/xfeatures2d.bib | 20 + .../include/opencv2/xfeatures2d.hpp | 45 ++ modules/xfeatures2d/perf/perf_beblid.cpp | 36 ++ modules/xfeatures2d/src/beblid.cpp | 403 ++++++++++++++++++ modules/xfeatures2d/src/beblid.p256.hpp | 81 ++++ modules/xfeatures2d/src/beblid.p512.hpp | 146 +++++++ modules/xfeatures2d/test/test_features2d.cpp | 7 + .../test_rotation_and_scale_invariance.cpp | 4 + 9 files changed, 743 insertions(+), 1 deletion(-) create mode 100644 modules/xfeatures2d/perf/perf_beblid.cpp create mode 100644 modules/xfeatures2d/src/beblid.cpp create mode 100644 modules/xfeatures2d/src/beblid.p256.hpp create mode 100644 modules/xfeatures2d/src/beblid.p512.hpp diff --git a/modules/xfeatures2d/README.md b/modules/xfeatures2d/README.md index 577b0436548..f2cba30154e 100644 --- a/modules/xfeatures2d/README.md +++ b/modules/xfeatures2d/README.md @@ -5,4 +5,4 @@ Extra 2D Features Framework 2. Non-free 2D feature algorithms Extra 2D Features Framework containing experimental and non-free 2D feature detector/descriptor algorithms: - SURF, BRIEF, Censure, Freak, LUCID, Daisy, Self-similar. + SURF, BRIEF, Censure, Freak, LUCID, Daisy, BEBLID, Self-similar. diff --git a/modules/xfeatures2d/doc/xfeatures2d.bib b/modules/xfeatures2d/doc/xfeatures2d.bib index 6337a606d17..2e950c46ae3 100644 --- a/modules/xfeatures2d/doc/xfeatures2d.bib +++ b/modules/xfeatures2d/doc/xfeatures2d.bib @@ -140,4 +140,24 @@ @incollection{LUCID pages = {1--9} year = {2012} publisher = {NIPS} +} + +@article{Suarez2020BEBLID, + title = {{BEBLID: Boosted Efficient Binary Local Image Descriptor}}, + journal = {Pattern Recognition Letters}, + volume = {133}, + pages = {366--372}, + year = {2020}, + issn = {0167-8655}, + doi = {https://doi.org/10.1016/j.patrec.2020.04.005}, + url = {https://raw.githubusercontent.com/iago-suarez/BEBLID/master/BEBLID_Boosted_Efficient_Binary_Local_Image_Descriptor.pdf}, + author = {Iago Su\'arez and Ghesn Sfeir and Jos\'e M. Buenaposada and Luis Baumela}, +} + +@inproceedings{winder2007learning, + title= {Learning Local Image Descriptors}, + author= {Winder, Simon AJ and Brown, Matthew}, + booktitle= {Computer Vision and Pattern Recognition}, + pages={1--8}, + year={2007}, } \ No newline at end of file diff --git a/modules/xfeatures2d/include/opencv2/xfeatures2d.hpp b/modules/xfeatures2d/include/opencv2/xfeatures2d.hpp index 103dbe8e78c..fa7c449273b 100644 --- a/modules/xfeatures2d/include/opencv2/xfeatures2d.hpp +++ b/modules/xfeatures2d/include/opencv2/xfeatures2d.hpp @@ -179,6 +179,51 @@ class CV_EXPORTS_W LATCH : public Feature2D CV_WRAP static Ptr create(int bytes = 32, bool rotationInvariance = true, int half_ssd_size = 3, double sigma = 2.0); }; +/** @brief Class implementing BEBLID (Boosted Efficient Binary Local Image Descriptor), + * described in @cite Suarez2020BEBLID . + +BEBLID \cite Suarez2020BEBLID is a efficient binary descriptor learned with boosting. +It is able to describe keypoints from any detector just by changing the scale_factor parameter. +In several benchmarks it has proved to largely improve other binary descriptors like ORB or +BRISK with the same efficiency. BEBLID describes using the difference of mean gray values in +different regions of the image around the KeyPoint, the descriptor is specifically optimized for +image matching and patch retrieval addressing the asymmetries of these problems. + +If you find this code useful, please add a reference to the following paper: +
Iago Suárez, Ghesn Sfeir, José M. Buenaposada, and Luis Baumela. +BEBLID: Boosted efficient binary local image descriptor. +Pattern Recognition Letters, 133:366–372, 2020.
+ +The descriptor was trained using 1 million of randomly sampled pairs of patches +(20% positives and 80% negatives) from the Liberty split of the UBC datasets +\cite winder2007learning as described in the paper @cite Suarez2020BEBLID. +You can check in the [AKAZE example](https://raw.githubusercontent.com/opencv/opencv/master/samples/cpp/tutorial_code/features2D/AKAZE_match.cpp) +how well BEBLID works. Detecting 10000 keypoints with ORB and describing with BEBLID obtains +561 inliers (75%) whereas describing with ORB obtains only 493 inliers (63%). +*/ +class CV_EXPORTS_W BEBLID : public Feature2D +{ +public: + /** + * @brief Descriptor number of bits, each bit is a boosting weak-learner. + * The user can choose between 512 or 256 bits. + */ + enum BeblidSize + { + SIZE_512_BITS = 100, SIZE_256_BITS = 101, + }; + /** @brief Creates the BEBLID descriptor. + @param scale_factor Adjust the sampling window around detected keypoints: + - 1.00f should be the scale for ORB keypoints + - 6.75f should be the scale for SIFT detected keypoints + - 6.25f is default and fits for KAZE, SURF detected keypoints + - 5.00f should be the scale for AKAZE, MSD, AGAST, FAST, BRISK keypoints + @param n_bits Determine the number of bits in the descriptor. Should be either + BEBLID::SIZE_512_BITS or BEBLID::SIZE_256_BITS. + */ + CV_WRAP static Ptr create(float scale_factor, int n_bits = BEBLID::SIZE_512_BITS); +}; + /** @brief Class implementing DAISY descriptor, described in @cite Tola10 @param radius radius of the descriptor at the initial scale diff --git a/modules/xfeatures2d/perf/perf_beblid.cpp b/modules/xfeatures2d/perf/perf_beblid.cpp new file mode 100644 index 00000000000..2e2d3eb546a --- /dev/null +++ b/modules/xfeatures2d/perf/perf_beblid.cpp @@ -0,0 +1,36 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +#include "perf_precomp.hpp" + +namespace opencv_test { namespace { + +typedef perf::TestBaseWithParam beblid; + +#define BEBLID_IMAGES \ + "cv/detectors_descriptors_evaluation/images_datasets/leuven/img1.png",\ + "stitching/a3.png" + +#ifdef OPENCV_ENABLE_NONFREE +PERF_TEST_P(beblid, extract, testing::Values(BEBLID_IMAGES)) +{ + string filename = getDataPath(GetParam()); + Mat frame = imread(filename, IMREAD_GRAYSCALE); + ASSERT_FALSE(frame.empty()) << "Unable to load source image " << filename; + + Mat mask; + declare.in(frame).time(90); + + Ptr detector = SURF::create(); + vector points; + detector->detect(frame, points, mask); + + Ptr descriptor = BEBLID::create(6.25f); + cv::Mat descriptors; + TEST_CYCLE() descriptor->compute(frame, points, descriptors); + + SANITY_CHECK_NOTHING(); +} +#endif // NONFREE + +}} // namespace diff --git a/modules/xfeatures2d/src/beblid.cpp b/modules/xfeatures2d/src/beblid.cpp new file mode 100644 index 00000000000..c0d38ed004e --- /dev/null +++ b/modules/xfeatures2d/src/beblid.cpp @@ -0,0 +1,403 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Author: Iago Suarez + +// Implementation of the article: +// Iago Suarez, Ghesn Sfeir, Jose M. Buenaposada, and Luis Baumela. +// BEBLID: Boosted Efficient Binary Local Image Descriptor. +// Pattern Recognition Letters, 133:366–372, 2020. + +#include "precomp.hpp" + +#define CV_BEBLID_PARALLEL + +#define CV_ROUNDNUM(x) ((int)(x + 0.5f)) +#define CV_DEGREES_TO_RADS 0.017453292519943295 // (M_PI / 180.0) +#define CV_BEBLID_EXTRA_RATIO_MARGIN 1.75f + +using namespace cv; +using namespace std; + + +namespace cv +{ +namespace xfeatures2d +{ + +// Struct containing the 6 parameters that define an Average Box weak-learner +struct ABWLParams +{ + int x1, y1, x2, y2, boxRadius, th; +}; + +// BEBLID implementation +class BEBLID_Impl CV_FINAL: public BEBLID +{ +public: + + // constructor + explicit BEBLID_Impl(float scale_factor, int n_bits = SIZE_512_BITS); + + // destructor + ~BEBLID_Impl() CV_OVERRIDE = default; + + // returns the descriptor length in bytes + int descriptorSize() const CV_OVERRIDE { return int(wl_params_.size() / 8); } + + // returns the descriptor type + int descriptorType() const CV_OVERRIDE { return CV_8UC1; } + + // returns the default norm type + int defaultNorm() const CV_OVERRIDE { return cv::NORM_HAMMING; } + + // compute descriptors given keypoints + void compute(InputArray image, vector &keypoints, OutputArray descriptors) CV_OVERRIDE; + +private: + std::vector wl_params_; + float scale_factor_; + cv::Size patch_size_; + + void computeBEBLID(const cv::Mat &integralImg, + const std::vector &keypoints, + cv::Mat &descriptors); +}; // END BEBLID_Impl CLASS + +/** + * @brief Function that determines if a keypoint is close to the image border. + * @param kp The detected keypoint + * @param imgSize The size of the image + * @param patchSize The size of the normalized patch where the measurement functions were learnt. + * @param scaleFactor A scale factor that magnifies the measurement functions w.r.t. the keypoint. + * @return true if the keypoint is in the border, false otherwise + */ +static inline bool isKeypointInTheBorder(const cv::KeyPoint &kp, + const cv::Size &imgSize, + const cv::Size &patchSize = {32, 32}, + float scaleFactor = 1) +{ + // This would be the correct measure but since we will compare with half of the size, use this as border size + float s = scaleFactor * kp.size / (patchSize.width + patchSize.height); + cv::Size2f border(patchSize.width * s * CV_BEBLID_EXTRA_RATIO_MARGIN, + patchSize.height * s * CV_BEBLID_EXTRA_RATIO_MARGIN); + + if (kp.pt.x < border.width || kp.pt.x + border.width >= imgSize.width) + return true; + + if (kp.pt.y < border.height || kp.pt.y + border.height >= imgSize.height) + return true; + + return false; +} + +/** + * @brief Rectifies the coordinates of the measurement functions that conform the descriptor + * with the keypoint location parameters. + * @param wlPatchParams The input weak learner parameters learnt for the normalized patch + * @param wlImageParams The output weak learner parameters adapted to the keypoint location + * @param kp The keypoint defining the offset, rotation and scale to be applied + * @param scaleFactor A scale factor that magnifies the measurement functions w.r.t. the keypoint. + * @param patchSize The size of the normalized patch where the measurement functions were learnt. + */ +static inline void rectifyABWL(const std::vector &wlPatchParams, + std::vector &wlImageParams, + const cv::KeyPoint &kp, + float scaleFactor = 1, + const cv::Size &patchSize = cv::Size(32, 32)) +{ + float m00, m01, m02, m10, m11, m12; + float s, cosine, sine; + + s = scaleFactor * kp.size / (0.5f * (patchSize.width + patchSize.height)); + wlImageParams.resize(wlPatchParams.size()); + + if (kp.angle == -1) + { + m00 = s; + m01 = 0.0f; + m02 = -0.5f * s * patchSize.width + kp.pt.x; + m10 = 0.0f; + m11 = s; + m12 = -s * 0.5f * patchSize.height + kp.pt.y; + } + else + { + cosine = (kp.angle >= 0) ? float(cos(kp.angle * CV_DEGREES_TO_RADS)) : 1.f; + sine = (kp.angle >= 0) ? float(sin(kp.angle * CV_DEGREES_TO_RADS)) : 0.f; + + m00 = s * cosine; + m01 = -s * sine; + m02 = (-s * cosine + s * sine) * patchSize.width * 0.5f + kp.pt.x; + m10 = s * sine; + m11 = s * cosine; + m12 = (-s * sine - s * cosine) * patchSize.height * 0.5f + kp.pt.y; + } + + for (size_t i = 0; i < wlPatchParams.size(); i++) + { + wlImageParams[i].x1 = CV_ROUNDNUM(m00 * wlPatchParams[i].x1 + m01 * wlPatchParams[i].y1 + m02); + wlImageParams[i].y1 = CV_ROUNDNUM(m10 * wlPatchParams[i].x1 + m11 * wlPatchParams[i].y1 + m12); + wlImageParams[i].x2 = CV_ROUNDNUM(m00 * wlPatchParams[i].x2 + m01 * wlPatchParams[i].y2 + m02); + wlImageParams[i].y2 = CV_ROUNDNUM(m10 * wlPatchParams[i].x2 + m11 * wlPatchParams[i].y2 + m12); + wlImageParams[i].boxRadius = CV_ROUNDNUM(s * wlPatchParams[i].boxRadius); + } +} + +/** + * @brief Computes the Average Box Weak-Learner response, measuring the difference of + * gray level in the two square regions. + * @param wlImageParams The weak-learner parameter defining the size and locations of each box. + * @param integralImage The integral image used to compute the average gray value in the square regions. + * @return The difference of gray level in the two squares defined by wlImageParams + */ +static inline float computeABWLResponse(const ABWLParams &wlImageParams, + const cv::Mat &integralImage) +{ + CV_DbgAssert(!integralImage.empty()); + CV_DbgAssert(integralImage.type() == CV_32SC1); + + int frameWidth, frameHeight, box1x1, box1y1, box1x2, box1y2, box2x1, box2y1, box2x2, box2y2; + int A, B, C, D; + int box_area1, box_area2; + float sum1, sum2, average1, average2; + // Since the integral image has one extra row and col, calculate the patch dimensions + frameWidth = integralImage.cols; + frameHeight = integralImage.rows; + + // For the first box, we calculate its margin coordinates + box1x1 = wlImageParams.x1 - wlImageParams.boxRadius; + if (box1x1 < 0) + box1x1 = 0; + else if (box1x1 >= frameWidth - 1) + box1x1 = frameWidth - 2; + box1y1 = wlImageParams.y1 - wlImageParams.boxRadius; + if (box1y1 < 0) + box1y1 = 0; + else if (box1y1 >= frameHeight - 1) + box1y1 = frameHeight - 2; + box1x2 = wlImageParams.x1 + wlImageParams.boxRadius + 1; + if (box1x2 <= 0) + box1x2 = 1; + else if (box1x2 >= frameWidth) + box1x2 = frameWidth - 1; + box1y2 = wlImageParams.y1 + wlImageParams.boxRadius + 1; + if (box1y2 <= 0) + box1y2 = 1; + else if (box1y2 >= frameHeight) + box1y2 = frameHeight - 1; + CV_DbgAssert((box1x1 < box1x2 && box1y1 < box1y2) && "Box 1 has size 0"); + + // For the second box, we calculate its margin coordinates + box2x1 = wlImageParams.x2 - wlImageParams.boxRadius; + if (box2x1 < 0) + box2x1 = 0; + else if (box2x1 >= frameWidth - 1) + box2x1 = frameWidth - 2; + box2y1 = wlImageParams.y2 - wlImageParams.boxRadius; + if (box2y1 < 0) + box2y1 = 0; + else if (box2y1 >= frameHeight - 1) + box2y1 = frameHeight - 2; + box2x2 = wlImageParams.x2 + wlImageParams.boxRadius + 1; + if (box2x2 <= 0) + box2x2 = 1; + else if (box2x2 >= frameWidth) + box2x2 = frameWidth - 1; + box2y2 = wlImageParams.y2 + wlImageParams.boxRadius + 1; + if (box2y2 <= 0) + box2y2 = 1; + else if (box2y2 >= frameHeight) + box2y2 = frameHeight - 1; + CV_DbgAssert((box2x1 < box2x2 && box2y1 < box2y2) && "Box 2 has size 0"); + + // Read the integral image values for the first box + A = integralImage.at(box1y1, box1x1); + B = integralImage.at(box1y1, box1x2); + C = integralImage.at(box1y2, box1x1); + D = integralImage.at(box1y2, box1x2); + + // Calculate the mean intensity value of the pixels in the box + sum1 = float(A + D - B - C); + box_area1 = (box1y2 - box1y1) * (box1x2 - box1x1); + CV_DbgAssert(box_area1 > 0); + average1 = sum1 / box_area1; + + // Calculate the indices on the integral image where the box falls + A = integralImage.at(box2y1, box2x1); + B = integralImage.at(box2y1, box2x2); + C = integralImage.at(box2y2, box2x1); + D = integralImage.at(box2y2, box2x2); + + // Calculate the mean intensity value of the pixels in the box + sum2 = float(A + D - B - C); + box_area2 = (box2y2 - box2y1) * (box2x2 - box2x1); + CV_DbgAssert(box_area2 > 0); + average2 = sum2 / box_area2; + + return average1 - average2; +} + +// descriptor computation using keypoints +void BEBLID_Impl::compute(InputArray _image, vector &keypoints, OutputArray _descriptors) +{ + Mat image = _image.getMat(); + + if (image.empty()) + return; + + if (keypoints.empty()) + { + // clean output buffer (it may be reused with "allocated" data) + _descriptors.release(); + return; + } + + Mat grayImage; + switch (image.type()) { + case CV_8UC1: + grayImage = image; + break; + case CV_8UC3: + cvtColor(image, grayImage, COLOR_BGR2GRAY); + break; + case CV_8UC4: + cvtColor(image, grayImage, COLOR_BGRA2GRAY); + break; + default: + CV_Error(Error::StsBadArg, "Image should be 8UC1, 8UC3 or 8UC4"); + } + + cv::Mat integralImg; + + // compute the integral image + cv::integral(grayImage, integralImg); + + // Create the output array of descriptors + _descriptors.create((int)keypoints.size(), descriptorSize(), descriptorType()); + + // descriptor storage + cv::Mat descriptors = _descriptors.getMat(); + CV_DbgAssert(descriptors.type() == CV_8UC1); + + // Compute the BEBLID descriptors + computeBEBLID(integralImg, keypoints, descriptors); +} + +// constructor +BEBLID_Impl::BEBLID_Impl(float scale_factor, int n_bits) + : scale_factor_(scale_factor), patch_size_(32, 32) +{ + #include "beblid.p512.hpp" + #include "beblid.p256.hpp" + if (n_bits == SIZE_512_BITS) + wl_params_.assign(wl_params_512, wl_params_512 + sizeof(wl_params_512) / sizeof(wl_params_512[0])); + else if(n_bits == SIZE_256_BITS) + wl_params_.assign(wl_params_256, wl_params_256 + sizeof(wl_params_256) / sizeof(wl_params_256[0])); + else + CV_Error(Error::StsBadArg, "n_wls should be either SIZE_512_BITS or SIZE_256_BITS"); +} + +// Internal function that implements the core of BEBLID descriptor +void BEBLID_Impl::computeBEBLID(const cv::Mat &integralImg, + const std::vector &keypoints, + cv::Mat &descriptors) +{ + CV_DbgAssert(!integralImg.empty()); + CV_DbgAssert(size_t(descriptors.rows) == keypoints.size()); + const int *integralPtr = integralImg.ptr(); + cv::Size frameSize(integralImg.cols - 1, integralImg.rows - 1); + + // Parallel Loop to process descriptors +#ifndef CV_BEBLID_PARALLEL + const cv::Range range(0, keypoints.size()); +#else + cv::parallel_for_(cv::Range(0, int(keypoints.size())), [&](const Range &range) +#endif + { + // Get a pointer to the first element in the range + ABWLParams *wl; + float responseFun; + int areaResponseFun, kpIdx; + size_t wlIdx; + int box1x1, box1y1, box1x2, box1y2, box2x1, box2y1, box2x2, box2y2, bit_idx, side; + uchar byte = 0; + std::vector imgWLParams(wl_params_.size()); + uchar *d = &descriptors.at(range.start, 0); + + for (kpIdx = range.start; kpIdx < range.end; kpIdx++) + { + // Rectify the weak learners coordinates using the keypoint information + rectifyABWL(wl_params_, imgWLParams, keypoints[kpIdx], scale_factor_, patch_size_); + if (isKeypointInTheBorder(keypoints[kpIdx], frameSize, patch_size_, scale_factor_)) + { + // Code to process the keypoints in the image margins + for (wlIdx = 0; wlIdx < wl_params_.size(); wlIdx++) { + bit_idx = 7 - int(wlIdx % 8); + responseFun = computeABWLResponse(imgWLParams[wlIdx], integralImg); + // Set the bit to 1 if the response function is less or equal to the threshod + byte |= (responseFun <= wl_params_[wlIdx].th) << bit_idx; + // If we filled the byte, save it + if (bit_idx == 0) + { + *d = byte; + byte = 0; + d++; + } + } + } + else + { + // Code to process the keypoints in the image center + wl = imgWLParams.data(); + for (wlIdx = 0; wlIdx < wl_params_.size(); wlIdx++) + { + bit_idx = 7 - int(wlIdx % 8); + + // For the first box, we calculate its margin coordinates + box1x1 = wl->x1 - wl->boxRadius; + box1y1 = (wl->y1 - wl->boxRadius) * integralImg.cols; + box1x2 = wl->x1 + wl->boxRadius + 1; + box1y2 = (wl->y1 + wl->boxRadius + 1) * integralImg.cols; + // For the second box, we calculate its margin coordinates + box2x1 = wl->x2 - wl->boxRadius; + box2y1 = (wl->y2 - wl->boxRadius) * integralImg.cols; + box2x2 = wl->x2 + wl->boxRadius + 1; + box2y2 = (wl->y2 + wl->boxRadius + 1) * integralImg.cols; + side = 1 + (wl->boxRadius << 1); + + // Get the difference between the average level of the two boxes + areaResponseFun = (integralPtr[box1y1 + box1x1] // A of Box1 + + integralPtr[box1y2 + box1x2] // D of Box1 + - integralPtr[box1y1 + box1x2] // B of Box1 + - integralPtr[box1y2 + box1x1] // C of Box1 + - integralPtr[box2y1 + box2x1] // A of Box2 + - integralPtr[box2y2 + box2x2] // D of Box2 + + integralPtr[box2y1 + box2x2] // B of Box2 + + integralPtr[box2y2 + box2x1]); // C of Box2 + + // Set the bit to 1 if the response function is less or equal to the threshod + byte |= (areaResponseFun <= (wl_params_[wlIdx].th * (side * side))) << bit_idx; + wl++; + // If we filled the byte, save it + if (bit_idx == 0) + { + *d = byte; + byte = 0; + d++; + } + } // End of for each dimension + } // End of else (of pixels in the image center) + } // End of for each keypoint + } // End of thread scope +#ifdef CV_BEBLID_PARALLEL + ); +#endif +} + +Ptr BEBLID::create(float scale_factor, int n_bits) +{ + return makePtr(scale_factor, n_bits); +} +} // END NAMESPACE XFEATURES2D +} // END NAMESPACE CV diff --git a/modules/xfeatures2d/src/beblid.p256.hpp b/modules/xfeatures2d/src/beblid.p256.hpp new file mode 100644 index 00000000000..15be1ba2f9e --- /dev/null +++ b/modules/xfeatures2d/src/beblid.p256.hpp @@ -0,0 +1,81 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Author: Iago Suarez + +// Implementation of the article: +// Iago Suarez, Ghesn Sfeir, Jose M. Buenaposada, and Luis Baumela. +// BEBLID: Boosted Efficient Binary Local Image Descriptor. +// Pattern Recognition Letters, 133:366–372, 2020. + +// ABWLParams: x1, y1, x2, y2, boxRadius, th + +// Pre-trained parameters of BEBLID-256 trained in Liberty data set with +// a million of patch pairs, 20% positives and 80% negatives +static const ABWLParams wl_params_256[] = { + {26, 20, 14, 16, 5, 16}, {17, 17, 15, 15, 2, 7}, {18, 16, 8, 13, 3, 18}, + {19, 15, 13, 14, 3, 17}, {16, 16, 5, 15, 4, 10}, {25, 10, 16, 16, 6, 11}, + {16, 15, 12, 15, 1, 12}, {18, 17, 14, 17, 1, 13}, {15, 14, 5, 21, 5, 6}, {14, 14, 11, 7, 4, 2}, + {23, 27, 16, 17, 4, 8}, {12, 17, 10, 24, 5, 0}, {15, 15, 13, 14, 1, 6}, {16, 16, 14, 16, 1, 7}, + {19, 18, 16, 15, 1, 6}, {24, 7, 19, 15, 6, 4}, {15, 16, 6, 8, 5, 6}, {24, 16, 8, 15, 7, 22}, + {15, 6, 13, 16, 4, 6}, {17, 19, 15, 15, 1, 6}, {17, 12, 16, 16, 1, 2}, {11, 15, 7, 25, 6, 0}, + {15, 15, 14, 10, 2, 2}, {26, 15, 18, 17, 4, 6}, {18, 12, 17, 27, 4, 3}, {9, 15, 6, 8, 6, 1}, + {15, 17, 14, 23, 3, 1}, {11, 17, 4, 14, 4, 1}, {22, 18, 19, 5, 5, 5}, {11, 18, 11, 5, 5, 3}, + {22, 5, 19, 19, 5, 2}, {12, 26, 6, 15, 3, 5}, {16, 16, 14, 18, 1, 7}, {22, 26, 22, 13, 5, 2}, + {18, 13, 16, 16, 1, 4}, {14, 26, 13, 10, 5, 3}, {17, 13, 14, 14, 1, 10}, {21, 16, 19, 7, 3, 4}, + {14, 15, 14, 13, 1, 0}, {26, 26, 20, 18, 5, 1}, {12, 10, 8, 21, 4, 3}, {14, 17, 13, 7, 3, 0}, + {13, 12, 10, 19, 2, 4}, {17, 20, 17, 13, 2, 0}, {8, 25, 6, 11, 6, 2}, {27, 11, 20, 24, 4, 3}, + {14, 18, 12, 14, 2, 5}, {22, 19, 18, 20, 2, 5}, {18, 4, 17, 14, 3, 1}, {13, 28, 13, 18, 3, 3}, + {15, 12, 14, 17, 1, 4}, {13, 20, 10, 11, 2, 3}, {10, 5, 4, 17, 4, 2}, {7, 18, 3, 18, 3, 2}, + {21, 11, 15, 2, 2, 11}, {20, 15, 17, 17, 1, 6}, {10, 20, 4, 27, 4, 3}, {24, 25, 23, 7, 6, 0}, + {18, 15, 18, 12, 2, 0}, {17, 16, 16, 13, 1, 3}, {14, 20, 14, 15, 1, 1}, {17, 17, 17, 14, 1, 0}, + {7, 15, 6, 5, 5, 3}, {11, 21, 11, 13, 2, 1}, {18, 16, 15, 9, 1, 7}, {19, 19, 18, 15, 1, 2}, + {28, 19, 20, 16, 3, 1}, {14, 16, 11, 10, 1, 3}, {22, 13, 19, 14, 1, 2}, {9, 10, 4, 4, 4, 3}, + {20, 26, 10, 29, 2, 12}, {14, 17, 12, 19, 1, 3}, {21, 18, 18, 24, 2, 6}, {16, 15, 15, 19, 1, 4}, + {27, 4, 24, 15, 4, 2}, {15, 22, 14, 6, 2, 2}, {13, 16, 9, 12, 1, 2}, {12, 12, 11, 18, 1, 2}, + {22, 17, 20, 11, 2, 2}, {18, 28, 17, 23, 3, 1}, {6, 9, 5, 21, 4, 0}, {12, 3, 8, 11, 3, 5}, + {21, 16, 19, 16, 1, 2}, {18, 16, 17, 19, 1, 2}, {27, 12, 22, 3, 3, 2}, {13, 27, 4, 26, 4, 3}, + {5, 22, 3, 26, 3, 2}, {24, 28, 23, 20, 3, 2}, {11, 17, 8, 19, 2, 0}, {13, 16, 11, 16, 1, 3}, + {18, 15, 18, 8, 2, 1}, {15, 17, 14, 14, 1, 3}, {19, 14, 17, 12, 1, 4}, {25, 10, 22, 20, 2, 0}, + {14, 12, 13, 9, 1, 1}, {9, 10, 3, 9, 3, 2}, {20, 22, 19, 17, 1, 0}, {16, 24, 16, 10, 2, 0}, + {15, 23, 13, 29, 2, 2}, {15, 20, 14, 17, 1, 4}, {27, 27, 22, 27, 4, 1}, {14, 7, 6, 3, 3, 3}, + {21, 3, 20, 7, 3, 0}, {29, 5, 25, 11, 2, 1}, {15, 21, 15, 20, 1, 0}, {8, 17, 8, 11, 2, 1}, + {17, 13, 17, 8, 1, 0}, {7, 25, 3, 21, 3, 0}, {7, 11, 7, 8, 3, 1}, {4, 11, 3, 26, 3, 2}, + {15, 18, 15, 11, 1, 1}, {23, 15, 20, 19, 2, 2}, {5, 9, 3, 4, 3, 2}, {28, 18, 25, 8, 3, 0}, + {20, 22, 17, 30, 1, 5}, {29, 29, 28, 16, 2, 1}, {28, 11, 24, 15, 2, 1}, {20, 7, 18, 9, 1, 2}, + {19, 12, 18, 16, 1, 2}, {11, 20, 11, 17, 2, 1}, {13, 16, 13, 13, 1, 0}, {29, 3, 23, 5, 2, 0}, + {19, 21, 17, 18, 1, 3}, {12, 8, 12, 3, 2, 2}, {14, 13, 13, 20, 1, 2}, {11, 21, 9, 29, 2, 3}, + {7, 30, 6, 22, 1, 2}, {11, 9, 10, 15, 1, 3}, {8, 3, 2, 9, 2, 0}, {19, 7, 18, 3, 3, 2}, + {21, 9, 19, 11, 1, 1}, {18, 10, 17, 13, 1, 2}, {6, 17, 1, 30, 1, 6}, {17, 29, 16, 28, 2, 1}, + {17, 20, 17, 18, 1, 0}, {15, 9, 13, 23, 1, 4}, {12, 14, 11, 16, 1, 1}, {7, 17, 5, 14, 2, 1}, + {30, 30, 23, 12, 1, 2}, {29, 18, 26, 20, 2, 0}, {10, 20, 9, 17, 2, 1}, {4, 15, 2, 8, 2, 2}, + {7, 7, 7, 3, 3, 1}, {9, 19, 8, 24, 1, 2}, {28, 25, 27, 25, 3, 0}, {13, 15, 12, 18, 1, 1}, + {25, 2, 19, 5, 2, 2}, {15, 4, 15, 3, 3, 0}, {25, 19, 24, 29, 2, 2}, {18, 24, 18, 20, 1, 1}, + {4, 10, 1, 2, 1, 3}, {5, 18, 1, 18, 1, 2}, {13, 22, 13, 19, 1, 1}, {10, 26, 8, 28, 2, 0}, + {24, 13, 24, 6, 1, 1}, {15, 19, 14, 15, 1, 4}, {5, 8, 2, 16, 2, 0}, {12, 4, 11, 2, 2, 0}, + {14, 29, 14, 24, 1, 1}, {3, 20, 1, 22, 1, 1}, {17, 5, 12, 1, 1, 5}, {21, 16, 20, 23, 1, 2}, + {25, 17, 22, 13, 1, 0}, {6, 21, 5, 16, 1, 0}, {7, 15, 6, 19, 1, 1}, {20, 17, 19, 15, 1, 1}, + {3, 29, 3, 23, 2, 1}, {16, 25, 16, 22, 1, 0}, {28, 20, 28, 12, 3, 0}, {27, 13, 23, 10, 1, 0}, + {24, 24, 17, 29, 1, 5}, {13, 2, 11, 4, 1, 2}, {22, 23, 21, 21, 1, 0}, {19, 30, 19, 24, 1, 1}, + {30, 30, 26, 27, 1, 0}, {17, 5, 17, 1, 1, 0}, {26, 7, 24, 1, 1, 1}, {28, 6, 28, 3, 3, 0}, + {3, 15, 1, 13, 1, 1}, {7, 8, 5, 6, 1, 1}, {19, 16, 19, 15, 1, 0}, {12, 9, 11, 7, 1, 0}, + {17, 22, 16, 20, 1, 2}, {12, 14, 12, 11, 1, 1}, {25, 29, 23, 26, 1, 0}, {15, 19, 15, 18, 1, 0}, + {13, 22, 12, 25, 1, 0}, {1, 22, 1, 11, 1, 0}, {14, 12, 14, 9, 1, 1}, {10, 27, 9, 23, 1, 2}, + {9, 4, 6, 1, 1, 1}, {22, 12, 21, 16, 1, 0}, {5, 27, 1, 28, 1, 1}, {30, 14, 28, 7, 1, 0}, + {17, 9, 16, 21, 1, 2}, {17, 9, 17, 6, 1, 0}, {4, 4, 1, 1, 1, 1}, {30, 2, 28, 5, 1, 0}, + {18, 4, 17, 7, 1, 1}, {15, 13, 15, 10, 1, 1}, {12, 30, 11, 26, 1, 2}, {16, 28, 15, 29, 1, 1}, + {30, 11, 28, 11, 1, 0}, {9, 12, 8, 10, 1, 1}, {22, 19, 21, 16, 1, 0}, {30, 20, 29, 26, 1, 0}, + {22, 10, 20, 7, 1, 2}, {2, 2, 1, 5, 1, 0}, {9, 9, 7, 9, 1, 0}, {27, 1, 25, 3, 1, 0}, + {21, 23, 20, 25, 1, 1}, {10, 3, 8, 5, 1, 1}, {24, 1, 23, 3, 1, 0}, {5, 29, 4, 28, 1, 0}, + {27, 23, 26, 18, 1, 1}, {22, 2, 22, 1, 1, 0}, {7, 20, 6, 19, 1, 0}, {12, 26, 9, 25, 1, 2}, + {7, 1, 5, 2, 1, 0}, {2, 21, 1, 18, 1, 0}, {2, 24, 1, 21, 1, 0}, {8, 17, 8, 14, 1, 0}, + {30, 1, 28, 2, 1, 0}, {15, 30, 15, 28, 1, 0}, {2, 5, 1, 9, 1, 0}, {18, 28, 17, 26, 1, 1}, + {7, 29, 1, 30, 1, 1}, {17, 2, 17, 1, 1, 0}, {21, 13, 21, 9, 1, 1}, {29, 15, 27, 15, 1, 0}, + {28, 8, 27, 7, 2, 0}, {29, 14, 28, 18, 1, 0}, {2, 26, 1, 30, 1, 1}, {16, 8, 16, 6, 1, 0}, + {30, 26, 26, 24, 1, 0}, {15, 17, 15, 16, 6, 0}, {30, 29, 27, 30, 1, 0}, {3, 30, 1, 28, 1, 0}, + {17, 1, 16, 2, 1, 1}, {14, 30, 12, 30, 1, 1}, {12, 17, 12, 16, 1, 0}, {4, 18, 4, 16, 1, 0}, + {11, 4, 11, 1, 1, 1}, {21, 2, 18, 1, 1, 2}, {16, 17, 16, 15, 5, 0}, {3, 1, 2, 2, 1, 0}, + {23, 17, 23, 16, 1, 0}, {18, 12, 18, 11, 1, 0}, {10, 28, 8, 30, 1, 0}, {12, 10, 12, 8, 1, 1}, + {2, 14, 1, 9, 1, 1}, {6, 25, 6, 21, 1, 1}, {6, 2, 2, 1, 1, 1}, {30, 19, 29, 20, 1, 0}, + {25, 21, 23, 20, 1, 0}, {16, 10, 16, 9, 1, 0} +}; diff --git a/modules/xfeatures2d/src/beblid.p512.hpp b/modules/xfeatures2d/src/beblid.p512.hpp new file mode 100644 index 00000000000..496d0bd2024 --- /dev/null +++ b/modules/xfeatures2d/src/beblid.p512.hpp @@ -0,0 +1,146 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Author: Iago Suarez + +// Implementation of the article: +// Iago Suarez, Ghesn Sfeir, Jose M. Buenaposada, and Luis Baumela. +// BEBLID: Boosted Efficient Binary Local Image Descriptor. +// Pattern Recognition Letters, 133:366–372, 2020. + +// ABWLParams: x1, y1, x2, y2, boxRadius, th + +// Pre-trained parameters of BEBLID-512 trained in Liberty data set with +// a million of patch pairs, 20% positives and 80% negatives +static const ABWLParams wl_params_512[] = { + {24, 18, 15, 17, 6, 13}, {19, 14, 13, 17, 2, 18}, {23, 19, 12, 15, 6, 19}, + {24, 14, 16, 16, 6, 11}, {16, 15, 12, 16, 1, 12}, {16, 15, 7, 10, 4, 10}, + {17, 12, 8, 17, 3, 16}, {24, 12, 11, 17, 7, 19}, {19, 17, 14, 11, 3, 13}, + {16, 15, 13, 15, 1, 10}, {16, 14, 6, 18, 5, 10}, {25, 5, 14, 15, 5, 15}, + {17, 18, 14, 16, 2, 10}, {17, 14, 14, 13, 2, 9}, {15, 14, 6, 22, 5, 7}, {14, 16, 5, 17, 5, 5}, + {16, 13, 15, 16, 1, 4}, {18, 17, 15, 15, 1, 9}, {26, 26, 15, 14, 5, 12}, {18, 18, 16, 16, 1, 4}, + {15, 14, 14, 27, 4, 0}, {17, 13, 15, 16, 1, 6}, {15, 15, 13, 14, 1, 6}, {18, 17, 16, 16, 1, 4}, + {14, 13, 6, 7, 5, 4}, {27, 12, 17, 15, 4, 8}, {12, 13, 7, 24, 7, 2}, {17, 18, 15, 15, 1, 6}, + {16, 16, 12, 17, 1, 12}, {27, 20, 16, 16, 4, 11}, {12, 14, 7, 5, 5, 0}, {12, 16, 7, 26, 5, 0}, + {15, 15, 15, 7, 4, -1}, {16, 17, 14, 17, 2, 6}, {16, 13, 10, 6, 4, 7}, {15, 26, 15, 19, 4, 1}, + {26, 5, 17, 13, 5, 7}, {15, 23, 5, 12, 5, 8}, {17, 14, 10, 11, 3, 14}, {21, 27, 17, 16, 4, 5}, + {15, 16, 14, 16, 1, 3}, {14, 11, 12, 26, 5, 1}, {12, 14, 12, 5, 4, -3}, {16, 16, 14, 12, 1, 7}, + {13, 20, 7, 13, 3, 4}, {19, 6, 17, 16, 6, 3}, {11, 9, 10, 19, 4, 2}, {14, 15, 13, 9, 3, 1}, + {16, 16, 14, 25, 3, 3}, {8, 26, 8, 13, 4, 3}, {16, 14, 15, 19, 2, 3}, {18, 15, 15, 16, 1, 9}, + {26, 23, 19, 16, 5, 4}, {11, 21, 4, 13, 4, 1}, {20, 16, 20, 5, 4, 2}, {15, 16, 15, 13, 1, 0}, + {16, 20, 16, 15, 2, 0}, {22, 13, 17, 14, 2, 8}, {18, 17, 14, 15, 1, 13}, {21, 12, 20, 26, 4, 3}, + {10, 7, 8, 18, 5, 3}, {11, 26, 11, 20, 5, 2}, {13, 21, 13, 17, 3, 1}, {10, 23, 6, 7, 6, 1}, + {10, 14, 5, 14, 5, 0}, {23, 25, 16, 6, 6, 8}, {18, 16, 18, 5, 4, 1}, {16, 16, 16, 14, 1, 0}, + {11, 15, 4, 23, 4, -2}, {17, 14, 16, 16, 1, 2}, {26, 4, 20, 24, 4, 2}, {20, 19, 18, 14, 2, 3}, + {14, 17, 10, 15, 2, 6}, {17, 13, 17, 9, 3, 0}, {26, 21, 5, 24, 5, 20}, {20, 15, 19, 25, 5, 3}, + {27, 15, 19, 5, 4, 5}, {10, 14, 10, 6, 6, -2}, {12, 22, 11, 10, 3, 2}, {17, 16, 16, 20, 2, 3}, + {15, 15, 12, 19, 1, 7}, {15, 11, 14, 17, 2, 4}, {14, 20, 10, 15, 2, 7}, {10, 14, 3, 7, 3, -5}, + {12, 16, 9, 11, 3, 1}, {19, 17, 17, 11, 2, 5}, {26, 7, 19, 26, 5, 4}, {20, 10, 19, 18, 3, 1}, + {17, 13, 16, 16, 1, 2}, {17, 11, 16, 4, 4, 2}, {15, 19, 14, 12, 2, 3}, {17, 18, 16, 13, 1, 3}, + {11, 9, 4, 27, 4, -1}, {21, 23, 18, 17, 3, 3}, {7, 21, 6, 7, 5, -1}, {25, 27, 21, 18, 4, -1}, + {14, 17, 14, 14, 2, 0}, {12, 11, 8, 19, 3, 3}, {14, 15, 13, 22, 2, 0}, {8, 23, 5, 17, 5, 1}, + {15, 16, 14, 8, 2, 1}, {16, 24, 15, 18, 3, 3}, {19, 25, 19, 18, 5, -1}, {11, 23, 10, 13, 2, 3}, + {19, 14, 18, 22, 2, 3}, {26, 15, 22, 6, 4, 2}, {24, 17, 19, 8, 3, 5}, {21, 15, 16, 15, 1, 10}, + {15, 14, 14, 20, 1, 2}, {16, 27, 13, 5, 4, 5}, {10, 4, 5, 13, 4, 3}, {12, 14, 10, 10, 2, 0}, + {14, 18, 14, 11, 1, -1}, {23, 6, 22, 20, 5, 0}, {14, 12, 10, 19, 2, 6}, {17, 18, 17, 15, 2, 0}, + {16, 15, 15, 18, 1, 4}, {11, 13, 3, 4, 3, -4}, {15, 14, 15, 8, 2, -1}, {11, 23, 5, 26, 5, 0}, + {20, 20, 19, 17, 2, 1}, {22, 19, 19, 20, 2, 3}, {16, 5, 15, 24, 4, 2}, {18, 15, 16, 12, 1, 5}, + {28, 27, 23, 15, 3, -2}, {7, 25, 6, 18, 6, 2}, {12, 19, 12, 13, 3, 0}, {9, 7, 4, 17, 4, 1}, + {14, 18, 13, 12, 1, 2}, {13, 16, 10, 23, 2, 1}, {24, 25, 23, 13, 6, -1}, {8, 13, 7, 4, 4, -3}, + {17, 15, 17, 11, 2, 0}, {20, 13, 18, 15, 1, 3}, {28, 3, 23, 15, 3, -2}, {13, 17, 12, 11, 1, 0}, + {16, 18, 16, 11, 1, 0}, {26, 16, 24, 26, 5, 2}, {14, 14, 11, 15, 1, 6}, {15, 9, 15, 3, 3, -1}, + {12, 28, 10, 19, 3, 6}, {18, 17, 18, 14, 2, 0}, {16, 14, 14, 15, 1, 7}, {20, 18, 19, 10, 2, 2}, + {27, 28, 18, 24, 3, 4}, {15, 11, 14, 25, 2, 1}, {16, 18, 15, 16, 1, 3}, {5, 27, 4, 6, 4, 0}, + {17, 20, 17, 14, 1, 0}, {13, 15, 9, 14, 1, 3}, {9, 23, 3, 23, 3, -1}, {9, 10, 3, 9, 3, -2}, + {16, 27, 16, 9, 3, 0}, {13, 17, 11, 15, 1, 3}, {14, 18, 14, 15, 1, 0}, {28, 12, 20, 21, 3, 2}, + {23, 7, 4, 27, 4, 16}, {16, 18, 16, 16, 1, -1}, {13, 16, 12, 19, 1, 1}, {20, 11, 19, 18, 2, 1}, + {23, 14, 19, 13, 1, 2}, {23, 10, 19, 3, 3, 5}, {15, 18, 13, 15, 1, 6}, {8, 14, 3, 19, 3, -3}, + {7, 18, 3, 17, 3, -2}, {22, 4, 21, 7, 4, 0}, {3, 28, 3, 18, 3, 2}, {19, 20, 17, 14, 1, 4}, + {16, 22, 15, 6, 2, 2}, {22, 20, 19, 29, 2, 5}, {11, 21, 9, 14, 2, 2}, {7, 9, 6, 4, 4, -2}, + {26, 19, 23, 9, 4, 1}, {16, 17, 16, 12, 2, 0}, {15, 5, 3, 4, 3, 4}, {18, 14, 17, 17, 1, 2}, + {19, 11, 17, 13, 1, 4}, {11, 17, 10, 10, 2, -1}, {15, 23, 12, 29, 2, 3}, + {28, 20, 24, 17, 3, -1}, {13, 10, 11, 2, 2, -1}, {28, 11, 23, 15, 3, -1}, + {16, 21, 16, 20, 2, 0}, {8, 8, 7, 17, 2, 2}, {15, 19, 14, 16, 1, 4}, {17, 11, 17, 10, 2, 0}, + {22, 21, 19, 16, 1, 1}, {13, 17, 13, 14, 1, 0}, {19, 13, 18, 16, 1, 2}, {6, 25, 5, 27, 4, -1}, + {16, 29, 16, 22, 2, 0}, {23, 27, 23, 22, 4, -1}, {29, 2, 22, 10, 2, -1}, {22, 10, 22, 5, 5, 1}, + {20, 16, 19, 15, 1, 1}, {20, 9, 19, 14, 1, 0}, {29, 29, 23, 22, 2, -1}, {12, 11, 10, 18, 1, 3}, + {4, 16, 4, 2, 2, -2}, {14, 8, 13, 2, 2, 0}, {16, 3, 15, 6, 3, 2}, {23, 8, 15, 2, 2, 10}, + {18, 19, 18, 16, 1, 0}, {12, 21, 6, 18, 1, 2}, {18, 15, 16, 19, 1, 5}, {16, 21, 16, 8, 2, 0}, + {18, 26, 17, 23, 2, 1}, {7, 8, 3, 3, 3, -3}, {6, 24, 3, 28, 3, -2}, {10, 19, 9, 26, 2, -3}, + {17, 9, 16, 13, 1, 2}, {13, 15, 13, 10, 1, -2}, {18, 16, 18, 12, 1, 0}, {17, 13, 17, 11, 1, 0}, + {6, 16, 3, 12, 3, -2}, {15, 21, 15, 20, 1, 0}, {23, 17, 20, 15, 2, 1}, {28, 22, 25, 8, 3, 0}, + {5, 16, 3, 25, 3, -3}, {14, 13, 13, 20, 1, 2}, {28, 28, 20, 27, 3, 2}, {15, 29, 8, 25, 2, 7}, + {10, 28, 5, 24, 3, 2}, {19, 14, 18, 13, 1, 2}, {19, 26, 14, 28, 3, 7}, {18, 21, 17, 18, 1, 2}, + {13, 17, 9, 20, 1, 2}, {15, 13, 13, 11, 1, 4}, {27, 7, 25, 15, 4, -1}, {12, 15, 11, 17, 1, 1}, + {13, 20, 12, 15, 1, 3}, {15, 20, 14, 22, 1, 2}, {19, 29, 17, 27, 2, 2}, {19, 3, 18, 5, 3, 1}, + {9, 21, 9, 17, 2, 1}, {19, 18, 17, 18, 1, 4}, {25, 13, 24, 18, 3, 0}, {11, 15, 10, 13, 1, 0}, + {9, 9, 8, 3, 2, -2}, {6, 8, 3, 8, 3, -1}, {28, 19, 23, 28, 3, 2}, {10, 30, 9, 23, 1, 3}, + {5, 5, 3, 18, 3, 1}, {14, 17, 12, 20, 1, 3}, {29, 16, 23, 15, 2, -1}, {23, 15, 21, 22, 2, 2}, + {28, 3, 25, 5, 3, 0}, {12, 20, 11, 17, 1, 2}, {20, 22, 18, 20, 1, 2}, {5, 9, 2, 2, 2, -3}, + {7, 27, 3, 19, 3, 1}, {13, 2, 7, 6, 2, 4}, {18, 29, 17, 25, 2, 1}, {15, 21, 14, 17, 1, 4}, + {13, 29, 12, 26, 2, 2}, {5, 22, 4, 12, 2, 0}, {16, 21, 16, 11, 1, 0}, {16, 23, 16, 10, 1, 0}, + {11, 5, 10, 11, 2, 3}, {15, 10, 14, 21, 1, 3}, {10, 18, 9, 18, 1, 0}, {17, 9, 16, 5, 2, 2}, + {19, 19, 19, 12, 1, 0}, {25, 12, 22, 4, 2, 2}, {6, 18, 1, 20, 1, -3}, {10, 13, 10, 10, 2, -1}, + {25, 16, 22, 16, 1, 0}, {18, 13, 18, 12, 1, 0}, {14, 13, 12, 11, 1, 3}, {10, 27, 1, 29, 1, -1}, + {13, 8, 11, 6, 1, 1}, {24, 24, 21, 28, 3, 2}, {22, 17, 20, 17, 1, 1}, {12, 13, 11, 18, 1, 1}, + {23, 3, 21, 7, 3, 0}, {18, 12, 17, 13, 1, 2}, {7, 28, 7, 25, 3, 1}, {28, 28, 28, 15, 3, -1}, + {17, 7, 17, 2, 2, 0}, {19, 9, 17, 11, 1, 3}, {14, 23, 14, 9, 1, 0}, {7, 22, 7, 19, 2, 1}, + {29, 24, 29, 2, 2, 0}, {28, 15, 25, 11, 3, 0}, {5, 11, 1, 10, 1, -2}, {2, 22, 2, 2, 2, -1}, + {22, 30, 16, 27, 1, 5}, {20, 15, 19, 13, 1, 1}, {23, 19, 22, 14, 2, 0}, {5, 7, 5, 3, 3, -1}, + {19, 20, 18, 18, 1, 1}, {29, 9, 25, 13, 2, -1}, {29, 23, 26, 23, 2, 0}, {9, 13, 8, 8, 1, -2}, + {21, 22, 21, 18, 2, -1}, {29, 12, 28, 20, 2, 0}, {18, 5, 1, 4, 1, 9}, {17, 4, 17, 2, 2, 0}, + {28, 29, 24, 25, 2, 0}, {14, 23, 13, 29, 1, 0}, {13, 5, 13, 1, 1, -1}, {20, 25, 20, 21, 1, -1}, + {6, 5, 2, 11, 2, 0}, {10, 14, 9, 21, 1, -1}, {13, 16, 13, 14, 1, 0}, {19, 17, 18, 14, 1, 2}, + {14, 21, 14, 17, 1, 1}, {20, 10, 18, 12, 1, 2}, {20, 4, 19, 3, 3, 1}, {3, 15, 1, 30, 1, -3}, + {13, 4, 8, 1, 1, 2}, {10, 18, 9, 14, 1, 0}, {6, 15, 1, 12, 1, -3}, {10, 25, 10, 20, 1, 2}, + {14, 11, 14, 7, 1, -1}, {22, 9, 20, 4, 1, 2}, {15, 27, 8, 30, 1, 4}, {10, 5, 10, 2, 2, -1}, + {17, 16, 16, 12, 1, 3}, {15, 18, 15, 10, 1, -1}, {20, 30, 20, 23, 1, -1}, {14, 9, 13, 22, 1, 2}, + {14, 22, 12, 25, 1, 2}, {5, 23, 2, 23, 2, -1}, {10, 16, 9, 16, 1, 0}, {26, 2, 19, 4, 1, 2}, + {3, 23, 2, 13, 2, 0}, {3, 17, 3, 7, 2, -1}, {15, 26, 15, 23, 1, 0}, {22, 14, 22, 8, 1, 1}, + {28, 9, 27, 6, 3, 0}, {26, 22, 25, 28, 3, 1}, {17, 10, 17, 5, 1, 1}, {11, 21, 10, 17, 1, 2}, + {20, 18, 20, 16, 1, 0}, {7, 20, 5, 20, 1, -1}, {17, 24, 17, 8, 1, 0}, {24, 9, 20, 9, 1, 1}, + {4, 13, 1, 16, 1, -1}, {30, 1, 28, 16, 1, -1}, {17, 21, 17, 17, 1, 0}, {19, 4, 11, 2, 1, 9}, + {30, 5, 24, 6, 1, 0}, {22, 19, 22, 12, 1, 0}, {9, 16, 9, 12, 1, -1}, {12, 16, 12, 12, 1, -1}, + {12, 24, 11, 29, 1, -1}, {3, 6, 1, 4, 1, -1}, {23, 29, 20, 27, 2, 1}, {23, 17, 22, 16, 1, 0}, + {30, 20, 26, 22, 1, 0}, {9, 2, 6, 5, 2, 1}, {20, 17, 19, 16, 1, 1}, {18, 26, 17, 30, 1, 1}, + {29, 14, 28, 14, 2, 0}, {20, 13, 19, 14, 1, 1}, {15, 23, 15, 21, 1, 0}, {8, 26, 2, 30, 1, -2}, + {4, 5, 3, 2, 2, -1}, {7, 16, 6, 12, 1, -1}, {29, 9, 23, 2, 2, 1}, {13, 2, 12, 5, 2, 2}, + {20, 18, 19, 21, 1, 2}, {7, 29, 2, 25, 2, 0}, {20, 3, 18, 8, 1, 1}, {14, 14, 14, 11, 1, -1}, + {12, 12, 12, 10, 1, -1}, {17, 27, 15, 30, 1, 2}, {22, 27, 20, 29, 2, 1}, {7, 12, 5, 9, 1, -2}, + {30, 30, 24, 24, 1, 0}, {19, 3, 19, 2, 2, 0}, {13, 19, 12, 18, 1, 2}, {3, 30, 2, 24, 1, 1}, + {9, 14, 7, 19, 1, -1}, {17, 22, 17, 18, 1, 0}, {18, 24, 17, 22, 1, 1}, {2, 18, 1, 23, 1, -1}, + {30, 23, 24, 19, 1, -1}, {11, 10, 11, 5, 1, -2}, {9, 30, 9, 27, 1, 1}, {21, 13, 20, 8, 1, 2}, + {6, 3, 2, 2, 2, -1}, {23, 22, 22, 26, 1, 1}, {12, 26, 11, 25, 1, 1}, {22, 1, 19, 5, 1, 1}, + {4, 24, 1, 25, 1, -1}, {5, 13, 5, 7, 1, -1}, {26, 22, 24, 16, 1, -1}, {27, 8, 27, 3, 2, 0}, + {13, 18, 13, 16, 1, 0}, {19, 15, 18, 17, 1, 2}, {30, 29, 26, 28, 1, 0}, {20, 15, 20, 14, 1, 0}, + {3, 18, 1, 15, 1, -1}, {18, 11, 17, 10, 1, 2}, {4, 18, 4, 16, 1, 0}, {8, 27, 5, 30, 1, -1}, + {30, 15, 28, 22, 1, 0}, {9, 19, 8, 22, 1, -1}, {30, 4, 29, 4, 1, 0}, {17, 10, 17, 8, 1, 0}, + {22, 6, 22, 1, 1, 1}, {2, 11, 1, 15, 1, 0}, {3, 16, 1, 17, 1, -1}, {9, 3, 8, 2, 2, 0}, + {3, 11, 1, 10, 1, -1}, {16, 29, 15, 28, 1, 1}, {15, 20, 15, 19, 1, 0}, {20, 17, 19, 17, 1, 1}, + {10, 3, 9, 8, 1, 2}, {10, 22, 7, 26, 1, -1}, {8, 16, 6, 16, 1, -1}, {16, 28, 16, 25, 1, 0}, + {12, 25, 10, 21, 1, 3}, {8, 9, 7, 7, 1, -1}, {3, 1, 1, 6, 1, 0}, {16, 7, 15, 9, 1, 2}, + {30, 23, 29, 23, 1, 0}, {22, 24, 21, 29, 1, 1}, {15, 1, 14, 3, 1, 1}, {18, 6, 17, 9, 1, 1}, + {26, 25, 25, 19, 1, -1}, {25, 13, 22, 18, 1, 0}, {11, 1, 10, 3, 1, 1}, {29, 28, 28, 30, 1, 0}, + {16, 17, 16, 13, 5, 0}, {28, 18, 28, 12, 2, 0}, {3, 22, 1, 23, 1, -1}, {10, 11, 10, 9, 1, -1}, + {7, 13, 6, 20, 1, -1}, {1, 15, 1, 6, 1, -1}, {16, 12, 16, 11, 1, 0}, {3, 26, 2, 30, 1, -1}, + {28, 30, 26, 23, 1, -1}, {17, 22, 16, 25, 1, 2}, {30, 13, 26, 7, 1, 0}, {10, 8, 7, 10, 1, 1}, + {2, 27, 1, 22, 1, 0}, {30, 7, 27, 8, 1, 0}, {22, 19, 21, 22, 1, 1}, {5, 19, 4, 21, 1, -1}, + {24, 6, 23, 11, 1, -1}, {24, 17, 23, 14, 1, 0}, {30, 7, 28, 1, 1, 0}, {11, 16, 11, 15, 1, 0}, + {29, 2, 26, 4, 1, 0}, {20, 4, 18, 1, 1, 2}, {18, 2, 17, 3, 1, 1}, {20, 30, 18, 29, 1, 1}, + {29, 15, 29, 9, 2, 0}, {14, 8, 14, 5, 1, -1}, {17, 15, 16, 18, 1, 3}, {12, 4, 11, 2, 2, 0}, + {23, 8, 21, 11, 1, 0}, {8, 30, 7, 24, 1, 2}, {2, 20, 1, 16, 1, 0}, {15, 26, 14, 29, 1, 1}, + {4, 30, 3, 29, 1, 0}, {19, 17, 19, 16, 1, 0}, {13, 17, 13, 15, 1, 0}, {2, 9, 1, 1, 1, -1}, + {30, 28, 27, 27, 1, 0}, {27, 4, 26, 1, 1, 0}, {19, 23, 19, 20, 1, -1}, {15, 24, 15, 23, 1, 0}, + {2, 29, 1, 28, 1, 0}, {2, 5, 1, 6, 1, 0}, {24, 29, 23, 26, 1, 0}, {13, 12, 12, 11, 1, 1}, + {12, 17, 12, 15, 1, 0}, {24, 26, 24, 22, 1, -1}, {11, 3, 10, 5, 1, 1}, {30, 2, 30, 1, 1, 0}, + {18, 30, 18, 29, 1, 0}, {30, 25, 29, 29, 1, 0}, {12, 30, 10, 28, 1, 1}, {24, 12, 22, 14, 1, 0}, + {6, 13, 4, 15, 1, -1}, {2, 26, 2, 23, 1, 0}, {8, 9, 7, 13, 1, 1}, {30, 1, 27, 1, 1, 0}, + {26, 29, 24, 30, 1, 0}, {18, 11, 18, 10, 1, 0}, {30, 19, 29, 17, 1, 0}, {20, 27, 19, 24, 1, 0}, + {28, 20, 26, 24, 1, 0}, {25, 9, 24, 9, 1, 0}, {27, 4, 24, 6, 1, 0}, {23, 21, 22, 19, 1, 0}, + {7, 13, 7, 10, 1, -1}, {12, 11, 11, 11, 1, 1}, {28, 26, 26, 26, 1, 0}, {8, 4, 6, 4, 1, 0}, + {15, 30, 15, 28, 1, 0}, {30, 14, 28, 14, 1, 0}, {17, 7, 17, 5, 1, 0}, {29, 10, 28, 6, 1, 0}, + {12, 17, 11, 17, 1, 1}, {16, 3, 16, 1, 1, 0}, {21, 3, 19, 3, 1, 1}, {12, 30, 11, 28, 1, 1}, + {18, 16, 18, 15, 1, 0}, {8, 18, 7, 20, 1, -1}, {5, 4, 1, 1, 1, -1}, {3, 27, 1, 30, 1, -1}, + {26, 4, 26, 1, 1, 0}, {5, 21, 2, 20, 1, -1}, {14, 1, 13, 3, 1, 1}, {30, 9, 28, 8, 1, 0}, + {13, 15, 12, 12, 1, 1}, {7, 23, 6, 25, 1, -1} +}; diff --git a/modules/xfeatures2d/test/test_features2d.cpp b/modules/xfeatures2d/test/test_features2d.cpp index e79e50370bb..6cf3bc09907 100644 --- a/modules/xfeatures2d/test/test_features2d.cpp +++ b/modules/xfeatures2d/test/test_features2d.cpp @@ -185,6 +185,13 @@ TEST( Features2d_DescriptorExtractor_LATCH, regression ) test.safe_run(); } +TEST(Features2d_DescriptorExtractor_BEBLID, regression ) +{ + CV_DescriptorExtractorTest test("descriptor-beblid", 1, + BEBLID::create(6.75)); + test.safe_run(); +} + TEST( Features2d_DescriptorExtractor_VGG, regression ) { CV_DescriptorExtractorTest > test( "descriptor-vgg", 0.03f, diff --git a/modules/xfeatures2d/test/test_rotation_and_scale_invariance.cpp b/modules/xfeatures2d/test/test_rotation_and_scale_invariance.cpp index df232a09a54..538fac526db 100644 --- a/modules/xfeatures2d/test/test_rotation_and_scale_invariance.cpp +++ b/modules/xfeatures2d/test/test_rotation_and_scale_invariance.cpp @@ -30,6 +30,10 @@ INSTANTIATE_TEST_CASE_P(LATCH, DescriptorRotationInvariance, Values( make_tuple(IMAGE_TSUKUBA, SIFT::create(), LATCH::create(), 0.98f) )); +INSTANTIATE_TEST_CASE_P(BEBLID, DescriptorRotationInvariance, Values( + make_tuple(IMAGE_TSUKUBA, SIFT::create(), BEBLID::create(6.75), 0.98f) +)); + INSTANTIATE_TEST_CASE_P(DAISY, DescriptorRotationInvariance, Values( make_tuple(IMAGE_TSUKUBA, BRISK::create(), From 4d9e025fca3681b911b25ea87b9b9be6cfb70cc9 Mon Sep 17 00:00:00 2001 From: Matthias Berberich Date: Thu, 10 Dec 2020 15:50:14 +0100 Subject: [PATCH 08/70] [aruco] do not copy image if already a grey-image Images are InputArrays and need not beeing copied here. --- modules/aruco/src/charuco.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/aruco/src/charuco.cpp b/modules/aruco/src/charuco.cpp index 61a1a81a247..691c7febb4d 100644 --- a/modules/aruco/src/charuco.cpp +++ b/modules/aruco/src/charuco.cpp @@ -348,7 +348,7 @@ static int _selectAndRefineChessboardCorners(InputArray _allCorners, InputArray if(_image.type() == CV_8UC3) cvtColor(_image, grey, COLOR_BGR2GRAY); else - _image.copyTo(grey); + grey = _image.getMat(); const Ptr params = DetectorParameters::create(); // use default params for corner refinement @@ -766,7 +766,7 @@ void detectCharucoDiamond(InputArray _image, InputArrayOfArrays _markerCorners, if(_image.type() == CV_8UC3) cvtColor(_image, grey, COLOR_BGR2GRAY); else - _image.copyTo(grey); + grey = _image.getMat(); // for each of the detected markers, try to find a diamond for(unsigned int i = 0; i < _markerIds.total(); i++) { From f10f4a64f70b0b1ef5b48a87e8dcdf96c91d429b Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Wed, 16 Dec 2020 16:19:55 +0100 Subject: [PATCH 09/70] Optimize calls to std::string::find() and friends for a single char. The character literal overload is more efficient. More info at: http://clang.llvm.org/extra/clang-tidy/checks/performance-faster-string-find.html --- modules/tracking/src/trackerFeature.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/tracking/src/trackerFeature.cpp b/modules/tracking/src/trackerFeature.cpp index a5b59bdcfa1..9610fe52836 100644 --- a/modules/tracking/src/trackerFeature.cpp +++ b/modules/tracking/src/trackerFeature.cpp @@ -65,8 +65,8 @@ Ptr TrackerFeature::create( const String& trackerFeatureType ) { if( trackerFeatureType.find( "FEATURE2D" ) == 0 ) { - size_t firstSep = trackerFeatureType.find_first_of( "." ); - size_t secondSep = trackerFeatureType.find_last_of( "." ); + size_t firstSep = trackerFeatureType.find_first_of('.'); + size_t secondSep = trackerFeatureType.find_last_of('.'); String detector = trackerFeatureType.substr( firstSep, secondSep - firstSep ); String descriptor = trackerFeatureType.substr( secondSep, trackerFeatureType.length() - secondSep ); From f01cc380d9617efe728b16cc2dd1cd4e20e9f1b1 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Sun, 20 Dec 2020 02:50:22 +0000 Subject: [PATCH 10/70] fix(aruco): add check to avoid infinite loop --- modules/aruco/src/aruco.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/aruco/src/aruco.cpp b/modules/aruco/src/aruco.cpp index 41b137d3c5b..0667e337555 100644 --- a/modules/aruco/src/aruco.cpp +++ b/modules/aruco/src/aruco.cpp @@ -914,6 +914,7 @@ static void _refineCandidateLines(std::vector& nContours, std::vector Date: Sat, 26 Dec 2020 14:54:03 +0100 Subject: [PATCH 11/70] ovis: implement real-time shadows --- modules/ovis/include/opencv2/ovis.hpp | 7 +++++-- modules/ovis/src/ovis.cpp | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/modules/ovis/include/opencv2/ovis.hpp b/modules/ovis/include/opencv2/ovis.hpp index c471b09e6a9..699e56e3689 100644 --- a/modules/ovis/include/opencv2/ovis.hpp +++ b/modules/ovis/include/opencv2/ovis.hpp @@ -52,7 +52,9 @@ enum SceneSettings /// Apply anti-aliasing. The first window determines the setting for all windows. SCENE_AA = 8, /// Render off-screen without a window. Allows separate AA setting. Requires manual update via @ref WindowScene::update - SCENE_OFFSCREEN = 16 + SCENE_OFFSCREEN = 16, + /// Enable real-time shadows in the scene. All entities cast shadows by default. Control via @ref ENTITY_CAST_SHADOWS + SCENE_SHADOWS = 32 }; enum MaterialProperty @@ -74,7 +76,8 @@ enum EntityProperty ENTITY_MATERIAL, ENTITY_SCALE, ENTITY_AABB_WORLD, - ENTITY_ANIMBLEND_MODE + ENTITY_ANIMBLEND_MODE, + ENTITY_CAST_SHADOWS }; /** diff --git a/modules/ovis/src/ovis.cpp b/modules/ovis/src/ovis.cpp index 1ad8bad2c63..2f9f69a6f73 100644 --- a/modules/ovis/src/ovis.cpp +++ b/modules/ovis/src/ovis.cpp @@ -187,6 +187,7 @@ struct Application : public OgreBites::ApplicationContext, public OgreBites::Inp uint32_t h; int key_pressed; int flags; + Ogre::MaterialPtr casterMat; Application(const Ogre::String& _title, const Size& sz, int _flags) : OgreBites::ApplicationContext("ovis"), mainWin(NULL), title(_title), w(sz.width), @@ -287,6 +288,9 @@ struct Application : public OgreBites::ApplicationContext, public OgreBites::Inp MaterialManager& matMgr = MaterialManager::getSingleton(); matMgr.setDefaultTextureFiltering(TFO_ANISOTROPIC); matMgr.setDefaultAnisotropy(16); + casterMat = matMgr.create("DepthCaster", Ogre::RGN_INTERNAL); + casterMat->setLightingEnabled(false); + casterMat->setDepthBias(-1, -1); } }; @@ -318,6 +322,21 @@ class WindowSceneImpl : public WindowScene RTShader::ShaderGenerator& shadergen = RTShader::ShaderGenerator::getSingleton(); shadergen.addSceneManager(sceneMgr); // must be done before we do anything with the scene + if (flags & SCENE_SHADOWS) + { + sceneMgr->setShadowTechnique(SHADOWTYPE_TEXTURE_MODULATIVE_INTEGRATED); + sceneMgr->setShadowTexturePixelFormat(PF_DEPTH32); + // arbitrary heuristic for shadowmap size + sceneMgr->setShadowTextureSize(std::max(sz.width, sz.height) * 2); + sceneMgr->setShadowCameraSetup(FocusedShadowCameraSetup::create()); + sceneMgr->setShadowTextureCasterMaterial(app->casterMat); + + // inject shadowmap into materials + const auto& schemeName = RTShader::ShaderGenerator::DEFAULT_SCHEME_NAME; + auto rs = shadergen.getRenderState(schemeName); + rs->addTemplateSubRenderState(shadergen.createSubRenderState("SGX_IntegratedPSSM3")); + } + sceneMgr->setAmbientLight(ColourValue(.1, .1, .1)); _createBackground(); } @@ -716,6 +735,13 @@ class WindowSceneImpl : public WindowScene SceneNode& node = _getSceneNode(sceneMgr, name); switch(prop) { + case ENTITY_CAST_SHADOWS: + { + Entity* ent = dynamic_cast(node.getAttachedObject(name)); + CV_Assert(ent && "invalid entity"); + ent->setCastShadows(bool(value[0])); + break; + } case ENTITY_SCALE: { node.setScale(value[0], value[1], value[2]); From 33ae078b0989b44ac8d262d210335b04bb268b4d Mon Sep 17 00:00:00 2001 From: Atlas42 Date: Thu, 31 Dec 2020 11:29:58 +0100 Subject: [PATCH 12/70] Merge pull request #2801 from Atlas42:cuda-hough-stream-fix Added stream support on hough circles, lines and segments * Added stream support on hough circles lines and segments - Passed the stream to the different cuda, OpenCV and thurst library calls - Replace all device by cuda synchronizes - Added extra synchronize calls after device to host transfers - Replaced the cuda globals by allocated values * Fixed missing include for CUDA 8 Co-authored-by: william.fink --- .../cudaimgproc/src/cuda/build_point_list.cu | 20 +++----- modules/cudaimgproc/src/cuda/hough_circles.cu | 46 ++++++++----------- modules/cudaimgproc/src/cuda/hough_lines.cu | 33 ++++++------- .../cudaimgproc/src/cuda/hough_segments.cu | 27 +++++------ modules/cudaimgproc/src/hough_circles.cpp | 45 +++++++++++------- modules/cudaimgproc/src/hough_lines.cpp | 39 ++++++++++------ modules/cudaimgproc/src/hough_segments.cpp | 36 ++++++++++----- 7 files changed, 127 insertions(+), 119 deletions(-) diff --git a/modules/cudaimgproc/src/cuda/build_point_list.cu b/modules/cudaimgproc/src/cuda/build_point_list.cu index addcabc2442..c30214d5d52 100644 --- a/modules/cudaimgproc/src/cuda/build_point_list.cu +++ b/modules/cudaimgproc/src/cuda/build_point_list.cu @@ -49,10 +49,8 @@ namespace cv { namespace cuda { namespace device { namespace hough { - __device__ int g_counter; - template - __global__ void buildPointList(const PtrStepSzb src, unsigned int* list) + __global__ void buildPointList(const PtrStepSzb src, unsigned int* list, int* counterPtr) { __shared__ unsigned int s_queues[4][32 * PIXELS_PER_THREAD]; __shared__ int s_qsize[4]; @@ -94,7 +92,7 @@ namespace cv { namespace cuda { namespace device } // calculate the offset in the global list - const int globalOffset = atomicAdd(&g_counter, totalSize); + const int globalOffset = atomicAdd(counterPtr, totalSize); for (int i = 0; i < blockDim.y; ++i) s_globStart[i] += globalOffset; } @@ -108,27 +106,23 @@ namespace cv { namespace cuda { namespace device list[gidx] = s_queues[threadIdx.y][i]; } - int buildPointList_gpu(PtrStepSzb src, unsigned int* list) + int buildPointList_gpu(PtrStepSzb src, unsigned int* list, int* counterPtr, cudaStream_t stream) { const int PIXELS_PER_THREAD = 16; - void* counterPtr; - cudaSafeCall( cudaGetSymbolAddress(&counterPtr, g_counter) ); - - cudaSafeCall( cudaMemset(counterPtr, 0, sizeof(int)) ); + cudaSafeCall( cudaMemsetAsync(counterPtr, 0, sizeof(int), stream) ); const dim3 block(32, 4); const dim3 grid(divUp(src.cols, block.x * PIXELS_PER_THREAD), divUp(src.rows, block.y)); cudaSafeCall( cudaFuncSetCacheConfig(buildPointList, cudaFuncCachePreferShared) ); - buildPointList<<>>(src, list); + buildPointList<<>>(src, list, counterPtr); cudaSafeCall( cudaGetLastError() ); - cudaSafeCall( cudaDeviceSynchronize() ); - int totalCount; - cudaSafeCall( cudaMemcpy(&totalCount, counterPtr, sizeof(int), cudaMemcpyDeviceToHost) ); + cudaSafeCall( cudaMemcpyAsync(&totalCount, counterPtr, sizeof(int), cudaMemcpyDeviceToHost, stream) ); + cudaSafeCall( cudaStreamSynchronize(stream) ); return totalCount; } diff --git a/modules/cudaimgproc/src/cuda/hough_circles.cu b/modules/cudaimgproc/src/cuda/hough_circles.cu index db1623eceb3..fcbf8c4122b 100644 --- a/modules/cudaimgproc/src/cuda/hough_circles.cu +++ b/modules/cudaimgproc/src/cuda/hough_circles.cu @@ -54,8 +54,6 @@ namespace cv { namespace cuda { namespace device { namespace hough_circles { - __device__ int g_counter; - //////////////////////////////////////////////////////////////////////// // circlesAccumCenters @@ -111,23 +109,22 @@ namespace cv { namespace cuda { namespace device } } - void circlesAccumCenters_gpu(const unsigned int* list, int count, PtrStepi dx, PtrStepi dy, PtrStepSzi accum, int minRadius, int maxRadius, float idp) + void circlesAccumCenters_gpu(const unsigned int* list, int count, PtrStepi dx, PtrStepi dy, PtrStepSzi accum, int minRadius, int maxRadius, float idp, cudaStream_t stream) { const dim3 block(256); const dim3 grid(divUp(count, block.x)); cudaSafeCall( cudaFuncSetCacheConfig(circlesAccumCenters, cudaFuncCachePreferL1) ); - circlesAccumCenters<<>>(list, count, dx, dy, accum, accum.cols - 2, accum.rows - 2, minRadius, maxRadius, idp); + circlesAccumCenters<<>>(list, count, dx, dy, accum, accum.cols - 2, accum.rows - 2, minRadius, maxRadius, idp); cudaSafeCall( cudaGetLastError() ); - cudaSafeCall( cudaDeviceSynchronize() ); + cudaSafeCall( cudaStreamSynchronize(stream) ); } //////////////////////////////////////////////////////////////////////// // buildCentersList - - __global__ void buildCentersList(const PtrStepSzi accum, unsigned int* centers, const int threshold) + __global__ void buildCentersList(const PtrStepSzi accum, unsigned int* centers, const int threshold, int* counterPtr) { const int x = blockIdx.x * blockDim.x + threadIdx.x; const int y = blockIdx.y * blockDim.y + threadIdx.y; @@ -145,31 +142,27 @@ namespace cv { namespace cuda { namespace device if (cur > threshold && cur > top && cur >= bottom && cur > left && cur >= right) { const unsigned int val = (y << 16) | x; - const int idx = ::atomicAdd(&g_counter, 1); + const int idx = ::atomicAdd(counterPtr, 1); centers[idx] = val; } } } - int buildCentersList_gpu(PtrStepSzi accum, unsigned int* centers, int threshold) + int buildCentersList_gpu(PtrStepSzi accum, unsigned int* centers, int threshold, int* counterPtr, cudaStream_t stream) { - void* counterPtr; - cudaSafeCall( cudaGetSymbolAddress(&counterPtr, g_counter) ); - - cudaSafeCall( cudaMemset(counterPtr, 0, sizeof(int)) ); + cudaSafeCall( cudaMemsetAsync(counterPtr, 0, sizeof(int), stream) ); const dim3 block(32, 8); const dim3 grid(divUp(accum.cols - 2, block.x), divUp(accum.rows - 2, block.y)); cudaSafeCall( cudaFuncSetCacheConfig(buildCentersList, cudaFuncCachePreferL1) ); - buildCentersList<<>>(accum, centers, threshold); + buildCentersList<<>>(accum, centers, threshold, counterPtr); cudaSafeCall( cudaGetLastError() ); - cudaSafeCall( cudaDeviceSynchronize() ); - int totalCount; - cudaSafeCall( cudaMemcpy(&totalCount, counterPtr, sizeof(int), cudaMemcpyDeviceToHost) ); + cudaSafeCall( cudaMemcpyAsync(&totalCount, counterPtr, sizeof(int), cudaMemcpyDeviceToHost, stream) ); + cudaSafeCall( cudaStreamSynchronize(stream) ); return totalCount; } @@ -179,7 +172,8 @@ namespace cv { namespace cuda { namespace device __global__ void circlesAccumRadius(const unsigned int* centers, const unsigned int* list, const int count, float3* circles, const int maxCircles, const float dp, - const int minRadius, const int maxRadius, const int histSize, const int threshold) + const int minRadius, const int maxRadius, const int histSize, const int threshold, + int* counterPtr) { int* smem = DynamicSharedMem(); @@ -219,7 +213,7 @@ namespace cv { namespace cuda { namespace device if (curVotes >= threshold && curVotes > smem[i] && curVotes >= smem[i + 2]) { - const int ind = ::atomicAdd(&g_counter, 1); + const int ind = ::atomicAdd(counterPtr, 1); if (ind < maxCircles) circles[ind] = make_float3(cx, cy, i + minRadius); } @@ -227,12 +221,9 @@ namespace cv { namespace cuda { namespace device } int circlesAccumRadius_gpu(const unsigned int* centers, int centersCount, const unsigned int* list, int count, - float3* circles, int maxCircles, float dp, int minRadius, int maxRadius, int threshold, bool has20) + float3* circles, int maxCircles, float dp, int minRadius, int maxRadius, int threshold, bool has20, int* counterPtr, cudaStream_t stream) { - void* counterPtr; - cudaSafeCall( cudaGetSymbolAddress(&counterPtr, g_counter) ); - - cudaSafeCall( cudaMemset(counterPtr, 0, sizeof(int)) ); + cudaSafeCall( cudaMemsetAsync(counterPtr, 0, sizeof(int), stream) ); const dim3 block(has20 ? 1024 : 512); const dim3 grid(centersCount); @@ -240,13 +231,12 @@ namespace cv { namespace cuda { namespace device const int histSize = maxRadius - minRadius + 1; size_t smemSize = (histSize + 2) * sizeof(int); - circlesAccumRadius<<>>(centers, list, count, circles, maxCircles, dp, minRadius, maxRadius, histSize, threshold); + circlesAccumRadius<<>>(centers, list, count, circles, maxCircles, dp, minRadius, maxRadius, histSize, threshold, counterPtr); cudaSafeCall( cudaGetLastError() ); - cudaSafeCall( cudaDeviceSynchronize() ); - int totalCount; - cudaSafeCall( cudaMemcpy(&totalCount, counterPtr, sizeof(int), cudaMemcpyDeviceToHost) ); + cudaSafeCall( cudaMemcpyAsync(&totalCount, counterPtr, sizeof(int), cudaMemcpyDeviceToHost, stream) ); + cudaSafeCall( cudaStreamSynchronize(stream) ); totalCount = ::min(totalCount, maxCircles); diff --git a/modules/cudaimgproc/src/cuda/hough_lines.cu b/modules/cudaimgproc/src/cuda/hough_lines.cu index 9a93cbf147a..6fa55791107 100644 --- a/modules/cudaimgproc/src/cuda/hough_lines.cu +++ b/modules/cudaimgproc/src/cuda/hough_lines.cu @@ -44,6 +44,7 @@ #include #include +#include #include "opencv2/core/cuda/common.hpp" #include "opencv2/core/cuda/emulation.hpp" @@ -53,8 +54,6 @@ namespace cv { namespace cuda { namespace device { namespace hough_lines { - __device__ int g_counter; - //////////////////////////////////////////////////////////////////////// // linesAccum @@ -126,7 +125,7 @@ namespace cv { namespace cuda { namespace device accumRow[i] = smem[i]; } - void linesAccum_gpu(const unsigned int* list, int count, PtrStepSzi accum, float rho, float theta, size_t sharedMemPerBlock, bool has20) + void linesAccum_gpu(const unsigned int* list, int count, PtrStepSzi accum, float rho, float theta, size_t sharedMemPerBlock, bool has20, cudaStream_t stream) { const dim3 block(has20 ? 1024 : 512); const dim3 grid(accum.rows - 2); @@ -134,19 +133,18 @@ namespace cv { namespace cuda { namespace device size_t smemSize = (accum.cols - 1) * sizeof(int); if (smemSize < sharedMemPerBlock - 1000) - linesAccumShared<<>>(list, count, accum, 1.0f / rho, theta, accum.cols - 2); + linesAccumShared<<>>(list, count, accum, 1.0f / rho, theta, accum.cols - 2); else - linesAccumGlobal<<>>(list, count, accum, 1.0f / rho, theta, accum.cols - 2); + linesAccumGlobal<<>>(list, count, accum, 1.0f / rho, theta, accum.cols - 2); cudaSafeCall( cudaGetLastError() ); - - cudaSafeCall( cudaDeviceSynchronize() ); + cudaSafeCall( cudaStreamSynchronize(stream) ); } //////////////////////////////////////////////////////////////////////// // linesGetResult - __global__ void linesGetResult(const PtrStepSzi accum, float2* out, int* votes, const int maxSize, const float rho, const float theta, const int threshold, const int numrho) + __global__ void linesGetResult(const PtrStepSzi accum, float2* out, int* votes, const int maxSize, const float rho, const float theta, const int threshold, const int numrho, int* counterPtr) { const int r = blockIdx.x * blockDim.x + threadIdx.x; const int n = blockIdx.y * blockDim.y + threadIdx.y; @@ -165,7 +163,7 @@ namespace cv { namespace cuda { namespace device const float radius = (r - (numrho - 1) * 0.5f) * rho; const float angle = n * theta; - const int ind = ::atomicAdd(&g_counter, 1); + const int ind = ::atomicAdd(counterPtr, 1); if (ind < maxSize) { out[ind] = make_float2(radius, angle); @@ -174,25 +172,22 @@ namespace cv { namespace cuda { namespace device } } - int linesGetResult_gpu(PtrStepSzi accum, float2* out, int* votes, int maxSize, float rho, float theta, int threshold, bool doSort) + int linesGetResult_gpu(PtrStepSzi accum, float2* out, int* votes, int maxSize, float rho, float theta, int threshold, bool doSort, int* counterPtr, cudaStream_t stream) { - void* counterPtr; - cudaSafeCall( cudaGetSymbolAddress(&counterPtr, g_counter) ); - - cudaSafeCall( cudaMemset(counterPtr, 0, sizeof(int)) ); + cudaSafeCall( cudaMemsetAsync(counterPtr, 0, sizeof(int), stream) ); const dim3 block(32, 8); const dim3 grid(divUp(accum.cols - 2, block.x), divUp(accum.rows - 2, block.y)); cudaSafeCall( cudaFuncSetCacheConfig(linesGetResult, cudaFuncCachePreferL1) ); - linesGetResult<<>>(accum, out, votes, maxSize, rho, theta, threshold, accum.cols - 2); + linesGetResult<<>>(accum, out, votes, maxSize, rho, theta, threshold, accum.cols - 2, counterPtr); cudaSafeCall( cudaGetLastError() ); - cudaSafeCall( cudaDeviceSynchronize() ); - int totalCount; - cudaSafeCall( cudaMemcpy(&totalCount, counterPtr, sizeof(int), cudaMemcpyDeviceToHost) ); + cudaSafeCall( cudaMemcpyAsync(&totalCount, counterPtr, sizeof(int), cudaMemcpyDeviceToHost, stream) ); + + cudaSafeCall( cudaStreamSynchronize(stream) ); totalCount = ::min(totalCount, maxSize); @@ -200,7 +195,7 @@ namespace cv { namespace cuda { namespace device { thrust::device_ptr outPtr(out); thrust::device_ptr votesPtr(votes); - thrust::sort_by_key(votesPtr, votesPtr + totalCount, outPtr, thrust::greater()); + thrust::sort_by_key(thrust::cuda::par.on(stream), votesPtr, votesPtr + totalCount, outPtr, thrust::greater()); } return totalCount; diff --git a/modules/cudaimgproc/src/cuda/hough_segments.cu b/modules/cudaimgproc/src/cuda/hough_segments.cu index ca433d30db3..988f14c1d13 100644 --- a/modules/cudaimgproc/src/cuda/hough_segments.cu +++ b/modules/cudaimgproc/src/cuda/hough_segments.cu @@ -49,15 +49,14 @@ namespace cv { namespace cuda { namespace device { namespace hough_segments { - __device__ int g_counter; - texture tex_mask(false, cudaFilterModePoint, cudaAddressModeClamp); __global__ void houghLinesProbabilistic(const PtrStepSzi accum, int4* out, const int maxSize, const float rho, const float theta, const int lineGap, const int lineLength, - const int rows, const int cols) + const int rows, const int cols, + int* counterPtr) { const int r = blockIdx.x * blockDim.x + threadIdx.x; const int n = blockIdx.y * blockDim.y + threadIdx.y; @@ -182,7 +181,7 @@ namespace cv { namespace cuda { namespace device if (good_line) { - const int ind = ::atomicAdd(&g_counter, 1); + const int ind = ::atomicAdd(counterPtr, 1); if (ind < maxSize) out[ind] = make_int4(line_end[0].x, line_end[0].y, line_end[1].x, line_end[1].y); } @@ -202,7 +201,7 @@ namespace cv { namespace cuda { namespace device if (good_line) { - const int ind = ::atomicAdd(&g_counter, 1); + const int ind = ::atomicAdd(counterPtr, 1); if (ind < maxSize) out[ind] = make_int4(line_end[0].x, line_end[0].y, line_end[1].x, line_end[1].y); } @@ -214,29 +213,27 @@ namespace cv { namespace cuda { namespace device } } - int houghLinesProbabilistic_gpu(PtrStepSzb mask, PtrStepSzi accum, int4* out, int maxSize, float rho, float theta, int lineGap, int lineLength) + int houghLinesProbabilistic_gpu(PtrStepSzb mask, PtrStepSzi accum, int4* out, int maxSize, float rho, float theta, int lineGap, int lineLength, int* counterPtr, cudaStream_t stream) { - void* counterPtr; - cudaSafeCall( cudaGetSymbolAddress(&counterPtr, g_counter) ); - - cudaSafeCall( cudaMemset(counterPtr, 0, sizeof(int)) ); + cudaSafeCall( cudaMemsetAsync(counterPtr, 0, sizeof(int), stream) ); const dim3 block(32, 8); const dim3 grid(divUp(accum.cols - 2, block.x), divUp(accum.rows - 2, block.y)); bindTexture(&tex_mask, mask); - houghLinesProbabilistic<<>>(accum, + houghLinesProbabilistic<<>>(accum, out, maxSize, rho, theta, lineGap, lineLength, - mask.rows, mask.cols); + mask.rows, mask.cols, + counterPtr); cudaSafeCall( cudaGetLastError() ); - cudaSafeCall( cudaDeviceSynchronize() ); - int totalCount; - cudaSafeCall( cudaMemcpy(&totalCount, counterPtr, sizeof(int), cudaMemcpyDeviceToHost) ); + cudaSafeCall( cudaMemcpyAsync(&totalCount, counterPtr, sizeof(int), cudaMemcpyDeviceToHost, stream) ); + + cudaSafeCall( cudaStreamSynchronize(stream) ); totalCount = ::min(totalCount, maxSize); diff --git a/modules/cudaimgproc/src/hough_circles.cpp b/modules/cudaimgproc/src/hough_circles.cpp index 0fa962d71b4..61d49d46bec 100644 --- a/modules/cudaimgproc/src/hough_circles.cpp +++ b/modules/cudaimgproc/src/hough_circles.cpp @@ -55,15 +55,15 @@ namespace cv { namespace cuda { namespace device { namespace hough { - int buildPointList_gpu(PtrStepSzb src, unsigned int* list); + int buildPointList_gpu(PtrStepSzb src, unsigned int* list, int* counterPtr, cudaStream_t stream); } namespace hough_circles { - void circlesAccumCenters_gpu(const unsigned int* list, int count, PtrStepi dx, PtrStepi dy, PtrStepSzi accum, int minRadius, int maxRadius, float idp); - int buildCentersList_gpu(PtrStepSzi accum, unsigned int* centers, int threshold); + void circlesAccumCenters_gpu(const unsigned int* list, int count, PtrStepi dx, PtrStepi dy, PtrStepSzi accum, int minRadius, int maxRadius, float idp, cudaStream_t stream); + int buildCentersList_gpu(PtrStepSzi accum, unsigned int* centers, int threshold, int* counterPtr, cudaStream_t stream); int circlesAccumRadius_gpu(const unsigned int* centers, int centersCount, const unsigned int* list, int count, - float3* circles, int maxCircles, float dp, int minRadius, int maxRadius, int threshold, bool has20); + float3* circles, int maxCircles, float dp, int minRadius, int maxRadius, int threshold, bool has20, int* counterPtr, cudaStream_t stream); } }}} @@ -73,6 +73,7 @@ namespace { public: HoughCirclesDetectorImpl(float dp, float minDist, int cannyThreshold, int votesThreshold, int minRadius, int maxRadius, int maxCircles); + ~HoughCirclesDetectorImpl(); void detect(InputArray src, OutputArray circles, Stream& stream); @@ -140,6 +141,8 @@ namespace Ptr filterDx_; Ptr filterDy_; Ptr canny_; + + int* counterPtr_; }; bool centersCompare(Vec3f a, Vec3f b) {return (a[2] > b[2]);} @@ -153,16 +156,22 @@ namespace filterDx_ = cuda::createSobelFilter(CV_8UC1, CV_32S, 1, 0); filterDy_ = cuda::createSobelFilter(CV_8UC1, CV_32S, 0, 1); + + cudaSafeCall(cudaMalloc(&counterPtr_, sizeof(int))); } - void HoughCirclesDetectorImpl::detect(InputArray _src, OutputArray circles, Stream& stream) + HoughCirclesDetectorImpl::~HoughCirclesDetectorImpl() { - // TODO : implement async version - CV_UNUSED(stream); + cudaSafeCall(cudaFree(counterPtr_)); + } + void HoughCirclesDetectorImpl::detect(InputArray _src, OutputArray circles, Stream& stream) + { using namespace cv::cuda::device::hough; using namespace cv::cuda::device::hough_circles; + auto cudaStream = StreamAccessor::getStream(stream); + GpuMat src = _src.getGpuMat(); CV_Assert( src.type() == CV_8UC1 ); @@ -182,13 +191,13 @@ namespace canny_->setLowThreshold(std::max(cannyThreshold_ / 2, 1)); canny_->setHighThreshold(cannyThreshold_); - canny_->detect(dx_, dy_, edges_); + canny_->detect(dx_, dy_, edges_, stream); ensureSizeIsEnough(2, src.size().area(), CV_32SC1, list_); unsigned int* srcPoints = list_.ptr(0); unsigned int* centers = list_.ptr(1); - const int pointsCount = buildPointList_gpu(edges_, srcPoints); + const int pointsCount = buildPointList_gpu(edges_, srcPoints, counterPtr_, cudaStream); if (pointsCount == 0) { circles.release(); @@ -196,13 +205,13 @@ namespace } ensureSizeIsEnough(cvCeil(src.rows * idp) + 2, cvCeil(src.cols * idp) + 2, CV_32SC1, accum_); - accum_.setTo(Scalar::all(0)); + accum_.setTo(Scalar::all(0), stream); - circlesAccumCenters_gpu(srcPoints, pointsCount, dx_, dy_, accum_, minRadius_, maxRadius_, idp); + circlesAccumCenters_gpu(srcPoints, pointsCount, dx_, dy_, accum_, minRadius_, maxRadius_, idp, cudaStream); - accum_.download(tt); + accum_.download(tt, stream); - int centersCount = buildCentersList_gpu(accum_, centers, votesThreshold_); + int centersCount = buildCentersList_gpu(accum_, centers, votesThreshold_, counterPtr_, cudaStream); if (centersCount == 0) { circles.release(); @@ -218,7 +227,8 @@ namespace ushort2* oldBuf = oldBuf_.data(); ushort2* newBuf = newBuf_.data(); - cudaSafeCall( cudaMemcpy(oldBuf, centers, centersCount * sizeof(ushort2), cudaMemcpyDeviceToHost) ); + cudaSafeCall( cudaMemcpyAsync(oldBuf, centers, centersCount * sizeof(ushort2), cudaMemcpyDeviceToHost, cudaStream) ); + cudaSafeCall( cudaStreamSynchronize(cudaStream) ); const int cellSize = cvRound(minDist_); const int gridWidth = (src.cols + cellSize - 1) / cellSize; @@ -290,14 +300,15 @@ namespace } } - cudaSafeCall( cudaMemcpy(centers, newBuf, newCount * sizeof(unsigned int), cudaMemcpyHostToDevice) ); + cudaSafeCall( cudaMemcpyAsync(centers, newBuf, newCount * sizeof(unsigned int), cudaMemcpyHostToDevice, cudaStream) ); centersCount = newCount; } ensureSizeIsEnough(1, maxCircles_, CV_32FC3, result_); int circlesCount = circlesAccumRadius_gpu(centers, centersCount, srcPoints, pointsCount, result_.ptr(), maxCircles_, - dp_, minRadius_, maxRadius_, votesThreshold_, deviceSupports(FEATURE_SET_COMPUTE_20)); + dp_, minRadius_, maxRadius_, votesThreshold_, deviceSupports(FEATURE_SET_COMPUTE_20), + counterPtr_, cudaStream); if (circlesCount == 0) { @@ -306,7 +317,7 @@ namespace } result_.cols = circlesCount; - result_.copyTo(circles); + result_.copyTo(circles, stream); } } diff --git a/modules/cudaimgproc/src/hough_lines.cpp b/modules/cudaimgproc/src/hough_lines.cpp index e112e09a3a8..06de8c0bebe 100644 --- a/modules/cudaimgproc/src/hough_lines.cpp +++ b/modules/cudaimgproc/src/hough_lines.cpp @@ -55,13 +55,13 @@ namespace cv { namespace cuda { namespace device { namespace hough { - int buildPointList_gpu(PtrStepSzb src, unsigned int* list); + int buildPointList_gpu(PtrStepSzb src, unsigned int* list, int* counterPtr, cudaStream_t stream); } namespace hough_lines { - void linesAccum_gpu(const unsigned int* list, int count, PtrStepSzi accum, float rho, float theta, size_t sharedMemPerBlock, bool has20); - int linesGetResult_gpu(PtrStepSzi accum, float2* out, int* votes, int maxSize, float rho, float theta, int threshold, bool doSort); + void linesAccum_gpu(const unsigned int* list, int count, PtrStepSzi accum, float rho, float theta, size_t sharedMemPerBlock, bool has20, cudaStream_t stream); + int linesGetResult_gpu(PtrStepSzi accum, float2* out, int* votes, int maxSize, float rho, float theta, int threshold, bool doSort, int* counterPtr, cudaStream_t stream); } }}} @@ -70,10 +70,8 @@ namespace class HoughLinesDetectorImpl : public HoughLinesDetector { public: - HoughLinesDetectorImpl(float rho, float theta, int threshold, bool doSort, int maxLines) : - rho_(rho), theta_(theta), threshold_(threshold), doSort_(doSort), maxLines_(maxLines) - { - } + HoughLinesDetectorImpl(float rho, float theta, int threshold, bool doSort, int maxLines); + ~HoughLinesDetectorImpl(); void detect(InputArray src, OutputArray lines, Stream& stream); void downloadResults(InputArray d_lines, OutputArray h_lines, OutputArray h_votes, Stream& stream); @@ -124,16 +122,27 @@ namespace GpuMat accum_; GpuMat list_; GpuMat result_; + + int* counterPtr_; }; - void HoughLinesDetectorImpl::detect(InputArray _src, OutputArray lines, Stream& stream) + HoughLinesDetectorImpl::HoughLinesDetectorImpl(float rho, float theta, int threshold, bool doSort, int maxLines) : + rho_(rho), theta_(theta), threshold_(threshold), doSort_(doSort), maxLines_(maxLines) + { + cudaSafeCall(cudaMalloc(&counterPtr_, sizeof(int))); + } + + HoughLinesDetectorImpl::~HoughLinesDetectorImpl() { - // TODO : implement async version - CV_UNUSED(stream); + cudaSafeCall(cudaFree(counterPtr_)); + } + void HoughLinesDetectorImpl::detect(InputArray _src, OutputArray lines, Stream& stream) + { using namespace cv::cuda::device::hough; using namespace cv::cuda::device::hough_lines; + auto cudaStream = StreamAccessor::getStream(stream); GpuMat src = _src.getGpuMat(); CV_Assert( src.type() == CV_8UC1 ); @@ -143,7 +152,7 @@ namespace ensureSizeIsEnough(1, src.size().area(), CV_32SC1, list_); unsigned int* srcPoints = list_.ptr(); - const int pointsCount = buildPointList_gpu(src, srcPoints); + const int pointsCount = buildPointList_gpu(src, srcPoints, counterPtr_, cudaStream); if (pointsCount == 0) { lines.release(); @@ -155,14 +164,14 @@ namespace CV_Assert( numangle > 0 && numrho > 0 ); ensureSizeIsEnough(numangle + 2, numrho + 2, CV_32SC1, accum_); - accum_.setTo(Scalar::all(0)); + accum_.setTo(Scalar::all(0), stream); DeviceInfo devInfo; - linesAccum_gpu(srcPoints, pointsCount, accum_, rho_, theta_, devInfo.sharedMemPerBlock(), devInfo.supports(FEATURE_SET_COMPUTE_20)); + linesAccum_gpu(srcPoints, pointsCount, accum_, rho_, theta_, devInfo.sharedMemPerBlock(), devInfo.supports(FEATURE_SET_COMPUTE_20), cudaStream); ensureSizeIsEnough(2, maxLines_, CV_32FC2, result_); - int linesCount = linesGetResult_gpu(accum_, result_.ptr(0), result_.ptr(1), maxLines_, rho_, theta_, threshold_, doSort_); + int linesCount = linesGetResult_gpu(accum_, result_.ptr(0), result_.ptr(1), maxLines_, rho_, theta_, threshold_, doSort_, counterPtr_, cudaStream); if (linesCount == 0) { @@ -171,7 +180,7 @@ namespace } result_.cols = linesCount; - result_.copyTo(lines); + result_.copyTo(lines, stream); } void HoughLinesDetectorImpl::downloadResults(InputArray _d_lines, OutputArray h_lines, OutputArray h_votes, Stream& stream) diff --git a/modules/cudaimgproc/src/hough_segments.cpp b/modules/cudaimgproc/src/hough_segments.cpp index 34ee4744619..f0eb0beecd7 100644 --- a/modules/cudaimgproc/src/hough_segments.cpp +++ b/modules/cudaimgproc/src/hough_segments.cpp @@ -55,17 +55,17 @@ namespace cv { namespace cuda { namespace device { namespace hough { - int buildPointList_gpu(PtrStepSzb src, unsigned int* list); + int buildPointList_gpu(PtrStepSzb src, unsigned int* list, int* counterPtr, cudaStream_t stream); } namespace hough_lines { - void linesAccum_gpu(const unsigned int* list, int count, PtrStepSzi accum, float rho, float theta, size_t sharedMemPerBlock, bool has20); + void linesAccum_gpu(const unsigned int* list, int count, PtrStepSzi accum, float rho, float theta, size_t sharedMemPerBlock, bool has20, cudaStream_t stream); } namespace hough_segments { - int houghLinesProbabilistic_gpu(PtrStepSzb mask, PtrStepSzi accum, int4* out, int maxSize, float rho, float theta, int lineGap, int lineLength); + int houghLinesProbabilistic_gpu(PtrStepSzb mask, PtrStepSzi accum, int4* out, int maxSize, float rho, float theta, int lineGap, int lineLength, int* counterPtr, cudaStream_t stream); } }}} @@ -74,10 +74,8 @@ namespace class HoughSegmentDetectorImpl : public HoughSegmentDetector { public: - HoughSegmentDetectorImpl(float rho, float theta, int minLineLength, int maxLineGap, int maxLines) : - rho_(rho), theta_(theta), minLineLength_(minLineLength), maxLineGap_(maxLineGap), maxLines_(maxLines) - { - } + HoughSegmentDetectorImpl(float rho, float theta, int minLineLength, int maxLineGap, int maxLines); + ~HoughSegmentDetectorImpl(); void detect(InputArray src, OutputArray lines, Stream& stream); @@ -127,8 +125,21 @@ namespace GpuMat accum_; GpuMat list_; GpuMat result_; + + int* counterPtr_; }; + HoughSegmentDetectorImpl::HoughSegmentDetectorImpl(float rho, float theta, int minLineLength, int maxLineGap, int maxLines) : + rho_(rho), theta_(theta), minLineLength_(minLineLength), maxLineGap_(maxLineGap), maxLines_(maxLines) + { + cudaSafeCall(cudaMalloc(&counterPtr_, sizeof(int))); + } + + HoughSegmentDetectorImpl::~HoughSegmentDetectorImpl() + { + cudaSafeCall(cudaFree(counterPtr_)); + } + void HoughSegmentDetectorImpl::detect(InputArray _src, OutputArray lines, Stream& stream) { // TODO : implement async version @@ -138,6 +149,7 @@ namespace using namespace cv::cuda::device::hough_lines; using namespace cv::cuda::device::hough_segments; + auto cudaStream = StreamAccessor::getStream(stream); GpuMat src = _src.getGpuMat(); CV_Assert( src.type() == CV_8UC1 ); @@ -147,7 +159,7 @@ namespace ensureSizeIsEnough(1, src.size().area(), CV_32SC1, list_); unsigned int* srcPoints = list_.ptr(); - const int pointsCount = buildPointList_gpu(src, srcPoints); + const int pointsCount = buildPointList_gpu(src, srcPoints, counterPtr_, cudaStream); if (pointsCount == 0) { lines.release(); @@ -159,14 +171,14 @@ namespace CV_Assert( numangle > 0 && numrho > 0 ); ensureSizeIsEnough(numangle + 2, numrho + 2, CV_32SC1, accum_); - accum_.setTo(Scalar::all(0)); + accum_.setTo(Scalar::all(0), stream); DeviceInfo devInfo; - linesAccum_gpu(srcPoints, pointsCount, accum_, rho_, theta_, devInfo.sharedMemPerBlock(), devInfo.supports(FEATURE_SET_COMPUTE_20)); + linesAccum_gpu(srcPoints, pointsCount, accum_, rho_, theta_, devInfo.sharedMemPerBlock(), devInfo.supports(FEATURE_SET_COMPUTE_20), cudaStream); ensureSizeIsEnough(1, maxLines_, CV_32SC4, result_); - int linesCount = houghLinesProbabilistic_gpu(src, accum_, result_.ptr(), maxLines_, rho_, theta_, maxLineGap_, minLineLength_); + int linesCount = houghLinesProbabilistic_gpu(src, accum_, result_.ptr(), maxLines_, rho_, theta_, maxLineGap_, minLineLength_, counterPtr_, cudaStream); if (linesCount == 0) { @@ -175,7 +187,7 @@ namespace } result_.cols = linesCount; - result_.copyTo(lines); + result_.copyTo(lines, stream); } } From d601d8933fd6f06e23a9a213cca8419e0ab6bf35 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 1 Jan 2021 13:45:11 +0000 Subject: [PATCH 13/70] copyright: 2021 --- LICENSE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index c196fe18c44..ebc39268e96 100644 --- a/LICENSE +++ b/LICENSE @@ -7,11 +7,11 @@ copy or use the software. For Open Source Computer Vision Library (3-clause BSD License) -Copyright (C) 2000-2018, Intel Corporation, all rights reserved. +Copyright (C) 2000-2021, Intel Corporation, all rights reserved. Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved. Copyright (C) 2009-2015, NVIDIA Corporation, all rights reserved. Copyright (C) 2010-2013, Advanced Micro Devices, Inc., all rights reserved. -Copyright (C) 2015-2018, OpenCV Foundation, all rights reserved. +Copyright (C) 2015-2021, OpenCV Foundation, all rights reserved. Copyright (C) 2015-2016, Itseez Inc., all rights reserved. Third party copyrights are property of their respective owners. From 55f5303cf3e4c094af7c6288e4944d364de9ea55 Mon Sep 17 00:00:00 2001 From: MrKepzie Date: Mon, 4 Jan 2021 15:14:24 +0100 Subject: [PATCH 14/70] rlof optflow: fix crash when no match are found --- modules/optflow/include/opencv2/optflow/rlofflow.hpp | 2 ++ modules/optflow/src/rlofflow.cpp | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/optflow/include/opencv2/optflow/rlofflow.hpp b/modules/optflow/include/opencv2/optflow/rlofflow.hpp index 58daf533750..48456d73311 100644 --- a/modules/optflow/include/opencv2/optflow/rlofflow.hpp +++ b/modules/optflow/include/opencv2/optflow/rlofflow.hpp @@ -227,6 +227,7 @@ class CV_EXPORTS_W RLOFOpticalFlowParameter{ * @note If the grid size is set to (1,1) and the forward backward threshold <= 0 than pixelwise dense optical flow field is * computed by RLOF without using interpolation. * + * @note Note that in output, if no correspondences are found between \a I0 and \a I1, the \a flow is set to 0. * @see optflow::calcOpticalFlowDenseRLOF(), optflow::RLOFOpticalFlowParameter */ class CV_EXPORTS_W DenseRLOFOpticalFlow : public DenseOpticalFlow @@ -493,6 +494,7 @@ class CV_EXPORTS_W SparseRLOFOpticalFlow : public SparseOpticalFlow * computed with the RLOF. * * @note SIMD parallelization is only available when compiling with SSE4.1. + * @note Note that in output, if no correspondences are found between \a I0 and \a I1, the \a flow is set to 0. * * @sa optflow::DenseRLOFOpticalFlow, optflow::RLOFOpticalFlowParameter */ diff --git a/modules/optflow/src/rlofflow.cpp b/modules/optflow/src/rlofflow.cpp index 5e756f39707..e9c4b129713 100644 --- a/modules/optflow/src/rlofflow.cpp +++ b/modules/optflow/src/rlofflow.cpp @@ -203,7 +203,11 @@ class DenseOpticalFlowRLOFImpl : public DenseRLOFOpticalFlow filtered_prevPoints = prevPoints; filtered_currPoints = currPoints; } - + // Interpolators below expect non empty matches + if (filtered_prevPoints.empty()) { + flow.setTo(0); + return; + } if (interp_type == InterpolationType::INTERP_EPIC) { Ptr gd = ximgproc::createEdgeAwareInterpolator(); From ebf745ee114e1a2100e52a21531fc30a245cf514 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 14 Jan 2021 19:57:04 +0000 Subject: [PATCH 15/70] xfeatures2d: disable BoostDesc/VGG in case of missing data --- modules/xfeatures2d/CMakeLists.txt | 36 ++++++++++++++----- modules/xfeatures2d/src/boostdesc.cpp | 9 +++++ modules/xfeatures2d/src/vgg.cpp | 10 ++++++ modules/xfeatures2d/test/test_features2d.cpp | 5 ++- .../test_rotation_and_scale_invariance.cpp | 8 +++++ 5 files changed, 59 insertions(+), 9 deletions(-) diff --git a/modules/xfeatures2d/CMakeLists.txt b/modules/xfeatures2d/CMakeLists.txt index 3c86ebc141b..bbc540e278c 100644 --- a/modules/xfeatures2d/CMakeLists.txt +++ b/modules/xfeatures2d/CMakeLists.txt @@ -5,13 +5,33 @@ if(HAVE_CUDA) endif() ocv_define_module(xfeatures2d opencv_core opencv_imgproc opencv_features2d opencv_calib3d OPTIONAL opencv_shape opencv_ml opencv_cudaarithm WRAP python java) -include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/download_vgg.cmake) -include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/download_boostdesc.cmake) -set(DOWNLOAD_DIR "${OpenCV_BINARY_DIR}/downloads/xfeatures2d") -download_boost_descriptors("${DOWNLOAD_DIR}" boost_status) -download_vgg_descriptors("${DOWNLOAD_DIR}" vgg_status) -if(NOT boost_status OR NOT vgg_status) - ocv_module_disable(xfeatures2d) +if(NOT OPENCV_SKIP_FEATURES2D_DOWNLOADING) + include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/download_vgg.cmake) + include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/download_boostdesc.cmake) + set(DOWNLOAD_DIR "${OpenCV_BINARY_DIR}/downloads/xfeatures2d") + download_boost_descriptors("${DOWNLOAD_DIR}" boost_status) + download_vgg_descriptors("${DOWNLOAD_DIR}" vgg_status) + if(boost_status) + ocv_append_source_file_compile_definitions(${CMAKE_CURRENT_SOURCE_DIR}/src/boostdesc.cpp "OPENCV_XFEATURES2D_HAS_BOOST_DATA=1") + else() + message(WARNING "features2d: Boost descriptor implementation is not available due to missing data (download failed: https://github.com/opencv/opencv_contrib/issues/1301)") + endif() + if(vgg_status) + ocv_append_source_file_compile_definitions(${CMAKE_CURRENT_SOURCE_DIR}/src/vgg.cpp "OPENCV_XFEATURES2D_HAS_VGG_DATA=1") + else() + message(WARNING "features2d: VGG descriptor implementation is not available due to missing data (download failed: https://github.com/opencv/opencv_contrib/issues/1301)") + endif() + + if(boost_status OR vgg_status) + ocv_module_include_directories("${DOWNLOAD_DIR}") + endif() endif() -ocv_module_include_directories("${DOWNLOAD_DIR}") +if(TARGET opencv_test_${name}) + if(boost_status) + ocv_target_compile_definitions(opencv_test_${name} PRIVATE "OPENCV_XFEATURES2D_HAS_BOOST_DATA=1") + endif() + if(vgg_status) + ocv_target_compile_definitions(opencv_test_${name} PRIVATE "OPENCV_XFEATURES2D_HAS_VGG_DATA=1") + endif() +endif() diff --git a/modules/xfeatures2d/src/boostdesc.cpp b/modules/xfeatures2d/src/boostdesc.cpp index 3f7ee225fc9..fe672a7324f 100644 --- a/modules/xfeatures2d/src/boostdesc.cpp +++ b/modules/xfeatures2d/src/boostdesc.cpp @@ -65,6 +65,8 @@ namespace cv namespace xfeatures2d { +#ifdef OPENCV_XFEATURES2D_HAS_BOOST_DATA + /* !BoostDesc implementation */ @@ -729,9 +731,16 @@ BoostDesc_Impl::~BoostDesc_Impl() { } +#endif // OPENCV_XFEATURES2D_HAS_BOOST_DATA + Ptr BoostDesc::create( int desc, bool use_scale_orientation, float scale_factor ) { +#ifdef OPENCV_XFEATURES2D_HAS_BOOST_DATA return makePtr( desc, use_scale_orientation, scale_factor ); +#else + CV_UNUSED(desc); CV_UNUSED(use_scale_orientation); CV_UNUSED(scale_factor); + CV_Error(Error::StsNotImplemented, "The OpenCV xfeatures2d binaries is built without downloaded Boost decriptor features: https://github.com/opencv/opencv_contrib/issues/1301"); +#endif } diff --git a/modules/xfeatures2d/src/vgg.cpp b/modules/xfeatures2d/src/vgg.cpp index a187f6b1d7a..506f5d0c13d 100644 --- a/modules/xfeatures2d/src/vgg.cpp +++ b/modules/xfeatures2d/src/vgg.cpp @@ -66,6 +66,8 @@ namespace cv namespace xfeatures2d { +#ifdef OPENCV_XFEATURES2D_HAS_VGG_DATA + /* !VGG implementation */ @@ -527,10 +529,18 @@ VGG_Impl::~VGG_Impl() { } +#endif + Ptr VGG::create( int desc, float isigma, bool img_normalize, bool use_scale_orientation, float scale_factor, bool dsc_normalize ) { +#ifdef OPENCV_XFEATURES2D_HAS_VGG_DATA return makePtr( desc, isigma, img_normalize, use_scale_orientation, scale_factor, dsc_normalize ); +#else + CV_UNUSED(desc); CV_UNUSED(isigma); CV_UNUSED(img_normalize); + CV_UNUSED(use_scale_orientation); CV_UNUSED(scale_factor); CV_UNUSED(dsc_normalize); + CV_Error(Error::StsNotImplemented, "The OpenCV xfeatures2d binaries is built without downloaded VGG decriptor features: https://github.com/opencv/opencv_contrib/issues/1301"); +#endif } diff --git a/modules/xfeatures2d/test/test_features2d.cpp b/modules/xfeatures2d/test/test_features2d.cpp index 2d6ec187757..90e7fe7d2ce 100644 --- a/modules/xfeatures2d/test/test_features2d.cpp +++ b/modules/xfeatures2d/test/test_features2d.cpp @@ -1114,13 +1114,16 @@ TEST( Features2d_DescriptorExtractor_LATCH, regression ) test.safe_run(); } +#ifdef OPENCV_XFEATURES2D_HAS_VGG_DATA TEST( Features2d_DescriptorExtractor_VGG, regression ) { CV_DescriptorExtractorTest > test( "descriptor-vgg", 0.03f, VGG::create() ); test.safe_run(); } +#endif // OPENCV_XFEATURES2D_HAS_VGG_DATA +#ifdef OPENCV_XFEATURES2D_HAS_BOOST_DATA TEST( Features2d_DescriptorExtractor_BGM, regression ) { CV_DescriptorExtractorTest test( "descriptor-boostdesc-bgm", @@ -1176,7 +1179,7 @@ TEST( Features2d_DescriptorExtractor_BINBOOST_256, regression ) BoostDesc::create(BoostDesc::BINBOOST_256) ); test.safe_run(); } - +#endif // OPENCV_XFEATURES2D_HAS_BOOST_DATA /*#if CV_SSE2 TEST( Features2d_DescriptorExtractor_Calonder_uchar, regression ) diff --git a/modules/xfeatures2d/test/test_rotation_and_scale_invariance.cpp b/modules/xfeatures2d/test/test_rotation_and_scale_invariance.cpp index 502c8ea44ed..f841d2cafa3 100644 --- a/modules/xfeatures2d/test/test_rotation_and_scale_invariance.cpp +++ b/modules/xfeatures2d/test/test_rotation_and_scale_invariance.cpp @@ -641,6 +641,7 @@ TEST(DISABLED_Features2d_RotationInvariance_Descriptor_DAISY, regression) test.safe_run(); } +#ifdef OPENCV_XFEATURES2D_HAS_VGG_DATA TEST(Features2d_RotationInvariance_Descriptor_VGG120, regression) { DescriptorRotationInvarianceTest test(KAZE::create(), @@ -676,6 +677,7 @@ TEST(Features2d_RotationInvariance_Descriptor_VGG48, regression) 0.98f); test.safe_run(); } +#endif // OPENCV_XFEATURES2D_HAS_VGG_DATA #ifdef OPENCV_ENABLE_NONFREE TEST(Features2d_RotationInvariance_Descriptor_BRIEF_64, regression) @@ -715,6 +717,7 @@ TEST(Features2d_RotationInvariance_Descriptor_FREAK, regression) test.safe_run(); } +#ifdef OPENCV_XFEATURES2D_HAS_BOOST_DATA TEST(Features2d_RotationInvariance_Descriptor_BoostDesc_BGM, regression) { DescriptorRotationInvarianceTest test(SURF::create(), @@ -777,6 +780,7 @@ TEST(Features2d_RotationInvariance_Descriptor_BoostDesc_BINBOOST_256, regression 0.98f); test.safe_run(); } +#endif // OPENCV_XFEATURES2D_HAS_BOOST_DATA /* * Detector's scale invariance check @@ -840,6 +844,7 @@ TEST(DISABLED_Features2d_ScaleInvariance_Descriptor_DAISY, regression) test.safe_run(); } +#ifdef OPENCV_XFEATURES2D_HAS_VGG_DATA TEST(Features2d_ScaleInvariance_Descriptor_VGG120, regression) { DescriptorScaleInvarianceTest test(KAZE::create(), @@ -875,8 +880,10 @@ TEST(Features2d_ScaleInvariance_Descriptor_VGG48, regression) 0.93f); test.safe_run(); } +#endif // OPENCV_XFEATURES2D_HAS_VGG_DATA #ifdef OPENCV_ENABLE_NONFREE +#ifdef OPENCV_XFEATURES2D_HAS_BOOST_DATA TEST(Features2d_ScaleInvariance_Descriptor_BoostDesc_BGM, regression) { DescriptorScaleInvarianceTest test(SURF::create(), @@ -939,6 +946,7 @@ TEST(Features2d_ScaleInvariance_Descriptor_BoostDesc_BINBOOST_256, regression) 0.98f); test.safe_run(); } +#endif // OPENCV_XFEATURES2D_HAS_BOOST_DATA #endif // NONFREE }} // namespace From 7e992b89f22a488cbdc0eb95a4f086906db19438 Mon Sep 17 00:00:00 2001 From: arsaratovtsev Date: Fri, 15 Jan 2021 17:35:47 +0300 Subject: [PATCH 16/70] replace lower with upper --- modules/rgbd/src/hash_tsdf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/rgbd/src/hash_tsdf.cpp b/modules/rgbd/src/hash_tsdf.cpp index b414500217f..b09dc63329f 100644 --- a/modules/rgbd/src/hash_tsdf.cpp +++ b/modules/rgbd/src/hash_tsdf.cpp @@ -99,7 +99,7 @@ void HashTSDFVolumeCPU::integrate(InputArray _depth, float depthFactor, const Ma for (int i = lower_bound[0]; i <= upper_bound[0]; i++) for (int j = lower_bound[1]; j <= upper_bound[1]; j++) - for (int k = lower_bound[2]; k <= lower_bound[2]; k++) + for (int k = lower_bound[2]; k <= upper_bound[2]; k++) { const Vec3i tsdf_idx = Vec3i(i, j, k); if (!localAccessVolUnits.count(tsdf_idx)) From 582fe44b7a41c5c866b211bcd66c8a03e6c9b68c Mon Sep 17 00:00:00 2001 From: Vishal Chiluka Date: Thu, 21 May 2020 02:31:02 +0530 Subject: [PATCH 17/70] NVIDIA_OPTICAL_FLOW_2_0_INTEGRATION --- modules/cudaoptflow/CMakeLists.txt | 20 +- .../include/opencv2/cudaoptflow.hpp | 150 +++- .../python/test/test_nvidiaopticalflow.py | 36 + modules/cudaoptflow/perf/perf_optflow.cpp | 70 +- .../samples/nvidia_optical_flow.cpp | 168 ++++- modules/cudaoptflow/samples/optical_flow.cpp | 33 +- .../cudaoptflow/src/cuda/nvidiaOpticalFlow.cu | 99 +++ modules/cudaoptflow/src/nvidiaOpticalFlow.cpp | 658 +++++++++++++++++- modules/cudaoptflow/test/test_optflow.cpp | 114 ++- 9 files changed, 1238 insertions(+), 110 deletions(-) create mode 100644 modules/cudaoptflow/misc/python/test/test_nvidiaopticalflow.py create mode 100644 modules/cudaoptflow/src/cuda/nvidiaOpticalFlow.cu diff --git a/modules/cudaoptflow/CMakeLists.txt b/modules/cudaoptflow/CMakeLists.txt index e5b823ab4aa..7d2d3e74eef 100644 --- a/modules/cudaoptflow/CMakeLists.txt +++ b/modules/cudaoptflow/CMakeLists.txt @@ -8,21 +8,21 @@ ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4127 /wd4324 /wd4512 -Wundef -Wmissing-d ocv_define_module(cudaoptflow opencv_video opencv_optflow opencv_cudaarithm opencv_cudawarping opencv_cudaimgproc OPTIONAL opencv_cudalegacy WRAP python) -set(NVIDIA_OPTICAL_FLOW_1_0_HEADERS_COMMIT "79c6cee80a2df9a196f20afd6b598a9810964c32") -set(NVIDIA_OPTICAL_FLOW_1_0_HEADERS_MD5 "ca5acedee6cb45d0ec610a6732de5c15") -set(NVIDIA_OPTICAL_FLOW_1_0_HEADERS_PATH "${OpenCV_BINARY_DIR}/3rdparty/NVIDIAOpticalFlowSDK_1_0_Headers") -ocv_download(FILENAME "${NVIDIA_OPTICAL_FLOW_1_0_HEADERS_COMMIT}.zip" - HASH ${NVIDIA_OPTICAL_FLOW_1_0_HEADERS_MD5} +set(NVIDIA_OPTICAL_FLOW_2_0_HEADERS_COMMIT "edb50da3cf849840d680249aa6dbef248ebce2ca") +set(NVIDIA_OPTICAL_FLOW_2_0_HEADERS_MD5 "a73cd48b18dcc0cc8933b30796074191") +set(NVIDIA_OPTICAL_FLOW_2_0_HEADERS_PATH "${OpenCV_BINARY_DIR}/3rdparty/NVIDIAOpticalFlowSDK_2_0_Headers") +ocv_download(FILENAME "${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_COMMIT}.zip" + HASH ${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_MD5} URL "https://github.com/NVIDIA/NVIDIAOpticalFlowSDK/archive/" - DESTINATION_DIR "${NVIDIA_OPTICAL_FLOW_1_0_HEADERS_PATH}" - STATUS NVIDIA_OPTICAL_FLOW_1_0_HEADERS_DOWNLOAD_SUCCESS + DESTINATION_DIR "${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_PATH}" + STATUS NVIDIA_OPTICAL_FLOW_2_0_HEADERS_DOWNLOAD_SUCCESS ID "NVIDIA_OPTICAL_FLOW" UNPACK RELATIVE_URL) -if(NOT NVIDIA_OPTICAL_FLOW_1_0_HEADERS_DOWNLOAD_SUCCESS) - message(STATUS "Failed to download NVIDIA_Optical_Flow_1_0 Headers") +if(NOT NVIDIA_OPTICAL_FLOW_2_0_HEADERS_DOWNLOAD_SUCCESS) + message(STATUS "Failed to download NVIDIA_Optical_Flow_2_0 Headers") else() add_definitions(-DHAVE_NVIDIA_OPTFLOW=1) - ocv_include_directories(SYSTEM "${NVIDIA_OPTICAL_FLOW_1_0_HEADERS_PATH}/NVIDIAOpticalFlowSDK-${NVIDIA_OPTICAL_FLOW_1_0_HEADERS_COMMIT}") + ocv_include_directories(SYSTEM "${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_PATH}/NVIDIAOpticalFlowSDK-${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_COMMIT}") endif() \ No newline at end of file diff --git a/modules/cudaoptflow/include/opencv2/cudaoptflow.hpp b/modules/cudaoptflow/include/opencv2/cudaoptflow.hpp index 9fde392523c..3221dbc7ef6 100644 --- a/modules/cudaoptflow/include/opencv2/cudaoptflow.hpp +++ b/modules/cudaoptflow/include/opencv2/cudaoptflow.hpp @@ -392,9 +392,9 @@ class CV_EXPORTS_W OpticalFlowDual_TVL1 : public DenseOpticalFlow /** @brief Class for computing the optical flow vectors between two images using NVIDIA Optical Flow hardware and Optical Flow SDK 1.0. @note - A sample application demonstrating the use of NVIDIA Optical Flow can be found at -opencv_source_code/samples/gpu/nvidia_optical_flow.cpp +opencv_contrib_source_code/modules/cudaoptflow/samples/nvidia_optical_flow.cpp - An example application comparing accuracy and performance of NVIDIA Optical Flow with other optical flow algorithms in OpenCV can be found at -opencv_source_code/samples/gpu/optical_flow.cpp +opencv_contrib_source_code/modules/cudaoptflow/samples/optical_flow.cpp */ class CV_EXPORTS_W NvidiaOpticalFlow_1_0 : public NvidiaHWOpticalFlow @@ -417,18 +417,16 @@ class CV_EXPORTS_W NvidiaOpticalFlow_1_0 : public NvidiaHWOpticalFlow * using nearest neighbour upsampling method. @param flow Buffer of type CV_16FC2 containing flow vectors generated by calc(). - @param width Width of the input image in pixels for which these flow vectors were generated. - @param height Height of the input image in pixels for which these flow vectors were generated. + @param imageSize Size of the input image in pixels for which these flow vectors were generated. @param gridSize Granularity of the optical flow vectors returned by calc() function. Can be queried using getGridSize(). @param upsampledFlow Buffer of type CV_32FC2, containing upsampled flow vectors, each flow vector for 1 pixel, in the pitch-linear layout. */ - CV_WRAP virtual void upSampler(InputArray flow, int width, int height, + CV_WRAP virtual void upSampler(InputArray flow, cv::Size imageSize, int gridSize, InputOutputArray upsampledFlow) = 0; /** @brief Instantiate NVIDIA Optical Flow - @param width Width of input image in pixels. - @param height Height of input image in pixels. + @param imageSize Size of input image in pixels. @param perfPreset Optional parameter. Refer [NV OF SDK documentation](https://developer.nvidia.com/opticalflow-sdk) for details about presets. Defaults to NV_OF_PERF_LEVEL_SLOW. @param enableTemporalHints Optional parameter. Flag to enable temporal hints. When set to true, the hardware uses the flow vectors @@ -445,10 +443,142 @@ class CV_EXPORTS_W NvidiaOpticalFlow_1_0 : public NvidiaHWOpticalFlow If output stream is not set, the execute function will use default stream which is NULL stream; */ CV_WRAP static Ptr create( - int width, - int height, + cv::Size imageSize, cv::cuda::NvidiaOpticalFlow_1_0::NVIDIA_OF_PERF_LEVEL perfPreset - = cv::cuda::NvidiaOpticalFlow_1_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_SLOW, + = cv::cuda::NvidiaOpticalFlow_1_0::NV_OF_PERF_LEVEL_SLOW, + bool enableTemporalHints = false, + bool enableExternalHints = false, + bool enableCostBuffer = false, + int gpuId = 0, + Stream& inputStream = Stream::Null(), + Stream& outputStream = Stream::Null()); +}; + +/** @brief Class for computing the optical flow vectors between two images using NVIDIA Optical Flow hardware and Optical Flow SDK 2.0. +@note +- A sample application demonstrating the use of NVIDIA Optical Flow can be found at +opencv_contrib_source_code/modules/cudaoptflow/samples/nvidia_optical_flow.cpp +- An example application comparing accuracy and performance of NVIDIA Optical Flow with other optical flow algorithms in OpenCV can be found at +opencv_contrib_source_code/modules/cudaoptflow/samples/optical_flow.cpp +*/ + +class CV_EXPORTS_W NvidiaOpticalFlow_2_0 : public NvidiaHWOpticalFlow +{ +public: + /** + * Supported optical flow performance levels. + */ + enum NVIDIA_OF_PERF_LEVEL + { + NV_OF_PERF_LEVEL_UNDEFINED, + NV_OF_PERF_LEVEL_SLOW = 5, /**< Slow perf level results in lowest performance and best quality */ + NV_OF_PERF_LEVEL_MEDIUM = 10, /**< Medium perf level results in low performance and medium quality */ + NV_OF_PERF_LEVEL_FAST = 20, /**< Fast perf level results in high performance and low quality */ + NV_OF_PERF_LEVEL_MAX + }; + + /** + * Supported grid size for output buffer. + */ + enum NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE + { + NV_OF_OUTPUT_VECTOR_GRID_SIZE_UNDEFINED, + NV_OF_OUTPUT_VECTOR_GRID_SIZE_1 = 1, /**< Output buffer grid size is 1x1 */ + NV_OF_OUTPUT_VECTOR_GRID_SIZE_2 = 2, /**< Output buffer grid size is 2x2 */ + NV_OF_OUTPUT_VECTOR_GRID_SIZE_4 = 4, /**< Output buffer grid size is 4x4 */ + NV_OF_OUTPUT_VECTOR_GRID_SIZE_MAX + }; + + /** + * Supported grid size for hint buffer. + */ + enum NVIDIA_OF_HINT_VECTOR_GRID_SIZE + { + NV_OF_HINT_VECTOR_GRID_SIZE_UNDEFINED, + NV_OF_HINT_VECTOR_GRID_SIZE_1 = 1, /**< Hint buffer grid size is 1x1.*/ + NV_OF_HINT_VECTOR_GRID_SIZE_2 = 2, /**< Hint buffer grid size is 2x2.*/ + NV_OF_HINT_VECTOR_GRID_SIZE_4 = 4, /**< Hint buffer grid size is 4x4.*/ + NV_OF_HINT_VECTOR_GRID_SIZE_8 = 8, /**< Hint buffer grid size is 8x8.*/ + NV_OF_HINT_VECTOR_GRID_SIZE_MAX + }; + + /** @brief convertToFloat() helper function converts the hardware-generated flow vectors to floating point representation (1 flow vector for gridSize). + * gridSize can be queried via function getGridSize(). + + @param flow Buffer of type CV_16FC2 containing flow vectors generated by calc(). + @param floatFlow Buffer of type CV_32FC2, containing flow vectors in floating point representation, each flow vector for 1 pixel per gridSize, in the pitch-linear layout. + */ + CV_WRAP virtual void convertToFloat(InputArray flow, InputOutputArray floatFlow) = 0; + + /** @brief Instantiate NVIDIA Optical Flow + + @param imageSize Size of input image in pixels. + @param perfPreset Optional parameter. Refer [NV OF SDK documentation](https://developer.nvidia.com/opticalflow-sdk) for details about presets. + Defaults to NV_OF_PERF_LEVEL_SLOW. + @param outputGridSize Optional parameter. Refer [NV OF SDK documentation](https://developer.nvidia.com/opticalflow-sdk) for details about output grid sizes. + Defaults to NV_OF_OUTPUT_VECTOR_GRID_SIZE_1. + @param hintGridSize Optional parameter. Refer [NV OF SDK documentation](https://developer.nvidia.com/opticalflow-sdk) for details about hint grid sizes. + Defaults to NV_OF_HINT_VECTOR_GRID_SIZE_1. + @param enableTemporalHints Optional parameter. Flag to enable temporal hints. When set to true, the hardware uses the flow vectors + generated in previous call to calc() as internal hints for the current call to calc(). + Useful when computing flow vectors between successive video frames. Defaults to false. + @param enableExternalHints Optional Parameter. Flag to enable passing external hints buffer to calc(). Defaults to false. + @param enableCostBuffer Optional Parameter. Flag to enable cost buffer output from calc(). Defaults to false. + @param gpuId Optional parameter to select the GPU ID on which the optical flow should be computed. Useful in multi-GPU systems. Defaults to 0. + @param inputStream Optical flow algorithm may optionally involve cuda preprocessing on the input buffers. + The input cuda stream can be used to pipeline and synchronize the cuda preprocessing tasks with OF HW engine. + If input stream is not set, the execute function will use default stream which is NULL stream; + @param outputStream Optical flow algorithm may optionally involve cuda post processing on the output flow vectors. + The output cuda stream can be used to pipeline and synchronize the cuda post processing tasks with OF HW engine. + If output stream is not set, the execute function will use default stream which is NULL stream; + */ + CV_WRAP static Ptr create( + cv::Size imageSize, + cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_PERF_LEVEL perfPreset + = cv::cuda::NvidiaOpticalFlow_2_0::NV_OF_PERF_LEVEL_SLOW, + cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE outputGridSize + = cv::cuda::NvidiaOpticalFlow_2_0::NV_OF_OUTPUT_VECTOR_GRID_SIZE_1, + cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_HINT_VECTOR_GRID_SIZE hintGridSize + = cv::cuda::NvidiaOpticalFlow_2_0::NV_OF_HINT_VECTOR_GRID_SIZE_1, + bool enableTemporalHints = false, + bool enableExternalHints = false, + bool enableCostBuffer = false, + int gpuId = 0, + Stream& inputStream = Stream::Null(), + Stream& outputStream = Stream::Null()); + + /** @brief Instantiate NVIDIA Optical Flow with ROI Feature + + @param imageSize Size of input image in pixels. + @param roiData Pointer to ROI data. + @param perfPreset Optional parameter. Refer [NV OF SDK documentation](https://developer.nvidia.com/opticalflow-sdk) for details about presets. + Defaults to NV_OF_PERF_LEVEL_SLOW. + @param outputGridSize Optional parameter. Refer [NV OF SDK documentation](https://developer.nvidia.com/opticalflow-sdk) for details about output grid sizes. + Defaults to NV_OF_OUTPUT_VECTOR_GRID_SIZE_1. + @param hintGridSize Optional parameter. Refer [NV OF SDK documentation](https://developer.nvidia.com/opticalflow-sdk) for details about hint grid sizes. + Defaults to NV_OF_HINT_VECTOR_GRID_SIZE_1. + @param enableTemporalHints Optional parameter. Flag to enable temporal hints. When set to true, the hardware uses the flow vectors + generated in previous call to calc() as internal hints for the current call to calc(). + Useful when computing flow vectors between successive video frames. Defaults to false. + @param enableExternalHints Optional Parameter. Flag to enable passing external hints buffer to calc(). Defaults to false. + @param enableCostBuffer Optional Parameter. Flag to enable cost buffer output from calc(). Defaults to false. + @param gpuId Optional parameter to select the GPU ID on which the optical flow should be computed. Useful in multi-GPU systems. Defaults to 0. + @param inputStream Optical flow algorithm may optionally involve cuda preprocessing on the input buffers. + The input cuda stream can be used to pipeline and synchronize the cuda preprocessing tasks with OF HW engine. + If input stream is not set, the execute function will use default stream which is NULL stream; + @param outputStream Optical flow algorithm may optionally involve cuda post processing on the output flow vectors. + The output cuda stream can be used to pipeline and synchronize the cuda post processing tasks with OF HW engine. + If output stream is not set, the execute function will use default stream which is NULL stream; + */ + CV_WRAP static Ptr create( + cv::Size imageSize, + std::vector roiData, + cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_PERF_LEVEL perfPreset + = cv::cuda::NvidiaOpticalFlow_2_0::NV_OF_PERF_LEVEL_SLOW, + cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE outputGridSize + = cv::cuda::NvidiaOpticalFlow_2_0::NV_OF_OUTPUT_VECTOR_GRID_SIZE_1, + cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_HINT_VECTOR_GRID_SIZE hintGridSize + = cv::cuda::NvidiaOpticalFlow_2_0::NV_OF_HINT_VECTOR_GRID_SIZE_1, bool enableTemporalHints = false, bool enableExternalHints = false, bool enableCostBuffer = false, diff --git a/modules/cudaoptflow/misc/python/test/test_nvidiaopticalflow.py b/modules/cudaoptflow/misc/python/test/test_nvidiaopticalflow.py new file mode 100644 index 00000000000..94822c40817 --- /dev/null +++ b/modules/cudaoptflow/misc/python/test/test_nvidiaopticalflow.py @@ -0,0 +1,36 @@ +import os +import cv2 as cv +import numpy as np + +from tests_common import NewOpenCVTests, unittest + +class nvidiaopticalflow_test(NewOpenCVTests): + def setUp(self): + super(nvidiaopticalflow_test, self).setUp() + if not cv.cuda.getCudaEnabledDeviceCount(): + self.skipTest("No CUDA-capable device is detected") + + @unittest.skipIf('OPENCV_TEST_DATA_PATH' not in os.environ, + "OPENCV_TEST_DATA_PATH is not defined") + def test_calc(self): + frame1 = os.environ['OPENCV_TEST_DATA_PATH'] + '/gpu/opticalflow/frame0.png' + frame2 = os.environ['OPENCV_TEST_DATA_PATH'] + '/gpu/opticalflow/frame1.png' + + npMat1 = cv.cvtColor(cv.imread(frame1),cv.COLOR_BGR2GRAY) + npMat2 = cv.cvtColor(cv.imread(frame2),cv.COLOR_BGR2GRAY) + + cuMat1 = cv.cuda_GpuMat(npMat1) + cuMat2 = cv.cuda_GpuMat(npMat2) + try: + nvof = cv.cuda_NvidiaOpticalFlow_1_0.create(cuMat1.shape[1], cuMat1.shape[0], 5, False, False, False, 0) + flow = nvof.calc(cuMat1, cuMat2, None) + self.assertTrue(flow.shape[1] > 0 and flow.shape[0] > 0) + flowUpSampled = nvof.upSampler(flow[0], cuMat1.shape[1], cuMat1.shape[0], nvof.getGridSize(), None) + nvof.collectGarbage() + except cv.error as e: + if e.code == cv.Error.StsBadFunc or e.code == cv.Error.StsBadArg or e.code == cv.Error.StsNullPtr: + self.skipTest("Algorithm is not supported in the current environment") + self.assertTrue(flowUpSampled.shape[1] > 0 and flowUpSampled.shape[0] > 0) + +if __name__ == '__main__': + NewOpenCVTests.bootstrap() \ No newline at end of file diff --git a/modules/cudaoptflow/perf/perf_optflow.cpp b/modules/cudaoptflow/perf/perf_optflow.cpp index ceb4811f8bf..a9342c05d5c 100644 --- a/modules/cudaoptflow/perf/perf_optflow.cpp +++ b/modules/cudaoptflow/perf/perf_optflow.cpp @@ -339,13 +339,8 @@ PERF_TEST_P(ImagePair, NvidiaOpticalFlow_1_0, const cv::Mat frame1 = readImage(GetParam().second, cv::IMREAD_GRAYSCALE); ASSERT_FALSE(frame1.empty()); - - const int width = frame0.size().width; - const int height = frame0.size().height; - const bool enableTemporalHints = false; - const bool enableExternalHints = false; - const bool enableCostBuffer = false; - const int gpuid = 0; + Stream inputStream; + Stream outputStream; if (PERF_RUN_CUDA()) { @@ -355,9 +350,9 @@ PERF_TEST_P(ImagePair, NvidiaOpticalFlow_1_0, cv::Ptr d_nvof; try { - d_nvof = cv::cuda::NvidiaOpticalFlow_1_0::create(width, height, + d_nvof = cv::cuda::NvidiaOpticalFlow_1_0::create(frame0.size(), cv::cuda::NvidiaOpticalFlow_1_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_FAST, - enableTemporalHints, enableExternalHints, enableCostBuffer, gpuid); + false, false, false, 0, inputStream, outputStream); } catch (const cv::Exception& e) { @@ -376,6 +371,63 @@ PERF_TEST_P(ImagePair, NvidiaOpticalFlow_1_0, CUDA_SANITY_CHECK(u, 1e-10); CUDA_SANITY_CHECK(v, 1e-10); + + d_nvof->collectGarbage(); + } +} + +////////////////////////////////////////////////////// +// NvidiaOpticalFlow_2_0 + +PERF_TEST_P(ImagePair, NvidiaOpticalFlow_2_0, + Values(make_pair("gpu/opticalflow/frame0.png", "gpu/opticalflow/frame1.png"))) +{ + declare.time(10); + + const cv::Mat frame0 = readImage(GetParam().first, cv::IMREAD_GRAYSCALE); + ASSERT_FALSE(frame0.empty()); + + const cv::Mat frame1 = readImage(GetParam().second, cv::IMREAD_GRAYSCALE); + ASSERT_FALSE(frame1.empty()); + + const cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE outGridSize + = cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE::NV_OF_OUTPUT_VECTOR_GRID_SIZE_1; + const cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_HINT_VECTOR_GRID_SIZE hintGridSize + = cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_HINT_VECTOR_GRID_SIZE::NV_OF_HINT_VECTOR_GRID_SIZE_1; + Stream inputStream; + Stream outputStream; + + if (PERF_RUN_CUDA()) + { + const cv::cuda::GpuMat d_frame0(frame0); + const cv::cuda::GpuMat d_frame1(frame1); + cv::cuda::GpuMat d_flow; + cv::Ptr d_nvof; + try + { + d_nvof = cv::cuda::NvidiaOpticalFlow_2_0::create(frame0.size(), + cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_FAST, outGridSize, hintGridSize, + false, false, false, 0, inputStream, outputStream); + } + catch (const cv::Exception& e) + { + if (e.code == Error::StsBadFunc || e.code == Error::StsBadArg || e.code == Error::StsNullPtr) + throw SkipTestException("Current configuration is not supported"); + throw; + } + + TEST_CYCLE() d_nvof->calc(d_frame0, d_frame1, d_flow); + + cv::cuda::GpuMat flow[2]; + cv::cuda::split(d_flow, flow); + + cv::cuda::GpuMat u = flow[0]; + cv::cuda::GpuMat v = flow[1]; + + CUDA_SANITY_CHECK(u, 1e-10); + CUDA_SANITY_CHECK(v, 1e-10); + + d_nvof->collectGarbage(); } } diff --git a/modules/cudaoptflow/samples/nvidia_optical_flow.cpp b/modules/cudaoptflow/samples/nvidia_optical_flow.cpp index 478a5f16184..5dd4a0345e6 100644 --- a/modules/cudaoptflow/samples/nvidia_optical_flow.cpp +++ b/modules/cudaoptflow/samples/nvidia_optical_flow.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include "opencv2/core.hpp" #include "opencv2/core/utility.hpp" @@ -11,7 +12,6 @@ #include "opencv2/cudaarithm.hpp" #include "opencv2/video/tracking.hpp" -using namespace std; using namespace cv; using namespace cv::cuda; @@ -131,12 +131,88 @@ static void drawOpticalFlow(const Mat_& flowx, const Mat_& flowy } } +/* +ROI config file format. +numrois 3 +roi0 640 96 1152 192 +roi1 640 64 896 864 +roi2 640 960 256 32 +*/ +bool parseROI(std::string ROIFileName, std::vector& roiData) +{ + std::string str; + uint32_t nRois = 0; + std::ifstream hRoiFile; + hRoiFile.open(ROIFileName, std::ios::in); + + if (hRoiFile.is_open()) + { + while (std::getline(hRoiFile, str)) + { + std::istringstream iss(str); + std::vector tokens{ std::istream_iterator{iss}, + std::istream_iterator{} }; + + if (tokens.size() == 0) continue; // if empty line, coninue + + transform(tokens[0].begin(), tokens[0].end(), tokens[0].begin(), ::tolower); + if (tokens[0] == "numrois") + { + nRois = atoi(tokens[1].data()); + } + else if (tokens[0].rfind("roi", 0) == 0) + { + cv::Rect roi; + roi.x = atoi(tokens[1].data()); + roi.y = atoi(tokens[2].data()); + roi.width = atoi(tokens[3].data()); + roi.height = atoi(tokens[4].data()); + roiData.push_back(roi); + } + else if (tokens[0].rfind("#", 0) == 0) + { + continue; + } + else + { + std::cout << "Unidentified keyword in roi config file " << tokens[0] << std::endl; + hRoiFile.close(); + return false; + } + } + } + else + { + std::cout << "Unable to open ROI file " << std::endl; + return false; + } + if (nRois != roiData.size()) + { + std::cout << "NumRois(" << nRois << ")and specified roi rects (" << roiData.size() << ")are not matching " << std::endl; + hRoiFile.close(); + return false; + } + hRoiFile.close(); + return true; +} + int main(int argc, char **argv) { - std::unordered_map presetMap = { - { "slow", NvidiaOpticalFlow_1_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_SLOW }, - { "medium", NvidiaOpticalFlow_1_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_MEDIUM }, - { "fast", NvidiaOpticalFlow_1_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_FAST } }; + std::unordered_map presetMap = { + { "slow", NvidiaOpticalFlow_2_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_SLOW }, + { "medium", NvidiaOpticalFlow_2_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_MEDIUM }, + { "fast", NvidiaOpticalFlow_2_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_FAST } }; + + std::unordered_map outputGridSize = { + { 1, NvidiaOpticalFlow_2_0::NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE::NV_OF_OUTPUT_VECTOR_GRID_SIZE_1 }, + { 2, NvidiaOpticalFlow_2_0::NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE::NV_OF_OUTPUT_VECTOR_GRID_SIZE_2 }, + { 4, NvidiaOpticalFlow_2_0::NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE::NV_OF_OUTPUT_VECTOR_GRID_SIZE_4 } }; + + std::unordered_map hintGridSize = { + { 1, NvidiaOpticalFlow_2_0::NVIDIA_OF_HINT_VECTOR_GRID_SIZE::NV_OF_HINT_VECTOR_GRID_SIZE_1 }, + { 2, NvidiaOpticalFlow_2_0::NVIDIA_OF_HINT_VECTOR_GRID_SIZE::NV_OF_HINT_VECTOR_GRID_SIZE_2 }, + { 4, NvidiaOpticalFlow_2_0::NVIDIA_OF_HINT_VECTOR_GRID_SIZE::NV_OF_HINT_VECTOR_GRID_SIZE_4 }, + { 8, NvidiaOpticalFlow_2_0::NVIDIA_OF_HINT_VECTOR_GRID_SIZE::NV_OF_HINT_VECTOR_GRID_SIZE_8 } }; try { @@ -145,7 +221,10 @@ int main(int argc, char **argv) "{ r right | ../data/basketball2.png | specify right image }" "{ g gpuid | 0 | cuda device index}" "{ p preset | slow | perf preset for OF algo [ options : slow, medium, fast ]}" + "{ og outputGridSize | 1 | Output grid size of OF vector [ options : 1, 2, 4 ]}" + "{ hg hintGridSize | 1 | Hint grid size of OF vector [ options : 1, 2, 4, 8 ]}" "{ o output | OpenCVNvOF.flo | output flow vector file in middlebury format}" + "{ rc roiConfigFile | | Region of Interest config file }" "{ th enableTemporalHints | false | Enable temporal hints}" "{ eh enableExternalHints | false | Enable external hints}" "{ cb enableCostBuffer | false | Enable output cost buffer}" @@ -159,60 +238,93 @@ int main(int argc, char **argv) return 0; } - string pathL = cmd.get("left"); - string pathR = cmd.get("right"); - string preset = cmd.get("preset"); - string output = cmd.get("output"); + std::string pathL = cmd.get("left"); + std::string pathR = cmd.get("right"); + std::string preset = cmd.get("preset"); + std::string output = cmd.get("output"); + std::string roiConfiFile = cmd.get("roiConfigFile"); bool enableExternalHints = cmd.get("enableExternalHints"); bool enableTemporalHints = cmd.get("enableTemporalHints"); bool enableCostBuffer = cmd.get("enableCostBuffer"); int gpuId = cmd.get("gpuid"); + int outputBufferGridSize = cmd.get("outputGridSize"); + int hintBufferGridSize = cmd.get("hintGridSize"); - if (pathL.empty()) cout << "Specify left image path\n"; - if (pathR.empty()) cout << "Specify right image path\n"; - if (preset.empty()) cout << "Specify perf preset for OpticalFlow algo\n"; + if (pathL.empty()) std::cout << "Specify left image path" << std::endl; + if (pathR.empty()) std::cout << "Specify right image path" << std::endl; + if (preset.empty()) std::cout << "Specify perf preset for OpticalFlow algo" << std::endl; if (pathL.empty() || pathR.empty()) return 0; - auto search = presetMap.find(preset); - if (search == presetMap.end()) + auto p = presetMap.find(preset); + if (p == presetMap.end()) { std::cout << "Invalid preset level : " << preset << std::endl; return 0; } - NvidiaOpticalFlow_1_0::NVIDIA_OF_PERF_LEVEL perfPreset = search->second; + NvidiaOpticalFlow_2_0::NVIDIA_OF_PERF_LEVEL perfPreset = p->second; + + auto o = outputGridSize.find(outputBufferGridSize); + if (o == outputGridSize.end()) + { + std::cout << "Invalid output grid size: " << outputBufferGridSize << std::endl; + return 0; + } + NvidiaOpticalFlow_2_0::NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE outBufGridSize = o->second; + + NvidiaOpticalFlow_2_0::NVIDIA_OF_HINT_VECTOR_GRID_SIZE hintBufGridSize = + NvidiaOpticalFlow_2_0::NV_OF_HINT_VECTOR_GRID_SIZE_UNDEFINED; + if (enableExternalHints) + { + auto h = hintGridSize.find(hintBufferGridSize); + if (h == hintGridSize.end()) + { + std::cout << "Invalid hint grid size: " << hintBufferGridSize << std::endl; + return 0; + } + hintBufGridSize = h->second; + } + + std::vector roiData; + + if (!roiConfiFile.empty()) + { + if (!parseROI(roiConfiFile, roiData)) + { + std::cout << "Wrong Region of Interest config file, proceeding without ROI" << std::endl; + } + } Mat frameL = imread(pathL, IMREAD_GRAYSCALE); Mat frameR = imread(pathR, IMREAD_GRAYSCALE); - if (frameL.empty()) cout << "Can't open '" << pathL << "'\n"; - if (frameR.empty()) cout << "Can't open '" << pathR << "'\n"; + if (frameL.empty()) std::cout << "Can't open '" << pathL << "'" << std::endl; + if (frameR.empty()) std::cout << "Can't open '" << pathR << "'" << std::endl; if (frameL.empty() || frameR.empty()) return -1; - Ptr nvof = NvidiaOpticalFlow_1_0::create( - frameL.size().width, frameL.size().height, perfPreset, + Ptr nvof = NvidiaOpticalFlow_2_0::create( + frameL.size(), roiData, perfPreset, outBufGridSize, hintBufGridSize, enableTemporalHints, enableExternalHints, enableCostBuffer, gpuId); - Mat flowx, flowy, flowxy, upsampledFlowXY, image; + Mat flowx, flowy, flowxy, floatFlow, image; nvof->calc(frameL, frameR, flowxy); - nvof->upSampler(flowxy, frameL.size().width, frameL.size().height, - nvof->getGridSize(), upsampledFlowXY); + nvof->convertToFloat(flowxy, floatFlow); - if (output.size() != 0) + if (!output.empty()) { - if (!writeOpticalFlow(output, upsampledFlowXY)) - cout << "Failed to save Flow Vector" << endl; + if (!writeOpticalFlow(output, floatFlow)) + std::cout << "Failed to save Flow Vector" << std::endl; else - cout << "Flow vector saved as '" << output << "'\n"; + std::cout << "Flow vector saved as '" << output << "'" << std::endl; } Mat planes[] = { flowx, flowy }; - split(upsampledFlowXY, planes); + split(floatFlow, planes); flowx = planes[0]; flowy = planes[1]; drawOpticalFlow(flowx, flowy, image, 10); - imshow("Colorize image",image); + imshow("Colorize image", image); waitKey(0); nvof->collectGarbage(); } diff --git a/modules/cudaoptflow/samples/optical_flow.cpp b/modules/cudaoptflow/samples/optical_flow.cpp index c1353614e4d..4afd2dc0a76 100644 --- a/modules/cudaoptflow/samples/optical_flow.cpp +++ b/modules/cudaoptflow/samples/optical_flow.cpp @@ -183,8 +183,11 @@ int main(int argc, const char* argv[]) Ptr lk = cuda::DensePyrLKOpticalFlow::create(Size(7, 7)); Ptr farn = cuda::FarnebackOpticalFlow::create(); Ptr tvl1 = cuda::OpticalFlowDual_TVL1::create(); - Ptr nvof = cuda::NvidiaOpticalFlow_1_0::create(frame0.size().width, frame0.size().height, + Ptr nvof_1_0 = cuda::NvidiaOpticalFlow_1_0::create(frame0.size(), NvidiaOpticalFlow_1_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_FAST, false, false, false, 0, inputStream, outputStream); + Ptr nvof_2_0 = cuda::NvidiaOpticalFlow_2_0::create(frame0.size(), + NvidiaOpticalFlow_2_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_FAST, NvidiaOpticalFlow_2_0::NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE::NV_OF_OUTPUT_VECTOR_GRID_SIZE_1, + NvidiaOpticalFlow_2_0::NVIDIA_OF_HINT_VECTOR_GRID_SIZE::NV_OF_HINT_VECTOR_GRID_SIZE_UNDEFINED, false, false, false, 0, inputStream, outputStream); { GpuMat d_frame0f; @@ -242,16 +245,32 @@ int main(int argc, const char* argv[]) //Hence it is expected to be more than what is displayed in the NVIDIA Optical Flow SDK documentation. const int64 start = getTickCount(); - nvof->calc(d_frame0, d_frame1, d_flowxy); + nvof_1_0->calc(d_frame0, d_frame1, d_flowxy); const double timeSec = (getTickCount() - start) / getTickFrequency(); - cout << "NVIDIAOpticalFlow : " << timeSec << " sec" << endl; + cout << "NVIDIAOpticalFlow_1_0 : " << timeSec << " sec" << endl; - nvof->upSampler(d_flowxy, frame0.size().width, frame0.size().height, - nvof->getGridSize(), d_flow); + nvof_1_0->upSampler(d_flowxy, frame0.size(), nvof_1_0->getGridSize(), d_flow); - showFlow("NVIDIAOpticalFlow", d_flow); - nvof->collectGarbage(); + showFlow("NVIDIAOpticalFlow_1_0", d_flow); + nvof_1_0->collectGarbage(); + } + + { + //The timing displayed below includes the time taken to copy the input buffers to the OF CUDA input buffers + //and to copy the output buffers from the OF CUDA output buffer to the output buffer. + //Hence it is expected to be more than what is displayed in the NVIDIA Optical Flow SDK documentation. + const int64 start = getTickCount(); + + nvof_2_0->calc(d_frame0, d_frame1, d_flowxy); + + const double timeSec = (getTickCount() - start) / getTickFrequency(); + cout << "NVIDIAOpticalFlow_2_0 : " << timeSec << " sec" << endl; + + nvof_2_0->convertToFloat(d_flowxy, d_flow); + + showFlow("NVIDIAOpticalFlow_2_0", d_flow); + nvof_2_0->collectGarbage(); } imshow("Frame 0", frame0); diff --git a/modules/cudaoptflow/src/cuda/nvidiaOpticalFlow.cu b/modules/cudaoptflow/src/cuda/nvidiaOpticalFlow.cu new file mode 100644 index 00000000000..1ad21143e70 --- /dev/null +++ b/modules/cudaoptflow/src/cuda/nvidiaOpticalFlow.cu @@ -0,0 +1,99 @@ +// +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +//M*/ +#if !defined CUDA_DISABLER + +#include +#include + +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +typedef signed short int16_t; +typedef signed int int32_t; + +#define BLOCKDIM_X 32 +#define BLOCKDIM_Y 16 + +// data required to do 2x upsampling. Same can be used for 4x upsampling also +#define SMEM_COLS ((BLOCKDIM_X)/2) +#define SMEM_ROWS ((BLOCKDIM_Y)/2) + +namespace cv { namespace cuda { namespace device { namespace optflow_nvidia +{ +static const char *_cudaGetErrorEnum(cudaError_t error) { return cudaGetErrorName(error); } + +template +void check(T result, char const *const func, const char *const file, + int const line) { + if (result) { + fprintf(stderr, "CUDA error at %s:%d code=%d(%s) \"%s\" \n", file, line, + static_cast(result), _cudaGetErrorEnum(result), func); + // Make sure we call CUDA Device Reset before exiting + exit(EXIT_FAILURE); + } +} +#define checkCudaErrors(val) check((val), #val, __FILE__, __LINE__) + +template +static __device__ void ReadDevPtrData(void* devptr, uint32_t x0, uint32_t y0, uint32_t src_w, uint32_t src_h, uint32_t src_pitch, + T src[][SMEM_COLS], uint32_t i, uint32_t j) +{ + uint32_t shift = (sizeof(T) == sizeof(int32_t)) ? 2 : 1; + src[j][i] = *(T*)((uint8_t*)devptr + y0 * src_pitch + (x0 << shift)); +} + + +extern "C" +__global__ void NearestNeighborFlowKernel(cudaSurfaceObject_t srcSurf, void* srcDevPtr, uint32_t src_w, uint32_t src_pitch, uint32_t src_h, + cudaSurfaceObject_t dstSurf, void* dstDevPtr, uint32_t dst_w, uint32_t dst_pitch, uint32_t dst_h, + uint32_t nScaleFactor) +{ + int x = blockDim.x * blockIdx.x + threadIdx.x; + int y = blockDim.y * blockIdx.y + threadIdx.y; + + int x0 = x / nScaleFactor; + int y0 = y / nScaleFactor; + + __shared__ short2 src[SMEM_ROWS][SMEM_COLS]; + + int i = threadIdx.x / nScaleFactor; + int j = threadIdx.y / nScaleFactor; + + if ((x % nScaleFactor == 0) && (y % nScaleFactor == 0)) + { + ReadDevPtrData(srcDevPtr, x0, y0, src_w, src_h, src_pitch, src, i, j); + } + __syncthreads(); + + if (x < dst_w && y < dst_h) + { + if (dstDevPtr == NULL) + { + surf2Dwrite(src[j][i], dstSurf, x * sizeof(short2), y, cudaBoundaryModeClamp); + } + else + { + *(short2*)((uint8_t*)dstDevPtr + y * dst_pitch + (x << 2)) = src[j][i]; + } + } +} + +void FlowUpsample(void* srcDevPtr, uint32_t nSrcWidth, uint32_t nSrcPitch, uint32_t nSrcHeight, + void* dstDevPtr, uint32_t nDstWidth, uint32_t nDstPitch, uint32_t nDstHeight, + uint32_t nScaleFactor) +{ + + dim3 blockDim(BLOCKDIM_X, BLOCKDIM_Y); + dim3 gridDim((nDstWidth + blockDim.x - 1) / blockDim.x, (nDstHeight + blockDim.y - 1) / blockDim.y); + NearestNeighborFlowKernel << > > (0, srcDevPtr, nSrcWidth, nSrcPitch, nSrcHeight, + 0, dstDevPtr, nDstWidth, nDstPitch, nDstHeight, + nScaleFactor); + + checkCudaErrors(cudaGetLastError()); +}}}}} + +#endif \ No newline at end of file diff --git a/modules/cudaoptflow/src/nvidiaOpticalFlow.cpp b/modules/cudaoptflow/src/nvidiaOpticalFlow.cpp index b5c760da23d..df715698933 100644 --- a/modules/cudaoptflow/src/nvidiaOpticalFlow.cpp +++ b/modules/cudaoptflow/src/nvidiaOpticalFlow.cpp @@ -8,11 +8,27 @@ #if !defined HAVE_CUDA || defined(CUDA_DISABLER) -cv::Ptr cv::cuda::NvidiaOpticalFlow_1_0::create(int, int, NVIDIA_OF_PERF_LEVEL, bool, bool, bool, int, Stream&, Stream&) { throw_no_cuda(); return cv::Ptr(); } +cv::Ptr cv::cuda::NvidiaOpticalFlow_1_0::create + (int, int, NVIDIA_OF_PERF_LEVEL, bool, bool, bool, int, Stream&, Stream&) { + throw_no_cuda(); return cv::Ptr(); } + +cv::Ptr cv::cuda::NvidiaOpticalFlow_2_0::create( + int, int, NVIDIA_OF_PERF_LEVEL, NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE, NVIDIA_OF_HINT_VECTOR_GRID_SIZE, + bool, int, cv::Rect*, bool, bool, bool, int, Stream&, Stream&) { + throw_no_cuda(); return cv::Ptr(); +} #elif !defined HAVE_NVIDIA_OPTFLOW -cv::Ptr cv::cuda::NvidiaOpticalFlow_1_0::create(int, int, NVIDIA_OF_PERF_LEVEL, bool, bool, bool, int, Stream&, Stream&) +cv::Ptr cv::cuda::NvidiaOpticalFlow_1_0::create( + int, int, NVIDIA_OF_PERF_LEVEL, bool, bool, bool, int, Stream&, Stream&) +{ + CV_Error(cv::Error::HeaderIsNull, "OpenCV was build without NVIDIA OpticalFlow support"); +} + +cv::Ptr cv::cuda::NvidiaOpticalFlow_2_0::create( + int, int, NVIDIA_OF_PERF_LEVEL, NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE, NVIDIA_OF_HINT_VECTOR_GRID_SIZE, + bool, int, cv::Rect*, bool, bool, bool, int, , Stream&, Stream&) { CV_Error(cv::Error::HeaderIsNull, "OpenCV was build without NVIDIA OpticalFlow support"); } @@ -22,6 +38,13 @@ cv::Ptr cv::cuda::NvidiaOpticalFlow_1_0::create #include "nvOpticalFlowCommon.h" #include "nvOpticalFlowCuda.h" +namespace cv { namespace cuda { namespace device { namespace optflow_nvidia +{ +void FlowUpsample(void* srcDevPtr, uint32_t nSrcWidth, uint32_t nSrcPitch, uint32_t nSrcHeight, + void* dstDevPtr, uint32_t nDstWidth, uint32_t nDstPitch, uint32_t nDstHeight, + uint32_t nScaleFactor); +}}}} + #if defined(_WIN32) || defined(_WIN64) #include #else @@ -123,7 +146,7 @@ class LoadNvidiaModules private: typedef int(*PFNCudaCuCtxGetCurrent)(CUcontext*); typedef NV_OF_STATUS(NVOFAPI *PFNNvOFAPICreateInstanceCuda) - (uint32_t apiVer, NV_OF_CUDA_API_FUNCTION_LIST* cudaOf); + (int apiVer, NV_OF_CUDA_API_FUNCTION_LIST* cudaOf); PFNCudaCuCtxGetCurrent m_cudaDriverAPIGetCurrentCtx; PFNNvOFAPICreateInstanceCuda m_NvOFAPICreateInstanceCuda; @@ -249,9 +272,9 @@ class NvidiaOpticalFlowImpl : public cv::cuda::NvidiaOpticalFlow_1_0 NV_OF_BUFFER_DESCRIPTOR m_hintBufferDesc; NV_OF_BUFFER_DESCRIPTOR m_costBufferDesc; - uint32_t m_outputElementSize; - uint32_t m_costBufElementSize; - uint32_t m_hintBufElementSize; + int m_outputElementSize; + int m_costBufElementSize; + int m_hintBufElementSize; NV_OF_INIT_PARAMS m_initParams; @@ -288,7 +311,7 @@ class NvidiaOpticalFlowImpl : public cv::cuda::NvidiaOpticalFlow_1_0 std::mutex m_lock; public: - NvidiaOpticalFlowImpl(int width, int height, NV_OF_PERF_LEVEL perfPreset, bool bEnableTemporalHints, + NvidiaOpticalFlowImpl(cv::Size imageSize, NV_OF_PERF_LEVEL perfPreset, bool bEnableTemporalHints, bool bEnableExternalHints, bool bEnableCostBuffer, int gpuId, Stream inputStream, Stream outputStream); virtual void calc(InputArray inputImage, InputArray referenceImage, @@ -298,17 +321,17 @@ class NvidiaOpticalFlowImpl : public cv::cuda::NvidiaOpticalFlow_1_0 virtual void collectGarbage(); - virtual void upSampler(InputArray flow, int width, int height, + virtual void upSampler(InputArray flow, cv::Size imageSize, int gridSize, InputOutputArray upsampledFlow); virtual int getGridSize() const { return m_gridSize; } }; NvidiaOpticalFlowImpl::NvidiaOpticalFlowImpl( - int width, int height, NV_OF_PERF_LEVEL perfPreset, bool bEnableTemporalHints, + cv::Size imageSize, NV_OF_PERF_LEVEL perfPreset, bool bEnableTemporalHints, bool bEnableExternalHints, bool bEnableCostBuffer, int gpuId, Stream inputStream, Stream outputStream) : - m_width(width), m_height(height), m_preset(perfPreset), + m_width(imageSize.width), m_height(imageSize.height), m_preset(perfPreset), m_enableTemporalHints((NV_OF_BOOL)bEnableTemporalHints), m_enableExternalHints((NV_OF_BOOL)bEnableExternalHints), m_enableCostBuffer((NV_OF_BOOL)bEnableCostBuffer), m_gpuId(gpuId), @@ -371,7 +394,7 @@ NvidiaOpticalFlowImpl::NvidiaOpticalFlowImpl( m_costBufferDesc.height = nOutHeight; m_costBufferDesc.bufferFormat = NV_OF_BUFFER_FORMAT_UINT; m_costBufferDesc.bufferUsage = NV_OF_BUFFER_USAGE_COST; - m_costBufElementSize = sizeof(uint32_t); + m_costBufElementSize = sizeof(int); } m_ofAPI.reset(new NV_OF_CUDA_API_FUNCTION_LIST()); @@ -600,7 +623,7 @@ void NvidiaOpticalFlowImpl::collectGarbage() } } -void NvidiaOpticalFlowImpl::upSampler(InputArray _flow, int width, int height, +void NvidiaOpticalFlowImpl::upSampler(InputArray _flow, cv::Size imageSize, int gridSize, InputOutputArray upsampledFlow) { Mat flow; @@ -622,26 +645,26 @@ void NvidiaOpticalFlowImpl::upSampler(InputArray _flow, int width, int height, std::unique_ptr flowVectors = nullptr; const NV_OF_FLOW_VECTOR* _flowVectors = static_cast((const void*)flow.data); - flowVectors.reset(new float[2 * width * height]); - for (int y = 0; y < height; ++y) + flowVectors.reset(new float[2 * imageSize.width * imageSize.height]); + for (int y = 0; y < imageSize.height; ++y) { - for (int x = 0; x < width; ++x) + for (int x = 0; x < imageSize.width; ++x) { - uint32_t blockIdX = x / gridSize; - uint32_t blockIdY = y / gridSize; - uint32_t widthInBlocks = ((width + gridSize - 1) / gridSize); - uint32_t heightInBlocks = ((height + gridSize - 1) / gridSize);; + int blockIdX = x / gridSize; + int blockIdY = y / gridSize; + int widthInBlocks = ((imageSize.width + gridSize - 1) / gridSize); + int heightInBlocks = ((imageSize.height + gridSize - 1) / gridSize);; if ((blockIdX < widthInBlocks) && (blockIdY < heightInBlocks)) { - flowVectors[(y * 2 * width) + 2 * x] = (float) + flowVectors[(y * 2 * imageSize.width) + 2 * x] = (float) (_flowVectors[blockIdX + (blockIdY * widthInBlocks)].flowx / (float)(1 << 5)); - flowVectors[(y * 2 * width) + 2 * x + 1] = (float) + flowVectors[(y * 2 * imageSize.width) + 2 * x + 1] = (float) (_flowVectors[blockIdX + (blockIdY * widthInBlocks)].flowy / (float)(1 << 5)); } } } - Mat output(Size(width, height), CV_32FC2, flowVectors.get()); + Mat output(Size(imageSize.width, imageSize.height), CV_32FC2, flowVectors.get()); if (upsampledFlow.isMat()) { output.copyTo(upsampledFlow); @@ -656,18 +679,603 @@ void NvidiaOpticalFlowImpl::upSampler(InputArray _flow, int width, int height, CV_Error(Error::StsBadArg, "Incorrect flow buffer passed for upsampled flow. Pass either Mat or GpuMat"); } +} + +class NvidiaOpticalFlowImpl_2 : public cv::cuda::NvidiaOpticalFlow_2_0 +{ +private: + int m_width; + int m_height; + NV_OF_PERF_LEVEL m_preset; + NV_OF_OUTPUT_VECTOR_GRID_SIZE m_gridSize; + NV_OF_HINT_VECTOR_GRID_SIZE m_hintGridSize; + bool m_enableROI; + std::vector m_roiDataRect; + NV_OF_ROI_RECT* m_roiData; + bool m_enableTemporalHints; + bool m_enableExternalHints; + bool m_enableCostBuffer; + int m_gpuId; + Stream m_inputStream; + Stream m_outputStream; + + CUcontext m_cuContext; + int m_scaleFactor; + NV_OF_BUFFER_FORMAT m_format; + NV_OF_OUTPUT_VECTOR_GRID_SIZE m_hwGridSize; + + NV_OF_BUFFER_DESCRIPTOR m_inputBufferDesc; + NV_OF_BUFFER_DESCRIPTOR m_outputBufferDesc; + NV_OF_BUFFER_DESCRIPTOR m_hintBufferDesc; + NV_OF_BUFFER_DESCRIPTOR m_costBufferDesc; + + int m_outputElementSize; + int m_costBufElementSize; + int m_hintBufElementSize; + + NV_OF_INIT_PARAMS m_initParams; + + std::unique_ptr m_ofAPI; + NvOFHandle m_hOF; //nvof handle + + NvOFGPUBufferHandle m_hInputBuffer; + NvOFGPUBufferHandle m_hReferenceBuffer; + NvOFGPUBufferHandle m_hOutputBuffer; + NvOFGPUBufferHandle m_hOutputUpScaledBuffer; + NvOFGPUBufferHandle m_hHintBuffer; + NvOFGPUBufferHandle m_hCostBuffer; + + CUdeviceptr m_frame0cuDevPtr; + CUdeviceptr m_frame1cuDevPtr; + CUdeviceptr m_flowXYcuDevPtr; + CUdeviceptr m_flowXYUpScaledcuDevPtr; + CUdeviceptr m_hintcuDevPtr; + CUdeviceptr m_costcuDevPtr; + + NV_OF_CUDA_BUFFER_STRIDE_INFO m_inputBufferStrideInfo; + NV_OF_CUDA_BUFFER_STRIDE_INFO m_referenceBufferStrideInfo; + NV_OF_CUDA_BUFFER_STRIDE_INFO m_outputBufferStrideInfo; + NV_OF_CUDA_BUFFER_STRIDE_INFO m_outputUpScaledBufferStrideInfo; + NV_OF_CUDA_BUFFER_STRIDE_INFO m_hintBufferStrideInfo; + NV_OF_CUDA_BUFFER_STRIDE_INFO m_costBufferStrideInfo; + + NV_OF_CUDA_API_FUNCTION_LIST* GetAPI() + { + std::lock_guard lock(m_lock); + return m_ofAPI.get(); + } + + NvOFHandle GetHandle() { return m_hOF; } + +protected: + std::mutex m_lock; + +public: + NvidiaOpticalFlowImpl_2(cv::Size imageSize, NV_OF_PERF_LEVEL perfPreset, + NV_OF_OUTPUT_VECTOR_GRID_SIZE outputGridSize, NV_OF_HINT_VECTOR_GRID_SIZE hintGridSize, + bool bEnableROI, std::vector roiData, bool bEnableTemporalHints, + bool bEnableExternalHints, bool bEnableCostBuffer, int gpuId, Stream inputStream, Stream outputStream); + + virtual void calc(InputArray inputImage, InputArray referenceImage, + InputOutputArray flow, Stream& stream = Stream::Null(), + InputArray hint = cv::noArray(), OutputArray cost = cv::noArray()); + + virtual void collectGarbage(); + + virtual void convertToFloat(InputArray flow, InputOutputArray floatFlow); + + virtual int getGridSize() const { return m_gridSize; } +}; + +NvidiaOpticalFlowImpl_2::NvidiaOpticalFlowImpl_2( + cv::Size imageSize, NV_OF_PERF_LEVEL perfPreset, + NV_OF_OUTPUT_VECTOR_GRID_SIZE outputGridSize, NV_OF_HINT_VECTOR_GRID_SIZE hintGridSize, + bool bEnableROI, std::vector roiData, bool bEnableTemporalHints, + bool bEnableExternalHints, bool bEnableCostBuffer, int gpuId, Stream inputStream, Stream outputStream) : + m_width(imageSize.width), m_height(imageSize.height), m_preset(perfPreset), + m_gridSize(outputGridSize), m_hintGridSize(hintGridSize), + m_enableROI(bEnableROI), m_roiDataRect(roiData), + m_enableTemporalHints((NV_OF_BOOL)bEnableTemporalHints), + m_enableExternalHints((NV_OF_BOOL)bEnableExternalHints), + m_enableCostBuffer((NV_OF_BOOL)bEnableCostBuffer), m_gpuId(gpuId), + m_inputStream(inputStream), m_outputStream(outputStream), + m_cuContext(nullptr), m_scaleFactor(1), m_format(NV_OF_BUFFER_FORMAT_GRAYSCALE8), + m_hwGridSize((NV_OF_OUTPUT_VECTOR_GRID_SIZE)0) +{ + LoadNvidiaModules& LoadNvidiaModulesObj = LoadNvidiaModules::Init(); + + int nGpu = 0; + + cuSafeCall(cudaGetDeviceCount(&nGpu)); + if (m_gpuId < 0 || m_gpuId >= nGpu) + { + CV_Error(Error::StsBadArg, "Invalid GPU Ordinal"); + } + + cuSafeCall(cudaSetDevice(m_gpuId)); + cuSafeCall(cudaFree(m_cuContext)); + + cuSafeCall(LoadNvidiaModulesObj.GetCudaLibraryFunctionPtr()(&m_cuContext)); + + if (m_gridSize != (NV_OF_OUTPUT_VECTOR_GRID_SIZE)NV_OF_OUTPUT_VECTOR_GRID_SIZE_1 && + m_gridSize != (NV_OF_OUTPUT_VECTOR_GRID_SIZE)NV_OF_OUTPUT_VECTOR_GRID_SIZE_2 && + m_gridSize != (NV_OF_OUTPUT_VECTOR_GRID_SIZE)NV_OF_OUTPUT_VECTOR_GRID_SIZE_4) + { + CV_Error(Error::StsBadArg, "Unsupported output grid size"); + } + + if (m_enableExternalHints) + { + if (m_hintGridSize != (NV_OF_HINT_VECTOR_GRID_SIZE)NV_OF_HINT_VECTOR_GRID_SIZE_1 && + m_hintGridSize != (NV_OF_HINT_VECTOR_GRID_SIZE)NV_OF_HINT_VECTOR_GRID_SIZE_2 && + m_hintGridSize != (NV_OF_HINT_VECTOR_GRID_SIZE)NV_OF_HINT_VECTOR_GRID_SIZE_4 && + m_hintGridSize != (NV_OF_HINT_VECTOR_GRID_SIZE)NV_OF_HINT_VECTOR_GRID_SIZE_8) + { + CV_Error(Error::StsBadArg, "Unsupported hint grid size"); + } + } + + m_ofAPI.reset(new NV_OF_CUDA_API_FUNCTION_LIST()); + + NVOF_API_CALL(LoadNvidiaModulesObj.GetOFLibraryFunctionPtr()(NV_OF_API_VERSION, m_ofAPI.get())); + NVOF_API_CALL(GetAPI()->nvCreateOpticalFlowCuda(m_cuContext, &m_hOF)); + + m_roiData = (NV_OF_ROI_RECT*)m_roiDataRect.data(); + + uint32_t size = 0; + if (m_enableROI) + { + NVOF_API_CALL(GetAPI()->nvOFGetCaps(GetHandle(), NV_OF_CAPS_SUPPORT_ROI, nullptr, &size)); + std::unique_ptr val1(new uint32_t[size]); + NVOF_API_CALL(GetAPI()->nvOFGetCaps(GetHandle(), NV_OF_CAPS_SUPPORT_ROI, val1.get(), &size)); + if (val1[0] != NV_OF_TRUE) + { + m_enableROI = false; + m_roiData = nullptr; + CV_Error(Error::StsBadFunc, "ROI not supported on this GPU"); + } + } + size = 0; + NVOF_API_CALL(GetAPI()->nvOFGetCaps(GetHandle(), NV_OF_CAPS_SUPPORTED_OUTPUT_GRID_SIZES, nullptr, &size)); + std::unique_ptr val2(new uint32_t[size]); + NVOF_API_CALL(GetAPI()->nvOFGetCaps(GetHandle(), NV_OF_CAPS_SUPPORTED_OUTPUT_GRID_SIZES, val2.get(), &size)); + for (uint32_t i = 0; i < size; i++) + { + if (m_gridSize != val2[i]) + { + size = 0; + NVOF_API_CALL(GetAPI()->nvOFGetCaps(GetHandle(), NV_OF_CAPS_SUPPORTED_OUTPUT_GRID_SIZES, nullptr, &size)); + std::unique_ptr val3(new uint32_t[size]); + NVOF_API_CALL(GetAPI()->nvOFGetCaps(GetHandle(), NV_OF_CAPS_SUPPORTED_OUTPUT_GRID_SIZES, val3.get(), &size)); + + m_hwGridSize = (NV_OF_OUTPUT_VECTOR_GRID_SIZE)NV_OF_OUTPUT_VECTOR_GRID_SIZE_MAX; + for (uint32_t i = 0; i < size; i++) + { + if (m_gridSize == val3[i]) + { + m_hwGridSize = m_gridSize; + break; + } + if (m_gridSize < val3[i] && val3[i] < m_hwGridSize) + { + m_hwGridSize = (NV_OF_OUTPUT_VECTOR_GRID_SIZE)val3[i]; + } + } + if (m_hwGridSize >= (NV_OF_OUTPUT_VECTOR_GRID_SIZE)NV_OF_OUTPUT_VECTOR_GRID_SIZE_MAX) + { + CV_Error(Error::StsBadArg, "Invalid Grid Size"); + } + else + { + m_scaleFactor = m_hwGridSize / m_gridSize; + } + } + else + { + m_hwGridSize = m_gridSize; + } + } + + auto nOutWidth = (m_width + m_hwGridSize - 1) / m_hwGridSize; + auto nOutHeight = (m_height + m_hwGridSize - 1) / m_hwGridSize; + + auto outBufFmt = NV_OF_BUFFER_FORMAT_SHORT2; + + memset(&m_inputBufferDesc, 0, sizeof(m_inputBufferDesc)); + m_inputBufferDesc.width = m_width; + m_inputBufferDesc.height = m_height; + m_inputBufferDesc.bufferFormat = m_format; + m_inputBufferDesc.bufferUsage = NV_OF_BUFFER_USAGE_INPUT; + + memset(&m_outputBufferDesc, 0, sizeof(m_outputBufferDesc)); + m_outputBufferDesc.width = nOutWidth; + m_outputBufferDesc.height = nOutHeight; + m_outputBufferDesc.bufferFormat = outBufFmt; + m_outputBufferDesc.bufferUsage = NV_OF_BUFFER_USAGE_OUTPUT; + m_outputElementSize = sizeof(NV_OF_FLOW_VECTOR); + + if (m_enableExternalHints) + { + memset(&m_hintBufferDesc, 0, sizeof(m_hintBufferDesc)); + m_hintBufferDesc.width = nOutWidth; + m_hintBufferDesc.height = nOutHeight; + m_hintBufferDesc.bufferFormat = outBufFmt; + m_hintBufferDesc.bufferUsage = NV_OF_BUFFER_USAGE_HINT; + m_hintBufElementSize = m_outputElementSize; + } + + if (m_enableCostBuffer) + { + memset(&m_costBufferDesc, 0, sizeof(m_costBufferDesc)); + m_costBufferDesc.width = nOutWidth; + m_costBufferDesc.height = nOutHeight; + m_costBufferDesc.bufferFormat = NV_OF_BUFFER_FORMAT_UINT8; + m_costBufferDesc.bufferUsage = NV_OF_BUFFER_USAGE_COST; + m_costBufElementSize = sizeof(int); + } + + memset(&m_initParams, 0, sizeof(m_initParams)); + m_initParams.width = m_inputBufferDesc.width; + m_initParams.height = m_inputBufferDesc.height; + m_initParams.enableExternalHints = (NV_OF_BOOL)m_enableExternalHints; + m_initParams.enableOutputCost = (NV_OF_BOOL)m_enableCostBuffer; + m_initParams.hintGridSize = (NV_OF_BOOL)m_enableExternalHints == NV_OF_TRUE ? + m_hintGridSize : (NV_OF_HINT_VECTOR_GRID_SIZE)NV_OF_HINT_VECTOR_GRID_SIZE_UNDEFINED; + m_initParams.outGridSize = (NV_OF_OUTPUT_VECTOR_GRID_SIZE)m_hwGridSize; + m_initParams.mode = NV_OF_MODE_OPTICALFLOW; + m_initParams.perfLevel = m_preset; + m_initParams.enableRoi = (NV_OF_BOOL)m_enableROI; + + NVOF_API_CALL(GetAPI()->nvOFInit(GetHandle(), &m_initParams)); + + if (m_inputStream || m_outputStream) + { + NVOF_API_CALL(GetAPI()->nvOFSetIOCudaStreams(GetHandle(), + StreamAccessor::getStream(m_inputStream), StreamAccessor::getStream(m_outputStream))); + } + + //Input Buffer 1 + NVOF_API_CALL(GetAPI()->nvOFCreateGPUBufferCuda(GetHandle(), + &m_inputBufferDesc, NV_OF_CUDA_BUFFER_TYPE_CUDEVICEPTR, &m_hInputBuffer)); + m_frame0cuDevPtr = GetAPI()->nvOFGPUBufferGetCUdeviceptr(m_hInputBuffer); + NVOF_API_CALL(GetAPI()->nvOFGPUBufferGetStrideInfo( + m_hInputBuffer, &m_inputBufferStrideInfo)); + + //Input Buffer 2 + NVOF_API_CALL(GetAPI()->nvOFCreateGPUBufferCuda(GetHandle(), + &m_inputBufferDesc, NV_OF_CUDA_BUFFER_TYPE_CUDEVICEPTR, &m_hReferenceBuffer)); + m_frame1cuDevPtr = GetAPI()->nvOFGPUBufferGetCUdeviceptr(m_hReferenceBuffer); + NVOF_API_CALL(GetAPI()->nvOFGPUBufferGetStrideInfo( + m_hReferenceBuffer, &m_referenceBufferStrideInfo)); + + //Output Buffer + NVOF_API_CALL(GetAPI()->nvOFCreateGPUBufferCuda(GetHandle(), + &m_outputBufferDesc, NV_OF_CUDA_BUFFER_TYPE_CUDEVICEPTR, &m_hOutputBuffer)); + m_flowXYcuDevPtr = GetAPI()->nvOFGPUBufferGetCUdeviceptr(m_hOutputBuffer); + NVOF_API_CALL(GetAPI()->nvOFGPUBufferGetStrideInfo( + m_hOutputBuffer, &m_outputBufferStrideInfo)); + + if (m_scaleFactor > 1) + { + m_outputBufferDesc.width = (m_width + m_gridSize - 1) / m_gridSize;; + m_outputBufferDesc.height = (m_height + m_gridSize - 1) / m_gridSize;; + + //Output UpScaled Buffer + NVOF_API_CALL(GetAPI()->nvOFCreateGPUBufferCuda(GetHandle(), + &m_outputBufferDesc, NV_OF_CUDA_BUFFER_TYPE_CUDEVICEPTR, &m_hOutputUpScaledBuffer)); + m_flowXYUpScaledcuDevPtr = GetAPI()->nvOFGPUBufferGetCUdeviceptr(m_hOutputUpScaledBuffer); + NVOF_API_CALL(GetAPI()->nvOFGPUBufferGetStrideInfo( + m_hOutputUpScaledBuffer, &m_outputUpScaledBufferStrideInfo)); + } + + //Hint Buffer + if (m_enableExternalHints) + { + NVOF_API_CALL(GetAPI()->nvOFCreateGPUBufferCuda(GetHandle(), + &m_hintBufferDesc, NV_OF_CUDA_BUFFER_TYPE_CUDEVICEPTR, &m_hHintBuffer)); + m_hintcuDevPtr = GetAPI()->nvOFGPUBufferGetCUdeviceptr(m_hHintBuffer); + NVOF_API_CALL(GetAPI()->nvOFGPUBufferGetStrideInfo( + m_hHintBuffer, &m_hintBufferStrideInfo)); + } + + //Cost Buffer + if (m_enableCostBuffer) + { + NVOF_API_CALL(GetAPI()->nvOFCreateGPUBufferCuda(GetHandle(), + &m_costBufferDesc, NV_OF_CUDA_BUFFER_TYPE_CUDEVICEPTR, &m_hCostBuffer)); + m_costcuDevPtr = GetAPI()->nvOFGPUBufferGetCUdeviceptr(m_hCostBuffer); + NVOF_API_CALL(GetAPI()->nvOFGPUBufferGetStrideInfo( + m_hCostBuffer, &m_costBufferStrideInfo)); + } +} + +void NvidiaOpticalFlowImpl_2::calc(InputArray _frame0, InputArray _frame1, InputOutputArray _flow, + Stream& stream, InputArray hint, OutputArray cost) +{ + CV_UNUSED(stream); + GpuMat frame0GpuMat(_frame0.size(), _frame0.type(), (void*)m_frame0cuDevPtr, + m_inputBufferStrideInfo.strideInfo[0].strideXInBytes); + GpuMat frame1GpuMat(_frame1.size(), _frame1.type(), (void*)m_frame1cuDevPtr, + m_referenceBufferStrideInfo.strideInfo[0].strideXInBytes); + GpuMat flowXYGpuMat(Size((m_width + m_hwGridSize - 1) / m_hwGridSize, + (m_height + m_hwGridSize - 1) / m_hwGridSize), CV_16SC2, + (void*)m_flowXYcuDevPtr, m_outputBufferStrideInfo.strideInfo[0].strideXInBytes); + GpuMat flowXYGpuMatUpScaled(Size((m_width + m_gridSize - 1) / m_gridSize, + (m_height + m_gridSize - 1) / m_gridSize), CV_16SC2, + (void*)m_flowXYUpScaledcuDevPtr, m_outputUpScaledBufferStrideInfo.strideInfo[0].strideXInBytes); + + //check whether frame0 is Mat or GpuMat + if (_frame0.isMat()) + { + //Get Mats from InputArrays + Mat __frame0 = _frame0.getMat(); + frame0GpuMat.upload(__frame0, m_inputStream); + } + else if (_frame0.isGpuMat()) + { + //Get GpuMats from InputArrays + GpuMat __frame0 = _frame0.getGpuMat(); + __frame0.copyTo(frame0GpuMat, m_inputStream); + } + else + { + CV_Error(Error::StsBadArg, + "Incorrect input. Pass input image (frame0) as Mat or GpuMat"); + } + + //check whether frame1 is Mat or GpuMat + if (_frame1.isMat()) + { + //Get Mats from InputArrays + Mat __frame1 = _frame1.getMat(); + frame1GpuMat.upload(__frame1, m_inputStream); + } + else if (_frame1.isGpuMat()) + { + //Get GpuMats from InputArrays + GpuMat __frame1 = _frame1.getGpuMat(); + __frame1.copyTo(frame1GpuMat, m_inputStream); + } + else + { + CV_Error(Error::StsBadArg, + "Incorrect input. Pass reference image (frame1) as Mat or GpuMat"); + } + + if (m_enableExternalHints) + { + GpuMat hintGpuMat(hint.size(), hint.type(), (void*)m_hintcuDevPtr, + m_hintBufferStrideInfo.strideInfo[0].strideXInBytes); + + if (hint.isMat()) + { + //Get Mat from InputArray hint + Mat _hint = hint.getMat(); + hintGpuMat.upload(_hint, m_inputStream); + } + else if (hint.isGpuMat()) + { + //Get GpuMat from InputArray hint + GpuMat _hint = hint.getGpuMat(); + _hint.copyTo(hintGpuMat, m_inputStream); + } + else + { + CV_Error(Error::StsBadArg, "Incorrect hint buffer passed. Pass Mat or GpuMat"); + } + } + + //Execute Call + NV_OF_EXECUTE_INPUT_PARAMS exeInParams; + NV_OF_EXECUTE_OUTPUT_PARAMS exeOutParams; + memset(&exeInParams, 0, sizeof(exeInParams)); + exeInParams.inputFrame = m_hInputBuffer; + exeInParams.referenceFrame = m_hReferenceBuffer; + exeInParams.disableTemporalHints = (NV_OF_BOOL)m_enableTemporalHints == NV_OF_TRUE ? + NV_OF_FALSE : NV_OF_TRUE; + exeInParams.externalHints = m_initParams.enableExternalHints == NV_OF_TRUE ? + m_hHintBuffer : nullptr; + exeInParams.numRois = m_initParams.enableRoi == NV_OF_TRUE ? m_roiDataRect.size() : 0; + exeInParams.roiData = m_initParams.enableRoi == NV_OF_TRUE ? m_roiData : nullptr; + memset(&exeOutParams, 0, sizeof(exeOutParams)); + exeOutParams.outputBuffer = m_hOutputBuffer; + exeOutParams.outputCostBuffer = m_initParams.enableOutputCost == NV_OF_TRUE ? + m_hCostBuffer : nullptr; + NVOF_API_CALL(GetAPI()->nvOFExecute(GetHandle(), &exeInParams, &exeOutParams)); + + if (m_scaleFactor > 1) + { + uint32_t nSrcWidth = flowXYGpuMat.size().width; + uint32_t nSrcHeight = flowXYGpuMat.size().height; + uint32_t nSrcPitch = m_outputBufferStrideInfo.strideInfo[0].strideXInBytes; + uint32_t nDstWidth = flowXYGpuMatUpScaled.size().width; + uint32_t nDstHeight = flowXYGpuMatUpScaled.size().height; + uint32_t nDstPitch = m_outputUpScaledBufferStrideInfo.strideInfo[0].strideXInBytes; + cv::cuda::device::optflow_nvidia::FlowUpsample((void*)m_flowXYcuDevPtr, nSrcWidth, nSrcPitch, + nSrcHeight, (void*)m_flowXYUpScaledcuDevPtr, nDstWidth, nDstPitch, nDstHeight, m_scaleFactor); + + if (_flow.isMat()) + flowXYGpuMatUpScaled.download(_flow, m_outputStream); + else if (_flow.isGpuMat()) + flowXYGpuMatUpScaled.copyTo(_flow, m_outputStream); + else + CV_Error(Error::StsBadArg, "Incorrect flow buffer passed. Pass Mat or GpuMat"); + } + else + { + if (_flow.isMat()) + flowXYGpuMat.download(_flow, m_outputStream); + else if (_flow.isGpuMat()) + flowXYGpuMat.copyTo(_flow, m_outputStream); + else + CV_Error(Error::StsBadArg, "Incorrect flow buffer passed. Pass Mat or GpuMat"); + } + + if (m_enableCostBuffer) + { + GpuMat costGpuMat(Size((m_width + m_hwGridSize - 1) / m_hwGridSize, + (m_height + m_hwGridSize - 1) / m_hwGridSize), CV_8SC1, (void*)m_costcuDevPtr, + m_costBufferStrideInfo.strideInfo[0].strideXInBytes); + + if (cost.isMat()) + costGpuMat.download(cost, m_outputStream); + else if (cost.isGpuMat()) + costGpuMat.copyTo(cost, m_outputStream); + else + CV_Error(Error::StsBadArg, "Incorrect cost buffer passed. Pass Mat or GpuMat"); + } + m_outputStream.waitForCompletion(); +} + +void NvidiaOpticalFlowImpl_2::collectGarbage() +{ + if (m_enableROI) + { + m_roiData = nullptr; + } + if (m_hInputBuffer) + { + NVOF_API_CALL(GetAPI()->nvOFDestroyGPUBufferCuda(m_hInputBuffer)); + } + if (m_hReferenceBuffer) + { + NVOF_API_CALL(GetAPI()->nvOFDestroyGPUBufferCuda(m_hReferenceBuffer)); + } + if (m_hOutputBuffer) + { + NVOF_API_CALL(GetAPI()->nvOFDestroyGPUBufferCuda(m_hOutputBuffer)); + } + if (m_scaleFactor > 1 && m_hOutputUpScaledBuffer) + { + NVOF_API_CALL(GetAPI()->nvOFDestroyGPUBufferCuda(m_hOutputUpScaledBuffer)); + } + if (m_enableExternalHints) + { + if (m_hHintBuffer) + { + NVOF_API_CALL(GetAPI()->nvOFDestroyGPUBufferCuda(m_hHintBuffer)); + } + } + if (m_enableCostBuffer) + { + if (m_hCostBuffer) + { + NVOF_API_CALL(GetAPI()->nvOFDestroyGPUBufferCuda(m_hCostBuffer)); + } + } + if (m_inputStream) + { + m_inputStream.waitForCompletion(); + } + if (m_outputStream) + { + m_outputStream.waitForCompletion(); + } + if (m_hOF) + { + NVOF_API_CALL(GetAPI()->nvOFDestroy(m_hOF)); + } +} + +void NvidiaOpticalFlowImpl_2::convertToFloat(InputArray _flow, InputOutputArray floatFlow) +{ + Mat flow; + if (_flow.isMat()) + { + Mat __flow = _flow.getMat(); + __flow.copyTo(flow); + } + else if (_flow.isGpuMat()) + { + GpuMat __flow = _flow.getGpuMat(); + __flow.download(flow); + } + else + { + CV_Error(Error::StsBadArg, + "Incorrect flow buffer passed. Pass either Mat or GpuMat"); + } + + int width = flow.size().width; + int height = flow.size().height; + + Mat output(Size(width, height), CV_32FC2); + for (int y = 0; y < height; ++y) + { + for (int x = 0; x < (int)(width * sizeof(int16_t)); ++x) + { + output.at(y, x) = (float)(flow.at(y, x) / (float)(1 << 5)); + } + } + + if (floatFlow.isMat()) + { + output.copyTo(floatFlow); + } + else if (floatFlow.isGpuMat()) + { + GpuMat _output(output); + _output.copyTo(floatFlow); + } + else + { + CV_Error(Error::StsBadArg, + "Incorrect flow buffer passed for upsampled flow. Pass either Mat or GpuMat"); + } }} Ptr cv::cuda::NvidiaOpticalFlow_1_0::create( - int width, int height, NVIDIA_OF_PERF_LEVEL perfPreset, + cv::Size imageSize, NVIDIA_OF_PERF_LEVEL perfPreset, bool bEnableTemporalHints, bool bEnableExternalHints, bool bEnableCostBuffer, int gpuId, Stream& inputStream, Stream& outputStream) { return makePtr( - width, - height, + imageSize, + (NV_OF_PERF_LEVEL)perfPreset, + bEnableTemporalHints, + bEnableExternalHints, + bEnableCostBuffer, + gpuId, + inputStream, + outputStream); +} + +Ptr cv::cuda::NvidiaOpticalFlow_2_0::create( + cv::Size imageSize, NVIDIA_OF_PERF_LEVEL perfPreset, + NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE outputGridSize, NVIDIA_OF_HINT_VECTOR_GRID_SIZE hintGridSize, + bool bEnableTemporalHints, bool bEnableExternalHints, bool bEnableCostBuffer, + int gpuId, Stream& inputStream, Stream& outputStream) +{ + std::vector roi(0); + return makePtr( + imageSize, + (NV_OF_PERF_LEVEL)perfPreset, + (NV_OF_OUTPUT_VECTOR_GRID_SIZE)outputGridSize, + (NV_OF_HINT_VECTOR_GRID_SIZE)hintGridSize, + false, + roi, + bEnableTemporalHints, + bEnableExternalHints, + bEnableCostBuffer, + gpuId, + inputStream, + outputStream); +} + +Ptr cv::cuda::NvidiaOpticalFlow_2_0::create( + cv::Size imageSize, std::vector roiData, NVIDIA_OF_PERF_LEVEL perfPreset, + NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE outputGridSize, NVIDIA_OF_HINT_VECTOR_GRID_SIZE hintGridSize, + bool bEnableTemporalHints, bool bEnableExternalHints, bool bEnableCostBuffer, + int gpuId, Stream& inputStream, Stream& outputStream) +{ + return makePtr( + imageSize, (NV_OF_PERF_LEVEL)perfPreset, + (NV_OF_OUTPUT_VECTOR_GRID_SIZE)outputGridSize, + (NV_OF_HINT_VECTOR_GRID_SIZE)hintGridSize, + true, + roiData, bEnableTemporalHints, bEnableExternalHints, bEnableCostBuffer, diff --git a/modules/cudaoptflow/test/test_optflow.cpp b/modules/cudaoptflow/test/test_optflow.cpp index 433c9ee9c6b..bbbec700e52 100644 --- a/modules/cudaoptflow/test/test_optflow.cpp +++ b/modules/cudaoptflow/test/test_optflow.cpp @@ -495,19 +495,11 @@ CUDA_TEST_P(NvidiaOpticalFlow_1_0, Regression) cv::Mat frame1 = readImage("opticalflow/frame1.png", cv::IMREAD_GRAYSCALE); ASSERT_FALSE(frame1.empty()); - const int width = frame0.size().width; - const int height = frame0.size().height; - const bool enableTemporalHints = false; - const bool enableExternalHints = false; - const bool enableCostBuffer = false; - const int gpuid = 0; - cv::Ptr d_nvof; try { - d_nvof = cv::cuda::NvidiaOpticalFlow_1_0::create(width, height, - cv::cuda::NvidiaOpticalFlow_1_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_SLOW, - enableTemporalHints, enableExternalHints, enableCostBuffer, gpuid); + d_nvof = cv::cuda::NvidiaOpticalFlow_1_0::create(frame0.size(), + cv::cuda::NvidiaOpticalFlow_1_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_SLOW); } catch (const cv::Exception& e) { @@ -519,7 +511,7 @@ CUDA_TEST_P(NvidiaOpticalFlow_1_0, Regression) Mat flow, upsampledFlow; d_nvof->calc(loadMat(frame0), loadMat(frame1), flow); - d_nvof->upSampler(flow, width, height, gridSize, upsampledFlow); + d_nvof->upSampler(flow, frame0.size(), gridSize, upsampledFlow); std::string fname(cvtest::TS::ptr()->get_data_path()); fname += "opticalflow/nvofGolden.flo"; @@ -527,6 +519,7 @@ CUDA_TEST_P(NvidiaOpticalFlow_1_0, Regression) ASSERT_FALSE(golden.empty()); EXPECT_MAT_SIMILAR(golden, upsampledFlow, 1e-10); + d_nvof->collectGarbage(); } CUDA_TEST_P(NvidiaOpticalFlow_1_0, OpticalFlowNan) @@ -539,19 +532,11 @@ CUDA_TEST_P(NvidiaOpticalFlow_1_0, OpticalFlowNan) cv::Mat r_frame0, r_frame1; - const int width = frame0.size().width; - const int height = frame0.size().height; - const bool enableTemporalHints = false; - const bool enableExternalHints = false; - const bool enableCostBuffer = false; - const int gpuid = 0; - cv::Ptr d_nvof; try { - d_nvof = cv::cuda::NvidiaOpticalFlow_1_0::create(width, height, - cv::cuda::NvidiaOpticalFlow_1_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_SLOW, - enableTemporalHints, enableExternalHints, enableCostBuffer, gpuid); + d_nvof = cv::cuda::NvidiaOpticalFlow_1_0::create(frame0.size(), + cv::cuda::NvidiaOpticalFlow_1_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_SLOW); } catch (const cv::Exception& e) { @@ -569,9 +554,96 @@ CUDA_TEST_P(NvidiaOpticalFlow_1_0, OpticalFlowNan) EXPECT_TRUE(cv::checkRange(flowx)); EXPECT_TRUE(cv::checkRange(flowy)); + d_nvof->collectGarbage(); }; INSTANTIATE_TEST_CASE_P(CUDA_OptFlow, NvidiaOpticalFlow_1_0, ALL_DEVICES); +////////////////////////////////////////////////////// +// NvidiaOpticalFlow_2_0 + +struct NvidiaOpticalFlow_2_0 : testing::TestWithParam +{ + cv::cuda::DeviceInfo devInfo; + + virtual void SetUp() + { + devInfo = GetParam(); + + cv::cuda::setDevice(devInfo.deviceID()); + } +}; + +CUDA_TEST_P(NvidiaOpticalFlow_2_0, Regression) +{ + cv::Mat frame0 = readImage("opticalflow/frame0.png", cv::IMREAD_GRAYSCALE); + ASSERT_FALSE(frame0.empty()); + + cv::Mat frame1 = readImage("opticalflow/frame1.png", cv::IMREAD_GRAYSCALE); + ASSERT_FALSE(frame1.empty()); + + cv::Ptr d_nvof; + try + { + d_nvof = cv::cuda::NvidiaOpticalFlow_2_0::create(frame0.size(), + cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_SLOW); + } + catch (const cv::Exception& e) + { + if (e.code == Error::StsBadFunc || e.code == Error::StsBadArg || e.code == Error::StsNullPtr) + throw SkipTestException("Current configuration is not supported"); + throw; + } + + Mat flow, upsampledFlow; + d_nvof->calc(loadMat(frame0), loadMat(frame1), flow); + d_nvof->convertToFloat(flow, upsampledFlow); + + std::string fname(cvtest::TS::ptr()->get_data_path()); + fname += "opticalflow/nvofGolden_2.flo"; + cv::Mat golden = cv::readOpticalFlow(fname.c_str()); + ASSERT_FALSE(golden.empty()); + + EXPECT_MAT_SIMILAR(golden, upsampledFlow, 1e-10); + d_nvof->collectGarbage(); +} + +CUDA_TEST_P(NvidiaOpticalFlow_2_0, OpticalFlowNan) +{ + cv::Mat frame0 = readImage("opticalflow/rubberwhale1.png", cv::IMREAD_GRAYSCALE); + ASSERT_FALSE(frame0.empty()); + + cv::Mat frame1 = readImage("opticalflow/rubberwhale2.png", cv::IMREAD_GRAYSCALE); + ASSERT_FALSE(frame1.empty()); + + cv::Mat r_frame0, r_frame1; + + cv::Ptr d_nvof; + try + { + d_nvof = cv::cuda::NvidiaOpticalFlow_2_0::create(frame0.size(), + cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_SLOW); + } + catch (const cv::Exception& e) + { + if (e.code == Error::StsBadFunc || e.code == Error::StsBadArg || e.code == Error::StsNullPtr) + throw SkipTestException("Current configuration is not supported"); + throw; + } + + Mat flow, flowx, flowy; + d_nvof->calc(loadMat(frame0), loadMat(frame1), flow); + + Mat planes[] = { flowx, flowy }; + split(flow, planes); + flowx = planes[0]; flowy = planes[1]; + + EXPECT_TRUE(cv::checkRange(flowx)); + EXPECT_TRUE(cv::checkRange(flowy)); + d_nvof->collectGarbage(); +}; + +INSTANTIATE_TEST_CASE_P(CUDA_OptFlow, NvidiaOpticalFlow_2_0, ALL_DEVICES); + }} // namespace #endif // HAVE_CUDA From f1c0b5eae639fd7616b6a717e298bbaff33bf7e2 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Fri, 1 Jan 2021 20:44:58 -0500 Subject: [PATCH 18/70] Implement cv::cuda::inRange (Fixes OpenCV #6295) --- .../cudaarithm/include/opencv2/cudaarithm.hpp | 25 +++ .../misc/python/test/test_cudaarithm.py | 21 ++- .../perf/perf_element_operations.cpp | 37 +++++ modules/cudaarithm/src/cuda/in_range.cu | 99 ++++++++++++ modules/cudaarithm/src/element_operations.cpp | 2 + .../test/test_element_operations.cpp | 58 +++++++ .../opencv2/cudev/functional/functional.hpp | 150 ++++++++++++++++++ 7 files changed, 391 insertions(+), 1 deletion(-) create mode 100644 modules/cudaarithm/src/cuda/in_range.cu diff --git a/modules/cudaarithm/include/opencv2/cudaarithm.hpp b/modules/cudaarithm/include/opencv2/cudaarithm.hpp index 4aa6582dcf1..246d0dfd313 100644 --- a/modules/cudaarithm/include/opencv2/cudaarithm.hpp +++ b/modules/cudaarithm/include/opencv2/cudaarithm.hpp @@ -358,6 +358,31 @@ threshold types are not supported. */ CV_EXPORTS_W double threshold(InputArray src, OutputArray dst, double thresh, double maxval, int type, Stream& stream = Stream::Null()); +/** @brief Checks if array elements lie between two scalars. + +The function checks the range as follows: +- For every element of a single-channel input array: + \f[\texttt{dst} (I)= \texttt{lowerb}_0 \leq \texttt{src} (I)_0 \leq \texttt{upperb}_0\f] +- For two-channel arrays: + \f[\texttt{dst} (I)= \texttt{lowerb}_0 \leq \texttt{src} (I)_0 \leq \texttt{upperb}_0 \land \texttt{lowerb}_1 \leq \texttt{src} (I)_1 \leq \texttt{upperb}_1\f] +- and so forth. + +That is, dst (I) is set to 255 (all 1 -bits) if src (I) is within the +specified 1D, 2D, 3D, ... box and 0 otherwise. + +Note that unlike the CPU inRange, this does NOT accept an array for lowerb or +upperb, only a cv::Scalar. + +@param src first input array. +@param lowerb inclusive lower boundary cv::Scalar. +@param upperb inclusive upper boundary cv::Scalar. +@param dst output array of the same size as src and CV_8U type. +@param stream Stream for the asynchronous version. + +@sa cv::inRange + */ +CV_EXPORTS_W void inRange(InputArray src, const Scalar& lowerb, const Scalar& upperb, OutputArray dst, Stream& stream = Stream::Null()); + /** @brief Computes magnitudes of complex matrix elements. @param xy Source complex matrix in the interleaved format ( CV_32FC2 ). diff --git a/modules/cudaarithm/misc/python/test/test_cudaarithm.py b/modules/cudaarithm/misc/python/test/test_cudaarithm.py index b068fae44bf..bbc9527a767 100644 --- a/modules/cudaarithm/misc/python/test/test_cudaarithm.py +++ b/modules/cudaarithm/misc/python/test/test_cudaarithm.py @@ -174,5 +174,24 @@ def test_convolution(self): self.assertTrue(np.allclose(cuMatDst.download(), cv.filter2D(npMat,-1,kernel,anchor=(-1,-1))[iS[0]:iE[0]+1,iS[1]:iE[1]+1])) + def test_inrange(self): + npMat = (np.random.random((128, 128, 3)) * 255).astype(np.float32) + + bound1 = np.random.random((4,)) * 255 + bound2 = np.random.random((4,)) * 255 + lowerb = np.minimum(bound1, bound2).tolist() + upperb = np.maximum(bound1, bound2).tolist() + + cuMat = cv.cuda_GpuMat() + cuMat.upload(npMat) + + self.assertTrue((cv.cuda.inRange(cuMat, lowerb, upperb).download() == + cv.inRange(npMat, np.array(lowerb), np.array(upperb))).all()) + + cuMatDst = cv.cuda_GpuMat(cuMat.size(), cv.CV_8UC1) + cv.cuda.inRange(cuMat, lowerb, upperb, cuMatDst) + self.assertTrue((cuMatDst.download() == + cv.inRange(npMat, np.array(lowerb), np.array(upperb))).all()) + if __name__ == '__main__': - NewOpenCVTests.bootstrap() \ No newline at end of file + NewOpenCVTests.bootstrap() diff --git a/modules/cudaarithm/perf/perf_element_operations.cpp b/modules/cudaarithm/perf/perf_element_operations.cpp index 9aa2d4e4e0f..df2146fc147 100644 --- a/modules/cudaarithm/perf/perf_element_operations.cpp +++ b/modules/cudaarithm/perf/perf_element_operations.cpp @@ -1501,4 +1501,41 @@ PERF_TEST_P(Sz_Depth_Op, Threshold, } } +////////////////////////////////////////////////////////////////////// +// InRange + +PERF_TEST_P(Sz_Depth_Cn, InRange, + Combine(CUDA_TYPICAL_MAT_SIZES, + Values(CV_8U, CV_16U, CV_32F, CV_64F), + CUDA_CHANNELS_1_3_4)) +{ + const cv::Size size = GET_PARAM(0); + const int depth = GET_PARAM(1); + const int channels = GET_PARAM(2); + + cv::Mat src(size, CV_MAKE_TYPE(depth, channels)); + declare.in(src, WARMUP_RNG); + + const cv::Scalar lowerb(10, 50, 100); + const cv::Scalar upperb(70, 85, 200); + + if (PERF_RUN_CUDA()) + { + const cv::cuda::GpuMat d_src(src); + cv::cuda::GpuMat dst; + + TEST_CYCLE() cv::cuda::inRange(d_src, lowerb, upperb, dst); + + CUDA_SANITY_CHECK(dst, 0); + } + else + { + cv::Mat dst; + + TEST_CYCLE() cv::inRange(src, lowerb, upperb, dst); + + CPU_SANITY_CHECK(dst); + } +} + }} // namespace diff --git a/modules/cudaarithm/src/cuda/in_range.cu b/modules/cudaarithm/src/cuda/in_range.cu new file mode 100644 index 00000000000..1902b49996b --- /dev/null +++ b/modules/cudaarithm/src/cuda/in_range.cu @@ -0,0 +1,99 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "opencv2/opencv_modules.hpp" + +#ifndef HAVE_OPENCV_CUDEV + +#error "opencv_cudev is required" + +#else + +#include "opencv2/core/private.cuda.hpp" +#include "opencv2/cudaarithm.hpp" +#include "opencv2/cudev.hpp" + +using namespace cv; +using namespace cv::cuda; +using namespace cv::cudev; + +namespace { + +template +void inRangeImpl(const GpuMat& src, + const Scalar& lowerb, + const Scalar& upperb, + GpuMat& dst, + Stream& stream) { + gridTransformUnary(globPtr::type>(src), + globPtr(dst), + InRangeFunc(lowerb, upperb), + stream); +} + +} // namespace + +void cv::cuda::inRange(InputArray _src, + const Scalar& _lowerb, + const Scalar& _upperb, + OutputArray _dst, + Stream& stream) { + const GpuMat src = getInputMat(_src, stream); + + typedef void (*func_t)(const GpuMat& src, + const Scalar& lowerb, + const Scalar& upperb, + GpuMat& dst, + Stream& stream); + + // Note: We cannot support 16F with the current implementation because we + // use a CUDA vector (e.g. int3) to store the bounds, and there is no CUDA + // vector type for float16 + static constexpr const int MAX_CHANNELS = 4; + static constexpr const int NUM_DEPTHS = CV_64F + 1; + + static const std::array, MAX_CHANNELS> + funcs = {std::array{inRangeImpl, + inRangeImpl, + inRangeImpl, + inRangeImpl, + inRangeImpl, + inRangeImpl, + inRangeImpl}, + std::array{inRangeImpl, + inRangeImpl, + inRangeImpl, + inRangeImpl, + inRangeImpl, + inRangeImpl, + inRangeImpl}, + std::array{inRangeImpl, + inRangeImpl, + inRangeImpl, + inRangeImpl, + inRangeImpl, + inRangeImpl, + inRangeImpl}, + std::array{inRangeImpl, + inRangeImpl, + inRangeImpl, + inRangeImpl, + inRangeImpl, + inRangeImpl, + inRangeImpl}}; + + CV_CheckLE(src.channels(), MAX_CHANNELS, "Src must have <= 4 channels"); + CV_CheckLE(src.depth(), + CV_64F, + "Src must have depth 8U, 8S, 16U, 16S, 32S, 32F, or 64F"); + + GpuMat dst = getOutputMat(_dst, src.size(), CV_8UC1, stream); + + const func_t func = funcs.at(src.channels() - 1).at(src.depth()); + func(src, _lowerb, _upperb, dst, stream); + + syncOutput(dst, _dst, stream); +} + +#endif diff --git a/modules/cudaarithm/src/element_operations.cpp b/modules/cudaarithm/src/element_operations.cpp index f88119502d1..1ad3c17c40f 100644 --- a/modules/cudaarithm/src/element_operations.cpp +++ b/modules/cudaarithm/src/element_operations.cpp @@ -77,6 +77,8 @@ void cv::cuda::addWeighted(InputArray, double, InputArray, double, double, Outpu double cv::cuda::threshold(InputArray, OutputArray, double, double, int, Stream&) {throw_no_cuda(); return 0.0;} +void cv::cuda::inRange(InputArray, const Scalar&, const Scalar&, OutputArray, Stream&) { throw_no_cuda(); } + void cv::cuda::magnitude(InputArray, OutputArray, Stream&) { throw_no_cuda(); } void cv::cuda::magnitude(InputArray, InputArray, OutputArray, Stream&) { throw_no_cuda(); } void cv::cuda::magnitudeSqr(InputArray, OutputArray, Stream&) { throw_no_cuda(); } diff --git a/modules/cudaarithm/test/test_element_operations.cpp b/modules/cudaarithm/test/test_element_operations.cpp index 848ab5ce740..d2e314b10d9 100644 --- a/modules/cudaarithm/test/test_element_operations.cpp +++ b/modules/cudaarithm/test/test_element_operations.cpp @@ -2577,6 +2577,64 @@ INSTANTIATE_TEST_CASE_P(CUDA_Arithm, Threshold, testing::Combine( ALL_THRESH_OPS, WHOLE_SUBMAT)); +//////////////////////////////////////////////////////////////////////////////// +// InRange + +PARAM_TEST_CASE(InRange, cv::cuda::DeviceInfo, cv::Size, MatDepth, Channels, UseRoi) +{ + cv::cuda::DeviceInfo devInfo; + cv::Size size; + int depth; + int channels; + bool useRoi; + + virtual void SetUp() + { + devInfo = GET_PARAM(0); + size = GET_PARAM(1); + depth = GET_PARAM(2); + channels = GET_PARAM(3); + useRoi = GET_PARAM(4); + + cv::cuda::setDevice(devInfo.deviceID()); + } +}; + +CUDA_TEST_P(InRange, Accuracy) +{ + // Set max value to 127 for signed char + const int max_bound = (depth == CV_8S) ? 127 : 255; + + // Create lower and upper bound scalars, and make sure lowerb[i] <= + // upperb[i] + const cv::Scalar bound1 = randomScalar(0, max_bound); + const cv::Scalar bound2 = randomScalar(0, max_bound); + + cv::Scalar lowerb, upperb; + for (int i = 0; i < 4; i++) { + lowerb[i] = std::min(bound1[i], bound2[i]); + upperb[i] = std::max(bound1[i], bound2[i]); + } + + // Create mats and run CPU and GPU versions + const cv::Mat src = randomMat(size, CV_MAKE_TYPE(depth, channels)); + + cv::cuda::GpuMat dst; + cv::cuda::inRange(loadMat(src, useRoi), lowerb, upperb, dst); + + cv::Mat dst_gold; + cv::inRange(src, lowerb, upperb, dst_gold); + + EXPECT_MAT_NEAR(dst_gold, dst, 0); +} + +INSTANTIATE_TEST_CASE_P(CUDA_Arithm, InRange, testing::Combine( + ALL_DEVICES, + DIFFERENT_SIZES, + ALL_DEPTH, + ALL_CHANNELS, + WHOLE_SUBMAT)); + //////////////////////////////////////////////////////////////////////////////// // Magnitude diff --git a/modules/cudev/include/opencv2/cudev/functional/functional.hpp b/modules/cudev/include/opencv2/cudev/functional/functional.hpp index f6569cf3d55..e4165358c28 100644 --- a/modules/cudev/include/opencv2/cudev/functional/functional.hpp +++ b/modules/cudev/include/opencv2/cudev/functional/functional.hpp @@ -786,6 +786,156 @@ __host__ __device__ ThreshToZeroInvFunc thresh_to_zero_inv_func(T thresh) return f; } +// InRange functors + +/** @brief Functor that checks if a CUDA vector v is in the range between lowerb and upperb + + Implemented as a recursive template + +@tparam T underlying floating point/integral type +@tparam cn total number of channels in the input arguments +@tparam i number of the channel to check (will check this channel and lower) +@param lowerb inclusive scalar lower bound, as a CUDA vector, e.g. a uchar3 +@param upperb inclusive scalar upper bound, as a CUDA vector, e.g. a uchar3 +@param v scalar to check, as a CUDA vector, e.g. a uchar3 + */ +template +struct InRangeComparator { + __device__ bool operator()(const typename MakeVec::type& lowerb, + const typename MakeVec::type& upperb, + const typename MakeVec::type& v) const; +}; + +// Specialize InRangeComparator for MakeVec +#define OPENCV_CUDEV_FUNCTIONAL_MAKE_IN_RANGE_COMPARATOR(i, field) \ + template \ + struct InRangeComparator { \ + __device__ bool operator()( \ + const typename MakeVec::type& lowerb, \ + const typename MakeVec::type& upperb, \ + const typename MakeVec::type& v) const { \ + const bool in_range = \ + lowerb.field <= v.field && v.field <= upperb.field; \ + return in_range \ + && InRangeComparator{}(lowerb, upperb, v); \ + } \ + }; + +OPENCV_CUDEV_FUNCTIONAL_MAKE_IN_RANGE_COMPARATOR(4, w) +OPENCV_CUDEV_FUNCTIONAL_MAKE_IN_RANGE_COMPARATOR(3, z) +OPENCV_CUDEV_FUNCTIONAL_MAKE_IN_RANGE_COMPARATOR(2, y) +OPENCV_CUDEV_FUNCTIONAL_MAKE_IN_RANGE_COMPARATOR(1, x) + +#undef OPENCV_CUDEV_FUNCTIONAL_MAKE_IN_RANGE_COMPARATOR + +// Specialize for the base case of i=0 +template +struct InRangeComparator { + __device__ bool operator()(const typename MakeVec::type&, + const typename MakeVec::type&, + const typename MakeVec::type&) const { + return true; + } +}; + +// Specialize for MakeVec::type, which is e.g. uchar instead of uchar1 +template +struct InRangeComparator { + static constexpr const int cn = 1; + + __device__ bool operator()(const typename MakeVec::type& lowerb, + const typename MakeVec::type& upperb, + const typename MakeVec::type& v) const { + return lowerb <= v && v <= upperb; + } +}; + +/** @brief Functor that copies a cv::Scalar into a CUDA vector, e.g. a uchar3 + + Implemented as a recursive template + +@tparam T underlying floating point/integral type +@tparam cn total number of channels in the input arguments +@tparam i number of the channel to check (will check this channel and lower) +@param in cv::Scalar to copy from +@param out CUDA vector to copy into, e.g. a uchar3 + */ +template +struct InRangeCopier { + void operator()(const Scalar& in, + typename MakeVec::type& out) const; +}; + +// Specialize InRangeCopier for MakeVec +#define OPENCV_CUDEV_FUNCTIONAL_MAKE_IN_RANGE_COPIER(i, field) \ + template \ + struct InRangeCopier { \ + void operator()(const Scalar& in, \ + typename MakeVec::type& out) const { \ + const double in_rounded = (std::is_same::value \ + || std::is_same::value) \ + ? in[i - 1] \ + : std::round(in[i - 1]); \ + out.field = static_cast(in_rounded); \ + InRangeCopier{}(in, out); \ + } \ + }; + +OPENCV_CUDEV_FUNCTIONAL_MAKE_IN_RANGE_COPIER(4, w) +OPENCV_CUDEV_FUNCTIONAL_MAKE_IN_RANGE_COPIER(3, z) +OPENCV_CUDEV_FUNCTIONAL_MAKE_IN_RANGE_COPIER(2, y) +OPENCV_CUDEV_FUNCTIONAL_MAKE_IN_RANGE_COPIER(1, x) + +#undef OPENCV_CUDEV_FUNCTIONAL_MAKE_IN_RANGE_COPIER + +// Specialize for the base case of i=0 +template +struct InRangeCopier { + void operator()(const Scalar&, typename MakeVec::type&) const { + return; + } +}; + +// Specialize for MakeVec::type, which is e.g. uchar instead of uchar1 +template +struct InRangeCopier { + void operator()(const Scalar& in, typename MakeVec::type& out) const { + const double in_rounded = (std::is_same::value + || std::is_same::value) + ? in[0] + : std::round(in[0]); + out = static_cast(in_rounded); + } +}; + +/** @brief unary_function implementation of inRange + + Intended to be used to create an Op for gridTransformUnary + +@tparam T underlying floating point/integral type +@tparam cn total number of channels in the source image + */ +template +struct InRangeFunc : unary_function::type, uchar> { + typename MakeVec::type lowerb; + typename MakeVec::type upperb; + + /** @brief Builds an InRangeFunc with the given lower and upper bound scalars + + @param lowerb_scalar inclusive lower bound + @param upperb_scalar inclusive upper bound + */ + __host__ InRangeFunc(const Scalar& lowerb_scalar, const Scalar& upperb_scalar) { + InRangeCopier{}(lowerb_scalar, lowerb); + InRangeCopier{}(upperb_scalar, upperb); + } + + __device__ uchar + operator()(const typename MakeVec::type& src) const { + return InRangeComparator{}(lowerb, upperb, src) ? 255 : 0; + } +}; + // Function Object Adaptors template struct UnaryNegate : unary_function From 98acd62e7ad4fac956e2c5043553a4ea3c4ebd18 Mon Sep 17 00:00:00 2001 From: dddzg Date: Wed, 20 Jan 2021 20:12:51 +0800 Subject: [PATCH 19/70] add wechat qrcode module --- modules/wechat_qrcode/CMakeLists.txt | 30 + modules/wechat_qrcode/LICENSE | 298 ++++ modules/wechat_qrcode/README.md | 12 + .../include/opencv2/wechat_qrcode.hpp | 62 + modules/wechat_qrcode/samples/qrcode.py | 60 + .../wechat_qrcode/samples/qrcode_example.cpp | 77 + modules/wechat_qrcode/src/binarizermgr.cpp | 70 + modules/wechat_qrcode/src/binarizermgr.hpp | 51 + modules/wechat_qrcode/src/decodermgr.cpp | 80 + modules/wechat_qrcode/src/decodermgr.hpp | 46 + modules/wechat_qrcode/src/detector/align.cpp | 66 + modules/wechat_qrcode/src/detector/align.hpp | 42 + .../src/detector/ssd_detector.cpp | 53 + .../src/detector/ssd_detector.hpp | 31 + modules/wechat_qrcode/src/imgsource.cpp | 188 ++ modules/wechat_qrcode/src/imgsource.hpp | 63 + modules/wechat_qrcode/src/precomp.hpp | 29 + .../wechat_qrcode/src/scale/super_scale.cpp | 63 + .../wechat_qrcode/src/scale/super_scale.hpp | 32 + modules/wechat_qrcode/src/wechat_qrcode.cpp | 211 +++ modules/wechat_qrcode/src/zxing/binarizer.cpp | 90 + modules/wechat_qrcode/src/zxing/binarizer.hpp | 88 + .../wechat_qrcode/src/zxing/binarybitmap.cpp | 66 + .../wechat_qrcode/src/zxing/binarybitmap.hpp | 53 + .../wechat_qrcode/src/zxing/common/array.hpp | 115 ++ .../adaptive_threshold_mean_binarizer.cpp | 100 ++ .../adaptive_threshold_mean_binarizer.hpp | 39 + .../binarizer/fast_window_binarizer.cpp | 286 ++++ .../binarizer/fast_window_binarizer.hpp | 56 + .../binarizer/global_histogram_binarizer.cpp | 341 ++++ .../binarizer/global_histogram_binarizer.hpp | 46 + .../common/binarizer/hybrid_binarizer.cpp | 431 +++++ .../common/binarizer/hybrid_binarizer.hpp | 83 + .../binarizer/simple_adaptive_binarizer.cpp | 159 ++ .../binarizer/simple_adaptive_binarizer.hpp | 40 + .../src/zxing/common/bitarray.cpp | 241 +++ .../src/zxing/common/bitarray.hpp | 91 + .../src/zxing/common/bitmatrix.cpp | 403 +++++ .../src/zxing/common/bitmatrix.hpp | 118 ++ .../src/zxing/common/bitsource.cpp | 62 + .../src/zxing/common/bitsource.hpp | 57 + .../src/zxing/common/bytematrix.cpp | 57 + .../src/zxing/common/bytematrix.hpp | 60 + .../src/zxing/common/character.hpp | 54 + .../src/zxing/common/characterseteci.cpp | 113 ++ .../src/zxing/common/characterseteci.hpp | 46 + .../src/zxing/common/counted.hpp | 110 ++ .../src/zxing/common/decoder_result.cpp | 64 + .../src/zxing/common/decoder_result.hpp | 79 + .../src/zxing/common/detector_result.cpp | 27 + .../src/zxing/common/detector_result.hpp | 42 + .../common/greyscale_luminance_source.cpp | 77 + .../common/greyscale_luminance_source.hpp | 44 + .../greyscale_rotated_luminance_source.cpp | 72 + .../greyscale_rotated_luminance_source.hpp | 39 + .../src/zxing/common/grid_sampler.cpp | 122 ++ .../src/zxing/common/grid_sampler.hpp | 34 + .../src/zxing/common/imagecut.cpp | 67 + .../src/zxing/common/imagecut.hpp | 35 + .../wechat_qrcode/src/zxing/common/kmeans.cpp | 96 ++ .../wechat_qrcode/src/zxing/common/kmeans.hpp | 26 + .../src/zxing/common/mathutils.hpp | 131 ++ .../zxing/common/perspective_transform.cpp | 122 ++ .../zxing/common/perspective_transform.hpp | 39 + .../zxing/common/reedsolomon/genericgf.cpp | 101 ++ .../zxing/common/reedsolomon/genericgf.hpp | 60 + .../common/reedsolomon/genericgfpoly.cpp | 233 +++ .../common/reedsolomon/genericgfpoly.hpp | 45 + .../reedsolomon/reed_solomon_decoder.cpp | 190 +++ .../reedsolomon/reed_solomon_decoder.hpp | 46 + .../wechat_qrcode/src/zxing/common/str.cpp | 102 ++ .../wechat_qrcode/src/zxing/common/str.hpp | 63 + .../src/zxing/common/stringutils.cpp | 687 ++++++++ .../src/zxing/common/stringutils.hpp | 65 + .../src/zxing/common/unicomblock.cpp | 130 ++ .../src/zxing/common/unicomblock.hpp | 50 + .../wechat_qrcode/src/zxing/decodehints.hpp | 30 + .../wechat_qrcode/src/zxing/errorhandler.cpp | 50 + .../wechat_qrcode/src/zxing/errorhandler.hpp | 89 + .../src/zxing/luminance_source.cpp | 59 + .../src/zxing/luminance_source.hpp | 57 + .../zxing/qrcode/decoder/bitmatrixparser.cpp | 241 +++ .../zxing/qrcode/decoder/bitmatrixparser.hpp | 53 + .../src/zxing/qrcode/decoder/datablock.cpp | 103 ++ .../src/zxing/qrcode/decoder/datablock.hpp | 44 + .../src/zxing/qrcode/decoder/datamask.cpp | 120 ++ .../src/zxing/qrcode/decoder/datamask.hpp | 40 + .../decoder/decoded_bit_stream_parser.cpp | 492 ++++++ .../decoder/decoded_bit_stream_parser.hpp | 69 + .../src/zxing/qrcode/decoder/decoder.cpp | 227 +++ .../src/zxing/qrcode/decoder/decoder.hpp | 65 + .../src/zxing/qrcode/decoder/mode.cpp | 91 + .../src/zxing/qrcode/decoder/mode.hpp | 52 + .../decoder/qrcode_decoder_metadata.hpp | 66 + .../qrcode/detector/alignment_pattern.cpp | 45 + .../qrcode/detector/alignment_pattern.hpp | 37 + .../detector/alignment_pattern_finder.cpp | 239 +++ .../detector/alignment_pattern_finder.hpp | 64 + .../src/zxing/qrcode/detector/detector.cpp | 1071 ++++++++++++ .../src/zxing/qrcode/detector/detector.hpp | 147 ++ .../zxing/qrcode/detector/finder_pattern.cpp | 94 + .../zxing/qrcode/detector/finder_pattern.hpp | 61 + .../qrcode/detector/finder_pattern_finder.cpp | 1515 +++++++++++++++++ .../qrcode/detector/finder_pattern_finder.hpp | 139 ++ .../qrcode/detector/finder_pattern_info.cpp | 99 ++ .../qrcode/detector/finder_pattern_info.hpp | 48 + .../zxing/qrcode/detector/pattern_result.cpp | 32 + .../zxing/qrcode/detector/pattern_result.hpp | 51 + .../zxing/qrcode/error_correction_level.cpp | 46 + .../zxing/qrcode/error_correction_level.hpp | 44 + .../src/zxing/qrcode/format_information.cpp | 114 ++ .../src/zxing/qrcode/format_information.hpp | 48 + .../src/zxing/qrcode/qrcode_reader.cpp | 491 ++++++ .../src/zxing/qrcode/qrcode_reader.hpp | 131 ++ .../src/zxing/qrcode/version.cpp | 504 ++++++ .../src/zxing/qrcode/version.hpp | 88 + modules/wechat_qrcode/src/zxing/reader.cpp | 28 + modules/wechat_qrcode/src/zxing/reader.hpp | 39 + modules/wechat_qrcode/src/zxing/result.cpp | 71 + modules/wechat_qrcode/src/zxing/result.hpp | 78 + .../wechat_qrcode/src/zxing/resultpoint.cpp | 101 ++ .../wechat_qrcode/src/zxing/resultpoint.hpp | 49 + modules/wechat_qrcode/src/zxing/zxing.hpp | 91 + modules/wechat_qrcode/test/test_main.cpp | 14 + modules/wechat_qrcode/test/test_precomp.hpp | 14 + modules/wechat_qrcode/test/test_qrcode.cpp | 293 ++++ 126 files changed, 15691 insertions(+) create mode 100644 modules/wechat_qrcode/CMakeLists.txt create mode 100644 modules/wechat_qrcode/LICENSE create mode 100644 modules/wechat_qrcode/README.md create mode 100644 modules/wechat_qrcode/include/opencv2/wechat_qrcode.hpp create mode 100644 modules/wechat_qrcode/samples/qrcode.py create mode 100644 modules/wechat_qrcode/samples/qrcode_example.cpp create mode 100644 modules/wechat_qrcode/src/binarizermgr.cpp create mode 100644 modules/wechat_qrcode/src/binarizermgr.hpp create mode 100644 modules/wechat_qrcode/src/decodermgr.cpp create mode 100644 modules/wechat_qrcode/src/decodermgr.hpp create mode 100644 modules/wechat_qrcode/src/detector/align.cpp create mode 100644 modules/wechat_qrcode/src/detector/align.hpp create mode 100644 modules/wechat_qrcode/src/detector/ssd_detector.cpp create mode 100644 modules/wechat_qrcode/src/detector/ssd_detector.hpp create mode 100644 modules/wechat_qrcode/src/imgsource.cpp create mode 100644 modules/wechat_qrcode/src/imgsource.hpp create mode 100644 modules/wechat_qrcode/src/precomp.hpp create mode 100644 modules/wechat_qrcode/src/scale/super_scale.cpp create mode 100644 modules/wechat_qrcode/src/scale/super_scale.hpp create mode 100644 modules/wechat_qrcode/src/wechat_qrcode.cpp create mode 100644 modules/wechat_qrcode/src/zxing/binarizer.cpp create mode 100644 modules/wechat_qrcode/src/zxing/binarizer.hpp create mode 100644 modules/wechat_qrcode/src/zxing/binarybitmap.cpp create mode 100644 modules/wechat_qrcode/src/zxing/binarybitmap.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/array.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/binarizer/adaptive_threshold_mean_binarizer.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/binarizer/adaptive_threshold_mean_binarizer.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/binarizer/fast_window_binarizer.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/binarizer/fast_window_binarizer.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/binarizer/global_histogram_binarizer.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/binarizer/global_histogram_binarizer.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/binarizer/hybrid_binarizer.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/binarizer/hybrid_binarizer.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/binarizer/simple_adaptive_binarizer.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/binarizer/simple_adaptive_binarizer.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/bitarray.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/bitarray.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/bitmatrix.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/bitmatrix.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/bitsource.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/bitsource.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/bytematrix.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/bytematrix.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/character.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/characterseteci.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/characterseteci.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/counted.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/decoder_result.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/decoder_result.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/detector_result.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/detector_result.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/greyscale_luminance_source.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/greyscale_luminance_source.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/greyscale_rotated_luminance_source.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/greyscale_rotated_luminance_source.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/grid_sampler.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/grid_sampler.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/imagecut.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/imagecut.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/kmeans.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/kmeans.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/mathutils.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/perspective_transform.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/perspective_transform.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgf.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgf.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgfpoly.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgfpoly.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/reedsolomon/reed_solomon_decoder.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/reedsolomon/reed_solomon_decoder.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/str.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/str.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/stringutils.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/stringutils.hpp create mode 100644 modules/wechat_qrcode/src/zxing/common/unicomblock.cpp create mode 100644 modules/wechat_qrcode/src/zxing/common/unicomblock.hpp create mode 100644 modules/wechat_qrcode/src/zxing/decodehints.hpp create mode 100644 modules/wechat_qrcode/src/zxing/errorhandler.cpp create mode 100644 modules/wechat_qrcode/src/zxing/errorhandler.hpp create mode 100644 modules/wechat_qrcode/src/zxing/luminance_source.cpp create mode 100644 modules/wechat_qrcode/src/zxing/luminance_source.hpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/decoder/bitmatrixparser.cpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/decoder/bitmatrixparser.hpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/decoder/datablock.cpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/decoder/datablock.hpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/decoder/datamask.cpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/decoder/datamask.hpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/decoder/decoded_bit_stream_parser.cpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/decoder/decoded_bit_stream_parser.hpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/decoder/decoder.cpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/decoder/decoder.hpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/decoder/mode.cpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/decoder/mode.hpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/decoder/qrcode_decoder_metadata.hpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern.cpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern.hpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern_finder.cpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern_finder.hpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/detector/detector.cpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/detector/detector.hpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern.cpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern.hpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_finder.cpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_finder.hpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_info.cpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_info.hpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/detector/pattern_result.cpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/detector/pattern_result.hpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/error_correction_level.cpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/error_correction_level.hpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/format_information.cpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/format_information.hpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/qrcode_reader.cpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/qrcode_reader.hpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/version.cpp create mode 100644 modules/wechat_qrcode/src/zxing/qrcode/version.hpp create mode 100644 modules/wechat_qrcode/src/zxing/reader.cpp create mode 100644 modules/wechat_qrcode/src/zxing/reader.hpp create mode 100644 modules/wechat_qrcode/src/zxing/result.cpp create mode 100644 modules/wechat_qrcode/src/zxing/result.hpp create mode 100644 modules/wechat_qrcode/src/zxing/resultpoint.cpp create mode 100644 modules/wechat_qrcode/src/zxing/resultpoint.hpp create mode 100644 modules/wechat_qrcode/src/zxing/zxing.hpp create mode 100644 modules/wechat_qrcode/test/test_main.cpp create mode 100644 modules/wechat_qrcode/test/test_precomp.hpp create mode 100644 modules/wechat_qrcode/test/test_qrcode.cpp diff --git a/modules/wechat_qrcode/CMakeLists.txt b/modules/wechat_qrcode/CMakeLists.txt new file mode 100644 index 00000000000..210f4a0d50a --- /dev/null +++ b/modules/wechat_qrcode/CMakeLists.txt @@ -0,0 +1,30 @@ +set(the_description "WeChat QR code Detector") +ocv_define_module(wechat_qrcode opencv_core opencv_imgproc opencv_dnn WRAP java objc python js) + +# need to change +set(wechat_qrcode_commit_hash "a8b69ccc738421293254aec5ddb38bd523503252") +set(hash_detect_caffemodel "238e2b2d6f3c18d6c3a30de0c31e23cf") +set(hash_detect_prototxt "6fb4976b32695f9f5c6305c19f12537d") +set(hash_sr_caffemodel "cbfcd60361a73beb8c583eea7e8e6664") +set(hash_sr_prototxt "69db99927a70df953b471daaba03fbef") + +set(model_types caffemodel prototxt) +set(model_names detect sr) + +foreach(model_name ${model_names}) + foreach(model_type ${model_types}) + ocv_download(FILENAME ${model_name}.${model_type} + HASH ${hash_${model_name}_${model_type}} + URL + "${OPENCV_WECHAT_QRCODE_URL}" + "$ENV{OPENCV_WECHAT_QRCODE_URL}" + "https://raw.githubusercontent.com/WeChatCV/opencv_3rdparty/${wechat_qrcode_commit_hash}/" + DESTINATION_DIR "${CMAKE_BINARY_DIR}/downloads/wechat_qrcode" + ID "wechat_qrcode" + RELATIVE_URL + STATUS res) + if(NOT res) + message(WARNING "WeChatQRCode: Can't get ${model_name} ${model_type} file for wechat qrcode.") + endif() + endforeach() +endforeach() diff --git a/modules/wechat_qrcode/LICENSE b/modules/wechat_qrcode/LICENSE new file mode 100644 index 00000000000..ce2fbff9fec --- /dev/null +++ b/modules/wechat_qrcode/LICENSE @@ -0,0 +1,298 @@ +Tencent is pleased to support the open source community by making WeChat QRCode available. + +Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +The below software in this distribution may have been modified by THL A29 Limited ("Tencent Modifications"). +All Tencent Modifications are Copyright (C) THL A29 Limited. + +WeChat QRCode is licensed under the Apache License Version 2.0, except for the third-party components listed below. + +Terms of the Apache License Version 2.0 +-------------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + + + +Other dependencies and licenses: + +Open Source Software Licensed under the Apache License Version 2.0: +-------------------------------------------------------------------- +1. zxing +Copyright (c) zxing authors and contributors +Please note this software may have been modified by Tencent. + +Terms of the Apache License Version 2.0: +-------------------------------------------------------------------- +Apache License + +Version 2.0, January 2004 + +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of this License; and +You must cause any modified files to carry prominent notices stating that You changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + +You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + + +======================================================================== +jai-imageio +======================================================================== + +Copyright (c) 2005 Sun Microsystems, Inc. +Copyright © 2010-2014 University of Manchester +Copyright © 2010-2015 Stian Soiland-Reyes +Copyright © 2015 Peter Hull +All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +- Redistribution of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +- Redistribution in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + +Neither the name of Sun Microsystems, Inc. or the names of +contributors may be used to endorse or promote products derived +from this software without specific prior written permission. + +This software is provided "AS IS," without a warranty of any +kind. ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND +WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT, ARE HEREBY +EXCLUDED. SUN MIDROSYSTEMS, INC. ("SUN") AND ITS LICENSORS SHALL +NOT BE LIABLE FOR ANY DAMAGES SUFFERED BY LICENSEE AS A RESULT OF +USING, MODIFYING OR DISTRIBUTING THIS SOFTWARE OR ITS +DERIVATIVES. IN NO EVENT WILL SUN OR ITS LICENSORS BE LIABLE FOR +ANY LOST REVENUE, PROFIT OR DATA, OR FOR DIRECT, INDIRECT, SPECIAL, +CONSEQUENTIAL, INCIDENTAL OR PUNITIVE DAMAGES, HOWEVER CAUSED AND +REGARDLESS OF THE THEORY OF LIABILITY, ARISING OUT OF THE USE OF OR +INABILITY TO USE THIS SOFTWARE, EVEN IF SUN HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +You acknowledge that this software is not designed or intended for +use in the design, construction, operation or maintenance of any +nuclear facility. \ No newline at end of file diff --git a/modules/wechat_qrcode/README.md b/modules/wechat_qrcode/README.md new file mode 100644 index 00000000000..1da3167e211 --- /dev/null +++ b/modules/wechat_qrcode/README.md @@ -0,0 +1,12 @@ +WeChat QR code detector for detecting and parsing QR code. +================================================ + +WeChat QR code detector is a high-performance and lightweight QR code detect and decode library, which is contributed by WeChat Computer Vision Team (WeChatCV). It has been widely used in various Tencent applications, including WeChat, WeCom, QQ, QQ Browser, and so on. There are four primary features of WeChat QR code detector: + +1. CNN-based QR code detector. Different from the traditional detector, we introduce a tiny CNN model for multiple code detection. The detector is based on SSD architecture with a MobileNetV2-like backbone, which is run on caffe inference framework. + +2. CNN-based QR code enhancement. To improve the performance of tiny QR code, we design a lighten super-resolution CNN model for QR code, called QRSR. Depth-wise convolution, DenseNet concat and deconvolution are the core techniques in the QRSR model. + +3. More robust finder pattern detection. Besides traditional horizontal line searching, we propose an area size based finder pattern detection method. we calculate the area size of black and white block to locate the finder pattern by the pre-computed connected cells. + +4. Massive engineering optimization. Based on [zing-cpp](https://github.com/glassechidna/zxing-cpp), we conduct massive engineering optimization to boost the decoding success rate, such as trying more binarization methods, supporting N:1:3:1:1 finder pattern detection, finding more alignment pattern, clustering similar size finder pattern, and etc. diff --git a/modules/wechat_qrcode/include/opencv2/wechat_qrcode.hpp b/modules/wechat_qrcode/include/opencv2/wechat_qrcode.hpp new file mode 100644 index 00000000000..69ac40b7bd7 --- /dev/null +++ b/modules/wechat_qrcode/include/opencv2/wechat_qrcode.hpp @@ -0,0 +1,62 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#ifndef __OPENCV_WECHAT_QRCODE_HPP__ +#define __OPENCV_WECHAT_QRCODE_HPP__ +#include "opencv2/core.hpp" +/** @defgroup wechat_qrcode WeChat QR code detector for detecting and parsing QR code. + */ +namespace cv { +namespace wechat_qrcode { +//! @addtogroup wechat_qrcode +//! @{ +/** + * @brief QRCode includes two CNN-based models: + * A object detection model and a super resolution model. + * Object detection model is applied to detect QRCode with the bounding box. + * super resolution model is applied to zoom in QRCode when it is small. + * + */ +class CV_EXPORTS_W WeChatQRCode { +public: + /** + * @brief Initialize the QRCode. + * Two models are packaged with caffe format. + * Therefore, there are prototxt and caffe model two files. + * + * @param detector_prototxt_path prototxt file path for the detector + * @param detector_caffe_model_path caffe model file path for the detector + * @param super_resolution_prototxt_path prototxt file path for the super resolution model + * @param super_resolution_caffe_model_path caffe file path for the super resolution model + */ + CV_WRAP WeChatQRCode(const std::string& detector_prototxt_path = "", + const std::string& detector_caffe_model_path = "", + const std::string& super_resolution_prototxt_path = "", + const std::string& super_resolution_caffe_model_path = ""); + ~WeChatQRCode(){}; + + /** + * @brief Both detects and decodes QR code. + * To simplify the usage, there is a only API: detectAndDecode + * + * @param img supports grayscale or color (BGR) image. + * @param points optional output array of vertices of the found QR code quadrangle. Will be + * empty if not found. + * @return list of decoded string. + */ + CV_WRAP std::vector detectAndDecode(InputArray img, + OutputArrayOfArrays points = noArray()); + +protected: + class Impl; + Ptr p; +}; + +//! @} +} // namespace wechat_qrcode +} // namespace cv +#endif // __OPENCV_WECHAT_QRCODE_HPP__ diff --git a/modules/wechat_qrcode/samples/qrcode.py b/modules/wechat_qrcode/samples/qrcode.py new file mode 100644 index 00000000000..e3cc003c110 --- /dev/null +++ b/modules/wechat_qrcode/samples/qrcode.py @@ -0,0 +1,60 @@ +# This file is part of OpenCV project. +# It is subject to the license terms in the LICENSE file found in the top-level directory +# of this distribution and at http://opencv.org/license.html. +# +# Tencent is pleased to support the open source community by making WeChat QRCode available. +# Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +import cv2 +import sys + +print(sys.argv[0]) +print('A demo program of WeChat QRCode Detector:') +camIdx = -1 +if len(sys.argv) > 1: + if sys.argv[1] == "-camera": + camIdx = int(sys.argv[2]) if len(sys.argv)>2 else 0 + img = cv2.imread(sys.argv[1]) +else: + print(" Usage: " + sys.argv[0] + " ") + exit(0) + +# For python API generator, it follows the template: {module_name}_{class_name}, +# so it is a little weird. +# The model is downloaded to ${CMAKE_BINARY_DIR}/downloads/wechat_qrcode if cmake runs without warnings, +# otherwise you can download them from https://github.com/WeChatCV/opencv_3rdparty/tree/wechat_qrcode. +try: + detector = cv2.wechat_qrcode_WeChatQRCode( + "detect.prototxt", "detect.caffemodel", "sr.prototxt", "sr.caffemodel") +except: + print("---------------------------------------------------------------") + print("Failed to initialize WeChatQRCode.") + print("Please, download 'detector.*' and 'sr.*' from") + print("https://github.com/WeChatCV/opencv_3rdparty/tree/wechat_qrcode") + print("and put them into the current directory.") + print("---------------------------------------------------------------") + exit(0) + +prevstr = "" + +if camIdx < 0: + res, points = detector.detectAndDecode(img) + print(res) +else: + cap = cv2.VideoCapture(camIdx) + while True: + res, img = cap.read() + if img.empty(): + break + res, points = detector.detectAndDecode(img) + for t in res: + if t != prevstr: + print(t) + if res: + prevstr = res[-1] + cv2.imshow("image", img) + if cv2.waitKey(30) >= 0: + break + # When everything done, release the capture + cap.release() + cv2.destroyAllWindows() diff --git a/modules/wechat_qrcode/samples/qrcode_example.cpp b/modules/wechat_qrcode/samples/qrcode_example.cpp new file mode 100644 index 00000000000..62217b258f1 --- /dev/null +++ b/modules/wechat_qrcode/samples/qrcode_example.cpp @@ -0,0 +1,77 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#include +#include +#include +#include + +using namespace std; +using namespace cv; + +#include +int main(int argc, char* argv[]) { + cout << endl << argv[0] << endl << endl; + cout << "A demo program of WeChat QRCode Detector: " << endl; + + Mat img; + int camIdx = -1; + if (argc > 1) { + bool live = strcmp(argv[1], "-camera") == 0; + if (live) { + camIdx = argc > 2 ? atoi(argv[2]) : 0; + } else { + img = imread(argv[1]); + } + } else { + cout << " Usage: " << argv[0] << " " << endl; + return 0; + } + // The model is downloaded to ${CMAKE_BINARY_DIR}/downloads/wechat_qrcode if cmake runs without warnings, + // otherwise you can download them from https://github.com/WeChatCV/opencv_3rdparty/tree/wechat_qrcode. + Ptr detector; + + try { + detector = makePtr("detect.prototxt", "detect.caffemodel", + "sr.prototxt", "sr.caffemodel"); + } catch (const std::exception& e) { + cout << + "\n---------------------------------------------------------------\n" + "Failed to initialize WeChatQRCode.\n" + "Please, download 'detector.*' and 'sr.*' from\n" + "https://github.com/WeChatCV/opencv_3rdparty/tree/wechat_qrcode\n" + "and put them into the current directory.\n" + "---------------------------------------------------------------\n"; + cout << e.what() << endl; + return 0; + } + string prevstr = ""; + vector points; + + if (camIdx < 0) { + auto res = detector->detectAndDecode(img, points); + for (const auto& t : res) cout << t << endl; + } else { + VideoCapture cap(camIdx); + for(;;) { + cap >> img; + if (img.empty()) + break; + auto res = detector->detectAndDecode(img, points); + for (const auto& t : res) { + if (t != prevstr) + cout << t << endl; + } + if (!res.empty()) + prevstr = res.back(); + imshow("image", img); + if (waitKey(30) >= 0) + break; + } + } + return 0; +} \ No newline at end of file diff --git a/modules/wechat_qrcode/src/binarizermgr.cpp b/modules/wechat_qrcode/src/binarizermgr.cpp new file mode 100644 index 00000000000..d651ff07be5 --- /dev/null +++ b/modules/wechat_qrcode/src/binarizermgr.cpp @@ -0,0 +1,70 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#include "binarizermgr.hpp" +#include "imgsource.hpp" +#include "precomp.hpp" + +using zxing::Binarizer; +using zxing::LuminanceSource; +namespace cv { +namespace wechat_qrcode { +BinarizerMgr::BinarizerMgr() : m_iNowRotateIndex(0), m_iNextOnceBinarizer(-1) { + m_vecRotateBinarizer.push_back(Hybrid); + m_vecRotateBinarizer.push_back(FastWindow); + m_vecRotateBinarizer.push_back(SimpleAdaptive); + m_vecRotateBinarizer.push_back(AdaptiveThreshold); +} + +BinarizerMgr::~BinarizerMgr() {} + +zxing::Ref BinarizerMgr::Binarize(zxing::Ref source) { + BINARIZER binarizerIdx = m_vecRotateBinarizer[m_iNowRotateIndex]; + if (m_iNextOnceBinarizer >= 0) { + binarizerIdx = (BINARIZER)m_iNextOnceBinarizer; + } + + zxing::Ref binarizer; + switch (binarizerIdx) { + case Hybrid: + binarizer = new zxing::HybridBinarizer(source); + break; + case FastWindow: + binarizer = new zxing::FastWindowBinarizer(source); + break; + case SimpleAdaptive: + binarizer = new zxing::SimpleAdaptiveBinarizer(source); + break; + case AdaptiveThreshold: + binarizer = new zxing::AdaptiveThresholdMeanBinarizer(source); + break; + default: + binarizer = new zxing::HybridBinarizer(source); + break; + } + + return binarizer; +} + +void BinarizerMgr::SwitchBinarizer() { + m_iNowRotateIndex = (m_iNowRotateIndex + 1) % m_vecRotateBinarizer.size(); +} + +int BinarizerMgr::GetCurBinarizer() { + if (m_iNextOnceBinarizer != -1) return m_iNextOnceBinarizer; + return m_vecRotateBinarizer[m_iNowRotateIndex]; +} + +void BinarizerMgr::SetNextOnceBinarizer(int iBinarizerIndex) { + m_iNextOnceBinarizer = iBinarizerIndex; +} + +void BinarizerMgr::SetBinarizer(vector vecRotateBinarizer) { + m_vecRotateBinarizer = vecRotateBinarizer; +} +} // namespace wechat_qrcode +} // namespace cv \ No newline at end of file diff --git a/modules/wechat_qrcode/src/binarizermgr.hpp b/modules/wechat_qrcode/src/binarizermgr.hpp new file mode 100644 index 00000000000..a8a1e4ba7cf --- /dev/null +++ b/modules/wechat_qrcode/src/binarizermgr.hpp @@ -0,0 +1,51 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#ifndef __OPENCV_WECHAT_QRCODE_BINARIZERMGR_HPP__ +#define __OPENCV_WECHAT_QRCODE_BINARIZERMGR_HPP__ + +#include "zxing/binarizer.hpp" +#include "zxing/common/binarizer/adaptive_threshold_mean_binarizer.hpp" +#include "zxing/common/counted.hpp" +#include "zxing/common/binarizer/fast_window_binarizer.hpp" +#include "zxing/common/binarizer/hybrid_binarizer.hpp" +#include "zxing/common/binarizer/simple_adaptive_binarizer.hpp" +#include "zxing/zxing.hpp" + +namespace cv { +namespace wechat_qrcode { +class BinarizerMgr { +public: + enum BINARIZER { + Hybrid = 0, + FastWindow = 1, + SimpleAdaptive = 2, + AdaptiveThreshold = 3 + }; + +public: + BinarizerMgr(); + ~BinarizerMgr(); + + zxing::Ref Binarize(zxing::Ref source); + + void SwitchBinarizer(); + + int GetCurBinarizer(); + + void SetNextOnceBinarizer(int iBinarizerIndex); + + void SetBinarizer(vector vecRotateBinarizer); + +private: + int m_iNowRotateIndex; + int m_iNextOnceBinarizer; + vector m_vecRotateBinarizer; +}; +} // namespace wechat_qrcode +} // namespace cv +#endif // __OPENCV_WECHAT_QRCODE_BINARIZERMGR_HPP__ diff --git a/modules/wechat_qrcode/src/decodermgr.cpp b/modules/wechat_qrcode/src/decodermgr.cpp new file mode 100644 index 00000000000..cf40b4fffbc --- /dev/null +++ b/modules/wechat_qrcode/src/decodermgr.cpp @@ -0,0 +1,80 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#include "decodermgr.hpp" +#include "precomp.hpp" + +using zxing::ArrayRef; +using zxing::BinaryBitmap; +using zxing::DecodeHints; +using zxing::ErrorHandler; +using zxing::LuminanceSource; +using zxing::Ref; +using zxing::Result; +using zxing::UnicomBlock; +namespace cv { +namespace wechat_qrcode { +int DecoderMgr::decodeImage(cv::Mat src, bool use_nn_detector, string& result) { + int width = src.cols; + int height = src.rows; + if (width <= 20 || height <= 20) + return -1; // image data is not enough for providing reliable results + + std::vector scaled_img_data(src.data, src.data + width * height); + zxing::ArrayRef scaled_img_zx = + zxing::ArrayRef(new zxing::Array(scaled_img_data)); + + zxing::Ref zx_result; + + decode_hints_.setUseNNDetector(use_nn_detector); + + Ref source; + qbarUicomBlock_ = new UnicomBlock(width, height); + + // Four Binarizers + int tryBinarizeTime = 4; + for (int tb = 0; tb < tryBinarizeTime; tb++) { + if (source == NULL || height * width > source->getMaxSize()) { + source = ImgSource::create(scaled_img_zx.data(), width, height); + } else { + source->reset(scaled_img_zx.data(), width, height); + } + int ret = TryDecode(source, zx_result); + if (!ret) { + result = zx_result->getText()->getText(); + return ret; + } + // try different binarizers + binarizer_mgr_.SwitchBinarizer(); + } + return -1; +} + +int DecoderMgr::TryDecode(Ref source, Ref& result) { + int res = -1; + string cell_result; + + // get binarizer + zxing::Ref binarizer = binarizer_mgr_.Binarize(source); + zxing::Ref binary_bitmap(new BinaryBitmap(binarizer)); + binary_bitmap->m_poUnicomBlock = qbarUicomBlock_; + + result = Decode(binary_bitmap, decode_hints_); + res = (result == NULL) ? 1 : 0; + + if (res == 0) { + result->setBinaryMethod(int(binarizer_mgr_.GetCurBinarizer())); + } + + return res; +} + +Ref DecoderMgr::Decode(Ref image, DecodeHints hints) { + return reader_->decode(image, hints); +} +} // namespace wechat_qrcode +} // namespace cv \ No newline at end of file diff --git a/modules/wechat_qrcode/src/decodermgr.hpp b/modules/wechat_qrcode/src/decodermgr.hpp new file mode 100644 index 00000000000..10ac16e7c72 --- /dev/null +++ b/modules/wechat_qrcode/src/decodermgr.hpp @@ -0,0 +1,46 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#ifndef __OPENCV_WECHAT_QRCODE_DECODERMGR_HPP__ +#define __OPENCV_WECHAT_QRCODE_DECODERMGR_HPP__ + +// zxing +#include "zxing/binarizer.hpp" +#include "zxing/binarybitmap.hpp" +#include "zxing/decodehints.hpp" +#include "zxing/qrcode/qrcode_reader.hpp" +#include "zxing/result.hpp" + +// qbar +#include "binarizermgr.hpp" +#include "imgsource.hpp" +namespace cv { +namespace wechat_qrcode { + +class DecoderMgr { +public: + DecoderMgr() { reader_ = new zxing::qrcode::QRCodeReader(); }; + ~DecoderMgr(){}; + + int decodeImage(cv::Mat src, bool use_nn_detector, string& result); + +private: + zxing::Ref qbarUicomBlock_; + zxing::DecodeHints decode_hints_; + + zxing::Ref reader_; + BinarizerMgr binarizer_mgr_; + + zxing::Ref Decode(zxing::Ref image, + zxing::DecodeHints hints); + + int TryDecode(zxing::Ref source, zxing::Ref& result); +}; + +} // namespace wechat_qrcode +} // namespace cv +#endif // __OPENCV_WECHAT_QRCODE_DECODERMGR_HPP__ diff --git a/modules/wechat_qrcode/src/detector/align.cpp b/modules/wechat_qrcode/src/detector/align.cpp new file mode 100644 index 00000000000..06fe6cf8fa1 --- /dev/null +++ b/modules/wechat_qrcode/src/detector/align.cpp @@ -0,0 +1,66 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#include "align.hpp" + +using std::max; +using std::min; +namespace cv { +namespace wechat_qrcode { +Align::Align() { rotate90_ = false; } + +Mat Align::calcWarpMatrix(const Mat src, const Mat dst) { + M_ = getPerspectiveTransform(src, dst); + M_inv_ = M_.inv(); + return M_; +} + +vector Align::warpBack(const vector &dst_pts) { + vector src_pts; + for (size_t j = 0; j < dst_pts.size(); j++) { + auto src_x = (rotate90_ ? dst_pts[j].y : dst_pts[j].x) + crop_x_; + auto src_y = (rotate90_ ? dst_pts[j].x : dst_pts[j].y) + crop_y_; + src_pts.push_back(Point2f(src_x, src_y)); + } + return src_pts; +} + +Mat Align::crop(const Mat &inputImg, const int width, const int height) { + Mat warp_dst = Mat::zeros(height, width, inputImg.type()); + + warpPerspective(inputImg, warp_dst, M_, warp_dst.size(), INTER_LINEAR, BORDER_CONSTANT, 255); + + return warp_dst; +} + +Mat Align::crop(const Mat &inputImg, const Mat &srcPts, const float paddingW, const float paddingH, + const int minPadding) { + int x0 = srcPts.at(0, 0); + int y0 = srcPts.at(0, 1); + int x2 = srcPts.at(2, 0); + int y2 = srcPts.at(2, 1); + + int width = x2 - x0 + 1; + int height = y2 - y0 + 1; + + int padx = max(paddingW * width, static_cast(minPadding)); + int pady = max(paddingH * height, static_cast(minPadding)); + + crop_x_ = max(x0 - padx, 0); + crop_y_ = max(y0 - pady, 0); + int end_x = min(x2 + padx, inputImg.cols - 1); + int end_y = min(y2 + pady, inputImg.rows - 1); + + Rect crop_roi(crop_x_, crop_y_, end_x - crop_x_ + 1, end_y - crop_y_ + 1); + + Mat dst = inputImg(crop_roi).clone(); + if (rotate90_) dst = dst.t(); // really is just transpose + return dst; +} + +} // namespace wechat_qrcode +} // namespace cv diff --git a/modules/wechat_qrcode/src/detector/align.hpp b/modules/wechat_qrcode/src/detector/align.hpp new file mode 100644 index 00000000000..c8cbc17849e --- /dev/null +++ b/modules/wechat_qrcode/src/detector/align.hpp @@ -0,0 +1,42 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#ifndef __DETECTOR_ALIGN_HPP_ +#define __DETECTOR_ALIGN_HPP_ + +#include +#include +#include "../precomp.hpp" +#include "opencv2/core.hpp" +#include "opencv2/imgproc.hpp" + +namespace cv { +namespace wechat_qrcode { + +class Align { +public: + Align(); + Mat calcWarpMatrix(const Mat src, const Mat dst); + std::vector warpBack(const std::vector &dst_pts); + Mat crop(const Mat &inputImg, const Mat &srcPts, const float paddingW, const float paddingH, + const int minPadding); + + void setRotate90(bool v) { rotate90_ = v; } + +private: + Mat crop(const Mat &inputImg, const int width, const int height); + Mat M_; + Mat M_inv_; + + int crop_x_; + int crop_y_; + bool rotate90_; +}; + +} // namespace wechat_qrcode +} // namespace cv +#endif // __DETECTOR_ALIGN_HPP_ diff --git a/modules/wechat_qrcode/src/detector/ssd_detector.cpp b/modules/wechat_qrcode/src/detector/ssd_detector.cpp new file mode 100644 index 00000000000..f0455534ffd --- /dev/null +++ b/modules/wechat_qrcode/src/detector/ssd_detector.cpp @@ -0,0 +1,53 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#include "ssd_detector.hpp" +#include "../precomp.hpp" +#define CLIP(x, x1, x2) max(x1, min(x, x2)) +namespace cv { +namespace wechat_qrcode { +int SSDDetector::init(const string& proto_path, const string& model_path) { + net_ = dnn::readNetFromCaffe(proto_path, model_path); + return 0; +} + +vector SSDDetector::forward(Mat img, const int target_width, const int target_height) { + int img_w = img.cols; + int img_h = img.rows; + Mat input; + resize(img, input, Size(target_width, target_height), 0, 0, INTER_CUBIC); + + dnn::blobFromImage(input, input, 1.0 / 255, Size(input.cols, input.rows), {0.0f, 0.0f, 0.0f}, + false, false); + net_.setInput(input, "data"); + + auto prob = net_.forward("detection_output"); + vector point_list; + for (int row = 0; row < prob.size[2]; row++) { + const float* prob_score = prob.ptr(0, 0, row); + if (prob_score[1] == 1) { + auto point = Mat(4, 2, CV_32FC1); + float x0 = CLIP(prob_score[3] * img_w, 0.0f, img_w - 1.0f); + float y0 = CLIP(prob_score[4] * img_h, 0.0f, img_h - 1.0f); + float x1 = CLIP(prob_score[5] * img_w, 0.0f, img_w - 1.0f); + float y1 = CLIP(prob_score[6] * img_h, 0.0f, img_h - 1.0f); + + point.at(0, 0) = x0; + point.at(0, 1) = y0; + point.at(1, 0) = x1; + point.at(1, 1) = y0; + point.at(2, 0) = x1; + point.at(2, 1) = y1; + point.at(3, 0) = x0; + point.at(3, 1) = y1; + point_list.push_back(point); + } + } + return point_list; +} +} // namespace wechat_qrcode +} // namespace cv \ No newline at end of file diff --git a/modules/wechat_qrcode/src/detector/ssd_detector.hpp b/modules/wechat_qrcode/src/detector/ssd_detector.hpp new file mode 100644 index 00000000000..e510cb32477 --- /dev/null +++ b/modules/wechat_qrcode/src/detector/ssd_detector.hpp @@ -0,0 +1,31 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#ifndef __DETECTOR_SSD_DETECTOR_HPP_ +#define __DETECTOR_SSD_DETECTOR_HPP_ + +#include + +#include "opencv2/dnn.hpp" +#include "opencv2/imgproc.hpp" +namespace cv { +namespace wechat_qrcode { + +class SSDDetector { +public: + SSDDetector(){}; + ~SSDDetector(){}; + int init(const std::string& proto_path, const std::string& model_path); + std::vector forward(Mat img, const int target_width, const int target_height); + +private: + dnn::Net net_; +}; + +} // namespace wechat_qrcode +} // namespace cv +#endif // __DETECTOR_SSD_DETECTOR_HPP_ diff --git a/modules/wechat_qrcode/src/imgsource.cpp b/modules/wechat_qrcode/src/imgsource.cpp new file mode 100644 index 00000000000..7bec2989b6c --- /dev/null +++ b/modules/wechat_qrcode/src/imgsource.cpp @@ -0,0 +1,188 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "imgsource.hpp" +#include "precomp.hpp" +using zxing::ArrayRef; +using zxing::ByteMatrix; +using zxing::ErrorHandler; +using zxing::LuminanceSource; +using zxing::Ref; +namespace cv { +namespace wechat_qrcode { + +// Initialize the ImgSource +ImgSource::ImgSource(unsigned char* pixels, int width, int height) + : Super(width, height) { + luminances = new unsigned char[width * height]; + memset(luminances, 0, width * height); + + rgbs = pixels; + + dataWidth = width; + dataHeight = height; + left = 0; + top = 0; + + // Make gray luminances first + makeGray(); +} + +// Added for crop function +ImgSource::ImgSource(unsigned char* pixels, int width, int height, int left_, int top_, + int cropWidth, int cropHeight, + ErrorHandler& err_handler) + : Super(cropWidth, cropHeight) { + rgbs = pixels; + + dataWidth = width; + dataHeight = height; + left = left_; + top = top_; + + // super(width, height); + if ((left_ + cropWidth) > dataWidth || (top_ + cropHeight) > dataHeight || top_ < 0 || + left_ < 0) { + err_handler = + zxing::IllegalArgumentErrorHandler("Crop rectangle does not fit within image data."); + return; + } + + luminances = new unsigned char[width * height]; + + // Make gray luminances first + makeGray(); +} + +ImgSource::~ImgSource() { + if (luminances != NULL) { + delete[] luminances; + } +} + +Ref ImgSource::create(unsigned char* pixels, int width, int height) { + return Ref(new ImgSource(pixels, width, height)); +} + +Ref ImgSource::create(unsigned char* pixels, int width, int height, int left, int top, + int cropWidth, int cropHeight, + zxing::ErrorHandler& err_handler) { + return Ref(new ImgSource(pixels, width, height, left, top, cropWidth, cropHeight, err_handler)); +} + +void ImgSource::reset(unsigned char* pixels, int width, int height) { + rgbs = pixels; + left = 0; + top = 0; + + setWidth(width); + setHeight(height); + dataWidth = width; + dataHeight = height; + makeGrayReset(); +} + +ArrayRef ImgSource::getRow(int y, zxing::ArrayRef row, + zxing::ErrorHandler& err_handler) const { + if (y < 0 || y >= getHeight()) { + err_handler = zxing::IllegalArgumentErrorHandler("Requested row is outside the image"); + return ArrayRef(); + } + + int width = getWidth(); + if (row->data() == NULL || row->empty() || row->size() < width) { + row = zxing::ArrayRef(width); + } + int offset = (y + top) * dataWidth + left; + + char* rowPtr = &row[0]; + arrayCopy(luminances, offset, rowPtr, 0, width); + + return row; +} + +/** This is a more efficient implementation. */ +ArrayRef ImgSource::getMatrix() const { + int width = getWidth(); + int height = getHeight(); + + int area = width * height; + + // If the caller asks for the entire underlying image, save the copy and + // give them the original data. The docs specifically warn that + // result.length must be ignored. + if (width == dataWidth && height == dataHeight) { + return _matrix; + } + + zxing::ArrayRef newMatrix = zxing::ArrayRef(area); + + int inputOffset = top * dataWidth + left; + + // If the width matches the full width of the underlying data, perform a + // single copy. + if (width == dataWidth) { + arrayCopy(luminances, inputOffset, &newMatrix[0], 0, area); + return newMatrix; + } + + // Otherwise copy one cropped row at a time. + for (int y = 0; y < height; y++) { + int outputOffset = y * width; + arrayCopy(luminances, inputOffset, &newMatrix[0], outputOffset, width); + inputOffset += dataWidth; + } + return newMatrix; +} + + +void ImgSource::makeGray() { + int area = dataWidth * dataHeight; + _matrix = zxing::ArrayRef(area); + arrayCopy(rgbs, 0, &_matrix[0], 0, area); +} + +void ImgSource::makeGrayReset() { + int area = dataWidth * dataHeight; + arrayCopy(rgbs, 0, &_matrix[0], 0, area); +} + +void ImgSource::arrayCopy(unsigned char* src, int inputOffset, char* dst, int outputOffset, + int length) const { + const unsigned char* srcPtr = src + inputOffset; + char* dstPtr = dst + outputOffset; + + memcpy(dstPtr, srcPtr, length * sizeof(unsigned char)); +} + +bool ImgSource::isCropSupported() const { return true; } + +Ref ImgSource::crop(int left_, int top_, int width, int height, + ErrorHandler& err_handler) const { + return ImgSource::create(rgbs, dataWidth, dataHeight, left + left_, top + top_, width, height, err_handler); +} + +bool ImgSource::isRotateSupported() const { return false; } + +Ref ImgSource::rotateCounterClockwise(ErrorHandler& err_handler) const { + // Intentionally flip the left, top, width, and height arguments as + // needed. dataWidth and dataHeight are always kept unrotated. + int width = getWidth(); + int height = getHeight(); + + return ImgSource::create(rgbs, dataWidth, dataHeight, top, left, height, width, err_handler); +} + + +Ref ImgSource::getByteMatrix() const { + return Ref(new ByteMatrix(getWidth(), getHeight(), getMatrix())); +} +} // namespace wechat_qrcode +} // namespace cv \ No newline at end of file diff --git a/modules/wechat_qrcode/src/imgsource.hpp b/modules/wechat_qrcode/src/imgsource.hpp new file mode 100644 index 00000000000..ec62dc6ab3f --- /dev/null +++ b/modules/wechat_qrcode/src/imgsource.hpp @@ -0,0 +1,63 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __OPENCV_WECHAT_QRCODE_IMGSOURCE_HPP__ +#define __OPENCV_WECHAT_QRCODE_IMGSOURCE_HPP__ +#include "zxing/common/bytematrix.hpp" +#include "zxing/errorhandler.hpp" +#include "zxing/luminance_source.hpp" +namespace cv { +namespace wechat_qrcode { +class ImgSource : public zxing::LuminanceSource { +private: + typedef LuminanceSource Super; + zxing::ArrayRef _matrix; + unsigned char* rgbs; + unsigned char* luminances; + int dataWidth; + int dataHeight; + int left; + int top; + void makeGray(); + void makeGrayReset(); + + void arrayCopy(unsigned char* src, int inputOffset, char* dst, int outputOffset, + int length) const; + + + ~ImgSource(); + +public: + ImgSource(unsigned char* pixels, int width, int height); + ImgSource(unsigned char* pixels, int width, int height, int left, int top, int cropWidth, + int cropHeight, zxing::ErrorHandler& err_handler); + + static zxing::Ref create(unsigned char* pixels, int width, int height); + static zxing::Ref create(unsigned char* pixels, int width, int height, int left, + int top, int cropWidth, int cropHeight, zxing::ErrorHandler& err_handler); + void reset(unsigned char* pixels, int width, int height); + zxing::ArrayRef getRow(int y, zxing::ArrayRef row, + zxing::ErrorHandler& err_handler) const override; + zxing::ArrayRef getMatrix() const override; + zxing::Ref getByteMatrix() const override; + + bool isCropSupported() const override; + zxing::Ref crop(int left, int top, int width, int height, + zxing::ErrorHandler& err_handler) const override; + + bool isRotateSupported() const override; + zxing::Ref rotateCounterClockwise( + zxing::ErrorHandler& err_handler) const override; + + int getMaxSize() { return dataHeight * dataWidth; } +}; +} // namespace wechat_qrcode +} // namespace cv +#endif // __OPENCV_WECHAT_QRCODE_IMGSOURCE_HPP__ diff --git a/modules/wechat_qrcode/src/precomp.hpp b/modules/wechat_qrcode/src/precomp.hpp new file mode 100644 index 00000000000..46c09c35c32 --- /dev/null +++ b/modules/wechat_qrcode/src/precomp.hpp @@ -0,0 +1,29 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#ifndef __OPENCV_WECHAT_QRCODE_PRECOMP_HPP__ +#define __OPENCV_WECHAT_QRCODE_PRECOMP_HPP__ +#ifdef _MSC_VER +#pragma warning(disable: 4244) +#pragma warning(disable: 4267) +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "imgsource.hpp" +using std::ostringstream; +using std::string; +using std::vector; +#endif // __OPENCV_WECHAT_QRCODE_PRECOMP_HPP__ diff --git a/modules/wechat_qrcode/src/scale/super_scale.cpp b/modules/wechat_qrcode/src/scale/super_scale.cpp new file mode 100644 index 00000000000..5a70975ec87 --- /dev/null +++ b/modules/wechat_qrcode/src/scale/super_scale.cpp @@ -0,0 +1,63 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#include "super_scale.hpp" +#include "../precomp.hpp" + +#define CLIP(x, x1, x2) max(x1, min(x, x2)) +namespace cv { +namespace wechat_qrcode { +int SuperScale::init(const std::string &proto_path, const std::string &model_path) { + srnet_ = dnn::readNetFromCaffe(proto_path, model_path); + net_loaded_ = true; + return 0; +} + +Mat SuperScale::processImageScale(const Mat &src, float scale, const bool &use_sr, + int sr_max_size) { + Mat dst = src; + if (scale == 1.0) { // src + return dst; + } + + int width = src.cols; + int height = src.rows; + if (scale == 2.0) { // upsample + int SR_TH = sr_max_size; + if (use_sr && (int)sqrt(width * height * 1.0) < SR_TH && net_loaded_) { + int ret = superResoutionScale(src, dst); + if (ret == 0) return dst; + } + + { resize(src, dst, Size(), scale, scale, INTER_CUBIC); } + } else if (scale < 1.0) { // downsample + resize(src, dst, Size(), scale, scale, INTER_AREA); + } + + return dst; +} + +int SuperScale::superResoutionScale(const Mat &src, Mat &dst) { + Mat blob; + dnn::blobFromImage(src, blob, 1.0 / 255, Size(src.cols, src.rows), {0.0f}, false, false); + + srnet_.setInput(blob); + auto prob = srnet_.forward(); + + dst = Mat(prob.size[2], prob.size[3], CV_8UC1); + + for (int row = 0; row < prob.size[2]; row++) { + const float *prob_score = prob.ptr(0, 0, row); + for (int col = 0; col < prob.size[3]; col++) { + float pixel = prob_score[col] * 255.0; + dst.at(row, col) = static_cast(CLIP(pixel, 0.0f, 255.0f)); + } + } + return 0; +} +} // namespace wechat_qrcode +} // namespace cv \ No newline at end of file diff --git a/modules/wechat_qrcode/src/scale/super_scale.hpp b/modules/wechat_qrcode/src/scale/super_scale.hpp new file mode 100644 index 00000000000..2717932c555 --- /dev/null +++ b/modules/wechat_qrcode/src/scale/super_scale.hpp @@ -0,0 +1,32 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#ifndef __SCALE_SUPER_SCALE_HPP_ +#define __SCALE_SUPER_SCALE_HPP_ + +#include +#include "opencv2/dnn.hpp" +#include "opencv2/imgproc.hpp" +namespace cv { +namespace wechat_qrcode { + +class SuperScale { +public: + SuperScale(){}; + ~SuperScale(){}; + int init(const std::string &proto_path, const std::string &model_path); + Mat processImageScale(const Mat &src, float scale, const bool &use_sr, int sr_max_size = 160); + +private: + dnn::Net srnet_; + bool net_loaded_ = false; + int superResoutionScale(const cv::Mat &src, cv::Mat &dst); +}; + +} // namespace wechat_qrcode +} // namespace cv +#endif // __SCALE_SUPER_SCALE_HPP_ diff --git a/modules/wechat_qrcode/src/wechat_qrcode.cpp b/modules/wechat_qrcode/src/wechat_qrcode.cpp new file mode 100644 index 00000000000..270f6920b8c --- /dev/null +++ b/modules/wechat_qrcode/src/wechat_qrcode.cpp @@ -0,0 +1,211 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#include "opencv2/wechat_qrcode.hpp" +#include "decodermgr.hpp" +#include "detector/align.hpp" +#include "detector/ssd_detector.hpp" +#include "opencv2/core.hpp" +#include "opencv2/core/utils/filesystem.hpp" +#include "precomp.hpp" +#include "scale/super_scale.hpp" +#include "zxing/result.hpp" +using cv::InputArray; +namespace cv { +namespace wechat_qrcode { +class WeChatQRCode::Impl { +public: + Impl() {} + ~Impl() {} + /** + * @brief detect QR codes from the given image + * + * @param img supports grayscale or color (BGR) image. + * @return vector detected QR code bounding boxes. + */ + std::vector detect(const Mat& img); + /** + * @brief decode QR codes from detected points + * + * @param img supports grayscale or color (BGR) image. + * @param candidate_points detected points. we name it "candidate points" which means no + * all the qrcode can be decoded. + * @param points succussfully decoded qrcode with bounding box points. + * @return vector + */ + std::vector decode(const Mat& img, std::vector& candidate_points, + std::vector& points); + int applyDetector(const Mat& img, std::vector& points); + Mat cropObj(const Mat& img, const Mat& point, Align& aligner); + std::vector getScaleList(const int width, const int height); + std::shared_ptr detector_; + std::shared_ptr super_resolution_model_; + bool use_nn_detector_, use_nn_sr_; +}; + +WeChatQRCode::WeChatQRCode(const String& detector_prototxt_path, + const String& detector_caffe_model_path, + const String& super_resolution_prototxt_path, + const String& super_resolution_caffe_model_path) { + p = makePtr(); + if (!detector_caffe_model_path.empty() && !detector_prototxt_path.empty()) { + // initialize detector model (caffe) + p->use_nn_detector_ = true; + CV_CheckEQ(utils::fs::exists(detector_prototxt_path), true, + "fail to find detector caffe prototxt file"); + CV_CheckEQ(utils::fs::exists(detector_caffe_model_path), true, + "fail to find detector caffe model file"); + p->detector_ = make_shared(); + auto ret = p->detector_->init(detector_prototxt_path, detector_caffe_model_path); + CV_CheckEQ(ret, 0, "fail to load the detector model."); + } else { + p->use_nn_detector_ = false; + p->detector_ = NULL; + } + // initialize super_resolution_model + // it could also support non model weights by cubic resizing + // so, we initialize it first. + p->super_resolution_model_ = make_shared(); + if (!super_resolution_prototxt_path.empty() && !super_resolution_caffe_model_path.empty()) { + p->use_nn_sr_ = true; + // initialize dnn model (onnx format) + CV_CheckEQ(utils::fs::exists(super_resolution_prototxt_path), true, + "fail to find super resolution prototxt model file"); + CV_CheckEQ(utils::fs::exists(super_resolution_caffe_model_path), true, + "fail to find super resolution caffe model file"); + auto ret = p->super_resolution_model_->init(super_resolution_prototxt_path, + super_resolution_caffe_model_path); + CV_CheckEQ(ret, 0, "fail to load the super resolution model."); + } else { + p->use_nn_sr_ = false; + } +} + +vector WeChatQRCode::detectAndDecode(InputArray img, OutputArrayOfArrays points) { + CV_Assert(!img.empty()); + CV_CheckDepthEQ(img.depth(), CV_8U, ""); + + if (img.cols() <= 20 || img.rows() <= 20) { + return vector(); // image data is not enough for providing reliable results + } + Mat input_img; + int incn = img.channels(); + CV_Check(incn, incn == 1 || incn == 3 || incn == 4, ""); + if (incn == 3 || incn == 4) { + cv::cvtColor(img, input_img, cv::COLOR_BGR2GRAY); + } else { + input_img = img.getMat(); + } + auto candidate_points = p->detect(input_img); + auto res_points = vector(); + auto ret = p->decode(input_img, candidate_points, res_points); + // opencv type convert + vector tmp_points; + if (points.needed()) { + for (size_t i = 0; i < res_points.size(); i++) { + Mat tmp_point; + tmp_points.push_back(tmp_point); + res_points[i].convertTo(((OutputArray)tmp_points[i]), + ((OutputArray)tmp_points[i]).fixedType() + ? ((OutputArray)tmp_points[i]).type() + : CV_32FC2); + } + points.createSameSize(tmp_points, CV_32FC2); + points.assign(tmp_points); + } + return ret; +}; + +vector WeChatQRCode::Impl::decode(const Mat& img, vector& candidate_points, + vector& points) { + if (candidate_points.size() == 0) { + return vector(); + } + vector decode_results; + for (auto& point : candidate_points) { + cv::Mat cropped_img; + if (use_nn_detector_) { + Align aligner; + cropped_img = cropObj(img, point, aligner); + } else { + cropped_img = img; + } + // scale_list contains different scale ratios + auto scale_list = getScaleList(cropped_img.cols, cropped_img.rows); + for (auto cur_scale : scale_list) { + cv::Mat scaled_img = + super_resolution_model_->processImageScale(cropped_img, cur_scale, use_nn_sr_); + string result; + DecoderMgr decodemgr; + auto ret = decodemgr.decodeImage(scaled_img, use_nn_detector_, result); + + if (ret == 0) { + decode_results.push_back(result); + points.push_back(point); + break; + } + } + } + + return decode_results; +} + +vector WeChatQRCode::Impl::detect(const Mat& img) { + auto points = vector(); + + if (use_nn_detector_) { + // use cnn detector + auto ret = applyDetector(img, points); + CV_CheckEQ(ret, 0, "fail to apply detector."); + } else { + auto width = img.cols, height = img.rows; + // if there is no detector, use the full image as input + auto point = Mat(4, 2, CV_32FC1); + point.at(0, 0) = 0; + point.at(0, 1) = 0; + point.at(1, 0) = width - 1; + point.at(1, 1) = 0; + point.at(2, 0) = width - 1; + point.at(2, 1) = height - 1; + point.at(3, 0) = 0; + point.at(3, 1) = height - 1; + points.push_back(point); + } + return points; +} + +int WeChatQRCode::Impl::applyDetector(const cv::Mat& img, vector& points) { + int img_w = img.cols; + int img_h = img.rows; + + // hard code input size + int minInputSize = 400; + float resizeRatio = sqrt(img_w * img_h * 1.0 / (minInputSize * minInputSize)); + int detect_width = img_w / resizeRatio; + int detect_height = img_h / resizeRatio; + + points = detector_->forward(img, detect_width, detect_height); + + return 0; +} + +cv::Mat WeChatQRCode::Impl::cropObj(const cv::Mat& img, const Mat& point, Align& aligner) { + // make some padding to boost the qrcode details recall. + float padding_w = 0.1f, padding_h = 0.1f; + auto min_padding = 15; + auto cropped = aligner.crop(img, point, padding_w, padding_h, min_padding); + return cropped; +} + +// empirical rules +vector WeChatQRCode::Impl::getScaleList(const int width, const int height) { + if (width < 320 || height < 320) return {1.0, 2.0, 0.5}; + if (width < 640 && height < 640) return {1.0, 0.5}; + return {0.5, 1.0}; +} +} // namespace wechat_qrcode +} // namespace cv \ No newline at end of file diff --git a/modules/wechat_qrcode/src/zxing/binarizer.cpp b/modules/wechat_qrcode/src/zxing/binarizer.cpp new file mode 100644 index 00000000000..758629fab13 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/binarizer.cpp @@ -0,0 +1,90 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "binarizer.hpp" + +#include + +namespace zxing { + +Binarizer::Binarizer(Ref source) : source_(source) { + dataWidth = source->getWidth(); + dataHeight = source->getHeight(); + + width = dataWidth; + height = dataHeight; + + matrix_ = NULL; + matrix0_ = NULL; + matrixInverted_ = NULL; + + histogramBinarized = false; + usingHistogram = false; +} + +Binarizer::~Binarizer() {} + +Ref Binarizer::getLuminanceSource() const { return source_; } + +int Binarizer::getWidth() const { + return width; +} + +int Binarizer::getHeight() const { + return height; +} + +int Binarizer::rotateCounterClockwise() { return 0; } + +int Binarizer::rotateCounterClockwise45() { return 0; } + +Ref Binarizer::getInvertedMatrix(ErrorHandler& err_handler) { + if (!matrix_) { + return Ref(); + } + + if (matrixInverted_ == NULL) { + matrixInverted_ = new BitMatrix(matrix_->getWidth(), matrix_->getHeight(), err_handler); + matrixInverted_->copyOf(matrix_, err_handler); + matrixInverted_->flipAll(); + } + + return matrixInverted_; +} + +// Return different black matrix according to cacheMode +Ref Binarizer::getBlackMatrix(ErrorHandler& err_handler) { + if (err_handler.ErrCode()) return Ref(); + matrix_ = matrix0_; + return matrix_; +} + +Ref Binarizer::getBlackRow(int y, Ref row, ErrorHandler& err_handler) { + if (!matrix_) { + matrix_ = getBlackMatrix(err_handler); + if (err_handler.ErrCode()) return Ref(); + } + + matrix_->getRow(y, row); + return row; +} + +ArrayRef Binarizer::getBlockArray(int size) { + ArrayRef blocks(new Array(size)); + + for (int i = 0; i < blocks->size(); i++) { + blocks[i].sum = 0; + blocks[i].min = 0xFF; + blocks[i].max = 0; + } + + return blocks; +} +} // namespace zxing diff --git a/modules/wechat_qrcode/src/zxing/binarizer.hpp b/modules/wechat_qrcode/src/zxing/binarizer.hpp new file mode 100644 index 00000000000..7e444c52c1f --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/binarizer.hpp @@ -0,0 +1,88 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_BINARIZER_HPP__ +#define __ZXING_BINARIZER_HPP__ + +#include "common/bitarray.hpp" +#include "common/bitmatrix.hpp" +#include "common/counted.hpp" +#include "errorhandler.hpp" +#include "luminance_source.hpp" + +#define ONED_ENABLE_LINE_BINARIZER + +namespace zxing { + +// typedef unsigned char uint8_t; + +struct BINARIZER_BLOCK { + int sum; + int min; + int max; + int threshold; + // int average; +}; + +#ifdef ONED_ENABLE_LINE_BINARIZER +struct DecodeTipInfo { + int class_id; +}; +#endif + +class Binarizer : public Counted { +private: + Ref source_; + bool histogramBinarized; + bool usingHistogram; + +public: + explicit Binarizer(Ref source); + virtual ~Binarizer(); + + // Added for store binarized result + + int dataWidth; + int dataHeight; + int width; + int height; + + // Store dynamicalli choice of which matrix is currently used + Ref matrix_; + + // Restore 0 degree result + Ref matrix0_; + + Ref matrixInverted_; + + bool isRotateSupported() const { return false; } + + // rotate counter clockwise 45 & 90 degree from binarized cache + int rotateCounterClockwise(); + int rotateCounterClockwise45(); + + virtual Ref getBlackMatrix(ErrorHandler& err_handler); + virtual Ref getInvertedMatrix(ErrorHandler& err_handler); + virtual Ref getBlackRow(int y, Ref row, ErrorHandler& err_handler); + + Ref getLuminanceSource() const; + // virtual Ref createBinarizer(Ref source) = 0; + virtual Ref createBinarizer(Ref source) { + return Ref(new Binarizer(source)); + }; + + int getWidth() const; + int getHeight() const; + + ArrayRef getBlockArray(int size); +}; + +} // namespace zxing +#endif // __ZXING_BINARIZER_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/binarybitmap.cpp b/modules/wechat_qrcode/src/zxing/binarybitmap.cpp new file mode 100644 index 00000000000..d617ac9ecee --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/binarybitmap.cpp @@ -0,0 +1,66 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "binarybitmap.hpp" + +using zxing::BinaryBitmap; +using zxing::BitArray; +using zxing::BitMatrix; +using zxing::ErrorHandler; +using zxing::LuminanceSource; +using zxing::Ref; + +// VC++ +using zxing::Binarizer; + +BinaryBitmap::BinaryBitmap(Ref binarizer) : binarizer_(binarizer) {} + +BinaryBitmap::~BinaryBitmap() {} + +Ref BinaryBitmap::getBlackRow(int y, Ref row, ErrorHandler& err_handler) { + Ref bitary = binarizer_->getBlackRow(y, row, err_handler); + if (err_handler.ErrCode()) return Ref(); + return bitary; +} + +Ref BinaryBitmap::getBlackMatrix(ErrorHandler& err_handler) { + Ref bitmtx = binarizer_->getBlackMatrix(err_handler); + if (err_handler.ErrCode()) return Ref(); + return bitmtx; +} + +Ref BinaryBitmap::getInvertedMatrix(ErrorHandler& err_handler) { + Ref bitmtx = binarizer_->getInvertedMatrix(err_handler); + if (err_handler.ErrCode()) return Ref(); + return bitmtx; +} + +int BinaryBitmap::getWidth() const { return binarizer_->getWidth(); } + +int BinaryBitmap::getHeight() const { return binarizer_->getHeight(); } + +Ref BinaryBitmap::getLuminanceSource() const { + return binarizer_->getLuminanceSource(); +} + +bool BinaryBitmap::isCropSupported() const { return getLuminanceSource()->isCropSupported(); } + +Ref BinaryBitmap::crop(int left, int top, int width, int height, + ErrorHandler& err_handler) { + return Ref(new BinaryBitmap(binarizer_->createBinarizer( + getLuminanceSource()->crop(left, top, width, height, err_handler)))); +} + +bool BinaryBitmap::isRotateSupported() const { return binarizer_->isRotateSupported(); } + +Ref BinaryBitmap::rotateCounterClockwise() { + binarizer_->rotateCounterClockwise(); + return Ref(new BinaryBitmap(binarizer_)); +} diff --git a/modules/wechat_qrcode/src/zxing/binarybitmap.hpp b/modules/wechat_qrcode/src/zxing/binarybitmap.hpp new file mode 100644 index 00000000000..d483fcd3efd --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/binarybitmap.hpp @@ -0,0 +1,53 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_BINARYBITMAP_HPP__ +#define __ZXING_BINARYBITMAP_HPP__ + +#include "binarizer.hpp" +#include "common/bitarray.hpp" +#include "common/bitmatrix.hpp" +#include "common/counted.hpp" +#include "common/unicomblock.hpp" +#include "errorhandler.hpp" + +namespace zxing { + +class BinaryBitmap : public Counted { +private: + Ref binarizer_; + +public: + explicit BinaryBitmap(Ref binarizer); + virtual ~BinaryBitmap(); + + Ref getBlackRow(int y, Ref row, ErrorHandler& err_handler); + Ref getBlackMatrix(ErrorHandler& err_handler); + Ref getInvertedMatrix(ErrorHandler& err_handler); + + Ref getLuminanceSource() const; + Ref m_poUnicomBlock; + + int getWidth() const; + int getHeight() const; + + bool isRotateSupported() const; + Ref rotateCounterClockwise(); + + bool isCropSupported() const; + Ref crop(int left, int top, int width, int height, ErrorHandler& err_handler); + + bool isHistogramBinarized() const; + bool ifUseHistogramBinarize() const; +}; + +} // namespace zxing + +#endif // __ZXING_BINARYBITMAP_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/array.hpp b/modules/wechat_qrcode/src/zxing/common/array.hpp new file mode 100644 index 00000000000..2edb946a23b --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/array.hpp @@ -0,0 +1,115 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_ARRAY_HPP__ +#define __ZXING_COMMON_ARRAY_HPP__ + +#include "counted.hpp" + +#include + +namespace zxing { + +template +class Array : public Counted { +protected: +public: + std::vector values_; + Array() {} + explicit Array(int n) : Counted(), values_(n, T()) {} + Array(T const *ts, int n) : Counted(), values_(ts, ts + n) {} + Array(T const *ts, T const *te) : Counted(), values_(ts, te) {} + Array(T v, int n) : Counted(), values_(n, v) {} + explicit Array(std::vector &v) : Counted(), values_(v) {} + Array(Array &other) : Counted(), values_(other.values_) {} + explicit Array(Array *other) : Counted(), values_(other->values_) {} + virtual ~Array() {} + Array &operator=(const Array &other) { + values_ = other.values_; + return *this; + } + Array &operator=(const std::vector &array) { + values_ = array; + return *this; + } + T const &operator[](int i) const { return values_[i]; } + T &operator[](int i) { return values_[i]; } + int size() const { return values_.size(); } + bool empty() const { return values_.size() == 0; } + std::vector const &values() const { return values_; } + std::vector &values() { return values_; } + + T *data() { + // return values_.data(); + return &values_[0]; + } + void append(T value) { values_.push_back(value); } +}; + +template +class ArrayRef : public Counted { +private: +public: + Array *array_; + ArrayRef() : array_(0) {} + explicit ArrayRef(int n) : array_(0) { reset(new Array(n)); } + ArrayRef(T *ts, int n) : array_(0) { reset(new Array(ts, n)); } + explicit ArrayRef(Array *a) : array_(0) { reset(a); } + ArrayRef(const ArrayRef &other) : Counted(), array_(0) { reset(other.array_); } + + ~ArrayRef() { + if (array_) { + array_->release(); + } + array_ = 0; + } + + T const &operator[](int i) const { return (*array_)[i]; } + + T &operator[](int i) { return (*array_)[i]; } + + void reset(Array *a) { + if (a) { + a->retain(); + } + if (array_) { + array_->release(); + } + array_ = a; + } + void reset(const ArrayRef &other) { reset(other.array_); } + ArrayRef &operator=(const ArrayRef &other) { + reset(other); + return *this; + } + ArrayRef &operator=(Array *a) { + reset(a); + return *this; + } + + Array &operator*() const { return *array_; } + + Array *operator->() const { return array_; } + + operator bool() const { return array_ != 0; } + bool operator!() const { return array_ == 0; } + + T *data() { return array_->data(); } + + void clear() { + T *ptr = array_->data(); + memset(ptr, 0, array_->size()); + } + void append(T value) { array_->append(value); } +}; + +} // namespace zxing + +#endif // __ZXING_COMMON_ARRAY_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/binarizer/adaptive_threshold_mean_binarizer.cpp b/modules/wechat_qrcode/src/zxing/common/binarizer/adaptive_threshold_mean_binarizer.cpp new file mode 100644 index 00000000000..57570941b02 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/binarizer/adaptive_threshold_mean_binarizer.cpp @@ -0,0 +1,100 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#include "adaptive_threshold_mean_binarizer.hpp" +using namespace std; +using namespace zxing; + +namespace { +const int BLOCK_SIZE = 25; +const int Bias = 10; +} // namespace + +AdaptiveThresholdMeanBinarizer::AdaptiveThresholdMeanBinarizer(Ref source) + : GlobalHistogramBinarizer(source) {} + +AdaptiveThresholdMeanBinarizer::~AdaptiveThresholdMeanBinarizer() {} + +Ref AdaptiveThresholdMeanBinarizer::createBinarizer(Ref source) { + return Ref(new AdaptiveThresholdMeanBinarizer(source)); +} + +Ref AdaptiveThresholdMeanBinarizer::getBlackRow(int y, Ref row, + ErrorHandler& err_handler) { + // First call binarize image in child class to get matrix0_ and binCache + if (!matrix0_) { + binarizeImage(err_handler); + if (err_handler.ErrCode()) return Ref(); + } + + // Call parent getBlackMatrix to get current matrix + return Binarizer::getBlackRow(y, row, err_handler); +} + +Ref AdaptiveThresholdMeanBinarizer::getBlackMatrix(ErrorHandler& err_handler) { + // First call binarize image in child class to get matrix0_ and binCache + if (!matrix0_) { + binarizeImage(err_handler); + if (err_handler.ErrCode()) return Ref(); + } + return Binarizer::getBlackMatrix(err_handler); +} + +int AdaptiveThresholdMeanBinarizer::binarizeImage(ErrorHandler& err_handler) { + if (width >= BLOCK_SIZE && height >= BLOCK_SIZE) { + LuminanceSource& source = *getLuminanceSource(); + Ref matrix(new BitMatrix(width, height, err_handler)); + if (err_handler.ErrCode()) return -1; + auto src = (unsigned char*)source.getMatrix()->data(); + auto dst = matrix->getPtr(); + cv::Mat mDst; + mDst = cv::Mat::zeros(cv::Size(width, height), CV_8UC1); + TransBufferToMat(src, mDst, width, height); + cv::Mat result; + int bs = width / 10; + bs = bs + bs % 2 - 1; + if (!(bs % 2 == 1 && bs > 1)) return -1; + cv::adaptiveThreshold(mDst, result, 255, cv::ADAPTIVE_THRESH_GAUSSIAN_C, cv::THRESH_BINARY, + bs, Bias); + TransMatToBuffer(result, dst, width, height); + if (err_handler.ErrCode()) return -1; + matrix0_ = matrix; + } else { + matrix0_ = GlobalHistogramBinarizer::getBlackMatrix(err_handler); + if (err_handler.ErrCode()) return 1; + } + return 0; +} + +int AdaptiveThresholdMeanBinarizer::TransBufferToMat(unsigned char* pBuffer, cv::Mat& mDst, + int nWidth, int nHeight) { + for (int j = 0; j < nHeight; ++j) { + unsigned char* data = mDst.ptr(j); + unsigned char* pSubBuffer = pBuffer + (nHeight - 1 - j) * nWidth; + memcpy(data, pSubBuffer, nWidth); + } + return 0; +} + +int AdaptiveThresholdMeanBinarizer::TransMatToBuffer(cv::Mat mSrc, unsigned char* ppBuffer, + int& nWidth, int& nHeight) { + nWidth = mSrc.cols; + // nWidth = ((nWidth + 3) / 4) * 4; + nHeight = mSrc.rows; + for (int j = 0; j < nHeight; ++j) { + unsigned char* pdi = ppBuffer + j * nWidth; + for (int z = 0; z < nWidth; ++z) { + int nj = nHeight - j - 1; + int value = *(uchar*)(mSrc.ptr(nj) + z); + if (value > 120) + pdi[z] = 0; + else + pdi[z] = 1; + } + } + return 0; +} \ No newline at end of file diff --git a/modules/wechat_qrcode/src/zxing/common/binarizer/adaptive_threshold_mean_binarizer.hpp b/modules/wechat_qrcode/src/zxing/common/binarizer/adaptive_threshold_mean_binarizer.hpp new file mode 100644 index 00000000000..aba64ea88c9 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/binarizer/adaptive_threshold_mean_binarizer.hpp @@ -0,0 +1,39 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#ifndef __ZXING_COMMON_ADAPTIVE_THRESHOLD_MEAN_BINARIZER_HPP__ +#define __ZXING_COMMON_ADAPTIVE_THRESHOLD_MEAN_BINARIZER_HPP__ +#include +#include +#include +#include "../../binarizer.hpp" +#include "../../errorhandler.hpp" +#include "../bitarray.hpp" +#include "../bitmatrix.hpp" +#include "../bytematrix.hpp" +#include "global_histogram_binarizer.hpp" + + +namespace zxing { + +class AdaptiveThresholdMeanBinarizer : public GlobalHistogramBinarizer { +public: + explicit AdaptiveThresholdMeanBinarizer(Ref source); + virtual ~AdaptiveThresholdMeanBinarizer(); + + virtual Ref getBlackMatrix(ErrorHandler& err_handler) override; + virtual Ref getBlackRow(int y, Ref row, ErrorHandler& err_handler) override; + Ref createBinarizer(Ref source) override; + +private: + int binarizeImage(ErrorHandler& err_handler); + int TransBufferToMat(unsigned char* pBuffer, cv::Mat& mDst, int nWidth, int nHeight); + int TransMatToBuffer(cv::Mat mSrc, unsigned char* ppBuffer, int& nWidth, int& nHeight); +}; + +} // namespace zxing +#endif // __ZXING_COMMON_ADAPTIVE_THRESHOLD_MEAN_BINARIZER_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/binarizer/fast_window_binarizer.cpp b/modules/wechat_qrcode/src/zxing/common/binarizer/fast_window_binarizer.cpp new file mode 100644 index 00000000000..d9a5e1edc36 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/binarizer/fast_window_binarizer.cpp @@ -0,0 +1,286 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "fast_window_binarizer.hpp" +using namespace std; +using namespace zxing; + + +namespace { +const int BLOCK_SIZE = 6; +// const int BLOCK_SIZE = 8; // not as good as BLOCK_SIZE = 6 +const float WINDOW_FRACTION = 0.13f; + +static int min(int a, int b) { return a < b ? a : b; } + +static int max(int a, int b) { return a > b ? a : b; } + +} // namespace + +FastWindowBinarizer::FastWindowBinarizer(Ref source) + : GlobalHistogramBinarizer(source), matrix_(NULL), cached_row_(NULL) { + width = source->getWidth(); + height = source->getHeight(); + int aw = width / BLOCK_SIZE; + int ah = height / BLOCK_SIZE; + + int ah2 = ah; + int ow2 = aw + 1; + + _luminancesInt = new int[width * height]; + _blockTotals = new int[ah * aw]; + _totals = new int[(ah + 1) * (aw + 1)]; + _rowTotals = new int[ah2 * ow2]; + + _internal = new unsigned int[(height + 1) * (width + 1)]; +} + +FastWindowBinarizer::~FastWindowBinarizer() { + delete[] _totals; + delete[] _blockTotals; + delete[] _luminancesInt; + delete[] _rowTotals; + + delete[] _internal; +} + +Ref FastWindowBinarizer::createBinarizer(Ref source) { + return Ref(new FastWindowBinarizer(source)); +} + +/** + * Calculates the final BitMatrix once for all requests. This could be called + * once from the constructor instead, but there are some advantages to doing it + * lazily, such as making profiling easier, and not doing heavy lifting when + * callers don't expect it. + */ +Ref FastWindowBinarizer::getBlackMatrix(ErrorHandler& err_handler) { + if (!matrix0_) { + binarizeImage1(err_handler); + if (err_handler.ErrCode()) return Ref(); + } + + return Binarizer::getBlackMatrix(err_handler); +} + +/** + * Calculate black row from BitMatrix + * If BitMatrix has been calculated then just get the row + * If BitMatrix has not been calculated then call getBlackMatrix first + */ + +Ref FastWindowBinarizer::getBlackRow(int y, Ref row, + ErrorHandler& err_handler) { + if (!matrix0_) { + binarizeImage1(err_handler); + if (err_handler.ErrCode()) return Ref(); + } + // Call parent getBlackMatrix to get current matrix + return Binarizer::getBlackRow(y, row, err_handler); +} + +void FastWindowBinarizer::calcBlockTotals(int* luminancesInt, int* output, int aw, int ah) { + for (int by = 0; by < ah; by++) { + int ey = (by + 1) * BLOCK_SIZE; + for (int bx = 0; bx < aw; bx++) { + int t = 0; + for (int y = by * BLOCK_SIZE; y < ey; y++) { + int offset = y * width + bx * BLOCK_SIZE; + int ex = offset + BLOCK_SIZE; + for (; offset < ex; offset++) { + // int v = luminancesInt[offset] & 0xff; + t += luminancesInt[offset]; + } + } + output[by * aw + bx] = t; + } + } +} + +void FastWindowBinarizer::cumulative(int* data, int* output, int _width, int _height) { + int ah = _height; + int aw = _width; + int ow = _width + 1; + // int[][] totals = new int[ah + 1][aw + 1]; + // int* rowTotals = new int[ah*ow]; + + for (int y = 0; y < ah; y++) { + int* row = _rowTotals + (y * ow); + int* rowdata = data + (y * aw); + int t = 0; + row[0] = t; + + for (int x = 0; x < aw; x++) { + t += rowdata[x]; + row[x + 1] = t; + } + } + + for (int x = 0; x <= aw; x++) { + output[x] = 0; // First row + int t = 0; + + for (int y = 0; y < ah; y++) { + t += _rowTotals[y * ow + x]; + output[(y + 1) * ow + x] = t; + } + } +} + +void FastWindowBinarizer::fastIntegral(const unsigned char* inputMatrix, + unsigned int* outputMatrix) { + // memset(outputMatrix,0,sizeof(int)*(height+1)*(width+1)); + // unsigned int *columnSum = new unsigned int[width]; // sum of each column + // calculate integral of the first line + outputMatrix[0] = outputMatrix[width + 1] = 0; + for (int i = 0; i < width; i++) { + // columnSum[i]=inputMatrix[i]; + outputMatrix[i + 1] = 0; + outputMatrix[width + 1 + i + 1] = outputMatrix[width + 1 + i] + inputMatrix[i]; + } + for (int i = 1; i < height; i++) { + const unsigned char* psi = inputMatrix + i * width; + unsigned int* pdi = outputMatrix + (i + 1) * (width + 1); + // first column of each line + pdi[0] = 0; + pdi[1] = psi[0]; + int row_sum = psi[0]; + // other columns + for (int j = 1; j < width; j++) { + row_sum += psi[j]; + pdi[j + 1] = pdi[j + 1 - width - 1] + row_sum; + } + } + return; +} + +int FastWindowBinarizer::binarizeImage1(ErrorHandler& err_handler) { + LuminanceSource& source = *getLuminanceSource(); + Ref matrix(new BitMatrix(width, height, err_handler)); + if (err_handler.ErrCode()) return -1; + + ArrayRef localLuminances = source.getMatrix(); + + unsigned char* src = (unsigned char*)localLuminances->data(); + unsigned char* dst = matrix->getPtr(); + fastWindow(src, dst, err_handler); + if (err_handler.ErrCode()) return -1; + + matrix0_ = matrix; + return 0; +} + +void FastWindowBinarizer::fastWindow(const unsigned char* src, unsigned char* dst, + ErrorHandler& err_handler) { + int r = (int)(min(width, height) * WINDOW_FRACTION / BLOCK_SIZE / 2 + 1); + const int NEWH_BLOCK_SIZE = BLOCK_SIZE * r; + if (height < NEWH_BLOCK_SIZE || width < NEWH_BLOCK_SIZE) { + matrix_ = GlobalHistogramBinarizer::getBlackMatrix(err_handler); + return; + } + const unsigned char* _img = src; + fastIntegral(_img, _internal); + int aw = width / BLOCK_SIZE; + int ah = height / BLOCK_SIZE; + memset(dst, 0, sizeof(char) * height * width); + for (int ai = 0; ai < ah; ai++) { + int top = max(0, ((ai - r + 1) * BLOCK_SIZE)); + int bottom = min(height, (ai + r) * BLOCK_SIZE); + unsigned int* pt = _internal + top * (width + 1); + unsigned int* pb = _internal + bottom * (width + 1); + for (int aj = 0; aj < aw; aj++) { + int left = max(0, (aj - r + 1) * BLOCK_SIZE); + int right = min(width, (aj + r) * BLOCK_SIZE); + unsigned int block = pb[right] + pt[left] - pt[right] - pb[left]; + int pixels = (bottom - top) * (right - left); + int avg = (int)block / pixels; + for (int bi = ai * BLOCK_SIZE; bi < height && bi < (ai + 1) * BLOCK_SIZE; bi++) { + const unsigned char* psi = src + bi * width; + unsigned char* pdi = dst + bi * width; + for (int bj = aj * BLOCK_SIZE; bj < width && bj < (aj + 1) * BLOCK_SIZE; bj++) { + if ((int)psi[bj] < avg) + pdi[bj] = 1; + else + pdi[bj] = 0; + } + } + } + } + // delete [] _internal; + return; +} + +int FastWindowBinarizer::binarizeImage0(ErrorHandler& err_handler) { + // if (matrix_) { + // return matrix_; + //} + + LuminanceSource& source = *getLuminanceSource(); + if (width >= BLOCK_SIZE && height >= BLOCK_SIZE) { + int r = (int)(min(width, height) * WINDOW_FRACTION / BLOCK_SIZE / 2 + 1); + + int aw = width / BLOCK_SIZE; + int ah = height / BLOCK_SIZE; + int ow = aw + 1; + + ArrayRef _luminances = source.getMatrix(); + + // Get luminances for int value first + for (int i = 0; i < width * height; i++) { + _luminancesInt[i] = _luminances[i] & 0xff; + } + + calcBlockTotals(_luminancesInt, _blockTotals, aw, ah); + + cumulative(_blockTotals, _totals, aw, ah); + + Ref newMatrix(new BitMatrix(width, height, err_handler)); + if (err_handler.ErrCode()) return -1; + unsigned char* newimg = newMatrix->getPtr(); + for (int by = 0; by < ah; by++) { + int top = max(0, by - r + 1); + int bottom = min(ah, by + r); + + for (int bx = 0; bx < aw; bx++) { + int left = max(0, bx - r + 1); + int right = min(aw, bx + r); + + int block = _totals[bottom * ow + right] + _totals[top * ow + left] - + _totals[top * ow + right] - _totals[bottom * ow + left]; + + int pixels = (bottom - top) * (right - left) * BLOCK_SIZE * BLOCK_SIZE; + int avg = block / pixels; + + for (int y = by * BLOCK_SIZE; y < (by + 1) * BLOCK_SIZE; y++) { + // int offset = y*width; + int* plumint = _luminancesInt + y * width; + unsigned char* pn = newimg + y * width; + for (int x = bx * BLOCK_SIZE; x < (bx + 1) * BLOCK_SIZE; x++) { + // int pixel = luminances[y*width + x] & 0xff; + // if(plumint[x] < avg) + // newMatrix->set(x, y); + if (plumint[x] < avg) + pn[x] = 1; + else + pn[x] = 0; + } + } + } + } + // delete[] data; + matrix_ = newMatrix; + } else { + // If the image is too small, fall back to the global histogram + // approach. + matrix_ = GlobalHistogramBinarizer::getBlackMatrix(err_handler); + } + + return 0; +} diff --git a/modules/wechat_qrcode/src/zxing/common/binarizer/fast_window_binarizer.hpp b/modules/wechat_qrcode/src/zxing/common/binarizer/fast_window_binarizer.hpp new file mode 100644 index 00000000000..a18c186e0b7 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/binarizer/fast_window_binarizer.hpp @@ -0,0 +1,56 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_DETECTOR_RESULT_HPP__ +#define __ZXING_COMMON_DETECTOR_RESULT_HPP__ + +#include "../../binarizer.hpp" +#include "../../errorhandler.hpp" +#include "../bitarray.hpp" +#include "../bitmatrix.hpp" +#include "global_histogram_binarizer.hpp" + +#include + +namespace zxing { + +class FastWindowBinarizer : public GlobalHistogramBinarizer { +private: + Ref matrix_; + Ref cached_row_; + + int* _luminancesInt; + int* _blockTotals; + int* _totals; + int* _rowTotals; + + unsigned int* _internal; + +public: + explicit FastWindowBinarizer(Ref source); + virtual ~FastWindowBinarizer(); + + virtual Ref getBlackMatrix(ErrorHandler& err_handler) override; + virtual Ref getBlackRow(int y, Ref row, ErrorHandler& err_handler) override; + + Ref createBinarizer(Ref source) override; + +private: + void calcBlockTotals(int* luminancesInt, int* output, int aw, int ah); + void cumulative(int* data, int* output, int _width, int _height); + int binarizeImage0(ErrorHandler& err_handler); + void fastIntegral(const unsigned char* inputMatrix, unsigned int* outputMatrix); + int binarizeImage1(ErrorHandler& err_handler); + void fastWindow(const unsigned char* src, unsigned char* dst, ErrorHandler& err_handler); +}; + +} // namespace zxing + +#endif // __ZXING_COMMON_DETECTOR_RESULT_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/binarizer/global_histogram_binarizer.cpp b/modules/wechat_qrcode/src/zxing/common/binarizer/global_histogram_binarizer.cpp new file mode 100644 index 00000000000..8c3bc60fa3d --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/binarizer/global_histogram_binarizer.cpp @@ -0,0 +1,341 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "global_histogram_binarizer.hpp" + +using zxing::ArrayRef; +using zxing::Binarizer; +using zxing::BitArray; +using zxing::BitMatrix; +using zxing::ErrorHandler; +using zxing::GlobalHistogramBinarizer; +using zxing::Ref; + +// VC++ +using zxing::LuminanceSource; + +namespace { +const int LUMINANCE_BITS = 5; +const int LUMINANCE_SHIFT = 8 - LUMINANCE_BITS; +const int LUMINANCE_BUCKETS = 1 << LUMINANCE_BITS; +const ArrayRef EMPTY(0); +} // namespace + +GlobalHistogramBinarizer::GlobalHistogramBinarizer(Ref source) + : Binarizer(source), luminances(EMPTY), buckets(LUMINANCE_BUCKETS) { + filtered = false; +} + +GlobalHistogramBinarizer::~GlobalHistogramBinarizer() {} + +void GlobalHistogramBinarizer::initArrays(int luminanceSize) { + if (luminances->size() < luminanceSize) { + luminances = ArrayRef(luminanceSize); + } + for (int x = 0; x < LUMINANCE_BUCKETS; x++) { + buckets[x] = 0; + } +} + +// Applies simple sharpening to the row data to improve performance of the 1D +// readers. +Ref GlobalHistogramBinarizer::getBlackRow(int y, Ref row, + ErrorHandler& err_handler) { + // First call binarize image in child class to get matrix0_ and binCache + if (!matrix0_) { + binarizeImage0(err_handler); + if (err_handler.ErrCode()) return Ref(); + } + // Call parent getBlackMatrix to get current matrix + return Binarizer::getBlackRow(y, row, err_handler); +} + +// Does not sharpen the data, as this call is intended to only be used by 2D +// readers. +Ref GlobalHistogramBinarizer::getBlackMatrix(ErrorHandler& err_handler) { + binarizeImage0(err_handler); + if (err_handler.ErrCode()) return Ref(); + // First call binarize image in child class to get matrix0_ and binCache + // Call parent getBlackMatrix to get current matrix + return Binarizer::getBlackMatrix(err_handler); +} + +using namespace std; + +int GlobalHistogramBinarizer::estimateBlackPoint(ArrayRef const& _buckets, + ErrorHandler& err_handler) { + // Find tallest peak in histogram + int numBuckets = _buckets->size(); + int maxBucketCount = 0; + int firstPeak = 0; + int firstPeakSize = 0; + if (false) { + for (int x = 0; x < numBuckets; x++) { + cerr << _buckets[x] << " "; + } + cerr << endl; + } + for (int x = 0; x < numBuckets; x++) { + if (_buckets[x] > firstPeakSize) { + firstPeak = x; + firstPeakSize = _buckets[x]; + } + if (_buckets[x] > maxBucketCount) { + maxBucketCount = _buckets[x]; + } + } + + // Find second-tallest peak -- well, another peak that is tall and not + // so close to the first one + int secondPeak = 0; + int secondPeakScore = 0; + for (int x = 0; x < numBuckets; x++) { + int distanceToBiggest = x - firstPeak; + // Encourage more distant second peaks by multiplying by square of + // distance + int score = _buckets[x] * distanceToBiggest * distanceToBiggest; + if (score > secondPeakScore) { + secondPeak = x; + secondPeakScore = score; + } + } + // Make sure firstPeak corresponds to the black peak. + if (firstPeak > secondPeak) { + int temp = firstPeak; + firstPeak = secondPeak; + secondPeak = temp; + } + + // Kind of arbitrary; if the two peaks are very close, then we figure there + // is so little dynamic range in the image, that discriminating black and + // white is too error-prone. Decoding the image/line is either pointless, or + // may in some cases lead to a false positive for 1D formats, which are + // relatively lenient. We arbitrarily say "close" is "<= 1/16 of the total + // histogram buckets apart" std::cerr << "! " << secondPeak << " " << + // firstPeak << " " << numBuckets << std::endl; + if (secondPeak - firstPeak <= numBuckets >> 4) { + err_handler = NotFoundErrorHandler("NotFound GlobalHistogramBinarizer"); + return -1; + } + + // Find a valley between them that is low and closer to the white peak + int bestValley = secondPeak - 1; + int bestValleyScore = -1; + for (int x = secondPeak - 1; x > firstPeak; x--) { + int fromFirst = x - firstPeak; + // Favor a "valley" that is not too close to either peak -- especially + // not the black peak -- and that has a low value of course + int score = fromFirst * fromFirst * (secondPeak - x) * (maxBucketCount - buckets[x]); + if (score > bestValleyScore) { + bestValley = x; + bestValleyScore = score; + } + } + + // std::cerr << "bps " << (bestValley << LUMINANCE_SHIFT) << std::endl; + return bestValley << LUMINANCE_SHIFT; +} + +// codes from sagazhou, only works well on one dataset +int GlobalHistogramBinarizer::estimateBlackPoint2(ArrayRef const& _buckets) { + int midValue = LUMINANCE_BUCKETS / 2 + 1; + // Find tallest and lowest peaks in histogram + // const int numBuckets = buckets->size(); + int maxPointArray[LUMINANCE_BUCKETS] = {0}; + int maxCrusor = 0; + int maxValue = 0, maxIndex = 0; + int minPointArray[LUMINANCE_BUCKETS] = {0}; + int minCrusor = 0; + + for (int i = 2; i < LUMINANCE_BUCKETS - 3; i++) { + if (_buckets[i] < _buckets[i + 1] && _buckets[i] < _buckets[i + 2] && + _buckets[i] < _buckets[i - 1] && _buckets[i] < _buckets[i - 2]) { + minPointArray[minCrusor++] = i; + } else if (_buckets[i] > _buckets[i + 1] && _buckets[i] > _buckets[i + 2] && + _buckets[i] > _buckets[i - 1] && _buckets[i] > _buckets[i - 2]) { + maxPointArray[maxCrusor++] = i; + if (_buckets[i] > maxValue) { + maxValue = _buckets[i]; + maxIndex = i; + } + } + } + bool bSlantBlack = true; + // most pixels are black + for (int i = 0; i < maxCrusor; ++i) { + if (maxPointArray[i] > midValue) { + bSlantBlack = false; + break; + } + } + + bool bSlantWhite = true; + // most pixels are white + for (int i = 0; i < maxCrusor; ++i) { + if (maxPointArray[i] < midValue) { + bSlantWhite = false; + break; + } + } + + if (bSlantBlack) { + int start = maxIndex + 30; + int end = midValue; + + if (minCrusor == 0) // unimodal + { + return 255; + } else { + int mostLeftIndex = 0; + bool bFind = false; + + for (int i = 0; i < minCrusor; ++i) // wave motion + { + if (minPointArray[i] > start && minPointArray[i] < end) { + mostLeftIndex = minPointArray[i]; + bFind = true; + break; + } + } + + if (bFind) { + return mostLeftIndex; + } else { + return 255; + } + } + } + + if (bSlantWhite) { + int start = midValue; + int end = maxIndex - 30; + + if (minCrusor == 0) // unimodal + { + return 0; + } else { + int mostRightIndex = 0; + bool bFind = false; + + for (int i = 0; i < minCrusor; ++i) // wave motion + { + if (minPointArray[i] > start && minPointArray[i] < end) { + mostRightIndex = i; + bFind = true; + } + } + + if (bFind) { + return mostRightIndex; + } else { + return 0; + } + } + } + + // balanced distribution + if (maxIndex < midValue) { + // the minest min value + if (minCrusor == 0) { + return 255; // all black + } else { + int start = maxIndex + 30; + int end = 253; + + for (int i = 0; i < minCrusor; ++i) // wave motion + { + if (minPointArray[i] > start && minPointArray[i] < end) { + return minPointArray[i]; + } + } + } + } else { + // maxest min value + if (minCrusor == 0) { + return 0; // white + } else { + int start = 0; + int end = maxIndex - 30; + int mostRightIndex = 0; + + for (int i = 0; i < minCrusor; ++i) // wave motion + { + if (minPointArray[i] > start && minPointArray[i] < end) { + mostRightIndex = minPointArray[i]; + } + } + + return mostRightIndex; + } + } + return 0; +} + +int GlobalHistogramBinarizer::binarizeImage0(ErrorHandler& err_handler) { + LuminanceSource& source = *getLuminanceSource(); + Ref matrix(new BitMatrix(width, height, err_handler)); + if (err_handler.ErrCode()) return -1; + +#ifdef INPUT_BINARIZED + { + ArrayRef localLuminances = source.getMatrix(); + for (int y = 0; y < height; y++) { + int offset = y * width; + for (int x = 0; x < width; x++) { + int pixel = localLuminances[offset + x] & 0xff; + if (pixel < 128) { + matrix->set(x, y); + } + } + } + } + matrix0_ = matrix; + return 0; +#endif + // Quickly calculates the histogram by sampling four rows from the image. + // This proved to be more robust on the blackbox tests than sampling a + // diagonal as we used to do. + initArrays(width); + ArrayRef localBuckets = buckets; + + for (int y = 1; y < 5; y++) { + int row = height * y / 5; + ArrayRef localLuminances = source.getRow(row, luminances, err_handler); + if (err_handler.ErrCode()) return -1; + int right = (width << 2) / 5; + for (int x = width / 5; x < right; x++) { + int pixel = localLuminances[x] & 0xff; + localBuckets[pixel >> LUMINANCE_SHIFT]++; + } + } + + + int blackPoint = estimateBlackPoint(localBuckets, err_handler); + if (err_handler.ErrCode()) return -1; + + ArrayRef localLuminances = source.getMatrix(); + for (int y = 0; y < height; y++) { + int offset = y * width; + for (int x = 0; x < width; x++) { + int pixel = localLuminances[offset + x] & 0xff; + if (pixel < blackPoint) { + matrix->set(x, y); + } + } + } + + matrix0_ = matrix; + + return 0; +} + +Ref GlobalHistogramBinarizer::createBinarizer(Ref source) { + return Ref(new GlobalHistogramBinarizer(source)); +} diff --git a/modules/wechat_qrcode/src/zxing/common/binarizer/global_histogram_binarizer.hpp b/modules/wechat_qrcode/src/zxing/common/binarizer/global_histogram_binarizer.hpp new file mode 100644 index 00000000000..4f090225663 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/binarizer/global_histogram_binarizer.hpp @@ -0,0 +1,46 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_GLOBAL_HISTOGRAM_BINARIZER_HPP__ +#define __ZXING_COMMON_GLOBAL_HISTOGRAM_BINARIZER_HPP__ + +#include "../../binarizer.hpp" +#include "../../errorhandler.hpp" +#include "../array.hpp" +#include "../bitarray.hpp" +#include "../bitmatrix.hpp" + +//#define INPUT_BINARIZED +namespace zxing { + +class GlobalHistogramBinarizer : public Binarizer { +protected: + ArrayRef luminances; + ArrayRef buckets; + +public: + explicit GlobalHistogramBinarizer(Ref source); + virtual ~GlobalHistogramBinarizer(); + + virtual Ref getBlackRow(int y, Ref row, ErrorHandler &err_handler) override; + virtual Ref getBlackMatrix(ErrorHandler &err_handler) override; + int estimateBlackPoint(ArrayRef const &buckets, ErrorHandler &err_handler); + int estimateBlackPoint2(ArrayRef const &buckets); + Ref createBinarizer(Ref source) override; + +private: + int binarizeImage0(ErrorHandler &err_handler); + void initArrays(int luminanceSize); + bool filtered; +}; + +} // namespace zxing + +#endif // __ZXING_COMMON_GLOBAL_HISTOGRAM_BINARIZER_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/binarizer/hybrid_binarizer.cpp b/modules/wechat_qrcode/src/zxing/common/binarizer/hybrid_binarizer.cpp new file mode 100644 index 00000000000..527f8b4e4ce --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/binarizer/hybrid_binarizer.cpp @@ -0,0 +1,431 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "hybrid_binarizer.hpp" +#include +#include +#include +#include + +using namespace std; +using namespace zxing; + +// This class uses 5*5 blocks to compute local luminance, where each block is +// 8*8 pixels So this is the smallest dimension in each axis we can accept. +namespace { +const int BLOCK_SIZE_POWER = 3; +const int BLOCK_SIZE = 1 << BLOCK_SIZE_POWER; // ...0100...00 +const int BLOCK_SIZE_MASK = BLOCK_SIZE - 1; // ...0011...11 +const int MINIMUM_DIMENSION = BLOCK_SIZE * 5; +#ifdef USE_SET_INT +const int BITS_PER_BYTE = 8; +const int BITS_PER_WORD = BitMatrix::bitsPerWord; +#endif +} // namespace + +HybridBinarizer::HybridBinarizer(Ref source) : GlobalHistogramBinarizer(source) { + int subWidth = width >> BLOCK_SIZE_POWER; + if ((width & BLOCK_SIZE_MASK) != 0) { + subWidth++; + } + int subHeight = height >> BLOCK_SIZE_POWER; + if ((height & BLOCK_SIZE_MASK) != 0) { + subHeight++; + } + + grayByte_ = source->getByteMatrix(); + + blocks_ = getBlockArray(subWidth * subHeight); + + subWidth_ = subWidth; + subHeight_ = subHeight; + + initBlocks(); + initBlockIntegral(); +} + +HybridBinarizer::~HybridBinarizer() { + // delete [] _bitCached; + // delete [] subSumPoints; + // delete [] subSumColumn; +} + +Ref HybridBinarizer::createBinarizer(Ref source) { + return Ref(new GlobalHistogramBinarizer(source)); +} + +int HybridBinarizer::initBlockIntegral() { + blockIntegral_ = new Array(width * height); + + int* integral = blockIntegral_->data(); + + // unsigned char* therow = grayByte_->getByteRow(0); + + // first row only + int rs = 0; + + for (int j = 0; j < width; j++) { + integral[j] = 0; + } + + for (int i = 0; i < height; i++) { + integral[i * width] = 0; + } + + for (int j = 0; j < subWidth_; j++) { + rs += blocks_[j].threshold; + integral[width + j + 1] = rs; + } + + // remaining cells are sum above and to the left + int offset = width; + int offsetBlock = 0; + + for (int i = 1; i < subHeight_; ++i) { + // therow = grayByte_->getByteRow(i); + offsetBlock = i * subWidth_; + + rs = 0; + + offset += width; + + for (int j = 0; j < subWidth_; ++j) { + rs += blocks_[offsetBlock + j].threshold; + integral[offset + j + 1] = rs + integral[offset - width + j + 1]; + } + } + + return 1; +} + +/** + * Calculates the final BitMatrix once for all requests. This could be called + * once from the constructor instead, but there are some advantages to doing it + * lazily, such as making profiling easier, and not doing heavy lifting when + * callers don't expect it. + */ +Ref HybridBinarizer::getBlackMatrix(ErrorHandler& err_handler) { + // First call binarize image in child class to get matrix0_ and binCache + if (!matrix0_) { + binarizeByBlock(err_handler); + if (err_handler.ErrCode()) return Ref(); + } + + // First call binarize image in child class to get matrix0_ and binCache + // Call parent getBlackMatrix to get current matrix + return Binarizer::getBlackMatrix(err_handler); +} + +#if 1 +/** + * Calculate black row from BitMatrix + * If BitMatrix has been calculated then just get the row + * If BitMatrix has not been calculated then call getBlackMatrix first + */ +Ref HybridBinarizer::getBlackRow(int y, Ref row, ErrorHandler& err_handler) { + // First call binarize image in child class to get matrix0_ and binCache + if (!matrix0_) { + binarizeByBlock(err_handler); + if (err_handler.ErrCode()) return Ref(); + } + + // Call parent getBlackMatrix to get current matrix + return Binarizer::getBlackRow(y, row, err_handler); +} +#endif + +namespace { +inline int cap(int value, int min, int max) { + return value < min ? min : value > max ? max : value; +} +} // namespace + + +// For each block in the image, calculates the average black point using a 5*5 +// grid of the blocks around it. Also handles the corner cases (fractional +// blocks are computed based on the last pixels in the row/column which are also +// used in the previous block.) + +#define THRES_BLOCKSIZE 2 + +// No use of level now +ArrayRef HybridBinarizer::getBlackPoints() { + int blackWidth, blackHeight; + + blackWidth = subWidth_; + blackHeight = subHeight_; + + ArrayRef blackPoints(blackWidth * blackHeight); + + int* blackArray = blackPoints->data(); + + int offset = 0; + for (int i = 0; i < blackHeight; i++) { + offset = i * blackWidth; + for (int j = 0; j < blackWidth; j++) { + blackArray[offset + j] = blocks_[offset + j].threshold; + } + } + + return blackPoints; +} + +// Original code 20140606 +void HybridBinarizer::calculateThresholdForBlock(Ref& _luminances, int subWidth, + int subHeight, int SIZE_POWER, + // ArrayRef &blackPoints, + Ref const& matrix, + ErrorHandler& err_handler) { + int block_size = 1 << SIZE_POWER; + + int maxYOffset = height - block_size; + int maxXOffset = width - block_size; + + int* blockIntegral = blockIntegral_->data(); + + int blockArea = ((2 * THRES_BLOCKSIZE + 1) * (2 * THRES_BLOCKSIZE + 1)); + + for (int y = 0; y < subHeight; y++) { + int yoffset = y << SIZE_POWER; + if (yoffset > maxYOffset) { + yoffset = maxYOffset; + } + for (int x = 0; x < subWidth; x++) { + int xoffset = x << SIZE_POWER; + if (xoffset > maxXOffset) { + xoffset = maxXOffset; + } + int left = cap(x, THRES_BLOCKSIZE, subWidth - THRES_BLOCKSIZE - 1); + int top = cap(y, THRES_BLOCKSIZE, subHeight - THRES_BLOCKSIZE - 1); + + int sum = 0; + // int sum2 = 0; + + int offset1 = (top - THRES_BLOCKSIZE) * (subWidth + 1) + left - THRES_BLOCKSIZE; + int offset2 = (top + THRES_BLOCKSIZE + 1) * (subWidth + 1) + left - THRES_BLOCKSIZE; + + int blocksize = THRES_BLOCKSIZE * 2 + 1; + + sum = blockIntegral[offset1] - blockIntegral[offset1 + blocksize] - + blockIntegral[offset2] + blockIntegral[offset2 + blocksize]; + + int average = sum / blockArea; + thresholdBlock(_luminances, xoffset, yoffset, average, matrix, err_handler); + if (err_handler.ErrCode()) return; + } + } +} + +#ifdef USE_SET_INT +void HybridBinarizer::thresholdFourBlocks(Ref& luminances, int xoffset, int yoffset, + int* thresholds, int stride, + Ref const& matrix) { + int setIntCircle = BITS_PER_WORD / BITS_PER_BYTE; + for (int y = 0; y < BLOCK_SIZE; y++) { + unsigned char* pTemp = luminances->getByteRow(yoffset + y); + pTemp = pTemp + xoffset; + unsigned int valueInt = 0; + int bitPosition = 0; + for (int k = 0; k < setIntCircle; k++) { + for (int x = 0; x < BLOCK_SIZE; x++) { + int pixel = *pTemp++; + if (pixel <= thresholds[k]) { + // bitPosition=(3-k)*8+x; + valueInt |= (unsigned int)1 << bitPosition; + } + bitPosition++; + } + } + matrix->setIntOneTime(xoffset, yoffset + y, valueInt); + } + return; +} +#endif + +// Applies a single threshold to a block of pixels +void HybridBinarizer::thresholdBlock(Ref& _luminances, int xoffset, int yoffset, + int threshold, Ref const& matrix, + ErrorHandler& err_handler) { + int rowBitsSize = matrix->getRowBitsSize(); + int rowSize = width; + + int rowBitStep = rowBitsSize - BLOCK_SIZE; + int rowStep = rowSize - BLOCK_SIZE; + + unsigned char* pTemp = _luminances->getByteRow(yoffset, err_handler); + if (err_handler.ErrCode()) return; + bool* bpTemp = matrix->getRowBoolPtr(yoffset); + + pTemp += xoffset; + bpTemp += xoffset; + + for (int y = 0; y < BLOCK_SIZE; y++) { + for (int x = 0; x < BLOCK_SIZE; x++) { + // comparison needs to be <= so that black == 0 pixels are black + // even if the threshold is 0. + *bpTemp++ = (*pTemp++ <= threshold) ? true : false; + } + + pTemp += rowBitStep; + bpTemp += rowStep; + } +} + +void HybridBinarizer::thresholdIrregularBlock(Ref& _luminances, int xoffset, + int yoffset, int blockWidth, int blockHeight, + int threshold, Ref const& matrix, + ErrorHandler& err_handler) { + for (int y = 0; y < blockHeight; y++) { + unsigned char* pTemp = _luminances->getByteRow(yoffset + y, err_handler); + if (err_handler.ErrCode()) return; + pTemp = pTemp + xoffset; + for (int x = 0; x < blockWidth; x++) { + // comparison needs to be <= so that black == 0 pixels are black + // even if the threshold is 0. + int pixel = *pTemp++; + if (pixel <= threshold) { + matrix->set(xoffset + x, yoffset + y); + } + } + } +} + +namespace { + +inline int getBlackPointFromNeighbors(ArrayRef block, int subWidth, int x, int y) { + return (block[(y - 1) * subWidth + x].threshold + 2 * block[y * subWidth + x - 1].threshold + + block[(y - 1) * subWidth + x - 1].threshold) >> + 2; +} + +} // namespace + + +#define MIN_DYNAMIC_RANGE 24 + +// Calculates a single black point for each block of pixels and saves it away. +int HybridBinarizer::initBlocks() { + Ref& _luminances = grayByte_; + int subWidth = subWidth_; + int subHeight = subHeight_; + + unsigned char* bytes = _luminances->bytes; + + const int minDynamicRange = 24; + + for (int y = 0; y < subHeight; y++) { + int yoffset = y << BLOCK_SIZE_POWER; + int maxYOffset = height - BLOCK_SIZE; + if (yoffset > maxYOffset) yoffset = maxYOffset; + for (int x = 0; x < subWidth; x++) { + int xoffset = x << BLOCK_SIZE_POWER; + int maxXOffset = width - BLOCK_SIZE; + if (xoffset > maxXOffset) xoffset = maxXOffset; + int sum = 0; + int min = 0xFF; + int max = 0; + for (int yy = 0, offset = yoffset * width + xoffset; yy < BLOCK_SIZE; + yy++, offset += width) { + for (int xx = 0; xx < BLOCK_SIZE; xx++) { + // int pixel = luminances->bytes[offset + xx] & 0xFF; + int pixel = bytes[offset + xx]; + sum += pixel; + + // still looking for good contrast + if (pixel < min) { + min = pixel; + } + if (pixel > max) { + max = pixel; + } + } + + // short-circuit min/max tests once dynamic range is met + if (max - min > minDynamicRange) { + // finish the rest of the rows quickly + for (yy++, offset += width; yy < BLOCK_SIZE; yy++, offset += width) { + for (int xx = 0; xx < BLOCK_SIZE; xx += 2) { + sum += bytes[offset + xx]; + sum += bytes[offset + xx + 1]; + } + } + } + } + + blocks_[y * subWidth + x].min = min; + blocks_[y * subWidth + x].max = max; + blocks_[y * subWidth + x].sum = sum; + blocks_[y * subWidth + x].threshold = + getBlockThreshold(x, y, subWidth, sum, min, max, minDynamicRange, BLOCK_SIZE_POWER); + } + } + + return 1; +} + +int HybridBinarizer::getBlockThreshold(int x, int y, int subWidth, int sum, int min, int max, + int minDynamicRange, int SIZE_POWER) { + // See + // http://groups.google.com/group/zxing/browse_thread/thread/d06efa2c35a7ddc0 + + // The default estimate is the average of the values in the block. + int average = sum >> (SIZE_POWER * 2); + if (max - min <= minDynamicRange) { + // If variation within the block is low, assume this is a block withe + // only light or only dark pixels. In that case we do not want to use + // the average, as it would divide this low contrast area into black and + // white pixels, essentially creating data out of noise. The default + // assumption is that the block is light/background. Since no estimate + // for the level of dark pixels exists locally, use half the min for the + // block. + average = min >> 1; + if (y > 0 && x > 0) { + // Correct the "white background" assumption for blocks that have + // neighbors by comparing the pixels in this block to the previously + // calculated black points. This is based on the fact that dark + // barcode symbology is always surrounded by some amout of light + // background for which reasonable black point estimates were made. + // The bp estimated at the boundaries is used for the interior. + int bp = getBlackPointFromNeighbors(blocks_, subWidth, x, y); + // The (min= MINIMUM_DIMENSION && height >= MINIMUM_DIMENSION) { + Ref newMatrix(new BitMatrix(width, height, err_handler)); + if (err_handler.ErrCode()) return -1; + + calculateThresholdForBlock(grayByte_, subWidth_, subHeight_, BLOCK_SIZE_POWER, newMatrix, + err_handler); + if (err_handler.ErrCode()) return -1; + + matrix0_ = newMatrix; + + } else { + // If the image is too small, fall back to the global histogram + // approach. + matrix0_ = GlobalHistogramBinarizer::getBlackMatrix(err_handler); + if (err_handler.ErrCode()) return 1; + } + // return matrix0_; + return 1; +} diff --git a/modules/wechat_qrcode/src/zxing/common/binarizer/hybrid_binarizer.hpp b/modules/wechat_qrcode/src/zxing/common/binarizer/hybrid_binarizer.hpp new file mode 100644 index 00000000000..758adc0b571 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/binarizer/hybrid_binarizer.hpp @@ -0,0 +1,83 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_HYBRID_BINARIZER_HPP__ +#define __ZXING_COMMON_HYBRID_BINARIZER_HPP__ + +#include "../../binarizer.hpp" +#include "../../errorhandler.hpp" +#include "../bitarray.hpp" +#include "../bitmatrix.hpp" +#include "../bytematrix.hpp" +#include "global_histogram_binarizer.hpp" + +#include + + +namespace zxing { + +class HybridBinarizer : public GlobalHistogramBinarizer { +private: + Ref grayByte_; + // ArrayRef integral_; + ArrayRef blockIntegral_; + ArrayRef blocks_; + + ArrayRef blackPoints_; + int level_; + + int subWidth_; + int subHeight_; + +public: + explicit HybridBinarizer(Ref source); + virtual ~HybridBinarizer(); + + virtual Ref getBlackMatrix(ErrorHandler& err_handler) override; + virtual Ref getBlackRow(int y, Ref row, ErrorHandler& err_handler) override; + + Ref createBinarizer(Ref source) override; + +private: + int initIntegral(); + int initBlockIntegral(); + int initBlocks(); + + // int calculateBlackPoints(); + ArrayRef getBlackPoints(); + int getBlockThreshold(int x, int y, int subWidth, int sum, int min, int max, + int minDynamicRange, int SIZE_POWER); + + + void calculateThresholdForBlock(Ref& luminances, int subWidth, int subHeight, + int SIZE_POWER, Ref const& matrix, + ErrorHandler& err_handler); + + + void thresholdBlock(Ref& luminances, int xoffset, int yoffset, int threshold, + Ref const& matrix, ErrorHandler& err_handler); + + void thresholdIrregularBlock(Ref& luminances, int xoffset, int yoffset, + int blockWidth, int blockHeight, int threshold, + Ref const& matrix, ErrorHandler& err_handler); + +#ifdef USE_SET_INT + void thresholdFourBlocks(Ref& luminances, int xoffset, int yoffset, int* thresholds, + int stride, Ref const& matrix); +#endif + + // Add for binarize image when call getBlackMatrix + // By Skylook + int binarizeByBlock(ErrorHandler& err_handler); +}; + +} // namespace zxing + +#endif // __ZXING_COMMON_HYBRID_BINARIZER_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/binarizer/simple_adaptive_binarizer.cpp b/modules/wechat_qrcode/src/zxing/common/binarizer/simple_adaptive_binarizer.cpp new file mode 100644 index 00000000000..8f610ca31d5 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/binarizer/simple_adaptive_binarizer.cpp @@ -0,0 +1,159 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "simple_adaptive_binarizer.hpp" +#include + +using namespace std; +using namespace zxing; + +// VC++ +using zxing::LuminanceSource; + +SimpleAdaptiveBinarizer::SimpleAdaptiveBinarizer(Ref source) + : GlobalHistogramBinarizer(source) { + filtered = false; +} + +SimpleAdaptiveBinarizer::~SimpleAdaptiveBinarizer() {} + +// Applies simple sharpening to the row data to improve performance of the 1D +// readers. +Ref SimpleAdaptiveBinarizer::getBlackRow(int y, Ref row, + ErrorHandler &err_handler) { + // First call binarize image in child class to get matrix0_ and binCache + if (!matrix0_) { + binarizeImage0(err_handler); + if (err_handler.ErrCode()) return Ref(); + } + // Call parent getBlackMatrix to get current matrix + return Binarizer::getBlackRow(y, row, err_handler); +} + +// Does not sharpen the data, as this call is intended to only be used by 2D +// readers. +Ref SimpleAdaptiveBinarizer::getBlackMatrix(ErrorHandler &err_handler) { + // First call binarize image in child class to get matrix0_ and binCache + if (!matrix0_) { + binarizeImage0(err_handler); + if (err_handler.ErrCode()) return Ref(); + } + + // First call binarize image in child class to get matrix0_ and binCache + // Call parent getBlackMatrix to get current matrix + return Binarizer::getBlackMatrix(err_handler); +} + +using namespace std; + +int SimpleAdaptiveBinarizer::binarizeImage0(ErrorHandler &err_handler) { + LuminanceSource &source = *getLuminanceSource(); + + Ref matrix(new BitMatrix(width, height, err_handler)); + if (err_handler.ErrCode()) return -1; + + ArrayRef localLuminances = source.getMatrix(); + + unsigned char *src = (unsigned char *)localLuminances->data(); + unsigned char *dst = matrix->getPtr(); + + qrBinarize(src, dst); + + matrix0_ = matrix; + + return 0; +} + +/*A simplified adaptive thresholder. + This compares the current pixel value to the mean value of a (large) window + surrounding it.*/ +int SimpleAdaptiveBinarizer::qrBinarize(const unsigned char *src, unsigned char *dst) { + unsigned char *mask = dst; + + if (width > 0 && height > 0) { + unsigned *col_sums; + int logwindw; + int logwindh; + int windw; + int windh; + int y0offs; + int y1offs; + unsigned g; + int x; + int y; + /*We keep the window size fairly large to ensure it doesn't fit + completely inside the center of a finder pattern of a version 1 QR + code at full resolution.*/ + for (logwindw = 4; logwindw < 8 && (1 << logwindw) < ((width + 7) >> 3); logwindw++) + ; + for (logwindh = 4; logwindh < 8 && (1 << logwindh) < ((height + 7) >> 3); logwindh++) + ; + windw = 1 << logwindw; + windh = 1 << logwindh; + + int logwinds = (logwindw + logwindh); + + col_sums = (unsigned *)malloc(width * sizeof(*col_sums)); + /*Initialize sums down each column.*/ + for (x = 0; x < width; x++) { + g = src[x]; + col_sums[x] = (g << (logwindh - 1)) + g; + } + for (y = 1; y < (windh >> 1); y++) { + y1offs = min(y, height - 1) * width; + for (x = 0; x < width; x++) { + g = src[y1offs + x]; + col_sums[x] += g; + } + } + for (y = 0; y < height; y++) { + unsigned m; + int x0; + int x1; + /*Initialize the sum over the window.*/ + m = (col_sums[0] << (logwindw - 1)) + col_sums[0]; + for (x = 1; x < (windw >> 1); x++) { + x1 = min(x, width - 1); + m += col_sums[x1]; + } + + int offset = y * width; + + for (x = 0; x < width; x++) { + /*Perform the test against the threshold T = (m/n)-D, + where n=windw*windh and D=3.*/ + g = src[offset + x]; + mask[offset + x] = ((g + 3) << (logwinds) < m); + /*Update the window sum.*/ + if (x + 1 < width) { + x0 = max(0, x - (windw >> 1)); + x1 = min(x + (windw >> 1), width - 1); + m += col_sums[x1] - col_sums[x0]; + } + } + /*Update the column sums.*/ + if (y + 1 < height) { + y0offs = max(0, y - (windh >> 1)) * width; + y1offs = min(y + (windh >> 1), height - 1) * width; + for (x = 0; x < width; x++) { + col_sums[x] -= src[y0offs + x]; + col_sums[x] += src[y1offs + x]; + } + } + } + free(col_sums); + } + + return 1; +} + +Ref SimpleAdaptiveBinarizer::createBinarizer(Ref source) { + return Ref(new SimpleAdaptiveBinarizer(source)); +} diff --git a/modules/wechat_qrcode/src/zxing/common/binarizer/simple_adaptive_binarizer.hpp b/modules/wechat_qrcode/src/zxing/common/binarizer/simple_adaptive_binarizer.hpp new file mode 100644 index 00000000000..956e87ee598 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/binarizer/simple_adaptive_binarizer.hpp @@ -0,0 +1,40 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_SIMPLEADAPTIVEBINARIZER_HPP__ +#define __ZXING_COMMON_SIMPLEADAPTIVEBINARIZER_HPP__ + +#include "../../binarizer.hpp" +#include "../array.hpp" +#include "../bitarray.hpp" +#include "../bitmatrix.hpp" +#include "global_histogram_binarizer.hpp" + + +namespace zxing { + +class SimpleAdaptiveBinarizer : public GlobalHistogramBinarizer { +public: + explicit SimpleAdaptiveBinarizer(Ref source); + virtual ~SimpleAdaptiveBinarizer(); + + virtual Ref getBlackRow(int y, Ref row, ErrorHandler &err_handler) override; + virtual Ref getBlackMatrix(ErrorHandler &err_handler) override; + Ref createBinarizer(Ref source) override; + +private: + int binarizeImage0(ErrorHandler &err_handler); + int qrBinarize(const unsigned char *src, unsigned char *dst); + bool filtered; +}; + +} // namespace zxing + +#endif // __ZXING_COMMON_SIMPLEADAPTIVEBINARIZER_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/bitarray.cpp b/modules/wechat_qrcode/src/zxing/common/bitarray.cpp new file mode 100644 index 00000000000..87e3aa3d0f6 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/bitarray.cpp @@ -0,0 +1,241 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "bitarray.hpp" + +using std::vector; +using zxing::ArrayRef; +using zxing::BitArray; +using zxing::ErrorHandler; +// VC++ +using zxing::Ref; + +#if __WORDSIZE == 64 +// typedef long int int64_t; +#else +typedef long long int int64_t; +#endif + +BitArray::BitArray(int size_) : size(size_), bits(size_), nextSets(size_), nextUnSets(size_) {} + +void BitArray::setUnchar(int i, unsigned char newBits) { bits[i] = newBits; } + +bool BitArray::isRange(int start, int end, bool value, ErrorHandler &err_handler) { + if (end < start) { + err_handler = IllegalArgumentErrorHandler("isRange"); + return false; + } + if (start < 0 || end >= bits->size()) { + err_handler = IllegalArgumentErrorHandler("isRange"); + return false; + } + if (end == start) { + return true; // empty range matches + } + + bool startBool = bits[start] != (unsigned char)0; + + int end2 = start; + + if (startBool) { + end2 = getNextUnset(start); + } else { + end2 = getNextSet(start); + } + + if (startBool == value) { + if (end2 < end) { + return false; + } + } else { + return false; + } + + return true; +} + +void BitArray::reverse() { + bool *rowBits = getRowBoolPtr(); + bool tempBit; + + for (int i = 0; i < size / 2; i++) { + tempBit = rowBits[i]; + rowBits[i] = rowBits[size - i - 1]; + rowBits[size - i - 1] = tempBit; + } +} + +void BitArray::initAllNextSets() { + bool *rowBits = getRowBoolPtr(); + + int *nextSetArray = nextSets->data(); + int *nextUnsetArray = nextUnSets->data(); + + // Init the last one + if (rowBits[size - 1]) { + nextSetArray[size - 1] = size - 1; + nextUnsetArray[size - 1] = size; + } else { + nextUnsetArray[size - 1] = size - 1; + nextSetArray[size - 1] = size; + } + + // do inits + for (int i = size - 2; i >= 0; i--) { + if (rowBits[i]) { + nextSetArray[i] = i; + nextUnsetArray[i] = nextUnsetArray[i + 1]; + } else { + nextUnsetArray[i] = i; + nextSetArray[i] = nextSetArray[i + 1]; + } + } +} + +void BitArray::initAllNextSetsFromCounters(std::vector counters) { + bool *rowBits = getRowBoolPtr(); + bool isWhite = rowBits[0]; + int c = 0; + int offset = 0; + int count = 0; + int prevCount = 0; + int currCount = 0; + int _size = counters.size(); + + int *nextSetArray = nextSets->data(); + int *nextUnsetArray = nextUnSets->data(); + + // int* countersArray = counters.data(); + int *countersArray = &counters[0]; + + while (c < _size) { + currCount = countersArray[c]; + + count += currCount; + + if (isWhite) { + for (int i = 0; i < currCount; i++) { + offset = prevCount + i; + nextSetArray[offset] = prevCount + i; + nextUnsetArray[offset] = count; + } + } else { + for (int i = 0; i < currCount; i++) { + offset = prevCount + i; + nextSetArray[offset] = count; + nextUnsetArray[offset] = prevCount + i; + } + } + + isWhite = !isWhite; + + prevCount += currCount; + + c++; + } +} + +int BitArray::getNextSet(int from) { + if (from >= size) { + return size; + } + return nextSets[from]; +} + +int BitArray::getNextUnset(int from) { + if (from >= size) { + return size; + } + return nextUnSets[from]; +} + +BitArray::~BitArray() {} + +int BitArray::getSize() const { return size; } + +void BitArray::clear() { + int max = bits->size(); + for (int i = 0; i < max; i++) { + bits[i] = 0; + } +} + +BitArray::Reverse::Reverse(Ref array_) : array(array_) { array->reverse(); } + +BitArray::Reverse::~Reverse() { array->reverse(); } + +void BitArray::appendBit(bool value) { + ArrayRef newBits(size + 1); + for (int i = 0; i < size; i++) { + newBits[i] = bits[i]; + } + bits = newBits; + if (value) { + set(size); + } + ++size; +} + +int BitArray::getSizeInBytes() const { return size; } + +// Appends the least-significant bits, from value, in order from +// most-significant to least-significant. For example, appending 6 bits +// from 0x000001E will append the bits 0, 1, 1, 1, 1, 0 in that order. +void BitArray::appendBits(int value, int numBits, ErrorHandler &err_handler) { + if (numBits < 0 || numBits > 32) { + err_handler = IllegalArgumentErrorHandler("Number of bits must be between 0 and 32"); + return; + } + ArrayRef newBits(size + numBits); + for (int i = 0; i < size; i++) newBits[i] = bits[i]; + bits = newBits; + for (int numBitsLeft = numBits; numBitsLeft > 0; numBitsLeft--) { + if (((value >> (numBitsLeft - 1)) & 0x01) == 1) { + set(size); + } + ++size; + } + return; +} + +void BitArray::appendBitArray(const BitArray &array) { + ArrayRef newBits(size + array.getSize()); + for (int i = 0; i < size; ++i) { + newBits[i] = bits[i]; + } + bits = newBits; + for (int i = 0; i < array.getSize(); ++i) { + if (array.get(i)) { + set(size); + } + ++size; + } +} + +void BitArray::toBytes(int bitOffset, ArrayRef &array, int offset, int numBytes) { + for (int i = 0; i < numBytes; i++) { + int theByte = 0; + if (get(bitOffset)) { + theByte = 1; + } + bitOffset++; + array[offset + i] = theByte; + } +} +void BitArray::bitXOR(const BitArray &other, ErrorHandler &err_handler) { + if (size != other.size) { + err_handler = IllegalArgumentErrorHandler("Sizes don't match"); + return; + } + + for (int i = 0; i < bits->size(); i++) { + bits[i] = bits[i] == other.bits[i] ? 0 : 1; + } +} diff --git a/modules/wechat_qrcode/src/zxing/common/bitarray.hpp b/modules/wechat_qrcode/src/zxing/common/bitarray.hpp new file mode 100644 index 00000000000..79c5280c60b --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/bitarray.hpp @@ -0,0 +1,91 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_BITARRAY_HPP__ +#define __ZXING_COMMON_BITARRAY_HPP__ + +#include "../errorhandler.hpp" +#include "../zxing.hpp" +#include "array.hpp" +#include "counted.hpp" +#include +#include +#include +#include + +namespace zxing { + +class BitArray : public Counted { +private: + int size; + ArrayRef bits; + ArrayRef nextSets; + ArrayRef nextUnSets; + // bool nextSetsInited; + +public: + explicit BitArray(int size); + ~BitArray(); + int getSize() const; + + bool get(int i) const { return bits[i] != 0; } + void set(int i) { + // bits[i] |= 0xFF; + bits[i] = true; + } + void setOneRow(unsigned char* rowBits, int length) { + unsigned char* dst = bits->data(); + memcpy(dst, rowBits, length); + } + + bool* getRowBoolPtr() { + // return (bool*)bits.data(); + return (bool*)bits->data(); + } + + // Init for next sets and unsets to speed up + void initAllNextSets(); + void initAllNextSetsFromCounters(std::vector counters); + + int getNextSet(int from); + int getNextUnset(int from); + + void setUnchar(int i, unsigned char newBist); + + void clear(); + bool isRange(int start, int end, bool value, ErrorHandler& err_handler); + + void reverse(); + + class Reverse { + private: + Ref array; + + public: + explicit Reverse(Ref array); + ~Reverse(); + }; + + void appendBit(bool value); + int getSizeInBytes() const; + void appendBits(int value, int numberOfBits, ErrorHandler& err_handler); + void appendBitArray(const BitArray& array); + void toBytes(int bitOffset, ArrayRef& array, int offset, int numBytes); + void bitXOR(const BitArray& other, ErrorHandler& err_handler); + +#ifndef USE_BYTE_FOR_BIT +private: + static int makeArraySize(int size); +#endif +}; + +} // namespace zxing + +#endif // __ZXING_COMMON_BITARRAY_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/bitmatrix.cpp b/modules/wechat_qrcode/src/zxing/common/bitmatrix.cpp new file mode 100644 index 00000000000..103140756a2 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/bitmatrix.cpp @@ -0,0 +1,403 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "bitmatrix.hpp" +#include +#include +#include + +using std::ostream; +using std::ostringstream; + +using zxing::ArrayRef; +using zxing::BitArray; +using zxing::BitMatrix; +using zxing::ErrorHandler; +using zxing::Ref; + +void BitMatrix::init(int _width, int _height, ErrorHandler& err_handler) { + if (_width < 1 || _height < 1) { + err_handler = IllegalArgumentErrorHandler("Both dimensions must be greater than 0"); + return; + } + width = _width; + height = _height; + this->rowBitsSize = width; + bits = ArrayRef(width * height); + rowOffsets = ArrayRef(height); + + // offsetRowSize = new int[height]; + rowOffsets[0] = 0; + for (int i = 1; i < height; i++) { + rowOffsets[i] = rowOffsets[i - 1] + width; + } + + isInitRowCounters = false; + isInitColsCounters = false; +} + +void BitMatrix::init(int _width, int _height, unsigned char* bitsPtr, ErrorHandler& err_handler) { + init(_width, _height, err_handler); + if (err_handler.ErrCode()) return; + memcpy(bits->data(), bitsPtr, width * height * sizeof(unsigned char)); +} + +void BitMatrix::initRowCounters() { + if (isInitRowCounters == true) { + return; + } + + row_counters = vector(width * height, 0); + row_counters_offset = vector(width * height, 0); + row_point_offset = vector(width * height, 0); + row_counter_offset_end = vector(height, 0); + + row_counters_recorded = vector(height, false); + + isInitRowCounters = true; +} +void BitMatrix::initColsCounters() { + if (isInitColsCounters == true) { + return; + } + + cols_counters = vector(width * height, 0); + cols_counters_offset = vector(width * height, 0); + cols_point_offset = vector(width * height, 0); + cols_counter_offset_end = vector(width, 0); + + cols_counters_recorded = vector(width, false); + + isInitColsCounters = true; +} + +BitMatrix::BitMatrix(int dimension, ErrorHandler& err_handler) { + init(dimension, dimension, err_handler); +} + +BitMatrix::BitMatrix(int _width, int _height, ErrorHandler& err_handler) { + init(_width, _height, err_handler); +} + +BitMatrix::BitMatrix(int _width, int _height, unsigned char* bitsPtr, ErrorHandler& err_handler) { + init(_width, _height, bitsPtr, err_handler); +} +// Copy bitMatrix +void BitMatrix::copyOf(Ref _bits, ErrorHandler& err_handler) { + int _width = _bits->getWidth(); + int _height = _bits->getHeight(); + init(_width, _height, err_handler); + + for (int y = 0; y < height; y++) { + bool* rowPtr = _bits->getRowBoolPtr(y); + setRowBool(y, rowPtr); + } +} + +void BitMatrix::xxor(Ref _bits) { + if (width != _bits->getWidth() || height != _bits->getHeight()) { + return; + } + + for (int y = 0; y < height && y < _bits->getHeight(); ++y) { + bool* rowPtrA = _bits->getRowBoolPtr(y); + bool* rowPtrB = getRowBoolPtr(y); + + for (int x = 0; x < width && x < _bits->getWidth(); ++x) { + rowPtrB[x] = rowPtrB[x] ^ rowPtrA[x]; + } + setRowBool(y, rowPtrB); + } +} + +BitMatrix::~BitMatrix() {} + +void BitMatrix::flip(int x, int y) { + bits[rowOffsets[y] + x] = (bits[rowOffsets[y] + x] == (unsigned char)0); +} + +void BitMatrix::flipAll() { + bool* matrixBits = (bool*)bits->data(); + for (int i = 0; i < bits->size(); i++) { + matrixBits[i] = !matrixBits[i]; + } +} + +void BitMatrix::flipRegion(int left, int top, int _width, int _height, ErrorHandler& err_handler) { + if (top < 0 || left < 0) { + err_handler = IllegalArgumentErrorHandler("Left and top must be nonnegative"); + return; + } + if (_height < 1 || _width < 1) { + err_handler = IllegalArgumentErrorHandler("Height and width must be at least 1"); + return; + } + int right = left + _width; + int bottom = top + _height; + if (bottom > this->height || right > this->width) { + err_handler = IllegalArgumentErrorHandler("The region must fit inside the matrix"); + return; + } + + for (int y = top; y < bottom; y++) { + for (int x = left; x < right; x++) { + bits[rowOffsets[y] + x] ^= 1; + } + } +} + +void BitMatrix::setRegion(int left, int top, int _width, int _height, ErrorHandler& err_handler) { + if (top < 0 || left < 0) { + err_handler = IllegalArgumentErrorHandler("Left and top must be nonnegative"); + return; + } + if (_height < 1 || _width < 1) { + err_handler = IllegalArgumentErrorHandler("Height and width must be at least 1"); + return; + } + int right = left + _width; + int bottom = top + _height; + if (bottom > this->height || right > this->width) { + err_handler = IllegalArgumentErrorHandler("The region must fit inside the matrix"); + return; + } + + for (int y = top; y < bottom; y++) { + for (int x = left; x < right; x++) { + bits[rowOffsets[y] + x] = true; + // bits[rowOffsets[y]+x] |= 0xFF; + } + } +} + +Ref BitMatrix::getRow(int y, Ref row) { + if (row.empty() || row->getSize() < width) { + row = new BitArray(width); + } + + // row-> + unsigned char* src = bits.data() + rowOffsets[y]; + row->setOneRow(src, width); + + return row; +} + +ArrayRef BitMatrix::getTopLeftOnBit() const { + int bitsOffset = 0; + while (bitsOffset < bits->size() && bits[bitsOffset] == 0) { + bitsOffset++; + } + if (bitsOffset == bits->size()) { + return ArrayRef(); + } + int y = bitsOffset / width; + int x = bitsOffset % width; + ArrayRef res(2); + res[0] = x; + res[1] = y; + return res; +} + +ArrayRef BitMatrix::getBottomRightOnBit() const { + int bitsOffset = bits->size() - 1; + while (bitsOffset >= 0 && bits[bitsOffset] == 0) { + bitsOffset--; + } + if (bitsOffset < 0) { + return ArrayRef(); + } + + int y = bitsOffset / width; + int x = bitsOffset % width; + ArrayRef res(2); + res[0] = x; + res[1] = y; + return res; +} + +void BitMatrix::getRowBool(int y, bool* getrow) { + int offset = rowOffsets[y]; + unsigned char* src = bits.data() + offset; + memcpy(getrow, src, rowBitsSize * sizeof(bool)); +} + +void BitMatrix::setRowBool(int y, bool* row) { + int offset = rowOffsets[y]; + unsigned char* dst = bits.data() + offset; + memcpy(dst, row, rowBitsSize * sizeof(bool)); + + return; +} + +bool* BitMatrix::getRowBoolPtr(int y) { + int offset = y * rowBitsSize; + unsigned char* src = bits.data() + offset; + return (bool*)src; +} + +void BitMatrix::clear() { + int size = bits->size(); + + unsigned char* dst = bits->data(); + memset(dst, 0, size * sizeof(unsigned char)); +} + +int BitMatrix::getWidth() const { return width; } + +int BitMatrix::getHeight() const { return height; } + +COUNTER_TYPE* BitMatrix::getRowPointInRecords(int y) { + if (!row_point_offset[y]) { + setRowRecords(y); + } + int offset = y * width; + COUNTER_TYPE* counters = &row_point_offset[0] + offset; + return (COUNTER_TYPE*)counters; +} + +COUNTER_TYPE* BitMatrix::getRowRecords(int y) { + if (!row_counters_recorded[y]) { + setRowRecords(y); + } + int offset = y * width; + COUNTER_TYPE* counters = &row_counters[0] + offset; + return (COUNTER_TYPE*)counters; +} + +COUNTER_TYPE* BitMatrix::getRowRecordsOffset(int y) { + if (!row_counters_recorded[y]) { + setRowRecords(y); + } + int offset = y * width; + COUNTER_TYPE* counters = &row_counters_offset[0] + offset; + return (COUNTER_TYPE*)counters; +} + +bool BitMatrix::getRowFirstIsWhite(int y) { + bool is_white = !get(0, y); + return is_white; +} + +bool BitMatrix::getRowLastIsWhite(int y) { + bool last_is_white = !get(width - 1, y); + return last_is_white; +} + +COUNTER_TYPE BitMatrix::getRowCounterOffsetEnd(int y) { + if (!row_counters_recorded[y]) { + setRowRecords(y); + } + return row_counter_offset_end[y]; +} + +void BitMatrix::setRowRecords(int y) { + COUNTER_TYPE* cur_row_counters = &row_counters[0] + y * width; + COUNTER_TYPE* cur_row_counters_offset = &row_counters_offset[0] + y * width; + COUNTER_TYPE* cur_row_point_in_counters = &row_point_offset[0] + y * width; + int end = width; + + bool* rowBit = getRowBoolPtr(y); + bool isWhite = !rowBit[0]; + int counterPosition = 0; + int i = 0; + cur_row_counters_offset[0] = 0; + while (i < end) { + if (rowBit[i] ^ isWhite) { // that is, exactly one is true + cur_row_counters[counterPosition]++; + } else { + counterPosition++; + if (counterPosition == end) { + break; + } else { + cur_row_counters[counterPosition] = 1; + isWhite = !isWhite; + cur_row_counters_offset[counterPosition] = i; + } + } + cur_row_point_in_counters[i] = counterPosition; + i++; + } + + // use the last row__onedReaderData->counter_size to record + // _onedReaderData->counter_size + row_counter_offset_end[y] = counterPosition < end ? (counterPosition + 1) : end; + + row_counters_recorded[y] = true; + return; +} + +COUNTER_TYPE* BitMatrix::getColsPointInRecords(int x) { + if (!cols_point_offset[x]) { + setColsRecords(x); + } + int offset = x * height; + COUNTER_TYPE* counters = &cols_point_offset[0] + offset; + return (COUNTER_TYPE*)counters; +} + +COUNTER_TYPE* BitMatrix::getColsRecords(int x) { + if (!cols_counters_recorded[x]) { + setColsRecords(x); + } + int offset = x * height; + COUNTER_TYPE* counters = &cols_counters[0] + offset; + return (COUNTER_TYPE*)counters; +} + +COUNTER_TYPE* BitMatrix::getColsRecordsOffset(int x) { + if (!cols_counters_recorded[x]) { + setColsRecords(x); + } + int offset = x * height; + COUNTER_TYPE* counters = &cols_counters_offset[0] + offset; + return (COUNTER_TYPE*)counters; +} + +COUNTER_TYPE BitMatrix::getColsCounterOffsetEnd(int x) { + if (!cols_counters_recorded[x]) { + setColsRecords(x); + } + return cols_counter_offset_end[x]; +} + +void BitMatrix::setColsRecords(int x) { + COUNTER_TYPE* cur_cols_counters = &cols_counters[0] + x * height; + COUNTER_TYPE* cur_cols_counters_offset = &cols_counters_offset[0] + x * height; + COUNTER_TYPE* cur_cols_point_in_counters = &cols_point_offset[0] + x * height; + int end = height; + + bool* rowBit = getRowBoolPtr(0); + bool isWhite = !rowBit[0]; + int counterPosition = 0; + int i = 0; + cur_cols_counters_offset[0] = 0; + while (i < end) { + if (rowBit[i] ^ isWhite) { // that is, exactly one is true + cur_cols_counters[counterPosition]++; + } else { + counterPosition++; + if (counterPosition == end) { + break; + } else { + cur_cols_counters[counterPosition] = 1; + isWhite = !isWhite; + cur_cols_counters_offset[counterPosition] = i; + } + } + cur_cols_point_in_counters[i] = counterPosition; + i++; + rowBit += width; + } + + cols_counter_offset_end[x] = counterPosition < end ? (counterPosition + 1) : end; + + cols_counters_recorded[x] = true; + return; +}; diff --git a/modules/wechat_qrcode/src/zxing/common/bitmatrix.hpp b/modules/wechat_qrcode/src/zxing/common/bitmatrix.hpp new file mode 100644 index 00000000000..624db9c5a0a --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/bitmatrix.hpp @@ -0,0 +1,118 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_BITMATRIX_HPP__ +#define __ZXING_COMMON_BITMATRIX_HPP__ + +#include "../errorhandler.hpp" +#include "array.hpp" +#include "bitarray.hpp" +#include "counted.hpp" + +#include +#include +using namespace std; + +namespace zxing { + +class BitMatrix : public Counted { +public: + static const int bitsPerWord = std::numeric_limits::digits; + +private: + int width; + int height; + int rowBitsSize; + + vector row_counters; + vector row_counters_offset; + vector row_counters_recorded; + vector row_counter_offset_end; + vector row_point_offset; + + vector cols_counters; + vector cols_counters_offset; + vector cols_counters_recorded; + vector cols_counter_offset_end; + vector cols_point_offset; + + ArrayRef bits; + ArrayRef rowOffsets; + +public: + BitMatrix(int _width, int _height, unsigned char* bitsPtr, ErrorHandler& err_handler); + BitMatrix(int dimension, ErrorHandler& err_handler); + BitMatrix(int _width, int _height, ErrorHandler& err_handler); + + void copyOf(Ref _bits, ErrorHandler& err_handler); + void xxor(Ref _bits); + + ~BitMatrix(); + + unsigned char get(int x, int y) const { return bits[width * y + x]; } + + void set(int x, int y) { bits[rowOffsets[y] + x] = (unsigned char)1; } + + void set(int x, int y, unsigned char value) { bits[rowOffsets[y] + x] = value; } + + void swap(int srcX, int srcY, int dstX, int dstY) { + auto temp = bits[width * srcY + srcX]; + bits[width * srcY + srcX] = bits[width * dstY + dstX]; + bits[width * dstY + dstX] = temp; + } + + void getRowBool(int y, bool* row); + bool* getRowBoolPtr(int y); + void setRowBool(int y, bool* row); + int getRowBitsSize() { return rowBitsSize; } + unsigned char* getPtr() { return bits->data(); } + + void flip(int x, int y); + void flipAll(); + void clear(); + void setRegion(int left, int top, int _width, int _height, ErrorHandler& err_handler); + void flipRegion(int left, int top, int _width, int _height, ErrorHandler& err_handler); + Ref getRow(int y, Ref row); + + int getWidth() const; + int getHeight() const; + + ArrayRef getTopLeftOnBit() const; + ArrayRef getBottomRightOnBit() const; + + bool isInitRowCounters; + void initRowCounters(); + COUNTER_TYPE* getRowRecords(int y); + COUNTER_TYPE* getRowRecordsOffset(int y); + bool getRowFirstIsWhite(int y); + COUNTER_TYPE getRowCounterOffsetEnd(int y); + bool getRowLastIsWhite(int y); + COUNTER_TYPE* getRowPointInRecords(int y); + + bool isInitColsCounters; + void initColsCounters(); + COUNTER_TYPE* getColsRecords(int x); + COUNTER_TYPE* getColsRecordsOffset(int x); + COUNTER_TYPE* getColsPointInRecords(int x); + COUNTER_TYPE getColsCounterOffsetEnd(int x); + +private: + inline void init(int, int, ErrorHandler& err_handler); + inline void init(int _width, int _height, unsigned char* bitsPtr, ErrorHandler& err_handler); + + void setRowRecords(int y); + void setColsRecords(int x); + + BitMatrix(const BitMatrix&, ErrorHandler& err_handler); +}; + +} // namespace zxing + +#endif // __ZXING_COMMON_BITMATRIX_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/bitsource.cpp b/modules/wechat_qrcode/src/zxing/common/bitsource.cpp new file mode 100644 index 00000000000..6ff5feaba3e --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/bitsource.cpp @@ -0,0 +1,62 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "bitsource.hpp" +#include + +namespace zxing { + +int BitSource::readBits(int numBits, ErrorHandler& err_handler) { + if (numBits < 0 || numBits > 32 || numBits > available()) { + std::ostringstream oss; + oss << numBits; + err_handler = IllegalArgumentErrorHandler(oss.str().c_str()); + return -1; + } + + int result = 0; + + // First, read remainder from current byte + if (bitOffset_ > 0) { + int bitsLeft = 8 - bitOffset_; + int toRead = numBits < bitsLeft ? numBits : bitsLeft; + int bitsToNotRead = bitsLeft - toRead; + int mask = (0xFF >> (8 - toRead)) << bitsToNotRead; + result = (bytes_[byteOffset_] & mask) >> bitsToNotRead; + numBits -= toRead; + bitOffset_ += toRead; + if (bitOffset_ == 8) { + bitOffset_ = 0; + byteOffset_++; + } + } + + // Next read whole bytes + if (numBits > 0) { + while (numBits >= 8) { + result = (result << 8) | (bytes_[byteOffset_] & 0xFF); + byteOffset_++; + numBits -= 8; + } + + // Finally read a partial byte + if (numBits > 0) { + int bitsToNotRead = 8 - numBits; + int mask = (0xFF >> bitsToNotRead) << bitsToNotRead; + result = (result << numBits) | ((bytes_[byteOffset_] & mask) >> bitsToNotRead); + bitOffset_ += numBits; + } + } + + return result; +} + +int BitSource::available() { return 8 * (bytes_->size() - byteOffset_) - bitOffset_; } +} // namespace zxing diff --git a/modules/wechat_qrcode/src/zxing/common/bitsource.hpp b/modules/wechat_qrcode/src/zxing/common/bitsource.hpp new file mode 100644 index 00000000000..797c2b5c9a1 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/bitsource.hpp @@ -0,0 +1,57 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_BITSOURCE_HPP__ +#define __ZXING_COMMON_BITSOURCE_HPP__ + +#include "../errorhandler.hpp" +#include "array.hpp" + +namespace zxing { +/** + *

This provides an easy abstraction to read bits at a time from a sequence + * of bytes, where the number of bits read is not often a multiple of 8.

+ * + *

This class is not thread-safe.

+ * + * @author srowen@google.com (Sean Owen) + * @author christian.brunschen@gmail.com (Christian Brunschen) + */ +class BitSource : public Counted { + typedef char byte; + +private: + ArrayRef bytes_; + int byteOffset_; + int bitOffset_; + +public: + /** + * @param bytes bytes from which this will read bits. Bits will be read from + * the first byte first. Bits are read within a byte from most-significant + * to least-significant bit. + */ + explicit BitSource(ArrayRef &bytes) : bytes_(bytes), byteOffset_(0), bitOffset_(0) {} + + int getBitOffset() { return bitOffset_; } + + int getByteOffset() { return byteOffset_; } + + int readBits(int numBits, ErrorHandler &err_handler); + + /** + * @return number of bits that can be read successfully + */ + int available(); +}; + +} // namespace zxing + +#endif // __ZXING_COMMON_BITSOURCE_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/bytematrix.cpp b/modules/wechat_qrcode/src/zxing/common/bytematrix.cpp new file mode 100644 index 00000000000..b5370671db3 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/bytematrix.cpp @@ -0,0 +1,57 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#include "bytematrix.hpp" +#include +#include +#include +#include + +using std::ostream; +using std::ostringstream; + +using zxing::ArrayRef; +using zxing::ByteMatrix; +using zxing::ErrorHandler; +using zxing::Ref; + +void ByteMatrix::init(int _width, int _height) { + if (_width < 1 || _height < 1) { + return; + } + this->width = _width; + this->height = _height; + bytes = new unsigned char[width * height]; + row_offsets = new int[height]; + row_offsets[0] = 0; + for (int i = 1; i < height; i++) { + row_offsets[i] = row_offsets[i - 1] + width; + } +} + +ByteMatrix::ByteMatrix(int dimension) { init(dimension, dimension); } + +ByteMatrix::ByteMatrix(int _width, int _height) { init(_width, _height); } + +ByteMatrix::ByteMatrix(int _width, int _height, ArrayRef source) { + init(_width, _height); + int size = _width * _height; + memcpy(&bytes[0], &source[0], size); +} + +ByteMatrix::~ByteMatrix() { + if (bytes) delete[] bytes; + if (row_offsets) delete[] row_offsets; +} + +unsigned char* ByteMatrix::getByteRow(int y, ErrorHandler& err_handler) { + if (y < 0 || y >= getHeight()) { + err_handler = IllegalArgumentErrorHandler("Requested row is outside the image."); + return NULL; + } + return &bytes[row_offsets[y]]; +} diff --git a/modules/wechat_qrcode/src/zxing/common/bytematrix.hpp b/modules/wechat_qrcode/src/zxing/common/bytematrix.hpp new file mode 100644 index 00000000000..d3bd0038103 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/bytematrix.hpp @@ -0,0 +1,60 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#ifndef __ZXING_COMMON_BYTEMATRIX_HPP__ +#define __ZXING_COMMON_BYTEMATRIX_HPP__ + +#include "../errorhandler.hpp" +#include "array.hpp" +#include "bitarray.hpp" +#include "counted.hpp" + +#include + +namespace zxing { + +class ByteMatrix : public Counted { +public: + explicit ByteMatrix(int dimension); + ByteMatrix(int _width, int _height); + ByteMatrix(int _width, int _height, ArrayRef source); + ~ByteMatrix(); + + char get(int x, int y) const { + int offset = row_offsets[y] + x; + return bytes[offset]; + } + + void set(int x, int y, char char_value) { + int offset = row_offsets[y] + x; + bytes[offset] = char_value & 0XFF; + } + + unsigned char* getByteRow(int y, ErrorHandler& err_handler); + + int getWidth() const { return width; } + int getHeight() const { return height; } + + unsigned char* bytes; + +private: + int width; + int height; + + // ArrayRef bytes; + // ArrayRef row_offsets; + int* row_offsets; + +private: + inline void init(int, int); + ByteMatrix(const ByteMatrix&); + ByteMatrix& operator=(const ByteMatrix&); +}; + +} // namespace zxing + +#endif // __ZXING_COMMON_BYTEMATRIX_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/character.hpp b/modules/wechat_qrcode/src/zxing/common/character.hpp new file mode 100644 index 00000000000..4059d8dcf5c --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/character.hpp @@ -0,0 +1,54 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#ifndef __ZXING_COMMON_CHARACTER_HPP___ +#define __ZXING_COMMON_CHARACTER_HPP___ + +#include +#include + +#include + +using namespace std; + +namespace zxing { + +class Character { +public: + static char toUpperCase(char c) { return toupper(c); }; + + static bool isDigit(char c) { + if (c < '0' || c > '9') { + return false; + } + + return true; + + // return isdigit(c); + }; + + static int digit(char c, int radix) { + // return digit(c, radix); + + if (c >= '0' && c <= '9') { + return (int)(c - '0'); + } + + if (c >= 'a' && c <= 'z' && c < (radix + 'a' - 10)) { + return (int)(c - 'a' + 10); + } + + if (c >= 'A' && c <= 'Z' && c < (radix + 'A' - 10)) { + return (int)(c - 'A' + 10); + } + + return -1; + } +}; +} // namespace zxing + +#endif // __ZXING_COMMON_CHARACTER_HPP___ diff --git a/modules/wechat_qrcode/src/zxing/common/characterseteci.cpp b/modules/wechat_qrcode/src/zxing/common/characterseteci.cpp new file mode 100644 index 00000000000..5ba94e22990 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/characterseteci.cpp @@ -0,0 +1,113 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "characterseteci.hpp" +using std::string; + +using zxing::common::CharacterSetECI; + +// Fix memory leak +// https://github.com/ukeller/zxing-cpp/commit/c632ffe47ca7342f894ae533263be249cbdfd37e +// std::map CharacterSetECI::VALUE_TO_ECI; +// std::map CharacterSetECI::NAME_TO_ECI; +std::map > CharacterSetECI::VALUE_TO_ECI; +std::map > CharacterSetECI::NAME_TO_ECI; + +const bool CharacterSetECI::inited = CharacterSetECI::init_tables(); + +#define ADD_CHARACTER_SET(VALUES, STRINGS) \ + { \ + static int values[] = {VALUES, -1}; \ + static char const* strings[] = {STRINGS, 0}; \ + addCharacterSet(values, strings); \ + } + +#define XC , + +bool CharacterSetECI::init_tables() { + ADD_CHARACTER_SET(0 XC 2, "Cp437"); + ADD_CHARACTER_SET(1 XC 3, "ISO8859_1" XC "ISO-8859-1"); + ADD_CHARACTER_SET(4, "ISO8859_2" XC "ISO-8859-2"); + ADD_CHARACTER_SET(5, "ISO8859_3" XC "ISO-8859-3"); + ADD_CHARACTER_SET(6, "ISO8859_4" XC "ISO-8859-4"); + ADD_CHARACTER_SET(7, "ISO8859_5" XC "ISO-8859-5"); + ADD_CHARACTER_SET(8, "ISO8859_6" XC "ISO-8859-6"); + ADD_CHARACTER_SET(9, "ISO8859_7" XC "ISO-8859-7"); + ADD_CHARACTER_SET(10, "ISO8859_8" XC "ISO-8859-8"); + ADD_CHARACTER_SET(11, "ISO8859_9" XC "ISO-8859-9"); + ADD_CHARACTER_SET(12, "ISO8859_10" XC "ISO-8859-10"); + ADD_CHARACTER_SET(13, "ISO8859_11" XC "ISO-8859-11"); + ADD_CHARACTER_SET(15, "ISO8859_13" XC "ISO-8859-13"); + ADD_CHARACTER_SET(16, "ISO8859_14" XC "ISO-8859-14"); + ADD_CHARACTER_SET(17, "ISO8859_15" XC "ISO-8859-15"); + ADD_CHARACTER_SET(18, "ISO8859_16" XC "ISO-8859-16"); + ADD_CHARACTER_SET(20, "SJIS" XC "Shift_JIS"); + ADD_CHARACTER_SET(21, "Cp1250" XC "windows-1250"); + ADD_CHARACTER_SET(22, "Cp1251" XC "windows-1251"); + ADD_CHARACTER_SET(23, "Cp1252" XC "windows-1252"); + ADD_CHARACTER_SET(24, "Cp1256" XC "windows-1256"); + ADD_CHARACTER_SET(25, "UnicodeBigUnmarked" XC "UTF-16BE" XC "UnicodeBig"); + ADD_CHARACTER_SET(26, "UTF8" XC "UTF-8"); + ADD_CHARACTER_SET(27 XC 170, "ASCII" XC "US-ASCII"); + ADD_CHARACTER_SET(28, "Big5"); + ADD_CHARACTER_SET(29, "GB18030" XC "GB2312" XC "EUC_CN" XC "GBK"); + ADD_CHARACTER_SET(30, "EUC_KR" XC "EUC-KR"); + return true; +} + +#undef XC + +CharacterSetECI::CharacterSetECI(int const* values, char const* const* names) + : values_(values), names_(names) { + zxing::Ref this_ref(this); + + for (int const* p_values = values_; *p_values != -1; p_values++) { + // VALUE_TO_ECI[*values] = this; + VALUE_TO_ECI[*p_values] = this_ref; + } + for (char const* const* p_names = names_; *p_names; p_names++) { + // NAME_TO_ECI[string(*names)] = this; + NAME_TO_ECI[string(*p_names)] = this_ref; + } +} + +char const* CharacterSetECI::name() const { return names_[0]; } + +int CharacterSetECI::getValue() const { return values_[0]; } + +void CharacterSetECI::addCharacterSet(int const* values, char const* const* names) { + new CharacterSetECI(values, names); +} + +CharacterSetECI* CharacterSetECI::getCharacterSetECIByValueFind(int value) { + if (value < 0 || value >= 900) { + return zxing::Ref(0); + } + + std::map >::iterator iter; + iter = VALUE_TO_ECI.find(value); + + if (iter != VALUE_TO_ECI.end()) { + return iter->second; + } else { + return zxing::Ref(0); + } +} + +CharacterSetECI* CharacterSetECI::getCharacterSetECIByName(string const& name) { + std::map >::iterator iter; + iter = NAME_TO_ECI.find(name); + + if (iter != NAME_TO_ECI.end()) { + return iter->second; + } else { + return zxing::Ref(0); + } +} diff --git a/modules/wechat_qrcode/src/zxing/common/characterseteci.hpp b/modules/wechat_qrcode/src/zxing/common/characterseteci.hpp new file mode 100644 index 00000000000..9b44f14dee9 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/characterseteci.hpp @@ -0,0 +1,46 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_CHARACTERSETECI_HPP__ +#define __ZXING_COMMON_CHARACTERSETECI_HPP__ + +#include +#include "../decodehints.hpp" +#include "counted.hpp" + +namespace zxing { +namespace common { + +class CharacterSetECI : public Counted { +private: + static std::map > VALUE_TO_ECI; + static std::map > NAME_TO_ECI; + static const bool inited; + static bool init_tables(); + + int const* const values_; + char const* const* const names_; + + CharacterSetECI(int const* values, char const* const* names); + + static void addCharacterSet(int const* value, char const* const* encodingNames); + +public: + char const* name() const; + int getValue() const; + + static CharacterSetECI* getCharacterSetECIByValueFind(int value); + static CharacterSetECI* getCharacterSetECIByName(std::string const& name); +}; + +} // namespace common +} // namespace zxing + +#endif // __ZXING_COMMON_CHARACTERSETECI_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/counted.hpp b/modules/wechat_qrcode/src/zxing/common/counted.hpp new file mode 100644 index 00000000000..d40d62a3f0d --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/counted.hpp @@ -0,0 +1,110 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_COUNTED_HPP__ +#define __ZXING_COMMON_COUNTED_HPP__ + +#include +#include +namespace zxing { + +/* base class for reference-counted objects */ +class Counted { +private: + unsigned int count_; + +public: + Counted() : count_(0) {} + virtual ~Counted() {} + Counted* retain() { + count_++; + return this; + } + void release() { + count_--; + if (count_ == 0) { + count_ = 0xDEADF001; + delete this; + } + } + + /* return the current count for denugging purposes or similar */ + int count() const { return count_; } +}; + +/* counting reference to reference-counted objects */ +template +class Ref { +private: +public: + T* object_; + explicit Ref(T* o = 0) : object_(0) { reset(o); } + Ref(const Ref& other) : object_(0) { reset(other.object_); } + + template + Ref(const Ref& other) : object_(0) { + reset(other.object_); + } + + ~Ref() { + if (object_) { + object_->release(); + } + } + + void reset(T* o) { + if (o) { + o->retain(); + } + if (object_ != 0) { + object_->release(); + } + object_ = o; + } + Ref& operator=(const Ref& other) { + reset(other.object_); + return *this; + } + template + Ref& operator=(const Ref& other) { + reset(other.object_); + return *this; + } + Ref& operator=(T* o) { + reset(o); + return *this; + } + template + Ref& operator=(Y* o) { + reset(o); + return *this; + } + + T& operator*() { return *object_; } + T* operator->() const { return object_; } + operator T*() const { return object_; } + + bool operator==(const T* that) { return object_ == that; } + bool operator==(const Ref& other) const { + return object_ == other.object_ || *object_ == *(other.object_); + } + template + bool operator==(const Ref& other) const { + return object_ == other.object_ || *object_ == *(other.object_); + } + + bool operator!=(const T* that) { return !(*this == that); } + + bool empty() const { return object_ == 0; } +}; + +} // namespace zxing + +#endif // __ZXING_COMMON_COUNTED_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/decoder_result.cpp b/modules/wechat_qrcode/src/zxing/common/decoder_result.cpp new file mode 100644 index 00000000000..1340084abcc --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/decoder_result.cpp @@ -0,0 +1,64 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "decoder_result.hpp" + +using namespace std; +using namespace zxing; + +DecoderResult::DecoderResult(ArrayRef rawBytes, Ref text, + ArrayRef >& byteSegments, string const& ecLevel) + : rawBytes_(rawBytes), text_(text), byteSegments_(byteSegments), ecLevel_(ecLevel) { + outputCharset_ = "UTF-8"; + otherClassName = ""; + qrcodeVersion_ = -1; +} + +DecoderResult::DecoderResult(ArrayRef rawBytes, Ref text, + ArrayRef >& byteSegments, string const& ecLevel, + string outputCharset) + : rawBytes_(rawBytes), + text_(text), + byteSegments_(byteSegments), + ecLevel_(ecLevel), + outputCharset_(outputCharset) { + otherClassName = ""; + qrcodeVersion_ = -1; +} + +DecoderResult::DecoderResult(ArrayRef rawBytes, Ref text, + ArrayRef >& byteSegments, string const& ecLevel, + string outputCharset, int qrcodeVersion, string& charsetMode) + : rawBytes_(rawBytes), + text_(text), + byteSegments_(byteSegments), + ecLevel_(ecLevel), + outputCharset_(outputCharset), + qrcodeVersion_(qrcodeVersion), + charsetMode_(charsetMode) { + otherClassName = ""; +} + +DecoderResult::DecoderResult(ArrayRef rawBytes, Ref text) + : rawBytes_(rawBytes), text_(text) { + outputCharset_ = "UTF-8"; + otherClassName = ""; +} + +DecoderResult::DecoderResult(ArrayRef rawBytes, Ref text, std::string outputCharset) + : rawBytes_(rawBytes), text_(text), outputCharset_(outputCharset) { + otherClassName = ""; +} + +ArrayRef DecoderResult::getRawBytes() { return rawBytes_; } + +Ref DecoderResult::getText() { return text_; } + +string DecoderResult::getCharset() { return outputCharset_; } diff --git a/modules/wechat_qrcode/src/zxing/common/decoder_result.hpp b/modules/wechat_qrcode/src/zxing/common/decoder_result.hpp new file mode 100644 index 00000000000..730b166baad --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/decoder_result.hpp @@ -0,0 +1,79 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_DECODER_RESULT_HPP__ +#define __ZXING_COMMON_DECODER_RESULT_HPP__ + +#include "../qrcode/decoder/qrcode_decoder_metadata.hpp" +#include "array.hpp" +#include "counted.hpp" +#include "str.hpp" + +#include + +namespace zxing { + +class DecoderResult : public Counted { +private: + ArrayRef rawBytes_; + Ref text_; + ArrayRef > byteSegments_; + std::string ecLevel_; + std::string outputCharset_; + int qrcodeVersion_; + std::string charsetMode_; + + Ref other_; + string otherClassName; + +public: + DecoderResult(ArrayRef rawBytes, Ref text, + ArrayRef >& byteSegments, std::string const& ecLevel); + + DecoderResult(ArrayRef rawBytes, Ref text, + ArrayRef >& byteSegments, std::string const& ecLevel, + std::string outputCharset); + + DecoderResult(ArrayRef rawBytes, Ref text, + ArrayRef >& byteSegments, std::string const& ecLevel, + std::string outputCharset, int qrcodeVersion, std::string& charsetMode); + + DecoderResult(ArrayRef rawBytes, Ref text); + + DecoderResult(ArrayRef rawBytes, Ref text, std::string outputCharset); + + ArrayRef getRawBytes(); + Ref getText(); + std::string getCharset(); + + void setOther(Ref other) { + other_ = other; + otherClassName = "QRCodeDecoderMetaData"; + }; + + Ref getOther() { + // className = otherClassName; + return other_; + }; + + string getOtherClassName() { return otherClassName; }; + + int getQRCodeVersion() const { return qrcodeVersion_; }; + + void setText(Ref text) { text_ = text; }; + + string getEcLevel() { return ecLevel_; } + + string getCharsetMode() { return charsetMode_; } +}; + +} // namespace zxing + +#endif // __ZXING_COMMON_DECODER_RESULT_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/detector_result.cpp b/modules/wechat_qrcode/src/zxing/common/detector_result.cpp new file mode 100644 index 00000000000..408ad2c3945 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/detector_result.cpp @@ -0,0 +1,27 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "detector_result.hpp" + +namespace zxing { + +DetectorResult::DetectorResult(Ref bits, ArrayRef > points, + int dimension, float modulesize) + : bits_(bits), points_(points), dimension_(dimension), modulesize_(modulesize) {} + +void DetectorResult::SetGray(Ref gray) { gray_ = gray; } + +Ref DetectorResult::getBits() { return bits_; } + +Ref DetectorResult::getGray() { return gray_; } + +ArrayRef > DetectorResult::getPoints() { return points_; } + +} // namespace zxing diff --git a/modules/wechat_qrcode/src/zxing/common/detector_result.hpp b/modules/wechat_qrcode/src/zxing/common/detector_result.hpp new file mode 100644 index 00000000000..c48e9efe031 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/detector_result.hpp @@ -0,0 +1,42 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_DETECTOR_RESULT_HPP__ +#define __ZXING_COMMON_DETECTOR_RESULT_HPP__ + +#include "../resultpoint.hpp" +#include "array.hpp" +#include "bitmatrix.hpp" +#include "bytematrix.hpp" +#include "counted.hpp" + +namespace zxing { + +class DetectorResult : public Counted { +private: + Ref bits_; + Ref gray_; + ArrayRef > points_; + +public: + DetectorResult(Ref bits, ArrayRef > points, int dimension = 0, + float modulesize = 0); + DetectorResult(Ref gray, ArrayRef > points, int dimension = 0, + float modulesize = 0); + Ref getBits(); + Ref getGray(); + void SetGray(Ref gray); + ArrayRef > getPoints(); + int dimension_; + float modulesize_; +}; +} // namespace zxing + +#endif // __ZXING_COMMON_DETECTOR_RESULT_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/greyscale_luminance_source.cpp b/modules/wechat_qrcode/src/zxing/common/greyscale_luminance_source.cpp new file mode 100644 index 00000000000..0efad7560e5 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/greyscale_luminance_source.cpp @@ -0,0 +1,77 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "greyscale_luminance_source.hpp" +#include "bytematrix.hpp" +#include "greyscale_rotated_luminance_source.hpp" +using zxing::ArrayRef; +using zxing::ByteMatrix; +using zxing::ErrorHandler; +using zxing::GreyscaleLuminanceSource; +using zxing::LuminanceSource; +using zxing::Ref; + +GreyscaleLuminanceSource::GreyscaleLuminanceSource(ArrayRef greyData, int dataWidth, + int dataHeight, int left, int top, int width, + int height, ErrorHandler& err_handler) + : Super(width, height), + greyData_(greyData), + dataWidth_(dataWidth), + dataHeight_(dataHeight), + left_(left), + top_(top) { + if (left + width > dataWidth || top + height > dataHeight || top < 0 || left < 0) { + err_handler = IllegalArgumentErrorHandler("Crop rectangle does not fit within image data."); + } +} + +ArrayRef GreyscaleLuminanceSource::getRow(int y, ArrayRef row, + ErrorHandler& err_handler) const { + if (y < 0 || y >= this->getHeight()) { + err_handler = IllegalArgumentErrorHandler("Requested row is outside the image."); + return ArrayRef(); + } + int width = getWidth(); + if (!row || row->size() < width) { + ArrayRef temp(width); + row = temp; + } + int offset = (y + top_) * dataWidth_ + left_; + memcpy(&row[0], &greyData_[offset], width); + return row; +} + +ArrayRef GreyscaleLuminanceSource::getMatrix() const { + int size = getWidth() * getHeight(); + ArrayRef result(size); + if (left_ == 0 && top_ == 0 && dataWidth_ == getWidth() && dataHeight_ == getHeight()) { + memcpy(&result[0], &greyData_[0], size); + } else { + for (int row = 0; row < getHeight(); row++) { + memcpy(&result[row * getWidth()], &greyData_[(top_ + row) * dataWidth_ + left_], + getWidth()); + } + } + return result; +} + +Ref GreyscaleLuminanceSource::rotateCounterClockwise( + ErrorHandler& err_handler) const { + // Intentionally flip the left, top, width, and height arguments as + // needed. dataWidth and dataHeight are always kept unrotated. + Ref result(new GreyscaleRotatedLuminanceSource( + greyData_, dataWidth_, dataHeight_, top_, left_, getHeight(), getWidth(), err_handler)); + if (err_handler.ErrCode()) return Ref(); + return result; +} + +Ref GreyscaleLuminanceSource::getByteMatrix() const { + return Ref(new ByteMatrix(getWidth(), getHeight(), getMatrix())); +} diff --git a/modules/wechat_qrcode/src/zxing/common/greyscale_luminance_source.hpp b/modules/wechat_qrcode/src/zxing/common/greyscale_luminance_source.hpp new file mode 100644 index 00000000000..438b14e3fad --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/greyscale_luminance_source.hpp @@ -0,0 +1,44 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_GREYSCALE_LUMINANCE_SOURCE_HPP__ +#define __ZXING_COMMON_GREYSCALE_LUMINANCE_SOURCE_HPP__ + +#include "../errorhandler.hpp" +#include "../luminance_source.hpp" +#include "bytematrix.hpp" + +namespace zxing { + +class GreyscaleLuminanceSource : public LuminanceSource { +private: + typedef LuminanceSource Super; + ArrayRef greyData_; + const int dataWidth_; + const int dataHeight_; + const int left_; + const int top_; + +public: + GreyscaleLuminanceSource(ArrayRef greyData, int dataWidth, int dataHeight, int left, + int top, int width, int height, ErrorHandler& err_handler); + + ArrayRef getRow(int y, ArrayRef row, ErrorHandler& err_handler) const override; + ArrayRef getMatrix() const override; + Ref getByteMatrix() const override; + + bool isRotateSupported() const override { return true; } + + Ref rotateCounterClockwise(ErrorHandler& err_handler) const override; +}; + +} // namespace zxing + +#endif // __ZXING_COMMON_GREYSCALE_LUMINANCE_SOURCE_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/greyscale_rotated_luminance_source.cpp b/modules/wechat_qrcode/src/zxing/common/greyscale_rotated_luminance_source.cpp new file mode 100644 index 00000000000..6e131d346f1 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/greyscale_rotated_luminance_source.cpp @@ -0,0 +1,72 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "greyscale_rotated_luminance_source.hpp" +#include "bytematrix.hpp" +using zxing::ArrayRef; +using zxing::ByteMatrix; +using zxing::ErrorHandler; +using zxing::GreyscaleRotatedLuminanceSource; +using zxing::Ref; + +// Note that dataWidth and dataHeight are not reversed, as we need to +// be able to traverse the greyData correctly, which does not get +// rotated. +GreyscaleRotatedLuminanceSource::GreyscaleRotatedLuminanceSource(ArrayRef greyData, + int dataWidth, int dataHeight, + int left, int top, int _width, + int _height, + ErrorHandler& err_handler) + : Super(_width, _height), greyData_(greyData), dataWidth_(dataWidth), left_(left), top_(top) { + // Intentionally comparing to the opposite dimension since we're rotated. + if (left + _width > dataHeight || top + _height > dataWidth) { + err_handler = IllegalArgumentErrorHandler("Crop rectangle does not fit within image data."); + } +} + +// The API asks for rows, but we're rotated, so we return columns. +ArrayRef GreyscaleRotatedLuminanceSource::getRow(int y, ArrayRef row, + ErrorHandler& err_handler) const { + if (y < 0 || y >= getHeight()) { + err_handler = IllegalArgumentErrorHandler("Requested row is outside the image."); + return ArrayRef(); + } + if (!row || row->size() < getWidth()) { + row = ArrayRef(getWidth()); + } + int offset = (left_ * dataWidth_) + (dataWidth_ - 1 - (y + top_)); + using namespace std; + if (false) { + cerr << offset << " = " << top_ << " " << left_ << " " << getHeight() << " " << getWidth() + << " " << y << endl; + } + for (int x = 0; x < getWidth(); x++) { + row[x] = greyData_[offset]; + offset += dataWidth_; + } + return row; +} + +ArrayRef GreyscaleRotatedLuminanceSource::getMatrix() const { + ArrayRef result(getWidth() * getHeight()); + for (int y = 0; y < getHeight(); y++) { + char* row = &result[y * getWidth()]; + int offset = (left_ * dataWidth_) + (dataWidth_ - 1 - (y + top_)); + for (int x = 0; x < getWidth(); x++) { + row[x] = greyData_[offset]; + offset += dataWidth_; + } + } + return result; +} + +Ref GreyscaleRotatedLuminanceSource::getByteMatrix() const { + return Ref(new ByteMatrix(getWidth(), getHeight(), getMatrix())); +} diff --git a/modules/wechat_qrcode/src/zxing/common/greyscale_rotated_luminance_source.hpp b/modules/wechat_qrcode/src/zxing/common/greyscale_rotated_luminance_source.hpp new file mode 100644 index 00000000000..2e1c8f67a2d --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/greyscale_rotated_luminance_source.hpp @@ -0,0 +1,39 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_GREYSCALE_ROTATED_LUMINANCE_SOURCE_HPP__ +#define __ZXING_COMMON_GREYSCALE_ROTATED_LUMINANCE_SOURCE_HPP__ + +#include "../errorhandler.hpp" +#include "../luminance_source.hpp" +#include "bytematrix.hpp" +namespace zxing { + +class GreyscaleRotatedLuminanceSource : public LuminanceSource { +private: + typedef LuminanceSource Super; + ArrayRef greyData_; + const int dataWidth_; + const int left_; + const int top_; + +public: + GreyscaleRotatedLuminanceSource(ArrayRef greyData, int dataWidth, int dataHeight, + int left, int top, int _width, int _height, + ErrorHandler& err_handler); + + ArrayRef getRow(int y, ArrayRef row, ErrorHandler& err_handler) const override; + ArrayRef getMatrix() const override; + Ref getByteMatrix() const override; +}; + +} // namespace zxing + +#endif // __ZXING_COMMON_GREYSCALE_ROTATED_LUMINANCE_SOURCE_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/grid_sampler.cpp b/modules/wechat_qrcode/src/zxing/common/grid_sampler.cpp new file mode 100644 index 00000000000..c99a99e3fb1 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/grid_sampler.cpp @@ -0,0 +1,122 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "grid_sampler.hpp" +#include "perspective_transform.hpp" + +#include +#include + +namespace zxing { +using namespace std; + +GridSampler GridSampler::gridSampler; + +GridSampler::GridSampler() {} + +// Samples an image for a rectangular matrix of bits of the given dimension. +Ref GridSampler::sampleGrid(Ref image, int dimension, + Ref transform, + ErrorHandler &err_handler) { + Ref bits(new BitMatrix(dimension, err_handler)); + if (err_handler.ErrCode()) return Ref(); + + vector points(dimension << 1, (const float)0.0f); + + int outlier = 0; + int maxOutlier = dimension * dimension * 3 / 10 - 1; + + for (int y = 0; y < dimension; y++) { + int max = points.size(); + float yValue = (float)y + 0.5f; + for (int x = 0; x < max; x += 2) { + points[x] = (float)(x >> 1) + 0.5f; + points[x + 1] = yValue; + } + transform->transformPoints(points); + // Quick check to see if points transformed to something inside the + // image; sufficient to check the endpoings + outlier += checkAndNudgePoints(image->getWidth(), image->getHeight(), points, err_handler); + if (err_handler.ErrCode()) return Ref(); + + if (outlier >= maxOutlier) { + ostringstream s; + s << "Over 30% points out of bounds."; + err_handler = ReaderErrorHandler(s.str().c_str()); + return Ref(); + } + + for (int x = 0; x < max; x += 2) { + if (image->get((int)points[x], (int)points[x + 1])) { + // Black (-ish) pixel + bits->set(x >> 1, y); + } + } + } + return bits; +} + +int GridSampler::checkAndNudgePoints(int width, int height, vector &points, + ErrorHandler &err_handler) { + // Modified to support stlport + float *pts = NULL; + + if (points.size() > 0) { + pts = &points[0]; + } else { + err_handler = ReaderErrorHandler("checkAndNudgePoints:: no points!"); + return -1; + } + + int size = (int)points.size() / 2; + + // The Java code assumes that if the start and end points are in bounds, the + // rest will also be. However, in some unusual cases points in the middle + // may also be out of bounds. Since we can't rely on an + // ArrayIndexOutOfBoundsException like Java, we check every point. + + int outCount = 0; + // int maxError = (int)(size/2/3 - 1); + + float maxborder = width / size * 3; + + for (size_t offset = 0; offset < points.size(); offset += 2) { + int x = (int)pts[offset]; + int y = (int)pts[offset + 1]; + // if((int)offset==0) + // cout<<"checkAndNudgePoints "<<(int)offset<<": ("< width || y < -1 || y > height) { + outCount++; + if (x > width + maxborder || y > height + maxborder || x < -maxborder || + y < -maxborder) { + err_handler = ReaderErrorHandler("checkAndNudgePoints::Out of bounds!"); + return -1; + } + } + + if (x <= -1) { + points[offset] = 0.0f; + } else if (x >= width) { + points[offset] = float(width - 1); + } + if (y <= -1) { + points[offset + 1] = 0.0f; + } else if (y >= height) { + points[offset + 1] = float(height - 1); + } + } + + return outCount; +} + +GridSampler &GridSampler::getInstance() { return gridSampler; } +} // namespace zxing diff --git a/modules/wechat_qrcode/src/zxing/common/grid_sampler.hpp b/modules/wechat_qrcode/src/zxing/common/grid_sampler.hpp new file mode 100644 index 00000000000..c697d1c03d7 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/grid_sampler.hpp @@ -0,0 +1,34 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_GRID_SAMPLER_HPP__ +#define __ZXING_COMMON_GRID_SAMPLER_HPP__ + +#include "bitmatrix.hpp" +#include "bytematrix.hpp" +#include "counted.hpp" +#include "perspective_transform.hpp" + +namespace zxing { +class GridSampler { +private: + static GridSampler gridSampler; + GridSampler(); + +public: + Ref sampleGrid(Ref image, int dimension, + Ref transform, ErrorHandler &err_handler); + static int checkAndNudgePoints(int width, int height, vector &points, + ErrorHandler &err_handler); + static GridSampler &getInstance(); +}; +} // namespace zxing + +#endif // __ZXING_COMMON_GRID_SAMPLER_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/imagecut.cpp b/modules/wechat_qrcode/src/zxing/common/imagecut.cpp new file mode 100644 index 00000000000..07300cc112b --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/imagecut.cpp @@ -0,0 +1,67 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#include "imagecut.hpp" + +#include + +namespace zxing { + +ImageCut::ImageCut() {} + +ImageCut::~ImageCut() {} + +int ImageCut::Cut(uint8_t* poImageData, int iWidth, int iHeight, int iTopLeftX, int iTopLeftY, + int iBottomRightX, int iBottomRightY, ImageCutResult& result) { + if (iTopLeftX < 0 || iTopLeftX > iBottomRightX || iBottomRightX >= iWidth) return -1; + if (iTopLeftY < 0 || iTopLeftY > iBottomRightY || iBottomRightY >= iHeight) return -1; + int iNewWidth = iBottomRightX - iTopLeftX + 1; + int iNewHeight = iBottomRightY - iTopLeftY + 1; + + result.arrImage = new Array(iNewWidth * iNewHeight); + result.iHeight = iNewHeight; + result.iWidth = iNewWidth; + + int idx = 0; + for (int y = 0; y < iHeight; ++y) { + if (y < iTopLeftY || y > iBottomRightY) continue; + for (int x = 0; x < iWidth; ++x) { + if (x < iTopLeftX || x > iBottomRightX) continue; + result.arrImage[idx++] = poImageData[y * iWidth + x]; + } + } + return 0; +} + +int ImageCut::Cut(Ref matrix, float fRatio, ImageCutResult& result) { + int iWidth = matrix->getWidth(); + int iHeight = matrix->getHeight(); + + int iMinX = iWidth * (1 - fRatio) / 2; + int iMinY = iHeight * (1 - fRatio) / 2; + int iMaxX = iWidth * (1 + fRatio) / 2 - 1; + int iMaxY = iHeight * (1 + fRatio) / 2 - 1; + + if (iMinY < 0 || iMinY > iMaxX || iMaxX >= iWidth) return -1; + if (iMinX < 0 || iMinX > iMaxX || iMaxX >= iWidth) return -1; + int iNewHeight = iMaxY - iMinY + 1; + int iNewWidth = iMaxX - iMinX + 1; + + result.arrImage = new Array(iNewWidth * iNewHeight); + result.iWidth = iNewWidth; + result.iHeight = iNewHeight; + + int idx = 0; + for (int y = 0; y < iNewHeight; ++y) { + for (int x = 0; x < iNewWidth; ++x) { + result.arrImage[idx++] = matrix->get(x + iMinX, y + iMinY); + } + } + return 0; +} + +} // namespace zxing diff --git a/modules/wechat_qrcode/src/zxing/common/imagecut.hpp b/modules/wechat_qrcode/src/zxing/common/imagecut.hpp new file mode 100644 index 00000000000..2b85d77e933 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/imagecut.hpp @@ -0,0 +1,35 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#ifndef __ZXING_COMMON_IMAGECUT_HPP__ +#define __ZXING_COMMON_IMAGECUT_HPP__ +#include +#include "bytematrix.hpp" +#include "counted.hpp" + +#include + +namespace zxing { + +typedef struct _ImageCutResult { + ArrayRef arrImage; + int iWidth; + int iHeight; +} ImageCutResult; + +class ImageCut { +public: + ImageCut(); + ~ImageCut(); + + static int Cut(uint8_t* poImageData, int iWidth, int iHeight, int iTopLeftX, int iTopLeftY, + int iBottomRightX, int iBottomRightY, ImageCutResult& result); + static int Cut(Ref matrix, float fRatio, ImageCutResult& result); +}; + +} // namespace zxing +#endif // __ZXING_COMMON_IMAGECUT_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/kmeans.cpp b/modules/wechat_qrcode/src/zxing/common/kmeans.cpp new file mode 100644 index 00000000000..f7e2446f11c --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/kmeans.cpp @@ -0,0 +1,96 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#include "kmeans.hpp" + +#include +#include +#include + +using namespace std; +typedef unsigned int uint; + +namespace zxing { + +double cal_distance(vector a, vector b) { + const float KMEANS_COUNT_FACTOR = 0; + const float KMEANS_MS_FACTOR = 1; + + uint da = a.size(); + uint db = b.size(); + if (da != db) cerr << "Dimensions of two vectors must be same!!\n"; + double val = 0.0; + for (uint i = 0; i < da; i++) { + if (i == 1) + val += KMEANS_MS_FACTOR * pow((a[i] - b[i]), 2); + else if (i == 0) + val += KMEANS_COUNT_FACTOR * pow((a[i] - b[i]), 2); + else + val += pow((a[i] - b[i]), 2); + } + return pow(val, 0.5); +} + +/* + * maxepoches max iteration epochs + * minchanged min central change times + */ +vector k_means(vector > trainX, uint k, uint maxepoches, uint minchanged) { + const uint row_num = trainX.size(); + const uint col_num = trainX[0].size(); + + // initialize the cluster central + vector clusters(k); + int step = trainX.size() / k; + + for (uint i = 0; i < k; i++) { + clusters[i].centroid = trainX[i * step]; + } + + // try max epochs times iteration untill convergence + for (uint it = 0; it < maxepoches; it++) { + for (uint i = 0; i < k; i++) { + clusters[i].samples.clear(); + } + for (uint j = 0; j < row_num; j++) { + uint c = 0; + double min_distance = cal_distance(trainX[j], clusters[c].centroid); + for (uint i = 1; i < k; i++) { + double distance = cal_distance(trainX[j], clusters[i].centroid); + if (distance < min_distance) { + min_distance = distance; + c = i; + } + } + clusters[c].samples.push_back(j); + } + + uint changed = 0; + // update cluster central + for (uint i = 0; i < k; i++) { + vector val(col_num, 0.0); + for (uint j = 0; j < clusters[i].samples.size(); j++) { + uint sample = clusters[i].samples[j]; + for (uint d = 0; d < col_num; d++) { + val[d] += trainX[sample][d]; + if (j == clusters[i].samples.size() - 1) { + double value = val[d] / clusters[i].samples.size(); + if (clusters[i].centroid[d] != value) { + clusters[i].centroid[d] = value; + changed++; + } + } + } + } + } + + if (changed <= minchanged) return clusters; + } + return clusters; +} + +} // namespace zxing diff --git a/modules/wechat_qrcode/src/zxing/common/kmeans.hpp b/modules/wechat_qrcode/src/zxing/common/kmeans.hpp new file mode 100644 index 00000000000..5d656dfb5d1 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/kmeans.hpp @@ -0,0 +1,26 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#ifndef __ZXING_COMMON_KMEANS_HPP__ +#define __ZXING_COMMON_KMEANS_HPP__ +#include + +namespace zxing { + +using namespace std; +typedef unsigned int uint; + +struct Cluster { + vector centroid; + vector samples; +}; + +double cal_distance(vector a, vector b); +vector k_means(vector > trainX, uint k, uint maxepoches, uint minchanged); + +} // namespace zxing +#endif // __ZXING_COMMON_KMEANS_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/mathutils.hpp b/modules/wechat_qrcode/src/zxing/common/mathutils.hpp new file mode 100644 index 00000000000..95ac189c4c2 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/mathutils.hpp @@ -0,0 +1,131 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_MATHUTILS_HPP__ +#define __ZXING_COMMON_MATHUTILS_HPP__ + +#include +#if (defined __GNUC__ && defined __x86_64__ && defined __SSE2__ && !defined __APPLE__ && \ + !defined __GXX_WEAK__) +#include +#endif + +#include +#include +#include + +namespace zxing { +namespace common { + +class MathUtils { +private: + MathUtils(); + ~MathUtils(); + +public: + /** + * Ends up being a bit faster than {@link Math#round(float)}. This merely + * rounds its argument to the nearest int, where x.5 rounds up to x+1. + */ + static inline int round(double value) { + // return (int) (d + 0.5f); + +#if (defined _MSC_VER && defined _M_X64) || \ + (defined __GNUC__ && defined __x86_64__ && defined __SSE2__ && !defined __APPLE__ && \ + !defined __GXX_WEAK__) + __m128d t = _mm_set_sd(value); + return _mm_cvtsd_si32(t); +#elif defined _MSC_VER && defined _M_IX86 + int t; + __asm + { + fld value; + fistp t; + } + return t; +#elif defined _MSC_VER && defined _M_ARM && defined HAVE_TEGRA_OPTIMIZATION + TEGRA_ROUND(value); +#elif defined HAVE_LRINT || defined CV_ICC || defined __GNUC__ +#ifdef HAVE_TEGRA_OPTIMIZATION + TEGRA_ROUND(value); +#else + return (int)lrint(value); +#endif +#else + double intpart, fractpart; + fractpart = modf(value, &intpart); + if ((fabs(fractpart) != 0.5) || ((((int)intpart) % 2) != 0)) + return (int)(value + (value >= 0 ? 0.5 : -0.5)); + else + return (int)intpart; +#endif + } + + static inline float distance(float aX, float aY, float bX, float bY) { + float xDiff = aX - bX; + float yDiff = aY - bY; + return sqrt(float(xDiff * xDiff + yDiff * yDiff)); + } + + static inline float distance_4_int(int aX, int aY, int bX, int bY) { + return sqrt(float((aX - bX) * (aX - bX) + (aY - bY) * (aY - bY))); + } + + static inline void getRangeValues(int& minValue, int& maxValue, int min, int max) { + int finalMinValue, finalMaxValue; + + if (minValue < maxValue) { + finalMinValue = minValue; + finalMaxValue = maxValue; + } else { + finalMinValue = maxValue; + finalMaxValue = minValue; + } + + finalMinValue = finalMinValue > min ? finalMinValue : min; + finalMaxValue = finalMaxValue < max ? finalMaxValue : max; + + minValue = finalMinValue; + maxValue = finalMaxValue; + } + + static inline bool isInRange(float x, float y, float width, float height) { + if ((x >= 0.0 && x <= (width - 1.0)) && (y >= 0.0 && y <= (height - 1.0))) { + return true; + } else { + return false; + } + } + + static inline float distance(int aX, int aY, int bX, int bY) { + int xDiff = aX - bX; + int yDiff = aY - bY; + return sqrt(float(xDiff * xDiff + yDiff * yDiff)); + } + + static inline float VecCross(float* v1, float* v2) { return v1[0] * v2[1] - v1[1] * v2[0]; } + + static inline void Stddev(std::vector& resultSet, float& avg, float& stddev) { + double sum = std::accumulate(resultSet.begin(), resultSet.end(), 0.0); + avg = sum / resultSet.size(); + + double accum = 0.0; + for (std::size_t i = 0; i < resultSet.size(); i++) { + accum += (resultSet[i] - avg) * (resultSet[i] - avg); + } + + stddev = sqrt(accum / (resultSet.size())); + } +}; + +} // namespace common +} // namespace zxing + +#endif // __ZXING_COMMON_MATHUTILS_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/perspective_transform.cpp b/modules/wechat_qrcode/src/zxing/common/perspective_transform.cpp new file mode 100644 index 00000000000..7779694f9e9 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/perspective_transform.cpp @@ -0,0 +1,122 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "perspective_transform.hpp" + +#include + +namespace zxing { +using namespace std; + +// This class implements a perspective transform in two dimensions. Given four +// source and four destination points, it will compute the transformation +// implied between them. The code is based directly upon section 3.4.2 of George +// Wolberg's "Digital Image Warping"; see pages 54-56 +PerspectiveTransform::PerspectiveTransform(float inA11, float inA21, float inA31, float inA12, + float inA22, float inA32, float inA13, float inA23, + float inA33) + : a11(inA11), + a12(inA12), + a13(inA13), + a21(inA21), + a22(inA22), + a23(inA23), + a31(inA31), + a32(inA32), + a33(inA33) {} + +Ref PerspectiveTransform::quadrilateralToQuadrilateral( + float x0, float y0, float x1, float y1, float x2, float y2, float x3, float y3, float x0p, + float y0p, float x1p, float y1p, float x2p, float y2p, float x3p, float y3p) { + Ref qToS = + PerspectiveTransform::quadrilateralToSquare(x0, y0, x1, y1, x2, y2, x3, y3); + Ref sToQ = + PerspectiveTransform::squareToQuadrilateral(x0p, y0p, x1p, y1p, x2p, y2p, x3p, y3p); + return sToQ->times(qToS); +} + +Ref PerspectiveTransform::squareToQuadrilateral(float x0, float y0, float x1, + float y1, float x2, float y2, + float x3, float y3) { + float dx3 = x0 - x1 + x2 - x3; + float dy3 = y0 - y1 + y2 - y3; + if (fabs(dx3) <= 1e-6 && fabs(dy3) <= 1e-6) { + Ref result( + new PerspectiveTransform(x1 - x0, x2 - x1, x0, y1 - y0, y2 - y1, y0, 0.0f, 0.0f, 1.0f)); + return result; + } else { + float dx1 = x1 - x2; + float dx2 = x3 - x2; + float dy1 = y1 - y2; + float dy2 = y3 - y2; + float denominator = dx1 * dy2 - dx2 * dy1; + float a13 = (dx3 * dy2 - dx2 * dy3) / denominator; + float a23 = (dx1 * dy3 - dx3 * dy1) / denominator; + Ref result( + new PerspectiveTransform(x1 - x0 + a13 * x1, x3 - x0 + a23 * x3, x0, y1 - y0 + a13 * y1, + y3 - y0 + a23 * y3, y0, a13, a23, 1.0f)); + return result; + } +} + +Ref PerspectiveTransform::quadrilateralToSquare(float x0, float y0, float x1, + float y1, float x2, float y2, + float x3, float y3) { + // Here, the adjoint serves as the inverse: + return squareToQuadrilateral(x0, y0, x1, y1, x2, y2, x3, y3)->buildAdjoint(); +} + +Ref PerspectiveTransform::buildAdjoint() { + // Adjoint is the transpose of the cofactor matrix: + Ref result(new PerspectiveTransform( + a22 * a33 - a23 * a32, a23 * a31 - a21 * a33, a21 * a32 - a22 * a31, a13 * a32 - a12 * a33, + a11 * a33 - a13 * a31, a12 * a31 - a11 * a32, a12 * a23 - a13 * a22, a13 * a21 - a11 * a23, + a11 * a22 - a12 * a21)); + return result; +} + +Ref PerspectiveTransform::times(Ref other) { + Ref result( + new PerspectiveTransform(a11 * other->a11 + a21 * other->a12 + a31 * other->a13, + a11 * other->a21 + a21 * other->a22 + a31 * other->a23, + a11 * other->a31 + a21 * other->a32 + a31 * other->a33, + a12 * other->a11 + a22 * other->a12 + a32 * other->a13, + a12 * other->a21 + a22 * other->a22 + a32 * other->a23, + a12 * other->a31 + a22 * other->a32 + a32 * other->a33, + a13 * other->a11 + a23 * other->a12 + a33 * other->a13, + a13 * other->a21 + a23 * other->a22 + a33 * other->a23, + a13 * other->a31 + a23 * other->a32 + a33 * other->a33)); + return result; +} + +void PerspectiveTransform::transformPoints(vector& points) { + int max = points.size(); + + // Modified to support stlport + float* pts = NULL; + + if (points.size() > 0) { + pts = &points[0]; + } + + for (int i = 0; i < max; i += 2) { + float x = pts[i]; + float y = pts[i + 1]; + + float denominator = a13 * x + a23 * y + a33; + + float w = 1.0f / denominator; + + pts[i] = (a11 * x + a21 * y + a31) * w; + pts[i + 1] = (a12 * x + a22 * y + a32) * w; + } +} + +} // namespace zxing diff --git a/modules/wechat_qrcode/src/zxing/common/perspective_transform.hpp b/modules/wechat_qrcode/src/zxing/common/perspective_transform.hpp new file mode 100644 index 00000000000..59e1db6fa22 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/perspective_transform.hpp @@ -0,0 +1,39 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_PERSPECTIVETRANSFORM_HPP__ +#define __ZXING_COMMON_PERSPECTIVETRANSFORM_HPP__ + +#include "counted.hpp" + +#include + +namespace zxing { +class PerspectiveTransform : public Counted { +private: + float a11, a12, a13, a21, a22, a23, a31, a32, a33; + PerspectiveTransform(float a11, float a21, float a31, float a12, float a22, float a32, + float a13, float a23, float a33); + +public: + static Ref quadrilateralToQuadrilateral( + float x0, float y0, float x1, float y1, float x2, float y2, float x3, float y3, float x0p, + float y0p, float x1p, float y1p, float x2p, float y2p, float x3p, float y3p); + static Ref squareToQuadrilateral(float x0, float y0, float x1, float y1, + float x2, float y2, float x3, float y3); + static Ref quadrilateralToSquare(float x0, float y0, float x1, float y1, + float x2, float y2, float x3, float y3); + Ref buildAdjoint(); + Ref times(Ref other); + void transformPoints(std::vector& points); +}; +} // namespace zxing + +#endif // __ZXING_COMMON_PERSPECTIVETRANSFORM_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgf.cpp b/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgf.cpp new file mode 100644 index 00000000000..51e1d99eac5 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgf.cpp @@ -0,0 +1,101 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "genericgf.hpp" +#include "genericgfpoly.hpp" + +#include + +using zxing::ErrorHandler; +using zxing::GenericGF; +using zxing::GenericGFPoly; +using zxing::Ref; + +GenericGF::GenericGF(int primitive_, int size_, int b, ErrorHandler &err_handler) + : size(size_), primitive(primitive_), generatorBase(b) { + expTable.resize(size); + logTable.resize(size); + + int x = 1; + + for (int i = 0; i < size; i++) { + expTable[i] = x; + x <<= 1; // x = x * 2; we're assuming the generator alpha is 2 + if (x >= size) { + x ^= primitive; + x &= size - 1; + } + } + for (int i = 0; i < size - 1; i++) { + logTable[expTable[i]] = i; + } + // logTable[0] == 0 but this should never be used + zero = + Ref(new GenericGFPoly(*this, ArrayRef(new Array(1)), err_handler)); + zero->getCoefficients()[0] = 0; + one = + Ref(new GenericGFPoly(*this, ArrayRef(new Array(1)), err_handler)); + one->getCoefficients()[0] = 1; + if (err_handler.ErrCode()) return; + // initialized = true; +} + +Ref GenericGF::getZero() { return zero; } + +Ref GenericGF::getOne() { return one; } + +Ref GenericGF::buildMonomial(int degree, int coefficient, + ErrorHandler &err_handler) { + if (degree < 0) { + err_handler = IllegalArgumentErrorHandler("Degree must be non-negative"); + return Ref(); + } + if (coefficient == 0) { + return zero; + } + ArrayRef coefficients(new Array(degree + 1)); + coefficients[0] = coefficient; + + Ref gfpoly(new GenericGFPoly(*this, coefficients, err_handler)); + if (err_handler.ErrCode()) return Ref(); + return gfpoly; +} + +int GenericGF::addOrSubtract(int a, int b) { return a ^ b; } + +int GenericGF::exp(int a) { return expTable[a]; } + +int GenericGF::log(int a, ErrorHandler &err_handler) { + if (a == 0) { + err_handler = IllegalArgumentErrorHandler("cannot give log(0)"); + return -1; + } + return logTable[a]; +} + +int GenericGF::inverse(int a, ErrorHandler &err_handler) { + if (a == 0) { + err_handler = IllegalArgumentErrorHandler("Cannot calculate the inverse of 0"); + return -1; + } + return expTable[size - logTable[a] - 1]; +} + +int GenericGF::multiply(int a, int b) { + if (a == 0 || b == 0) { + return 0; + } + + return expTable[(logTable[a] + logTable[b]) % (size - 1)]; +} + +int GenericGF::getSize() { return size; } + +int GenericGF::getGeneratorBase() { return generatorBase; } diff --git a/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgf.hpp b/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgf.hpp new file mode 100644 index 00000000000..7029af2b4ee --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgf.hpp @@ -0,0 +1,60 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_REEDSOLOMON_GENERICGF_HPP__ +#define __ZXING_COMMON_REEDSOLOMON_GENERICGF_HPP__ + +#include "../../errorhandler.hpp" +#include "../counted.hpp" + +#include + +namespace zxing { +class GenericGFPoly; + +static zxing::ErrorHandler gf_err_handler_; +#define GF_AZTEC_DATA_12 (new GenericGF(0x1069, 4096, 1, gf_err_handler_)) +#define GF_AZTEC_DATA_10 (new GenericGF(0x409, 1024, 1, gf_err_handler_)) +#define GF_AZTEC_DATA_6 (new GenericGF(0x43, 64, 1, gf_err_handler_)) +#define GF_AZTEC_PARAM (new GenericGF(0x13, 16, 1, gf_err_handler_)) +#define GF_QR_CODE_FIELD_256 (new GenericGF(0x011D, 256, 0, gf_err_handler_)) +#define GF_DATA_MATRIX_FIELD_256 (new GenericGF(0x012D, 256, 1, gf_err_handler_)) +#define GF_AZTEC_DATA_8 (new GenericGF(0x012D, 256, 1, gf_err_handler_)) +#define GF_MAXICODE_FIELD_64 (new GenericGF(0x43, 64, 1, gf_err_handler_)) +// #define GF_WXCODE (new GenericGF(0x011D, 256, 0, gf_err_handler_)) + +class GenericGF : public Counted { +private: + std::vector expTable; + std::vector logTable; + Ref zero; + Ref one; + int size; + int primitive; + int generatorBase; + +public: + GenericGF(int primitive, int size, int b, ErrorHandler &err_handler); + + Ref getZero(); + Ref getOne(); + int getSize(); + int getGeneratorBase(); + Ref buildMonomial(int degree, int coefficient, ErrorHandler &err_handler); + + static int addOrSubtract(int a, int b); + int exp(int a); + int log(int a, ErrorHandler &err_handler); + int inverse(int a, ErrorHandler &err_handler); + int multiply(int a, int b); +}; +} // namespace zxing + +#endif // __ZXING_COMMON_REEDSOLOMON_GENERICGF_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgfpoly.cpp b/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgfpoly.cpp new file mode 100644 index 00000000000..96f9fece18c --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgfpoly.cpp @@ -0,0 +1,233 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "genericgfpoly.hpp" +#include "genericgf.hpp" + +#include + +using zxing::ArrayRef; +using zxing::ErrorHandler; +using zxing::GenericGFPoly; +using zxing::Ref; + +// VC++ +using zxing::GenericGF; + +GenericGFPoly::GenericGFPoly(GenericGF &field, ArrayRef coefficients, + ErrorHandler &err_handler) + : field_(field) { + if (coefficients->size() == 0) { + err_handler = IllegalArgumentErrorHandler("need coefficients"); + return; + } + int coefficientsLength = coefficients->size(); + if (coefficientsLength > 1 && coefficients[0] == 0) { + // Leading term must be non-zero for anything except the constant + // polynomial "0" + int firstNonZero = 1; + while (firstNonZero < coefficientsLength && coefficients[firstNonZero] == 0) { + firstNonZero++; + } + if (firstNonZero == coefficientsLength) { + coefficients_ = field.getZero()->getCoefficients(); + } else { + coefficients_ = ArrayRef(new Array(coefficientsLength - firstNonZero)); + for (int i = 0; i < (int)coefficients_->size(); i++) { + coefficients_[i] = coefficients[i + firstNonZero]; + } + } + } else { + coefficients_ = coefficients; + } +} + +ArrayRef GenericGFPoly::getCoefficients() { return coefficients_; } + +int GenericGFPoly::getDegree() { return coefficients_->size() - 1; } + +bool GenericGFPoly::isZero() { return coefficients_[0] == 0; } + +int GenericGFPoly::getCoefficient(int degree) { + return coefficients_[coefficients_->size() - 1 - degree]; +} + +int GenericGFPoly::evaluateAt(int a) { + if (a == 0) { + // Just return the x^0 coefficient + return getCoefficient(0); + } + + int size = coefficients_->size(); + if (a == 1) { + // Just the sum of the coefficients + int result = 0; + for (int i = 0; i < size; i++) { + result = GenericGF::addOrSubtract(result, coefficients_[i]); + } + return result; + } + int result = coefficients_[0]; + for (int i = 1; i < size; i++) { + result = GenericGF::addOrSubtract(field_.multiply(a, result), coefficients_[i]); + } + return result; +} + +Ref GenericGFPoly::addOrSubtract(Ref other, + ErrorHandler &err_handler) { + if (!(&field_ == &other->field_)) { + err_handler = + IllegalArgumentErrorHandler("GenericGFPolys do not have same GenericGF field"); + return Ref(); + } + if (isZero()) { + return other; + } + if (other->isZero()) { + return Ref(this); + } + + ArrayRef smallerCoefficients = coefficients_; + ArrayRef largerCoefficients = other->getCoefficients(); + if (smallerCoefficients->size() > largerCoefficients->size()) { + ArrayRef temp = smallerCoefficients; + smallerCoefficients = largerCoefficients; + largerCoefficients = temp; + } + + ArrayRef sumDiff(new Array(largerCoefficients->size())); + int lengthDiff = largerCoefficients->size() - smallerCoefficients->size(); + // Copy high-order terms only found in higher-degree polynomial's + // coefficients + for (int i = 0; i < lengthDiff; i++) { + sumDiff[i] = largerCoefficients[i]; + } + + for (int i = lengthDiff; i < (int)largerCoefficients->size(); i++) { + sumDiff[i] = + GenericGF::addOrSubtract(smallerCoefficients[i - lengthDiff], largerCoefficients[i]); + } + + // return Ref(new GenericGFPoly(field_, sumDiff)); + Ref gfpoly(new GenericGFPoly(field_, sumDiff, err_handler)); + if (err_handler.ErrCode()) return Ref(); + return gfpoly; +} + +Ref GenericGFPoly::multiply(Ref other, + ErrorHandler &err_handler) { + if (!(&field_ == &other->field_)) { + err_handler = + IllegalArgumentErrorHandler("GenericGFPolys do not have same GenericGF field"); + return Ref(); + } + + if (isZero() || other->isZero()) { + return field_.getZero(); + } + + ArrayRef aCoefficients = coefficients_; + int aLength = aCoefficients->size(); + + ArrayRef bCoefficients = other->getCoefficients(); + int bLength = bCoefficients->size(); + + ArrayRef product(new Array(aLength + bLength - 1)); + for (int i = 0; i < aLength; i++) { + int aCoeff = aCoefficients[i]; + for (int j = 0; j < bLength; j++) { + product[i + j] = + GenericGF::addOrSubtract(product[i + j], field_.multiply(aCoeff, bCoefficients[j])); + } + } + + // return Ref(new GenericGFPoly(field_, product)); + Ref gfpoly(new GenericGFPoly(field_, product, err_handler)); + if (err_handler.ErrCode()) return Ref(); + return gfpoly; +} + +Ref GenericGFPoly::multiply(int scalar, ErrorHandler &err_handler) { + if (scalar == 0) { + return field_.getZero(); + } + if (scalar == 1) { + return Ref(this); + } + int size = coefficients_->size(); + ArrayRef product(new Array(size)); + for (int i = 0; i < size; i++) { + product[i] = field_.multiply(coefficients_[i], scalar); + } + // return Ref(new GenericGFPoly(field_, product)); + Ref gfpoly(new GenericGFPoly(field_, product, err_handler)); + if (err_handler.ErrCode()) return Ref(); + return gfpoly; +} + +Ref GenericGFPoly::multiplyByMonomial(int degree, int coefficient, + ErrorHandler &err_handler) { + if (degree < 0) { + err_handler = IllegalArgumentErrorHandler("degree must not be less then 0"); + return Ref(); + } + if (coefficient == 0) { + return field_.getZero(); + } + int size = coefficients_->size(); + ArrayRef product(new Array(size + degree)); + for (int i = 0; i < size; i++) { + product[i] = field_.multiply(coefficients_[i], coefficient); + } + // return Ref(new GenericGFPoly(field_, product)); + Ref gfpoly(new GenericGFPoly(field_, product, err_handler)); + if (err_handler.ErrCode()) return Ref(); + return gfpoly; +} + +std::vector> GenericGFPoly::divide(Ref other, + ErrorHandler &err_handler) { + if (!(&field_ == &other->field_)) { + err_handler = + IllegalArgumentErrorHandler("GenericGFPolys do not have same GenericGF field"); + return std::vector>(); + } + if (other->isZero()) { + err_handler = IllegalArgumentErrorHandler("divide by 0"); + return std::vector>(); + } + + Ref quotient = field_.getZero(); + Ref remainder = Ref(this); + + int denominatorLeadingTerm = other->getCoefficient(other->getDegree()); + int inverseDenominatorLeadingTerm = field_.inverse(denominatorLeadingTerm, err_handler); + if (err_handler.ErrCode()) return std::vector>(); + + while (remainder->getDegree() >= other->getDegree() && !remainder->isZero()) { + int degreeDifference = remainder->getDegree() - other->getDegree(); + int scale = field_.multiply(remainder->getCoefficient(remainder->getDegree()), + inverseDenominatorLeadingTerm); + Ref term = other->multiplyByMonomial(degreeDifference, scale, err_handler); + if (err_handler.ErrCode()) return std::vector>(); + Ref iterationQuotiont = + field_.buildMonomial(degreeDifference, scale, err_handler); + if (err_handler.ErrCode()) return std::vector>(); + quotient = quotient->addOrSubtract(iterationQuotiont, err_handler); + remainder = remainder->addOrSubtract(term, err_handler); + if (err_handler.ErrCode()) return std::vector>(); + } + + std::vector> returnValue(2); + returnValue[0] = quotient; + returnValue[1] = remainder; + return returnValue; +} diff --git a/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgfpoly.hpp b/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgfpoly.hpp new file mode 100644 index 00000000000..c6c17bcb804 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgfpoly.hpp @@ -0,0 +1,45 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_REEDSOLOMON_GENERICGFPOLY_HPP__ +#define __ZXING_COMMON_REEDSOLOMON_GENERICGFPOLY_HPP__ + +#include "../../errorhandler.hpp" +#include "../array.hpp" +#include "../counted.hpp" + +#include + +namespace zxing { + +class GenericGF; + +class GenericGFPoly : public Counted { +private: + GenericGF &field_; + ArrayRef coefficients_; + +public: + GenericGFPoly(GenericGF &field, ArrayRef coefficients, ErrorHandler &err_handler); + ArrayRef getCoefficients(); + int getDegree(); + bool isZero(); + int getCoefficient(int degree); + int evaluateAt(int a); + Ref addOrSubtract(Ref other, ErrorHandler &err_handler); + Ref multiply(Ref other, ErrorHandler &err_handler); + Ref multiply(int scalar, ErrorHandler &err_handler); + Ref multiplyByMonomial(int degree, int coefficient, ErrorHandler &err_handler); + std::vector> divide(Ref other, ErrorHandler &err_handler); +}; + +} // namespace zxing + +#endif // __ZXING_COMMON_REEDSOLOMON_GENERICGFPOLY_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/reedsolomon/reed_solomon_decoder.cpp b/modules/wechat_qrcode/src/zxing/common/reedsolomon/reed_solomon_decoder.cpp new file mode 100644 index 00000000000..dfaed4f4b14 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/reedsolomon/reed_solomon_decoder.cpp @@ -0,0 +1,190 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "reed_solomon_decoder.hpp" +#include +#include + +using std::vector; +using zxing::ArrayRef; +using zxing::ErrorHandler; +using zxing::GenericGFPoly; +using zxing::ReedSolomonDecoder; +using zxing::Ref; + +// VC++ +using zxing::GenericGF; + +ReedSolomonDecoder::ReedSolomonDecoder(Ref field_) : field(field_) {} + +ReedSolomonDecoder::~ReedSolomonDecoder() {} + +void ReedSolomonDecoder::decode(ArrayRef received, int twoS, ErrorHandler &err_handler) { + Ref poly(new GenericGFPoly(*field, received, err_handler)); + if (err_handler.ErrCode()) return; + ArrayRef syndromeCoefficients(twoS); + bool noError = true; + for (int i = 0; i < twoS; i++) { + int eval = poly->evaluateAt(field->exp(i + field->getGeneratorBase())); + syndromeCoefficients[syndromeCoefficients->size() - 1 - i] = eval; + if (eval != 0) { + noError = false; + } + } + if (noError) { + return; + } + Ref syndrome(new GenericGFPoly(*field, syndromeCoefficients, err_handler)); + Ref monomial = field->buildMonomial(twoS, 1, err_handler); + if (!monomial || err_handler.ErrCode()) { + err_handler = ErrorHandler("buildMonomial was zero"); + return; + } + vector> sigmaOmega = + runEuclideanAlgorithm(monomial, syndrome, twoS, err_handler); + if (err_handler.ErrCode()) return; + + Ref sigma = sigmaOmega[0]; + Ref omega = sigmaOmega[1]; + ArrayRef errorLocations = findErrorLocations(sigma, err_handler); + if (err_handler.ErrCode()) return; + + ArrayRef errorMagitudes = findErrorMagnitudes(omega, errorLocations, err_handler); + if (err_handler.ErrCode()) return; + for (int i = 0; i < errorLocations->size(); i++) { + int position = received->size() - 1 - field->log(errorLocations[i], err_handler); + if (position < 0 || err_handler.ErrCode()) { + err_handler = ErrorHandler("Bad error location"); + return; + } + received[position] = GenericGF::addOrSubtract(received[position], errorMagitudes[i]); + } +} + +vector> ReedSolomonDecoder::runEuclideanAlgorithm(Ref a, + Ref b, int R, + ErrorHandler &err_handler) { + vector> result(2); + + // Assume a's degree is >= b's + if (a->getDegree() < b->getDegree()) { + Ref tmp = a; + a = b; + b = tmp; + } + + Ref rLast(a); + Ref r(b); + Ref tLast(field->getZero()); + Ref t(field->getOne()); + + // Run Euclidean algorithm until r's degree is less than R/2 + while (r->getDegree() >= R / 2) { + Ref rLastLast(rLast); + Ref tLastLast(tLast); + rLast = r; + tLast = t; + + // Divide rLastLast by rLast, with quotient q and remainder r + if (rLast->isZero()) { + // Oops, Euclidean algorithm already terminated? + err_handler = ErrorHandler("r_{i-1} was zero"); + return vector>(); + } + r = rLastLast; + Ref q = field->getZero(); + int denominatorLeadingTerm = rLast->getCoefficient(rLast->getDegree()); + int dltInverse = field->inverse(denominatorLeadingTerm, err_handler); + if (err_handler.ErrCode()) return vector>(); + while (r->getDegree() >= rLast->getDegree() && !r->isZero()) { + int degreeDiff = r->getDegree() - rLast->getDegree(); + int scale = field->multiply(r->getCoefficient(r->getDegree()), dltInverse); + q = q->addOrSubtract(field->buildMonomial(degreeDiff, scale, err_handler), err_handler); + r = r->addOrSubtract(rLast->multiplyByMonomial(degreeDiff, scale, err_handler), + err_handler); + if (err_handler.ErrCode()) return vector>(); + } + + Ref tmp = q->multiply(tLast, err_handler); + if (err_handler.ErrCode()) return vector>(); + t = tmp->addOrSubtract(tLastLast, err_handler); + if (err_handler.ErrCode()) return vector>(); + + if (r->getDegree() >= rLast->getDegree()) { + err_handler = ErrorHandler("Division algorithm failed to reduce polynomial?"); + return vector>(); + } + } + + int sigmaTildeAtZero = t->getCoefficient(0); + if (sigmaTildeAtZero == 0) { + err_handler = ErrorHandler("sigmaTilde(0) was zero"); + return vector>(); + } + + int inverse = field->inverse(sigmaTildeAtZero, err_handler); + Ref sigma(t->multiply(inverse, err_handler)); + Ref omega(r->multiply(inverse, err_handler)); + if (err_handler.ErrCode()) return vector>(); + + result[0] = sigma; + result[1] = omega; + return result; +} + +ArrayRef ReedSolomonDecoder::findErrorLocations(Ref errorLocator, + ErrorHandler &err_handler) { + // This is a direct application of Chien's search + int numErrors = errorLocator->getDegree(); + if (numErrors == 1) { // shortcut + ArrayRef result(new Array(1)); + result[0] = errorLocator->getCoefficient(1); + return result; + } + ArrayRef result(new Array(numErrors)); + int e = 0; + for (int i = 1; i < field->getSize() && e < numErrors; i++) { + if (errorLocator->evaluateAt(i) == 0) { + result[e] = field->inverse(i, err_handler); + e++; + } + } + if (e != numErrors || err_handler.ErrCode()) { + err_handler = ErrorHandler("Error locator degree does not match number of root"); + return ArrayRef(); + } + return result; +} + +ArrayRef ReedSolomonDecoder::findErrorMagnitudes(Ref errorEvaluator, + ArrayRef errorLocations, + ErrorHandler &err_handler) { + // This is directly applying Forney's Formula + int s = errorLocations->size(); + ArrayRef result(new Array(s)); + for (int i = 0; i < s; i++) { + int xiInverse = field->inverse(errorLocations[i], err_handler); + int denominator = 1; + for (int j = 0; j < s; j++) { + if (i != j) { + int term = field->multiply(errorLocations[j], xiInverse); + int termPlus1 = (term & 0x1) == 0 ? term | 1 : term & ~1; + denominator = field->multiply(denominator, termPlus1); + } + } + result[i] = field->multiply(errorEvaluator->evaluateAt(xiInverse), + field->inverse(denominator, err_handler)); + if (field->getGeneratorBase() != 0) { + result[i] = field->multiply(result[i], xiInverse); + } + } + if (err_handler.ErrCode()) return ArrayRef(); + return result; +} diff --git a/modules/wechat_qrcode/src/zxing/common/reedsolomon/reed_solomon_decoder.hpp b/modules/wechat_qrcode/src/zxing/common/reedsolomon/reed_solomon_decoder.hpp new file mode 100644 index 00000000000..26daa79d2b5 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/reedsolomon/reed_solomon_decoder.hpp @@ -0,0 +1,46 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_REEDSOLOMON_REEDSOLOMONDECODER_HPP__ +#define __ZXING_COMMON_REEDSOLOMON_REEDSOLOMONDECODER_HPP__ + +#include "../../errorhandler.hpp" +#include "../array.hpp" +#include "../counted.hpp" +#include "genericgf.hpp" +#include "genericgfpoly.hpp" + +#include +#include + +namespace zxing { +class GenericGFPoly; +class GenericGF; + +class ReedSolomonDecoder { +private: + Ref field; + +public: + explicit ReedSolomonDecoder(Ref fld); + ~ReedSolomonDecoder(); + void decode(ArrayRef received, int twoS, ErrorHandler &err_handler); + std::vector> runEuclideanAlgorithm(Ref a, + Ref b, int R, + ErrorHandler &err_handler); + +private: + ArrayRef findErrorLocations(Ref errorLocator, ErrorHandler &err_handler); + ArrayRef findErrorMagnitudes(Ref errorEvaluator, + ArrayRef errorLocations, ErrorHandler &err_handler); +}; +} // namespace zxing + +#endif // __ZXING_COMMON_REEDSOLOMON_REEDSOLOMONDECODER_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/str.cpp b/modules/wechat_qrcode/src/zxing/common/str.cpp new file mode 100644 index 00000000000..82e1155ff1e --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/str.cpp @@ -0,0 +1,102 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "str.hpp" + +#include + +using std::string; +using zxing::Ref; +using zxing::String; +using zxing::StrUtil; + +using namespace std; + +String::String(const std::string& text) : text_(text) {} + +String::String(int capacity) { text_.reserve(capacity); } + +const std::string& String::getText() const { return text_; } + +char String::charAt(int i) const { return text_[i]; } + +int String::size() const { return text_.size(); } + +int String::length() const { return text_.size(); } + +Ref String::substring(int i) const { return Ref(new String(text_.substr(i))); } + +Ref String::substring(int start, int end) const { + return Ref(new String(text_.substr(start, (end - start)))); +} + +void String::append(const std::string& tail) { text_.append(tail); } + +void String::append(char c) { text_.append(1, c); } + +void String::append(int d) { + string str = StrUtil::numberToString(d); + text_.append(str); +} + +void String::append(Ref str) { append(str->getText()); } + +string StrUtil::COMBINE_STRING(string str1, string str2) { + string str = str1; + str += str2; + return str; +} + +string StrUtil::COMBINE_STRING(string str1, char c) { + string str = str1; + str += c; + return str; +} + +string StrUtil::COMBINE_STRING(string str1, int d) { + string str = str1; + str += numberToString(d); + return str; +} + +Ref StrUtil::COMBINE_STRING(char c1, Ref content, char c2) { + Ref str(new String(0)); + str->append(c1); + str->append(content); + str->append(c2); + + return str; +} + +template +string StrUtil::numberToString(T Number) { + ostringstream ss; + ss << Number; + return ss.str(); +} + +template +T StrUtil::stringToNumber(const string& Text) { + istringstream ss(Text); + T result; + return ss >> result ? result : 0; +} + +int StrUtil::indexOf(const char* str, char c) { + int len = strlen(str); + + for (int i = 0; i < len; i++) { + if (str[i] == c) { + return i; + } + } + + return -1; +} diff --git a/modules/wechat_qrcode/src/zxing/common/str.hpp b/modules/wechat_qrcode/src/zxing/common/str.hpp new file mode 100644 index 00000000000..46cb9dcd712 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/str.hpp @@ -0,0 +1,63 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_STR_HPP__ +#define __ZXING_COMMON_STR_HPP__ + +#include +#include +#include "counted.hpp" + +#include +#include +#include + +using namespace std; + +namespace zxing { + +class String : public Counted { +private: + std::string text_; + +public: + explicit String(const std::string& text); + explicit String(int); + char charAt(int) const; + Ref substring(int) const; + Ref substring(int, int) const; + const std::string& getText() const; + int size() const; + void append(std::string const& tail); + void append(char c); + void append(int d); + void append(Ref str); + int length() const; +}; + +class StrUtil { +public: + static string COMBINE_STRING(string str1, string str2); + static string COMBINE_STRING(string str1, char c); + static string COMBINE_STRING(string str1, int d); + static Ref COMBINE_STRING(char c1, Ref content, char c2); + + template + static string numberToString(T Number); + + template + static T stringToNumber(const string& Text); + + static int indexOf(const char* str, char c); +}; + +} // namespace zxing + +#endif // __ZXING_COMMON_STR_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/stringutils.cpp b/modules/wechat_qrcode/src/zxing/common/stringutils.cpp new file mode 100644 index 00000000000..2a4a01868f4 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/stringutils.cpp @@ -0,0 +1,687 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "../common/stringutils.hpp" +#include "../decodehints.hpp" + +#include + +using namespace std; +using namespace zxing; +using namespace zxing::common; + +// N.B.: these are the iconv strings for at least some versions of iconv + +char const* const StringUtils::PLATFORM_DEFAULT_ENCODING = "ANY"; +char const* const StringUtils::ASCII = "ASCII"; +char const* const StringUtils::SHIFT_JIS = "SHIFT-JIS"; +char const* const StringUtils::GBK = "GBK"; +char const* const StringUtils::EUC_JP = "EUC-JP"; +char const* const StringUtils::UTF8 = "UTF-8"; +char const* const StringUtils::ISO88591 = "ISO8859-1"; +char const* const StringUtils::GB2312 = "GB2312"; +char const* const StringUtils::BIG5 = "BIG5"; +char const* const StringUtils::GB18030 = "GB18030"; + +const bool StringUtils::ASSUME_SHIFT_JIS = false; + +#ifdef USE_UCHARDET +#include "uchardet/uchardet.h" +#endif + +// Added convertString +#ifndef NO_ICONV +#include + +// Required for compatibility. TODO: test on Symbian +//#ifdef ZXING_ICONV_CONST +#undef ICONV_CONST +#define ICONV_CONST const +//#endif + +#ifndef ICONV_CONST +#define ICONV_CONST /**/ +#endif + +// Add this to fix both Mac and Windows compilers +// by Skylook +template +class sloppy {}; + +// convert between T** and const T** +template +class sloppy { + T** t; + +public: + sloppy(T** mt) : t(mt) {} + sloppy(const T** mt) : t(const_cast(mt)) {} + + operator T* *() const { return t; } + operator const T* *() const { return const_cast(t); } +}; +#endif + +string StringUtils::convertString(const char* rawData, int length, const char* fromCharset, + const char* toCharset) { + string result; + const char* bufIn = rawData; + int nIn = length; + + // If from and to charset are the same, return + int ret = strcmp(fromCharset, toCharset); + if (ret == 0) { + result.append((const char*)bufIn, nIn); + return result; + } + +#ifndef NO_ICONV + if (nIn == 0) { + return ""; + } + iconv_t cd; + // cout< 0) { + // size_t oneway = iconv(cd, &fromPtr, &nFrom, &toPtr, &nTo); + oneway = iconv(cd, sloppy(&fromPtr), &nFrom, sloppy(&toPtr), &nTo); + } + iconv_close(cd); + + int nResult = maxOut - nTo; + bufOut[nResult] = '\0'; + result.append((const char*)bufOut); + delete[] bufOut; + + // Cannot convert string + if (oneway == (size_t)(-1)) { + // result.append((const char *)bufIn, nIn); + result = ""; + } +#else + result.append((const char*)bufIn, nIn); +#endif + + return result; +} + +string StringUtils::guessEncoding(char* bytes, int length) { +#ifdef USE_UCHARDET + if (length < 10) { + return guessEncodingZXing(bytes, length); + } else { + return guessEncodingUCharDet(bytes, length); + } +#else + return guessEncodingZXing(bytes, length); +#endif +} + +#ifdef USE_UCHARDET + +string StringUtils::guessEncodingUCharDet(char* bytes, int length) { + uchardet_t handle = uchardet_new(); + + int retval = uchardet_handle_data(handle, bytes, length); + + if (retval != 0) { + fprintf(stderr, "Handle data error.\n"); + exit(0); + } + + uchardet_data_end(handle); + + const char* charset = uchardet_get_charset(handle); + + string charsetStr(charset); + + uchardet_delete(handle); + + if (charsetStr.size() != 0) { + return charsetStr; + } else { + return guessEncodingZXing(bytes, length); + } + + // Otherwise, we take a wild guess with platform encoding + // return PLATFORM_DEFAULT_ENCODING; +} +#endif + +string StringUtils::guessEncodingZXing(char* bytes, int length) { + // + // typedef bool boolean; + // For now, merely tries to distinguish ISO-8859-1, UTF-8 and Shift_JIS, + // which should be by far the most common encodings. + bool canBeISO88591 = true; + bool canBeShiftJIS = true; + bool canBeUTF8 = true; + bool canBeGB2312 = true; + bool canBeGBK = true; + bool canBeBIG5 = true; + bool canBeASCII = true; + + int utf8BytesLeft = 0; + int utf2BytesChars = 0; + int utf3BytesChars = 0; + int utf4BytesChars = 0; + int sjisBytesLeft = 0; + int sjisKatakanaChars = 0; + int sjisCurKatakanaWordLength = 0; + int sjisCurDoubleBytesWordLength = 0; + int sjisMaxKatakanaWordLength = 0; + int sjisMaxDoubleBytesWordLength = 0; + int isoHighOther = 0; + + int gb2312SCByteChars = 0; + int big5TWBytesChars = 0; + + bool utf8bom = length > 3 && (unsigned char)bytes[0] == 0xEF && + (unsigned char)bytes[1] == 0xBB && (unsigned char)bytes[2] == 0xBF; + + for (int i = 0; i < length && (canBeISO88591 || canBeShiftJIS || canBeUTF8 || canBeGBK); i++) { + int value = bytes[i] & 0xFF; + + // UTF-8 stuff + if (canBeUTF8) { + if (utf8BytesLeft > 0) { + if ((value & 0x80) == 0) { + canBeUTF8 = false; + } else { + utf8BytesLeft--; + } + } else if ((value & 0x80) != 0) { + if ((value & 0x40) == 0) { + canBeUTF8 = false; + } else { + utf8BytesLeft++; + if ((value & 0x20) == 0) { + utf2BytesChars++; + } else { + utf8BytesLeft++; + if ((value & 0x10) == 0) { + utf3BytesChars++; + } else { + utf8BytesLeft++; + if ((value & 0x08) == 0) { + utf4BytesChars++; + } else { + canBeUTF8 = false; + } + } + } + } + } + } + // Shift_JIS stuff + if (canBeShiftJIS) { + if (sjisBytesLeft > 0) { + if (value < 0x40 || value == 0x7F || value > 0xFC) { + canBeShiftJIS = false; + } else { + sjisBytesLeft--; + } + } else if (value == 0x80 || value == 0xA0 || value > 0xEF) { + canBeShiftJIS = false; + } else if (value > 0xA0 && value < 0xE0) { + sjisKatakanaChars++; + sjisCurDoubleBytesWordLength = 0; + sjisCurKatakanaWordLength++; + if (sjisCurKatakanaWordLength > sjisMaxKatakanaWordLength) { + sjisMaxKatakanaWordLength = sjisCurKatakanaWordLength; + } + } else if (value > 0x7F) { + sjisBytesLeft++; + // sjisDoubleBytesChars++; + sjisCurKatakanaWordLength = 0; + sjisCurDoubleBytesWordLength++; + if (sjisCurDoubleBytesWordLength > sjisMaxDoubleBytesWordLength) { + sjisMaxDoubleBytesWordLength = sjisCurDoubleBytesWordLength; + } + } else { + // sjisLowChars++; + sjisCurKatakanaWordLength = 0; + sjisCurDoubleBytesWordLength = 0; + } + } + + // ISO-8859-1 stuff + if (canBeISO88591) { + if (value > 0x7F && value < 0xA0) { + canBeISO88591 = false; + } else if (value > 0x9F) { + if (value < 0xC0 || value == 0xD7 || value == 0xF7) { + isoHighOther++; + } + } + } + } + + // Get how many chinese sc & tw words + gb2312SCByteChars = is_gb2312_code(bytes, length); + big5TWBytesChars = is_big5_code(bytes, length); + + if (gb2312SCByteChars <= 0) { + canBeGB2312 = false; + } + + if (big5TWBytesChars <= 0) { + canBeBIG5 = false; + } + + if (!is_gbk_code(bytes, length)) { + canBeGBK = false; + } + + if (canBeUTF8 && utf8BytesLeft > 0) { + canBeUTF8 = false; + } + if (canBeShiftJIS && sjisBytesLeft > 0) { + canBeShiftJIS = false; + } + + if (is_ascii_code(bytes, length) <= 0) { + canBeASCII = false; + } + + // Easy -- if there is BOM or at least 1 valid not-single byte character + // (and no evidence it can't be UTF-8), done + if (canBeUTF8 && (utf8bom || utf2BytesChars + utf3BytesChars + utf4BytesChars > 0)) { + return UTF8; + } + + // if ( canBeBIG5 == false && canBeGB2312 == false ) + int chineseWordLen = + gb2312SCByteChars > big5TWBytesChars ? gb2312SCByteChars : big5TWBytesChars; + int chineseByteLen = chineseWordLen * 2; + int japaneseByteLen = sjisMaxKatakanaWordLength + sjisMaxDoubleBytesWordLength * 2; + + // if ( chineseByteLen < japaneseByteLen || (japaneseByteLen == 0 && + // chineseByteLen == 0) ) if ( (gb2312SCByteChars < sjisKatakanaChars) && + // (big5TWBytesChars < sjisKatakanaChars) ) + //{ + // Easy -- if assuming Shift_JIS or at least 3 valid consecutive not-ascii + // characters (and no evidence it can't be), done + if (canBeShiftJIS && + (ASSUME_SHIFT_JIS || sjisMaxKatakanaWordLength >= 3 || sjisMaxDoubleBytesWordLength >= 3)) { + // return SHIFT_JIS; + if (chineseByteLen <= japaneseByteLen) { + if (chineseByteLen == japaneseByteLen) { + if (chineseWordLen < sjisKatakanaChars) { + return SHIFT_JIS; + } + } else { + return SHIFT_JIS; + } + } + } + + // Distinguishing Shift_JIS and ISO-8859-1 can be a little tough for short + // words. The crude heuristic is: + // - If we saw + // - only two consecutive katakana chars in the whole text, or + // - at least 10% of bytes that could be "upper" not-alphanumeric Latin1, + // - then we conclude Shift_JIS, else ISO-8859-1 + if (canBeISO88591 && canBeShiftJIS) { + if ((sjisMaxKatakanaWordLength == 2 && sjisKatakanaChars == 2) || + isoHighOther * 10 >= length) { + /* + if ( chineseByteLen < japaneseByteLen ) + { + return SHIFT_JIS; + } + */ + if (chineseByteLen <= japaneseByteLen) { + if (chineseByteLen == japaneseByteLen) { + if (chineseWordLen < sjisKatakanaChars) { + return SHIFT_JIS; + } + } else { + return SHIFT_JIS; + } + } + } else { + if (chineseByteLen <= 0 && !canBeGB2312 && !canBeBIG5) { + return ISO88591; + } + } + } + //} + + // Otherwise, try in order ISO-8859-1, Shift JIS, UTF-8 and fall back to + // default platform encoding + if (canBeGB2312) { + return GB2312; + } + + if (canBeBIG5) { + return BIG5; + } + + if (canBeShiftJIS) { + return SHIFT_JIS; + } + + if (canBeGBK) { + return GBK; + } + + if (canBeISO88591) { + return ISO88591; + } + + if (canBeUTF8) { + return UTF8; + } + + if (canBeASCII) { + return ASCII; + } + + // Otherwise, we take a wild guess with platform encoding + return PLATFORM_DEFAULT_ENCODING; +} + +// judge the byte whether begin with binary 10 +int StringUtils::is_utf8_special_byte(unsigned char c) { + unsigned char special_byte = 0X02; // binary 00000010 + if (c >> 6 == special_byte) { + return 1; + } else { + return 0; + } +} + +int StringUtils::is_utf8_code(char* str, int length) { + unsigned char one_byte = 0X00; // binary 00000000 + unsigned char two_byte = 0X06; // binary 00000110 + unsigned char three_byte = 0X0E; // binary 00001110 + unsigned char four_byte = 0X1E; // binary 00011110 + unsigned char five_byte = 0X3E; // binary 00111110 + unsigned char six_byte = 0X7E; // binary 01111110 + + int utf8_yes = 0; + int utf8_no = 0; + + unsigned char k = 0; + unsigned char m = 0; + unsigned char n = 0; + unsigned char p = 0; + unsigned char q = 0; + + unsigned char c = 0; + for (int i = 0; i < length;) { + c = (unsigned char)str[i]; + if (c >> 7 == one_byte) { + i++; + continue; + } else if (c >> 5 == two_byte) { + k = (unsigned char)str[i + 1]; + if (is_utf8_special_byte(k)) { + utf8_yes++; + i += 2; + continue; + } + } else if (c >> 4 == three_byte) { + m = (unsigned char)str[i + 1]; + n = (unsigned char)str[i + 2]; + if (is_utf8_special_byte(m) && is_utf8_special_byte(n)) { + utf8_yes++; + i += 3; + continue; + } + } else if (c >> 3 == four_byte) { + k = (unsigned char)str[i + 1]; + m = (unsigned char)str[i + 2]; + n = (unsigned char)str[i + 3]; + if (is_utf8_special_byte(k) && is_utf8_special_byte(m) && is_utf8_special_byte(n)) { + utf8_yes++; + i += 4; + continue; + } + } else if (c >> 2 == five_byte) { + k = (unsigned char)str[i + 1]; + m = (unsigned char)str[i + 2]; + n = (unsigned char)str[i + 3]; + p = (unsigned char)str[i + 4]; + if (is_utf8_special_byte(k) && is_utf8_special_byte(m) && is_utf8_special_byte(n) && + is_utf8_special_byte(p)) { + utf8_yes++; + i += 5; + continue; + } + } else if (c >> 1 == six_byte) { + k = (unsigned char)str[i + 1]; + m = (unsigned char)str[i + 2]; + n = (unsigned char)str[i + 3]; + p = (unsigned char)str[i + 4]; + q = (unsigned char)str[i + 5]; + if (is_utf8_special_byte(k) && is_utf8_special_byte(m) && is_utf8_special_byte(n) && + is_utf8_special_byte(p) && is_utf8_special_byte(q)) { + utf8_yes++; + i += 6; + continue; + } + } + + utf8_no++; + i++; + } + + // printf("uft8_yes: %d utf8_no:%d\n", utf8_yes, utf8_no); + if ((utf8_yes + utf8_no) != 0) { + int ret = (100 * utf8_yes) / (utf8_yes + utf8_no); + if (ret > 90) { + return 1; + } else { + return 0; + } + } + return 0; +} +int StringUtils::is_gb2312_code(char* str, int length) { + unsigned char one_byte = 0X00; // binary 00000000 + + int gb2312_yes = 0; + int gb2312_no = 0; + + unsigned char k = 0; + + unsigned char c = 0; + for (int i = 0; i < length;) { + c = (unsigned char)str[i]; + if (c >> 7 == one_byte) { + i++; + continue; + } else if (c >= 0XA1 && c <= 0XF7) { + k = (unsigned char)str[i + 1]; + if (k >= 0XA1 && k <= 0XFE) { + gb2312_yes++; + i += 2; + continue; + } + } + + gb2312_no++; + i += 2; + } + + // printf("gb2312_yes: %d gb2312_no:%d\n", gb2312_yes, gb2312_no); + if ((gb2312_yes + gb2312_no) > 0) { + int ret = (100 * gb2312_yes) / (gb2312_yes + gb2312_no); + if (ret == 100) { + // if (ret > 90) { + // gb2312SCByteChars = gb2312_yes; + return gb2312_yes; + } else { + return 0; + } + } + return 0; +} + +int StringUtils::is_big5_code(char* str, int length) { + unsigned char one_byte = 0X00; // binary 00000000 + + int big5_yes = 0; + int big5_no = 0; + + unsigned char k = 0; + + unsigned char c = 0; + for (int i = 0; i < length;) { + c = (unsigned char)str[i]; + if (c >> 7 == one_byte) { + i++; + continue; + } else if (c >= 0XA1 && c <= 0XF9) { + k = (unsigned char)str[i + 1]; + if ((k >= 0X40 && k <= 0X7E) || (k >= 0XA1 && k <= 0XFE)) { + big5_yes++; + i += 2; + continue; + } + } + + big5_no++; + i += 2; + } + + // printf("%d %d\n", big5_yes, big5_no); + if ((big5_yes + big5_no) > 0) { + int ret = (100 * big5_yes) / (big5_yes + big5_no); + if (ret == 100) { + // if (ret > 90) { + // big5TWBytesChars = big5_yes; + return big5_yes; + } else { + return 0; + } + } + return 0; +} + +int StringUtils::is_gbk_code(char* str, int length) { + unsigned char one_byte = 0X00; // binary 00000000 + + int gbk_yes = 0; + int gbk_no = 0; + + unsigned char k = 0; + + unsigned char c = 0; + for (int i = 0; i < length;) { + c = (unsigned char)str[i]; + if (c >> 7 == one_byte) { + i++; + continue; + } else if (c >= 0X81 && c <= 0XFE) { + k = (unsigned char)str[i + 1]; + if (k >= 0X40 && k <= 0XFE) { + gbk_yes++; + i += 2; + continue; + } + } + + gbk_no++; + i += 2; + } + + // printf("gbk_yes: %d gbk_no:%d\n", gbk_yes, gbk_no); + if ((gbk_yes + gbk_no) > 0) { + int ret = (100 * gbk_yes) / (gbk_yes + gbk_no); + if (ret == 100) { + // if (ret > 90) { + return 1; + } else { + return 0; + } + } + return 0; +} + +int StringUtils::is_ascii_code(char* str, int length) { + unsigned char c = 0; + + bool isASCII = true; + + for (int i = 0; i < length; i++) { + c = (unsigned char)str[i]; + + if ((c > 127)) { + isASCII = false; + } + } + return (isASCII ? 1 : -1); +} + +//#define DEBUG + +int StringUtils::shift_jis_to_jis(const unsigned char* may_be_shift_jis, int* jis_first_ptr, + int* jis_second_ptr) { + int status = 0; + unsigned char first = may_be_shift_jis[0]; + unsigned char second = may_be_shift_jis[1]; + int jis_first = 0; + int jis_second = 0; + /* Check first byte is valid shift JIS. */ + if ((first >= 0x81 && first <= 0x84) || (first >= 0x87 && first <= 0x9f)) { + jis_first = 2 * (first - 0x70) - 1; + if (second >= 0x40 && second <= 0x9e) { + jis_second = second - 31; + if (jis_second > 95) { + jis_second -= 1; + } + + status = 1; + } else if (second >= 0x9f && second <= 0xfc) { + jis_second = second - 126; + jis_first += 1; + status = 1; + } else { + } + } else if (first >= 0xe0 && first <= 0xef) { + jis_first = 2 * (first - 0xb0) - 1; + if (second >= 0x40 && second <= 0x9e) { + jis_second = second - 31; + if (jis_second > 95) { + jis_second -= 1; + } + status = 1; + } else if (second >= 0x9f && second <= 0xfc) { + jis_second = second - 126; + jis_first += 1; + status = 1; + } + } else { + } + *jis_first_ptr = jis_first; + *jis_second_ptr = jis_second; + return status; +} diff --git a/modules/wechat_qrcode/src/zxing/common/stringutils.hpp b/modules/wechat_qrcode/src/zxing/common/stringutils.hpp new file mode 100644 index 00000000000..cf58cbd4ce8 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/stringutils.hpp @@ -0,0 +1,65 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_COMMON_STRINGUTILS_HPP__ +#define __ZXING_COMMON_STRINGUTILS_HPP__ + +#include "../decodehints.hpp" +#include "../zxing.hpp" + +#include +#include + +namespace zxing { +namespace common { +class StringUtils; +} +} // namespace zxing +using namespace std; + +class zxing::common::StringUtils { +private: + static char const* const PLATFORM_DEFAULT_ENCODING; + +public: + static char const* const ASCII; + static char const* const SHIFT_JIS; + static char const* const GB2312; + static char const* const EUC_JP; + static char const* const UTF8; + static char const* const ISO88591; + static char const* const GBK; + static char const* const GB18030; + static char const* const BIG5; + + static const bool ASSUME_SHIFT_JIS; + + static std::string guessEncoding(char* bytes, int length); + static std::string guessEncodingZXing(char* bytes, int length); + +#ifdef USE_UCHARDET + static std::string guessEncodingUCharDet(char* bytes, int length); +#endif + + static int is_utf8_special_byte(unsigned char c); + // static int is_utf8_code(const string& str); + static int is_utf8_code(char* str, int length); + static int is_gb2312_code(char* str, int length); + static int is_big5_code(char* str, int length); + static int is_gbk_code(char* str, int length); + static int is_ascii_code(char* str, int length); + static int shift_jis_to_jis(const unsigned char* may_be_shift_jis, int* jis_first_ptr, + int* jis_second_ptr); + + static std::string convertString(const char* rawData, int length, const char* fromCharset, + const char* toCharset); +}; + +#endif // __ZXING_COMMON_STRINGUTILS_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/common/unicomblock.cpp b/modules/wechat_qrcode/src/zxing/common/unicomblock.cpp new file mode 100644 index 00000000000..ae51cb742b3 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/unicomblock.cpp @@ -0,0 +1,130 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#include "unicomblock.hpp" +#include + +#include + +namespace zxing { +short UnicomBlock::SEARCH_POS[4][2] = {{1, 0}, {-1, 0}, {0, 1}, {0, -1}}; +UnicomBlock::UnicomBlock(int iMaxHeight, int iMaxWidth) + : m_iHeight(iMaxHeight), m_iWidth(iMaxWidth), m_bInit(false) {} + +UnicomBlock::~UnicomBlock() {} + +void UnicomBlock::Init() { + if (m_bInit) return; + m_vcIndex = std::vector(m_iHeight * m_iWidth, 0); + m_vcCount = std::vector(m_iHeight * m_iWidth, 0); + m_vcMinPnt = std::vector(m_iHeight * m_iWidth, 0); + m_vcMaxPnt = std::vector(m_iHeight * m_iWidth, 0); + m_vcQueue = std::vector(m_iHeight * m_iWidth, 0); + m_bInit = true; +} + +void UnicomBlock::Reset(Ref poImage) { + m_poImage = poImage; + memset(&m_vcIndex[0], 0, m_vcIndex.size() * sizeof(short)); + m_iNowIdx = 0; +} + +unsigned short UnicomBlock::GetUnicomBlockIndex(int y, int x) { + if (y >= m_iHeight || x >= m_iWidth) return 0; + if (m_vcIndex[y * m_iWidth + x]) return m_vcIndex[y * m_iWidth + x]; + Bfs(y, x); + return m_vcIndex[y * m_iWidth + x]; +} + +int UnicomBlock::GetUnicomBlockSize(int y, int x) { + if (y >= m_iHeight || x >= m_iWidth) return 0; + if (m_vcIndex[y * m_iWidth + x]) return m_vcCount[y * m_iWidth + x]; + Bfs(y, x); + return m_vcCount[y * m_iWidth + x]; +} + +int UnicomBlock::GetMinPoint(int y, int x, int &iMinY, int &iMinX) { + if (y >= m_iHeight || x >= m_iWidth) return -1; + if (m_vcIndex[y * m_iWidth + x]) { + iMinY = m_vcMinPnt[y * m_iWidth + x] >> 16; + iMinX = m_vcMinPnt[y * m_iWidth + x] & (0xFFFF); + return 0; + } + Bfs(y, x); + iMinY = m_vcMinPnt[y * m_iWidth + x] >> 16; + iMinX = m_vcMinPnt[y * m_iWidth + x] & (0xFFFF); + return 0; +} + +int UnicomBlock::GetMaxPoint(int y, int x, int &iMaxY, int &iMaxX) { + if (y >= m_iHeight || x >= m_iWidth) return -1; + if (m_vcIndex[y * m_iWidth + x]) { + iMaxY = m_vcMaxPnt[y * m_iWidth + x] >> 16; + iMaxX = m_vcMaxPnt[y * m_iWidth + x] & (0xFFFF); + return 0; + } + Bfs(y, x); + iMaxY = m_vcMaxPnt[y * m_iWidth + x] >> 16; + iMaxX = m_vcMaxPnt[y * m_iWidth + x] & (0xFFFF); + return 0; +} + +void UnicomBlock::Bfs(int y, int x) { + m_iNowIdx++; + + int iFront = 0; + int iTail = 0; + int iCount = 1; + + int iMaxX = x, iMaxY = y; + int iMinX = x, iMinY = y; + + const bool bValue = (m_poImage->get(x, y) != (unsigned char)0); + + m_vcIndex[y * m_iWidth + x] = m_iNowIdx; + m_vcQueue[iTail++] = y << 16 | x; + + while (iFront < iTail) { + int iNode = m_vcQueue[iFront++]; + int iX = iNode & (0xFFFF); + int iY = iNode >> 16; + iMaxX = max(iX, iMaxX); + iMaxY = max(iY, iMaxY); + iMinX = min(iX, iMinX); + iMinY = min(iY, iMinY); + + iCount++; + + for (int i = 0; i < 4; ++i) { + const int iNextX = iX + SEARCH_POS[i][0], iNextY = iY + SEARCH_POS[i][1]; + const int iPosition = iNextY * m_iWidth + iNextX; + + if (iPosition >= 0 && iPosition < int(m_vcIndex.size()) && 0 == m_vcIndex[iPosition]) { + if (iNextX < 0 || iNextX >= m_poImage->getWidth() || iNextY < 0 || + iNextY >= m_poImage->getHeight() || + bValue != (m_poImage->get(iNextX, iNextY) != (unsigned char)0)) + continue; + + m_vcIndex[iPosition] = m_iNowIdx; + m_vcQueue[iTail++] = iNextY << 16 | iNextX; + } + } + } + + if (iCount >= (1 << 16) - 1) iCount = 0xFFFF; + + const int iMinCombine = iMinY << 16 | iMinX; + const int iMaxCombine = iMaxY << 16 | iMaxX; + for (int i = 0; i < iTail; ++i) { + const int iPosition = (m_vcQueue[i] >> 16) * m_iWidth + (m_vcQueue[i] & (0xFFFF)); + + m_vcCount[iPosition] = iCount; + m_vcMinPnt[iPosition] = iMinCombine; + m_vcMaxPnt[iPosition] = iMaxCombine; + } +} +} // namespace zxing diff --git a/modules/wechat_qrcode/src/zxing/common/unicomblock.hpp b/modules/wechat_qrcode/src/zxing/common/unicomblock.hpp new file mode 100644 index 00000000000..858e03f018f --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/common/unicomblock.hpp @@ -0,0 +1,50 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#ifndef __ZXING_COMMON_UNICOMBLOCK_HPP__ +#define __ZXING_COMMON_UNICOMBLOCK_HPP__ +#include "bitmatrix.hpp" +#include "counted.hpp" + +#include +#include + +namespace zxing { +class UnicomBlock : public Counted { +public: + UnicomBlock(int iMaxHeight, int iMaxWidth); + ~UnicomBlock(); + + void Init(); + void Reset(Ref poImage); + + unsigned short GetUnicomBlockIndex(int y, int x); + + int GetUnicomBlockSize(int y, int x); + + int GetMinPoint(int y, int x, int &iMinY, int &iMinX); + int GetMaxPoint(int y, int x, int &iMaxY, int &iMaxX); + +private: + void Bfs(int y, int x); + + int m_iHeight; + int m_iWidth; + + unsigned short m_iNowIdx; + bool m_bInit; + std::vector m_vcIndex; + std::vector m_vcCount; + std::vector m_vcMinPnt; + std::vector m_vcMaxPnt; + std::vector m_vcQueue; + static short SEARCH_POS[4][2]; + + Ref m_poImage; +}; +} // namespace zxing +#endif // __ZXING_COMMON_UNICOMBLOCK_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/decodehints.hpp b/modules/wechat_qrcode/src/zxing/decodehints.hpp new file mode 100644 index 00000000000..ed1830b5d07 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/decodehints.hpp @@ -0,0 +1,30 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_DECODEHINTS_HPP__ +#define __ZXING_DECODEHINTS_HPP__ + +#include "errorhandler.hpp" + +namespace zxing { +class DecodeHints { +private: + bool use_nn_detector_; + +public: + explicit DecodeHints(bool use_nn_detector = false) : use_nn_detector_(use_nn_detector){}; + + bool getUseNNDetector() const { return use_nn_detector_; } + void setUseNNDetector(bool use_nn_detector) { use_nn_detector_ = use_nn_detector; } +}; + +} // namespace zxing + +#endif // __ZXING_DECODEHINTS_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/errorhandler.cpp b/modules/wechat_qrcode/src/zxing/errorhandler.cpp new file mode 100644 index 00000000000..466dd346af9 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/errorhandler.cpp @@ -0,0 +1,50 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#include "errorhandler.hpp" + +#include +namespace zxing { + +ErrorHandler::ErrorHandler() : err_code_(0), err_msg_("") { Init(); } + +ErrorHandler::ErrorHandler(const char* err_msg) : err_code_(-1), err_msg_(err_msg) { Init(); } + +ErrorHandler::ErrorHandler(std::string& err_msg) : err_code_(-1), err_msg_(err_msg) { Init(); } + +ErrorHandler::ErrorHandler(int err_code) : err_code_(err_code), err_msg_("error") { Init(); } + +ErrorHandler::ErrorHandler(int err_code, const char* err_msg) + : err_code_(err_code), err_msg_(err_msg) { + Init(); +} + +ErrorHandler::ErrorHandler(const ErrorHandler& other) { + err_code_ = other.ErrCode(); + err_msg_.assign(other.ErrMsg()); + Init(); +} + +ErrorHandler& ErrorHandler::operator=(const ErrorHandler& other) { + err_code_ = other.ErrCode(); + err_msg_.assign(other.ErrMsg()); + Init(); + return *this; +} + +void ErrorHandler::Init() { handler_type_ = KErrorHandler; } + +void ErrorHandler::Reset() { + err_code_ = 0; + err_msg_.assign(""); +} + +void ErrorHandler::PrintInfo() { + printf("handler_tpye %d, error code %d, errmsg %s\n", handler_type_, err_code_, + err_msg_.c_str()); +} +} // namespace zxing diff --git a/modules/wechat_qrcode/src/zxing/errorhandler.hpp b/modules/wechat_qrcode/src/zxing/errorhandler.hpp new file mode 100644 index 00000000000..486786b073c --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/errorhandler.hpp @@ -0,0 +1,89 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#ifndef __ZXING_ERRORHANDLER_HPP__ +#define __ZXING_ERRORHANDLER_HPP__ + +#include + +namespace zxing { + +enum { + KErrorHandler = 0, + KErrorHandler_NotFound = 1, + KErrorHandler_CheckSum = 2, + KErrorHandler_Reader = 3, + KErrorHandler_IllegalArgument = 4, + KErrorHandler_ReedSolomon = 5, + KErrorHandler_Format = 6, + KErrorHandler_Detector = 7, + KErrorHandler_IllegalState = 8, +}; + +class ErrorHandler { +public: + ErrorHandler(); + explicit ErrorHandler(std::string& err_msg); + explicit ErrorHandler(const char* err_msg); + explicit ErrorHandler(int err_code); + ErrorHandler(int err_code, std::string& err_msg); + ErrorHandler(int err_code, const char* err_msg); + + virtual ~ErrorHandler(){}; + + virtual inline int ErrCode() const { return err_code_; } + virtual inline const std::string& ErrMsg() const { return err_msg_; } + virtual inline int HandlerType() const { return handler_type_; } + + virtual void Init(); + ErrorHandler(const ErrorHandler& other); + ErrorHandler& operator=(const ErrorHandler& other); + + virtual void PrintInfo(); + virtual void Reset(); + +protected: + int handler_type_; + +private: + int err_code_; + std::string err_msg_; +}; + +#define DECLARE_ERROR_HANDLER(__HANDLER__) \ + class __HANDLER__##ErrorHandler : public ErrorHandler { \ + public: \ + __HANDLER__##ErrorHandler() : ErrorHandler() { Init(); }; \ + __HANDLER__##ErrorHandler(std::string& err_msg) : ErrorHandler(err_msg) { Init(); }; \ + __HANDLER__##ErrorHandler(const char* err_msg) : ErrorHandler(err_msg) { Init(); }; \ + __HANDLER__##ErrorHandler(int err_code) : ErrorHandler(err_code) { Init(); }; \ + __HANDLER__##ErrorHandler(int err_code, std::string& err_msg) \ + : ErrorHandler(err_code, err_msg) { \ + Init(); \ + }; \ + __HANDLER__##ErrorHandler(int err_code, const char* err_msg) \ + : ErrorHandler(err_code, err_msg) { \ + Init(); \ + }; \ + __HANDLER__##ErrorHandler(const ErrorHandler& other) : ErrorHandler(other) { Init(); }; \ + void Init() override { handler_type_ = KErrorHandler_##__HANDLER__; } \ + }; + +DECLARE_ERROR_HANDLER(Reader) +DECLARE_ERROR_HANDLER(IllegalArgument) +DECLARE_ERROR_HANDLER(ReedSolomon) +DECLARE_ERROR_HANDLER(Format) +DECLARE_ERROR_HANDLER(Detector) +DECLARE_ERROR_HANDLER(NotFound) +DECLARE_ERROR_HANDLER(CheckSum) +DECLARE_ERROR_HANDLER(IllegalState) + +#undef DECLARE_ERROR_HANDLER + +} // namespace zxing + +#endif // __ZXING_ERRORHANDLER_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/luminance_source.cpp b/modules/wechat_qrcode/src/zxing/luminance_source.cpp new file mode 100644 index 00000000000..e67052409f2 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/luminance_source.cpp @@ -0,0 +1,59 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "luminance_source.hpp" +#include + +using zxing::LuminanceSource; +using zxing::Ref; + +LuminanceSource::LuminanceSource(int width, int height) + : width_(width), height_(height) {} + +LuminanceSource::~LuminanceSource() {} + +bool LuminanceSource::isCropSupported() const { return false; } + +Ref LuminanceSource::crop(int, int, int, int, zxing::ErrorHandler&) const { + return Ref(); +} + +bool LuminanceSource::isRotateSupported() const { return false; } + +Ref LuminanceSource::rotateCounterClockwise(zxing::ErrorHandler&) const { + return Ref(); +} + +LuminanceSource::operator std::string() const { + ArrayRef row; + std::ostringstream oss; + zxing::ErrorHandler err_handler; + for (int y = 0; y < getHeight(); y++) { + err_handler.Reset(); + row = getRow(y, row, err_handler); + if (err_handler.ErrCode()) continue; + for (int x = 0; x < getWidth(); x++) { + int luminance = row[x] & 0xFF; + char c; + if (luminance < 0x40) { + c = '#'; + } else if (luminance < 0x80) { + c = '+'; + } else if (luminance < 0xC0) { + c = '.'; + } else { + c = ' '; + } + oss << c; + } + oss << '\n'; + } + return oss.str(); +} diff --git a/modules/wechat_qrcode/src/zxing/luminance_source.hpp b/modules/wechat_qrcode/src/zxing/luminance_source.hpp new file mode 100644 index 00000000000..e3435723656 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/luminance_source.hpp @@ -0,0 +1,57 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_LUMINANCE_SOURCE_HPP__ +#define __ZXING_LUMINANCE_SOURCE_HPP__ + +#include +#include "common/array.hpp" +#include "common/bytematrix.hpp" +#include "common/counted.hpp" +#include "errorhandler.hpp" + +namespace zxing { + +class LuminanceSource : public Counted { +protected: + int width_; + int height_; + +public: + LuminanceSource(int width, int height); + virtual ~LuminanceSource(); + + int getWidth() const { return width_; } + int getHeight() const { return height_; } + void setWidth(int w) { width_ = w; } + void setHeight(int h) { height_ = h; } + void filter(); + + // Callers take ownership of the returned memory and must call delete [] on + // it themselves. + virtual ArrayRef getRow(int y, ArrayRef row, + zxing::ErrorHandler& err_handler) const = 0; + virtual ArrayRef getMatrix() const = 0; + virtual Ref getByteMatrix() const = 0; + + virtual bool isCropSupported() const; + virtual Ref crop(int left, int top, int width, int height, + zxing::ErrorHandler& err_handler) const; + + virtual bool isRotateSupported() const; + + virtual Ref rotateCounterClockwise(zxing::ErrorHandler& err_handler) const; + + operator std::string() const; +}; + +} // namespace zxing + +#endif // __ZXING_LUMINANCE_SOURCE_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/bitmatrixparser.cpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/bitmatrixparser.cpp new file mode 100644 index 00000000000..98189c005a0 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/bitmatrixparser.cpp @@ -0,0 +1,241 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "bitmatrixparser.hpp" +#include +#include "datamask.hpp" + +using zxing::ErrorHandler; + +namespace zxing { +namespace qrcode { + +int BitMatrixParser::copyBit(size_t x, size_t y, int versionBits) { + bool bit = ((mirror_ ? bitMatrix_->get(y, x) : bitMatrix_->get(x, y)) != (unsigned char)0); + return bit ? (versionBits << 1) | 0x1 : versionBits << 1; +} + +BitMatrixParser::BitMatrixParser(Ref bitMatrix, ErrorHandler &err_handler) + : bitMatrix_(bitMatrix), parsedVersion_(0), parsedFormatInfo_() { + mirror_ = false; + size_t dimension = bitMatrix->getHeight(); + + if ((dimension < 21) || (dimension & 0x03) != 1) { + err_handler = zxing::ReaderErrorHandler("Dimension must be 1 mod 4 and >= 21"); + return; + } +} + +Ref BitMatrixParser::readFormatInformation(ErrorHandler &err_handler) { + if (parsedFormatInfo_ != 0) { + return parsedFormatInfo_; + } + + // Read top-left format info bits + int formatInfoBits1 = 0; + for (int i = 0; i < 6; i++) { + formatInfoBits1 = copyBit(i, 8, formatInfoBits1); + } + // .. and skip a bit in the timing pattern ... + formatInfoBits1 = copyBit(7, 8, formatInfoBits1); + formatInfoBits1 = copyBit(8, 8, formatInfoBits1); + formatInfoBits1 = copyBit(8, 7, formatInfoBits1); + // .. and skip a bit in the timing pattern ... + for (int j = 5; j >= 0; j--) { + formatInfoBits1 = copyBit(8, j, formatInfoBits1); + } + + // Read the top-right/bottom-left pattern + int dimension = bitMatrix_->getHeight(); + int formatInfoBits2 = 0; + int jMin = dimension - 7; + for (int j = dimension - 1; j >= jMin; j--) { + formatInfoBits2 = copyBit(8, j, formatInfoBits2); + } + for (int i = dimension - 8; i < dimension; i++) { + formatInfoBits2 = copyBit(i, 8, formatInfoBits2); + } + + parsedFormatInfo_ = + FormatInformation::decodeFormatInformation(formatInfoBits1, formatInfoBits2); + if (parsedFormatInfo_ != 0) { + return parsedFormatInfo_; + } + err_handler = zxing::ReaderErrorHandler("Could not decode format information"); + return Ref(); +} + +Version *BitMatrixParser::readVersion(ErrorHandler &err_handler) { + if (parsedVersion_ != 0) { + return parsedVersion_; + } + + int dimension = bitMatrix_->getHeight(); + + int provisionalVersion = (dimension - 17) >> 2; + if (provisionalVersion <= 6) { + Version *version = Version::getVersionForNumber(provisionalVersion, err_handler); + if (err_handler.ErrCode()) return NULL; + return version; + } + + // Read top-right version info: 3 wide by 6 tall + int versionBits = 0; + for (int y = 5; y >= 0; y--) { + int xMin = dimension - 11; + for (int x = dimension - 9; x >= xMin; x--) { + versionBits = copyBit(x, y, versionBits); + } + } + + parsedVersion_ = Version::decodeVersionInformation(versionBits); + if (parsedVersion_ != 0 && parsedVersion_->getDimensionForVersion(err_handler) == dimension) { + return parsedVersion_; + } + + // Hmm, failed. Try bottom left: 6 wide by 3 tall + versionBits = 0; + for (int x = 5; x >= 0; x--) { + int yMin = dimension - 11; + for (int y = dimension - 9; y >= yMin; y--) { + versionBits = copyBit(x, y, versionBits); + } + } + + parsedVersion_ = Version::decodeVersionInformation(versionBits); + if (parsedVersion_ == NULL) { + err_handler = zxing::ReaderErrorHandler("Could not decode version"); + return NULL; + } + + if (parsedVersion_ != 0 && parsedVersion_->getDimensionForVersion(err_handler) == dimension) { + return parsedVersion_; + } + + err_handler = zxing::ReaderErrorHandler("Could not decode version"); + return NULL; +} + +/** + *

Reads the bits in the {@link BitMatrix} representing the finder pattern in + * the correct order in order to reconstruct the codewords bytes contained + * within the QR Code.

+ * + * @return bytes encoded within the QR Code + */ +ArrayRef BitMatrixParser::readCodewords(ErrorHandler &err_handler) { + Ref formatInfo = readFormatInformation(err_handler); + if (err_handler.ErrCode()) return ArrayRef(); + + Version *version = readVersion(err_handler); + if (err_handler.ErrCode()) return ArrayRef(); + + DataMask &dataMask = DataMask::forReference((int)formatInfo->getDataMask(), err_handler); + if (err_handler.ErrCode()) return ArrayRef(); + // cout << (int)formatInfo->getDataMask() << endl; + int dimension = bitMatrix_->getHeight(); + + dataMask.unmaskBitMatrix(*bitMatrix_, dimension); + + // cerr << *bitMatrix_ << endl; + // cerr << version->getTotalCodewords() << endl; + + Ref functionPattern = version->buildFunctionPattern(err_handler); + if (err_handler.ErrCode()) return ArrayRef(); + + // cout << *functionPattern << endl; + + bool readingUp = true; + ArrayRef result(version->getTotalCodewords()); + int resultOffset = 0; + int currentByte = 0; + int bitsRead = 0; + // Read columns in pairs, from right to left + for (int x = dimension - 1; x > 0; x -= 2) { + if (x == 6) { + // Skip whole column with vertical alignment pattern; + // saves time and makes the other code proceed more cleanly + x--; + } + // Read alternatingly from bottom to top then top to bottom + for (int counter = 0; counter < dimension; counter++) { + int y = readingUp ? dimension - 1 - counter : counter; + for (int col = 0; col < 2; col++) { + // Ignore bits covered by the function pattern + if (!functionPattern->get(x - col, y)) { + // Read a bit + bitsRead++; + currentByte <<= 1; + if (bitMatrix_->get(x - col, y)) { + currentByte |= 1; + } + // If we've made a whole byte, save it off + if (bitsRead == 8) { + result[resultOffset++] = (char)currentByte; + bitsRead = 0; + currentByte = 0; + } + } + } + } + readingUp = !readingUp; // switch directions + } + + if (resultOffset != version->getTotalCodewords()) { + err_handler = zxing::ReaderErrorHandler("Did not read all codewords"); + return ArrayRef(); + } + + return result; +} + +/** + * Revert the mask removal done while reading the code words. The bit matrix + * should revert to its original state. + */ +void BitMatrixParser::remask() { + if (parsedFormatInfo_ == NULL) { + return; // We have no format information, and have no data mask + } + ErrorHandler err_handler; + DataMask &dataMask = DataMask::forReference(parsedFormatInfo_->getDataMask(), err_handler); + if (err_handler.ErrCode()) return; + int dimension = bitMatrix_->getHeight(); + dataMask.unmaskBitMatrix(*bitMatrix_, dimension); +} + +/** + * Prepare the parser for a mirrored operation. + * This flag has effect only on the {@link #readFormatInformation()} and the + * {@link #readVersion()}. Before proceeding with {@link #readCodewords()} the + * {@link #mirror()} method should be called. + * + * @param mirror Whether to read version and format information mirrored. + */ +void BitMatrixParser::setMirror(bool mirror) { + parsedVersion_ = NULL; + parsedFormatInfo_ = NULL; + mirror_ = mirror; +} + +/** Mirror the bit matrix in order to attempt a second reading. */ +void BitMatrixParser::mirror() { + for (int x = 0; x < bitMatrix_->getWidth(); x++) { + for (int y = x + 1; y < bitMatrix_->getHeight(); y++) { + if (bitMatrix_->get(x, y) != bitMatrix_->get(y, x)) { + bitMatrix_->flip(y, x); + bitMatrix_->flip(x, y); + } + } + } +} + +} // namespace qrcode +} // namespace zxing diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/bitmatrixparser.hpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/bitmatrixparser.hpp new file mode 100644 index 00000000000..ff5beec1726 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/bitmatrixparser.hpp @@ -0,0 +1,53 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_QRCODE_DECODER_BITMATRIXPARSER_HPP__ +#define __ZXING_QRCODE_DECODER_BITMATRIXPARSER_HPP__ + +#include "../../common/array.hpp" +#include "../../common/bitmatrix.hpp" +#include "../../common/counted.hpp" +#include "../../errorhandler.hpp" +#include "../format_information.hpp" +#include "../version.hpp" + +namespace zxing { +namespace qrcode { + +class BitMatrixParser : public Counted { +private: + Ref bitMatrix_; + Version *parsedVersion_; + Ref parsedFormatInfo_; + bool mirror_; + + int copyBit(size_t x, size_t y, int versionBits); + +public: + BitMatrixParser(Ref bitMatrix, ErrorHandler &err_handler); + Ref readFormatInformation(ErrorHandler &err_handler); + Version *readVersion(ErrorHandler &err_handler); + ArrayRef readCodewords(ErrorHandler &err_handler); + +public: + void remask(); + void setMirror(bool mirror); + void mirror(); + void mirrorH(); + +private: + BitMatrixParser(const BitMatrixParser &); + BitMatrixParser &operator=(const BitMatrixParser &); +}; + +} // namespace qrcode +} // namespace zxing + +#endif // __ZXING_QRCODE_DECODER_BITMATRIXPARSER_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/datablock.cpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/datablock.cpp new file mode 100644 index 00000000000..1b329cf579c --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/datablock.cpp @@ -0,0 +1,103 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "datablock.hpp" +namespace zxing { +namespace qrcode { + +using zxing::ErrorHandler; + +DataBlock::DataBlock(int numDataCodewords, ArrayRef codewords) + : numDataCodewords_(numDataCodewords), codewords_(codewords) {} + +int DataBlock::getNumDataCodewords() { return numDataCodewords_; } + +ArrayRef DataBlock::getCodewords() { return codewords_; } + +std::vector > DataBlock::getDataBlocks(ArrayRef rawCodewords, Version *version, + ErrorCorrectionLevel &ecLevel, + ErrorHandler &err_handler) { + // Figure out the number and size of data blocks used by this version and + // error correction level + ECBlocks &ecBlocks = version->getECBlocksForLevel(ecLevel); + + // First count the total number of data blocks + int totalBlocks = 0; + vector ecBlockArray = ecBlocks.getECBlocks(); + for (size_t i = 0; i < ecBlockArray.size(); i++) { + totalBlocks += ecBlockArray[i]->getCount(); + } + + // Now establish DataBlocks of the appropriate size and number of data + // codewords + std::vector > result(totalBlocks); + int numResultBlocks = 0; + for (size_t j = 0; j < ecBlockArray.size(); j++) { + ECB *ecBlock = ecBlockArray[j]; + for (int i = 0; i < ecBlock->getCount(); i++) { + int numDataCodewords = ecBlock->getDataCodewords(); + int numBlockCodewords = ecBlocks.getECCodewords() + numDataCodewords; + ArrayRef buffer(numBlockCodewords); + Ref blockRef(new DataBlock(numDataCodewords, buffer)); + result[numResultBlocks++] = blockRef; + } + } + + // All blocks have the same amount of data, except that the last n + // (where n may be 0) have 1 more byte. Figure out where these start. + int shorterBlocksTotalCodewords = result[0]->codewords_->size(); + int longerBlocksStartAt = result.size() - 1; + while (longerBlocksStartAt >= 0) { + int numCodewords = result[longerBlocksStartAt]->codewords_->size(); + if (numCodewords == shorterBlocksTotalCodewords) { + break; + } + if (numCodewords != shorterBlocksTotalCodewords + 1) { + err_handler = + zxing::IllegalArgumentErrorHandler("Data block sizes differ by more than 1"); + return std::vector >(); + } + longerBlocksStartAt--; + } + longerBlocksStartAt++; + + int shorterBlocksNumDataCodewords = shorterBlocksTotalCodewords - ecBlocks.getECCodewords(); + // The last elements of result may be 1 element longer; + // first fill out as many elements as all of them have + int rawCodewordsOffset = 0; + for (int i = 0; i < shorterBlocksNumDataCodewords; i++) { + for (int j = 0; j < numResultBlocks; j++) { + result[j]->codewords_[i] = rawCodewords[rawCodewordsOffset++]; + } + } + // Fill out the last data block in the longer ones + for (int j = longerBlocksStartAt; j < numResultBlocks; j++) { + result[j]->codewords_[shorterBlocksNumDataCodewords] = rawCodewords[rawCodewordsOffset++]; + } + // Now add in error correction blocks + int max = result[0]->codewords_->size(); + for (int i = shorterBlocksNumDataCodewords; i < max; i++) { + for (int j = 0; j < numResultBlocks; j++) { + int iOffset = j < longerBlocksStartAt ? i : i + 1; + result[j]->codewords_[iOffset] = rawCodewords[rawCodewordsOffset++]; + } + } + + if (rawCodewordsOffset != rawCodewords->size()) { + err_handler = + zxing::IllegalArgumentErrorHandler("rawCodewordsOffset != rawCodewords.length"); + return std::vector >(); + } + + return result; +} + +} // namespace qrcode +} // namespace zxing diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/datablock.hpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/datablock.hpp new file mode 100644 index 00000000000..8b23e6e6c61 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/datablock.hpp @@ -0,0 +1,44 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_QRCODE_DECODER_DATABLOCK_HPP__ +#define __ZXING_QRCODE_DECODER_DATABLOCK_HPP__ + +#include "../../common/array.hpp" +#include "../../common/counted.hpp" +#include "../../errorhandler.hpp" +#include "../error_correction_level.hpp" +#include "../version.hpp" + +#include + +namespace zxing { +namespace qrcode { + +class DataBlock : public Counted { +private: + int numDataCodewords_; + ArrayRef codewords_; + + DataBlock(int numDataCodewords, ArrayRef codewords); + +public: + static std::vector > getDataBlocks(ArrayRef rawCodewords, Version *version, + ErrorCorrectionLevel &ecLevel, + ErrorHandler &err_handler); + + int getNumDataCodewords(); + ArrayRef getCodewords(); +}; + +} // namespace qrcode +} // namespace zxing + +#endif // __ZXING_QRCODE_DECODER_DATABLOCK_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/datamask.cpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/datamask.cpp new file mode 100644 index 00000000000..56c42b5bc24 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/datamask.cpp @@ -0,0 +1,120 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "datamask.hpp" +namespace zxing { +namespace qrcode { + +using zxing::ErrorHandler; + +DataMask::DataMask() {} + +DataMask::~DataMask() {} + +DataMask& DataMask::forReference(int reference, ErrorHandler& err_handler) { + if (reference < 0 || reference > 7) { + err_handler = zxing::IllegalArgumentErrorHandler("reference must be between 0 and 7"); + return *DATA_MASKS[0]; + } + return *DATA_MASKS[reference]; +} + +void DataMask::unmaskBitMatrix(BitMatrix& bits, size_t dimension) { + for (size_t y = 0; y < dimension; y++) { + for (size_t x = 0; x < dimension; x++) { + // TODO: check why the coordinates have to be swapped + if (isMasked(y, x)) { + bits.flip(x, y); + } + } + } +} + +/** + * 000: mask bits for which (x + y) mod 2 == 0 + */ +class DataMask000 : public DataMask { +public: + bool isMasked(size_t x, size_t y) override { return ((x + y) % 2) == 0; } +}; + +/** + * 001: mask bits for which x mod 2 == 0 + */ +class DataMask001 : public DataMask { +public: + bool isMasked(size_t x, size_t) override { return (x % 2) == 0; } +}; + +/** + * 010: mask bits for which y mod 3 == 0 + */ +class DataMask010 : public DataMask { +public: + bool isMasked(size_t, size_t y) override { return y % 3 == 0; } +}; + +/** + * 011: mask bits for which (x + y) mod 3 == 0 + */ +class DataMask011 : public DataMask { +public: + bool isMasked(size_t x, size_t y) override { return (x + y) % 3 == 0; } +}; + +/** + * 100: mask bits for which (x/2 + y/3) mod 2 == 0 + */ +class DataMask100 : public DataMask { +public: + bool isMasked(size_t x, size_t y) override { return (((x >> 1) + (y / 3)) % 2) == 0; } +}; + +/** + * 101: mask bits for which xy mod 2 + xy mod 3 == 0 + */ +class DataMask101 : public DataMask { +public: + bool isMasked(size_t x, size_t y) override { + size_t temp = x * y; + return (temp % 2) + (temp % 3) == 0; + } +}; + +/** + * 110: mask bits for which (xy mod 2 + xy mod 3) mod 2 == 0 + */ +class DataMask110 : public DataMask { +public: + bool isMasked(size_t x, size_t y) override { + size_t temp = x * y; + return (((temp % 2) + (temp % 3)) % 2) == 0; + } +}; + +/** + * 111: mask bits for which ((x+y)mod 2 + xy mod 3) mod 2 == 0 + */ +class DataMask111 : public DataMask { +public: + bool isMasked(size_t x, size_t y) override { + return ((((x + y) % 2) + ((x * y) % 3)) % 2) == 0; + } +}; + +vector > DataMask::DATA_MASKS = { + Ref(new DataMask000()), Ref(new DataMask001()), + Ref(new DataMask010()), Ref(new DataMask011()), + Ref(new DataMask100()), Ref(new DataMask101()), + Ref(new DataMask110()), Ref(new DataMask111()), +}; + +} // namespace qrcode +} // namespace zxing diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/datamask.hpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/datamask.hpp new file mode 100644 index 00000000000..98b7f9e8569 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/datamask.hpp @@ -0,0 +1,40 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_QRCODE_DECODER_DATAMASK_HPP__ +#define __ZXING_QRCODE_DECODER_DATAMASK_HPP__ + +#include "../../common/array.hpp" +#include "../../common/bitmatrix.hpp" +#include "../../common/counted.hpp" +#include "../../errorhandler.hpp" + +#include + +namespace zxing { +namespace qrcode { + +class DataMask : public Counted { +private: + static std::vector > DATA_MASKS; + +protected: +public: + DataMask(); + virtual ~DataMask(); + void unmaskBitMatrix(BitMatrix& matrix, size_t dimension); + virtual bool isMasked(size_t x, size_t y) = 0; + static DataMask& forReference(int reference, ErrorHandler& err_handler); +}; + +} // namespace qrcode +} // namespace zxing + +#endif // __ZXING_QRCODE_DECODER_DATAMASK_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoded_bit_stream_parser.cpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoded_bit_stream_parser.cpp new file mode 100644 index 00000000000..86c8c43b207 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoded_bit_stream_parser.cpp @@ -0,0 +1,492 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "decoded_bit_stream_parser.hpp" +#include "../../common/stringutils.hpp" +#include "../../zxing.hpp" + +#include +#ifndef NO_ICONV_INSIDE +#include +#endif +#include + +#undef ICONV_CONST +#define ICONV_CONST const + +#ifndef ICONV_CONST +#define ICONV_CONST /**/ +#endif + +using zxing::ErrorHandler; + +// Add this to fix both Mac and Windows compilers +template +class sloppy {}; + +// convert between T** and const T** +template +class sloppy { + T** t; + +public: + explicit sloppy(T** mt) : t(mt) {} + explicit sloppy(const T** mt) : t(const_cast(mt)) {} + + operator T* *() const { return t; } + operator const T* *() const { return const_cast(t); } +}; + +using namespace std; +using namespace zxing; +using namespace zxing::qrcode; +using namespace zxing::common; + +const char DecodedBitStreamParser::ALPHANUMERIC_CHARS[] = { + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', + 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', + 'U', 'V', 'W', 'X', 'Y', 'Z', ' ', '$', '%', '*', '+', '-', '.', '/', ':'}; + +// string DecodedBitStreamParser::outputCharset = "UTF-8"; + +namespace { +int GB2312_SUBSET = 1; +} + +void DecodedBitStreamParser::append(std::string& result, string const& in, + ErrorHandler& err_handler) { + append(result, (char const*)in.c_str(), in.length(), err_handler); +} + +void DecodedBitStreamParser::append(std::string& result, const char* bufIn, size_t nIn, + ErrorHandler& err_handler) { + if (err_handler.ErrCode()) return; +#ifndef NO_ICONV_INSIDE + if (nIn == 0) { + return; + } + iconv_t cd; + // cout< 0) { + size_t oneway = iconv(cd, sloppy(&fromPtr), &nFrom, sloppy(&toPtr), &nTo); + + if (oneway == (size_t)(-1)) { + iconv_close(cd); + delete[] bufOut; + err_handler = zxing::ReaderErrorHandler("error converting characters"); + return; + } + } + iconv_close(cd); + + int nResult = maxOut - nTo; + bufOut[nResult] = '\0'; + result.append((const char*)bufOut); + delete[] bufOut; +#else + result.append((const char*)bufIn, nIn); +#endif +} + +void DecodedBitStreamParser::decodeHanziSegment(Ref bits_, string& result, int count, + ErrorHandler& err_handler) { + BitSource& bits(*bits_); + // Don't crash trying to read more bits than we have available. + if (count * 13 > bits.available()) { + err_handler = zxing::FormatErrorHandler("decodeKanjiSegment"); + return; + } + + // Each character will require 2 bytes. Read the characters as 2-byte pairs + // and decode as GB2312 afterwards + size_t nBytes = 2 * count; + char* buffer = new char[nBytes]; + int offset = 0; + while (count > 0) { + // Each 13 bits encodes a 2-byte character + int twoBytes = bits.readBits(13, err_handler); + if (err_handler.ErrCode()) return; + int assembledTwoBytes = ((twoBytes / 0x060) << 8) | (twoBytes % 0x060); + if (assembledTwoBytes < 0x003BF) { + // In the 0xA1A1 to 0xAAFE range + assembledTwoBytes += 0x0A1A1; + } else { + // In the 0xB0A1 to 0xFAFE range + assembledTwoBytes += 0x0A6A1; + } + buffer[offset] = (char)((assembledTwoBytes >> 8) & 0xFF); + buffer[offset + 1] = (char)(assembledTwoBytes & 0xFF); + offset += 2; + count--; + } + // for(int i=0;i bits, std::string& result, int count, + ErrorHandler& err_handler) { + // Each character will require 2 bytes. Read the characters as 2-byte pairs + // and decode as Shift_JIS afterwards + size_t nBytes = 2 * count; + char* buffer = new char[nBytes]; + int offset = 0; + while (count > 0) { + // Each 13 bits encodes a 2-byte character + + int twoBytes = bits->readBits(13, err_handler); + if (err_handler.ErrCode()) return; + int assembledTwoBytes = ((twoBytes / 0x0C0) << 8) | (twoBytes % 0x0C0); + if (assembledTwoBytes < 0x01F00) { + // In the 0x8140 to 0x9FFC range + assembledTwoBytes += 0x08140; + } else { + // In the 0xE040 to 0xEBBF range + assembledTwoBytes += 0x0C140; + } + buffer[offset] = (char)(assembledTwoBytes >> 8); + buffer[offset + 1] = (char)assembledTwoBytes; + offset += 2; + count--; + } + + append(result, buffer, nBytes, err_handler); + if (err_handler.ErrCode()) { + delete[] buffer; + return; + } + // cout< bits_, string& result, int count, + CharacterSetECI* currentCharacterSetECI, + ArrayRef >& byteSegments, + ErrorHandler& err_handler) { + int nBytes = count; + BitSource& bits(*bits_); + // Don't crash trying to read more bits than we have available. + int available = bits.available(); + // try to repair count data if count data is invalid + if (count * 8 > available) { + count = (available + 7 / 8); + } + + ArrayRef bytes_(count); + char* readBytes = &(*bytes_)[0]; + for (int i = 0; i < count; i++) { + // readBytes[i] = (char) bits.readBits(8); + int readBits = available < 8 ? available : 8; + readBytes[i] = (char)bits.readBits(readBits, err_handler); + } + if (err_handler.ErrCode()) return; + // vector encoding; + string encoding; + + if (currentCharacterSetECI == 0) { + // The spec isn't clear on this mode; see + // section 6.4.5: t does not say which encoding to assuming + // upon decoding. I have seen ISO-8859-1 used as well as + // Shift_JIS -- without anything like an ECI designator to + // give a hint. + encoding = outputCharset; + + } else { + // encoding .push_back(currentCharacterSetECI->name()); + encoding = currentCharacterSetECI->name(); + } + // cout<<"encoding: "<values().push_back(bytes_); +} + +void DecodedBitStreamParser::decodeNumericSegment(Ref bits, std::string& result, + int count, ErrorHandler& err_handler) { + int nBytes = count; + // char* bytes = new char[nBytes]; + ArrayRef bytes = ArrayRef(new Array(nBytes)); + int i = 0; + // Read three digits at a time + while (count >= 3) { + // Each 10 bits encodes three digits + if (bits->available() < 10) { + err_handler = zxing::ReaderErrorHandler("format exception"); + return; + } + int threeDigitsBits = bits->readBits(10, err_handler); + if (err_handler.ErrCode()) return; + if (threeDigitsBits >= 1000) { + ostringstream s; + s << "Illegal value for 3-digit unit: " << threeDigitsBits; + err_handler = zxing::ReaderErrorHandler(s.str().c_str()); + return; + } + bytes[i++] = ALPHANUMERIC_CHARS[threeDigitsBits / 100]; + bytes[i++] = ALPHANUMERIC_CHARS[(threeDigitsBits / 10) % 10]; + bytes[i++] = ALPHANUMERIC_CHARS[threeDigitsBits % 10]; + count -= 3; + } + if (count == 2) { + if (bits->available() < 7) { + err_handler = zxing::ReaderErrorHandler("format exception"); + return; + } + // Two digits left over to read, encoded in 7 bits + int twoDigitsBits = bits->readBits(7, err_handler); + if (err_handler.ErrCode()) return; + if (twoDigitsBits >= 100) { + ostringstream s; + s << "Illegal value for 2-digit unit: " << twoDigitsBits; + err_handler = zxing::ReaderErrorHandler(s.str().c_str()); + return; + } + bytes[i++] = ALPHANUMERIC_CHARS[twoDigitsBits / 10]; + bytes[i++] = ALPHANUMERIC_CHARS[twoDigitsBits % 10]; + } else if (count == 1) { + if (bits->available() < 4) { + err_handler = zxing::ReaderErrorHandler("format exception"); + return; + } + // One digit left over to read + int digitBits = bits->readBits(4, err_handler); + if (err_handler.ErrCode()) return; + if (digitBits >= 10) { + ostringstream s; + s << "Illegal value for digit unit: " << digitBits; + err_handler = zxing::ReaderErrorHandler(s.str().c_str()); + return; + } + bytes[i++] = ALPHANUMERIC_CHARS[digitBits]; + } + append(result, bytes->data(), nBytes, err_handler); + if (err_handler.ErrCode()) return; +} + +char DecodedBitStreamParser::toAlphaNumericChar(size_t value, ErrorHandler& err_handler) { + if (value >= sizeof(DecodedBitStreamParser::ALPHANUMERIC_CHARS)) { + err_handler = zxing::FormatErrorHandler("toAlphaNumericChar"); + return 0; + } + return ALPHANUMERIC_CHARS[value]; +} + +void DecodedBitStreamParser::decodeAlphanumericSegment(Ref bits_, string& result, + int count, bool fc1InEffect, + ErrorHandler& err_handler) { + BitSource& bits(*bits_); + ostringstream bytes; + // Read two characters at a time + while (count > 1) { + if (bits.available() < 11) { + err_handler = zxing::FormatErrorHandler("decodeAlphanumericSegment"); + return; + } + int nextTwoCharsBits = bits.readBits(11, err_handler); + bytes << toAlphaNumericChar(nextTwoCharsBits / 45, err_handler); + bytes << toAlphaNumericChar(nextTwoCharsBits % 45, err_handler); + if (err_handler.ErrCode()) return; + count -= 2; + } + if (count == 1) { + // special case: one character left + if (bits.available() < 6) { + err_handler = zxing::FormatErrorHandler("decodeAlphanumericSegment"); + return; + } + bytes << toAlphaNumericChar(bits.readBits(6, err_handler), err_handler); + if (err_handler.ErrCode()) return; + } + // See section 6.4.8.1, 6.4.8.2 + string s = bytes.str(); + if (fc1InEffect) { + // We need to massage the result a bit if in an FNC1 mode: + ostringstream r; + for (size_t i = 0; i < s.length(); i++) { + if (s[i] != '%') { + r << s[i]; + } else { + if (i < s.length() - 1 && s[i + 1] == '%') { + // %% is rendered as % + r << s[i++]; + } else { + // In alpha mode, % should be converted to FNC1 separator + // 0x1D + r << (char)0x1D; + } + } + } + s = r.str(); + } + append(result, s, err_handler); + if (err_handler.ErrCode()) return; +} + +namespace { +int parseECIValue(BitSource& bits, ErrorHandler& err_handler) { + int firstByte = bits.readBits(8, err_handler); + if (err_handler.ErrCode()) return 0; + if ((firstByte & 0x80) == 0) { + // just one byte + return firstByte & 0x7F; + } + if ((firstByte & 0xC0) == 0x80) { + // two bytes + int secondByte = bits.readBits(8, err_handler); + if (err_handler.ErrCode()) return 0; + return ((firstByte & 0x3F) << 8) | secondByte; + } + if ((firstByte & 0xE0) == 0xC0) { + // three bytes + int secondThirdBytes = bits.readBits(16, err_handler); + if (err_handler.ErrCode()) return 0; + return ((firstByte & 0x1F) << 16) | secondThirdBytes; + } + err_handler = zxing::FormatErrorHandler("parseECIValue"); + return 0; +} +} // namespace + +Ref DecodedBitStreamParser::decode(ArrayRef bytes, Version* version, + ErrorCorrectionLevel const& ecLevel, + ErrorHandler& err_handler, int iVersion) { + Ref bits_(new BitSource(bytes)); + BitSource& bits(*bits_); + string result; + result.reserve(50); + Mode* mode = 0; + string modeName; + ArrayRef > byteSegments(0); + + CharacterSetECI* currentCharacterSetECI = 0; + bool fc1InEffect = false; + + outputCharset = "UTF-8"; + do { + // While still another segment to read... + if (bits.available() < 4) { + // OK, assume we're done. Really, a TERMINATOR mode should have been + // recorded here + mode = &Mode::TERMINATOR; + } else { + mode = &Mode::forBits(bits.readBits(4, err_handler), + err_handler); // mode is encoded by 4 bits + if (err_handler.ErrCode()) return Ref(); + } + + if (mode != &Mode::TERMINATOR) { + if ((mode == &Mode::FNC1_FIRST_POSITION) || (mode == &Mode::FNC1_SECOND_POSITION)) { + // We do little with FNC1 except alter the parsed result a bit + // according to the spec + fc1InEffect = true; + } else if (mode == &Mode::STRUCTURED_APPEND) { + if (bits.available() < 16) { + err_handler = zxing::FormatErrorHandler("decode"); + return Ref(); + } + // not really supported; all we do is ignore it + // Read next 8 bits (symbol sequence #) and 8 bits (parity + // data), then continue + bits.readBits(16, err_handler); + if (err_handler.ErrCode()) return Ref(); + } else if (mode == &Mode::ECI) { + // Count doesn't apply to ECI + int value = parseECIValue(bits, err_handler); + if (err_handler.ErrCode()) Ref(); + currentCharacterSetECI = CharacterSetECI::getCharacterSetECIByValueFind(value); + if (currentCharacterSetECI == 0) { + err_handler = zxing::FormatErrorHandler("decode"); + return Ref(); + } + } else { + // First handle Hanzi mode which does not start with character + // count + if (mode == &Mode::HANZI) { + // chinese mode contains a sub set indicator right after + // mode indicator + int subset = bits.readBits(4, err_handler); + int countHanzi = + bits.readBits(mode->getCharacterCountBits(version), err_handler); + if (err_handler.ErrCode()) return Ref(); + if (subset == GB2312_SUBSET) { + decodeHanziSegment(bits_, result, countHanzi, err_handler); + if (err_handler.ErrCode()) Ref(); + outputCharset = "GB2312"; + modeName = mode->getName(); + } + } else { + // "Normal" QR code modes: + // How many characters will follow, encoded in this mode? + int count = bits.readBits(mode->getCharacterCountBits(version), err_handler); + if (err_handler.ErrCode()) return Ref(); + if (mode == &Mode::NUMERIC) { + decodeNumericSegment(bits_, result, count, err_handler); + if (err_handler.ErrCode()) { + err_handler = zxing::FormatErrorHandler("decode"); + return Ref(); + } + modeName = mode->getName(); + } else if (mode == &Mode::ALPHANUMERIC) { + decodeAlphanumericSegment(bits_, result, count, fc1InEffect, err_handler); + if (err_handler.ErrCode()) Ref(); + modeName = mode->getName(); + } else if (mode == &Mode::BYTE) { + decodeByteSegment(bits_, result, count, currentCharacterSetECI, + byteSegments, err_handler); + if (err_handler.ErrCode()) { + err_handler = zxing::FormatErrorHandler("decode"); + return Ref(); + } + modeName = mode->getName(); + // outputCharset = getResultCharset(); + } else if (mode == &Mode::KANJI) { + // int countKanji = + // bits.readBits(mode->getCharacterCountBits(version)); + // cout<<"countKanji: "<(); + modeName = mode->getName(); + } else { + err_handler = zxing::FormatErrorHandler("decode"); + return Ref(); + } + } + } + } + } while (mode != &Mode::TERMINATOR); + return Ref(new DecoderResult(bytes, Ref(new String(result)), + byteSegments, (string)ecLevel, + (string)outputCharset, iVersion, modeName)); +} diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoded_bit_stream_parser.hpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoded_bit_stream_parser.hpp new file mode 100644 index 00000000000..3c047545fce --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoded_bit_stream_parser.hpp @@ -0,0 +1,69 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_QRCODE_DECODER_DECODEDBITSTREAMPARSER_HPP__ +#define __ZXING_QRCODE_DECODER_DECODEDBITSTREAMPARSER_HPP__ + +#include "../../common/array.hpp" +#include "../../common/bitsource.hpp" +#include "../../common/characterseteci.hpp" +#include "../../common/counted.hpp" +#include "../../common/decoder_result.hpp" +#include "../../decodehints.hpp" +#include "../../errorhandler.hpp" +#include "mode.hpp" + +#include +#include +#include + +namespace zxing { +namespace qrcode { + +class DecodedBitStreamParser { +public: + DecodedBitStreamParser() : outputCharset("UTF-8") {} + +private: + static char const ALPHANUMERIC_CHARS[]; + + string outputCharset; + // string outputCharset; + + char toAlphaNumericChar(size_t value, ErrorHandler& err_handler); + + void decodeHanziSegment(Ref bits, std::string& result, int count, + ErrorHandler& err_handler); + void decodeKanjiSegment(Ref bits, std::string& result, int count, + ErrorHandler& err_handler); + void decodeByteSegment(Ref bits, std::string& result, int count); + void decodeByteSegment(Ref bits_, std::string& result, int count, + zxing::common::CharacterSetECI* currentCharacterSetECI, + ArrayRef >& byteSegments, ErrorHandler& err_handler); + void decodeAlphanumericSegment(Ref bits, std::string& result, int count, + bool fc1InEffect, ErrorHandler& err_handler); + void decodeNumericSegment(Ref bits, std::string& result, int count, + ErrorHandler& err_handler); + + void append(std::string& ost, const char* bufIn, size_t nIn, ErrorHandler& err_handler); + void append(std::string& ost, std::string const& in, ErrorHandler& err_handler); + +public: + Ref decode(ArrayRef bytes, Version* version, + ErrorCorrectionLevel const& ecLevel, ErrorHandler& err_handler, + int iVersion = -1); + + // string getResultCharset(); +}; + +} // namespace qrcode +} // namespace zxing + +#endif // __ZXING_QRCODE_DECODER_DECODEDBITSTREAMPARSER_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoder.cpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoder.cpp new file mode 100644 index 00000000000..d0e68a8f888 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoder.cpp @@ -0,0 +1,227 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "decoder.hpp" +#include "../error_correction_level.hpp" +#include "../version.hpp" +#include "datablock.hpp" +#include "decoded_bit_stream_parser.hpp" +#include "qrcode_decoder_metadata.hpp" + +#include + +using std::cout; +using std::endl; +using zxing::DecoderResult; +using zxing::Ref; +using zxing::qrcode::Decoder; + +// VC++ +// The main class which implements QR Code decoding -- as opposed to locating +// and extracting the QR Code from an image. +using zxing::ArrayRef; +using zxing::BitMatrix; +using zxing::DetectorResult; +using zxing::ErrorHandler; + +Decoder::Decoder() : rsDecoder_(Ref(GF_QR_CODE_FIELD_256)) { + possibleVersion_ = 0; + possibleFix_ = 0; + decoderState_ = NOTSTART; +} + +// Convenience method that can decode a QR Code represented as a 2D array of +// booleans. "true" is taken to mean a black module. +Ref Decoder::decode(Ref bits, ErrorHandler &err_handler) { + string errMsg = ""; + + // Used for mirrored qrcode + int width = bits->getWidth(); + int height = bits->getHeight(); + + Ref bits2(new BitMatrix(width, height, bits->getPtr(), err_handler)); + if (err_handler.ErrCode()) return Ref(); + Ref rst = decode(bits, false, err_handler); + if (err_handler.ErrCode() || rst == NULL) { + errMsg = err_handler.ErrMsg(); + } else { + return rst; + } + + err_handler.Reset(); + Ref result = decode(bits2, true, err_handler); + if (err_handler.ErrCode()) { + return Ref(); + } else { + // Success! Notify the caller that the code was mirrored. + result->setOther(Ref(new QRCodeDecoderMetaData(true))); + return result; + } +}; + +Ref Decoder::decode(Ref bits, bool isMirror, ErrorHandler &err_handler) { + // Ref Decoder::decode(BitMatrixParser& parser) { + // Construct a parser and read version, error-correction level + BitMatrixParser parser(bits, err_handler); + if (err_handler.ErrCode()) return Ref(); + + if (isMirror == true) { + // Revert the bit matrix + parser.remask(); + + // Will be attempting a mirrored reading of the version and format info. + parser.setMirror(true); + + // Preemptively read the version. + parser.readVersion(err_handler); + if (err_handler.ErrCode()) { + err_handler = zxing::ReaderErrorHandler("Decoder::decode mirror & no mirror"); + return Ref(); + } + + // Preemptively read the format information. + parser.readFormatInformation(err_handler); + if (err_handler.ErrCode()) return Ref(); + + /* + * Since we're here, this means we have successfully detected some kind + * of version and format information when mirrored. This is a good sign, + * that the QR code may be mirrored, and we should try once more with a + * mirrored content. + */ + // Prepare for a mirrored reading. + parser.mirror(); + } + + decoderState_ = START; + possibleFix_ = 0; + Version *version = parser.readVersion(err_handler); + if (err_handler.ErrCode() || version == NULL) { + err_handler = ReaderErrorHandler("Decoder::decode mirror & no mirror"); + return Ref(); + } + decoderState_ = READVERSION; + float fixedPatternScore = estimateFixedPattern(bits, version, err_handler); + if (err_handler.ErrCode()) return Ref(); + + Ref formatInfo = parser.readFormatInformation(err_handler); + if (err_handler.ErrCode()) return Ref(); + ErrorCorrectionLevel &ecLevel = formatInfo->getErrorCorrectionLevel(); + + decoderState_ = READERRORCORRECTIONLEVEL; + + // Read codewords + ArrayRef codewords(parser.readCodewords(err_handler)); + if (err_handler.ErrCode()) { + err_handler = zxing::ReaderErrorHandler("Decoder::decode mirror & no mirror"); + return Ref(); + } + + decoderState_ = READCODEWORDSORRECTIONLEVEL; + possibleFix_ = fixedPatternScore; + + // Separate into data blocks + std::vector > dataBlocks( + DataBlock::getDataBlocks(codewords, version, ecLevel, err_handler)); + if (err_handler.ErrCode()) return Ref(); + + // Count total number of data bytes + int totalBytes = 0; + for (size_t i = 0; i < dataBlocks.size(); i++) { + totalBytes += dataBlocks[i]->getNumDataCodewords(); + } + ArrayRef resultBytes(totalBytes); + int resultOffset = 0; + + // Error-correct and copy data blocks together into a stream of bytes + for (size_t j = 0; j < dataBlocks.size(); j++) { + err_handler.Reset(); + Ref dataBlock(dataBlocks[j]); + ArrayRef codewordBytes = dataBlock->getCodewords(); + int numDataCodewords = dataBlock->getNumDataCodewords(); + + correctErrors(codewordBytes, numDataCodewords, err_handler); + if (err_handler.ErrCode()) return Ref(); + + for (int i = 0; i < numDataCodewords; i++) { + resultBytes[resultOffset++] = codewordBytes[i]; + } + } + + decoderState_ = FINISH; + // return DecodedBitStreamParser::decode(resultBytes, + DecodedBitStreamParser dbs_parser; + Ref rst = + dbs_parser.decode(resultBytes, version, ecLevel, err_handler, version->getVersionNumber()); + + if (err_handler.ErrCode()) return Ref(); + return rst; +} + +// Given data and error-correction codewords received, possibly corrupted by +// errors, attempts to correct the errors in-place using Reed-Solomon error +// correction.

codewordBytes: data and error correction codewords +// numDataCodewords: number of codewords that are data bytes +void Decoder::correctErrors(ArrayRef codewordBytes, int numDataCodewords, + ErrorHandler &err_handler) { + // First read into an arrya of ints + int numCodewords = codewordBytes->size(); + ArrayRef codewordInts(numCodewords); + for (int i = 0; i < numCodewords; i++) { + codewordInts[i] = codewordBytes[i] & 0xff; + } + int numECCodewords = numCodewords - numDataCodewords; + bool correctErrorsFinishished = false; + + rsDecoder_.decode(codewordInts, numECCodewords, err_handler); + if (err_handler.ErrCode()) { + return; + } + + correctErrorsFinishished = true; + + // Copy back into array of bytes -- only need to worry about the bytes that + // were data We don't care about errors in the error-correction codewords + if (correctErrorsFinishished) { + for (int i = 0; i < numDataCodewords; i++) { + codewordBytes[i] = (char)codewordInts[i]; + } + } +} + +unsigned int Decoder::getPossibleVersion() { return possibleVersion_; } + +float Decoder::estimateFixedPattern(Ref bits, zxing::qrcode::Version *version, + ErrorHandler &err_handler) { + Ref fixedPatternValue = version->buildFixedPatternValue(err_handler); + if (err_handler.ErrCode()) { + err_handler = zxing::ReaderErrorHandler("Decoder::decode mirror & no mirror"); + return -1.0; + } + Ref fixedPatternTemplate = version->buildFixedPatternTemplate(err_handler); + if (err_handler.ErrCode()) { + err_handler = zxing::ReaderErrorHandler("Decoder::decode mirror & no mirror"); + return -1.0; + } + + int iSum = 0; + int iCount = 0; + for (int i = 0; i < bits->getHeight(); ++i) { + for (int j = 0; j < bits->getWidth(); ++j) { + if (fixedPatternTemplate->get(i, j)) { + iSum++; + if (bits->get(i, j) == fixedPatternValue->get(i, j)) iCount++; + } + } + } + + float possbielFix = 2.0 * iCount / iSum - 1; + return possbielFix > 0 ? possbielFix : 0; +} diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoder.hpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoder.hpp new file mode 100644 index 00000000000..3ce67f7765e --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoder.hpp @@ -0,0 +1,65 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_QRCODE_DECODER_DECODER_HPP__ +#define __ZXING_QRCODE_DECODER_DECODER_HPP__ + +#include "../../common/array.hpp" +#include "../../common/bitmatrix.hpp" +#include "../../common/counted.hpp" +#include "../../common/decoder_result.hpp" +#include "../../common/detector_result.hpp" +#include "../../common/reedsolomon/reed_solomon_decoder.hpp" +#include "../../errorhandler.hpp" +#include "../version.hpp" +#include "bitmatrixparser.hpp" + +namespace zxing { +namespace qrcode { + +class Decoder { +public: + enum DecoderState { + NOTSTART = 19, + START = 20, + READVERSION = 21, + READERRORCORRECTIONLEVEL = 22, + READCODEWORDSORRECTIONLEVEL = 23, + FINISH = 24 + }; + +private: + DecoderState decoderState_; + float possibleFix_; + ReedSolomonDecoder rsDecoder_; + void correctErrors(ArrayRef bytes, int numDataCodewords, ErrorHandler& err_handler); + +public: + Decoder(); + Ref decode(Ref bits, ErrorHandler& err_handler); + +private: + Ref decode(Ref bits, bool isMirror, ErrorHandler& err_handler); + + float estimateFixedPattern(Ref bits, Version* version, ErrorHandler& err_handler); + +private: + unsigned int possibleVersion_; + +public: + unsigned int getPossibleVersion(); + DecoderState getState() { return decoderState_; } + float getPossibleFix() { return possibleFix_; } +}; + +} // namespace qrcode +} // namespace zxing + +#endif // __ZXING_QRCODE_DECODER_DECODER_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/mode.cpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/mode.cpp new file mode 100644 index 00000000000..5ba6ab91d7c --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/mode.cpp @@ -0,0 +1,91 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "mode.hpp" +#include "../../common/counted.hpp" +#include "../../zxing.hpp" +#include "../version.hpp" + +#include + +using std::ostringstream; +using zxing::qrcode::Mode; + +// VC++ +using zxing::ErrorHandler; +using zxing::qrcode::Version; + +Mode Mode::TERMINATOR(0, 0, 0, 0x00, "TERMINATOR"); +Mode Mode::NUMERIC(10, 12, 14, 0x01, "NUMERIC"); +Mode Mode::ALPHANUMERIC(9, 11, 13, 0x02, "ALPHANUMERIC"); +Mode Mode::STRUCTURED_APPEND(0, 0, 0, 0x03, "STRUCTURED_APPEND"); +Mode Mode::BYTE(8, 16, 16, 0x04, "BYTE"); +Mode Mode::ECI(0, 0, 0, 0x07, "ECI"); +Mode Mode::KANJI(8, 10, 12, 0x08, "KANJI"); +Mode Mode::FNC1_FIRST_POSITION(0, 0, 0, 0x05, "FNC1_FIRST_POSITION"); +Mode Mode::FNC1_SECOND_POSITION(0, 0, 0, 0x09, "FNC1_SECOND_POSITION"); +Mode Mode::HANZI(8, 10, 12, 0x0D, "HANZI"); + +// Mode::Mode(int cbv0_9, int cbv10_26, int cbv27, int /* bits */, char const* +// name) : +Mode::Mode(int cbv0_9, int cbv10_26, int cbv27, int bits, char const* name) + : characterCountBitsForVersions0To9_(cbv0_9), + characterCountBitsForVersions10To26_(cbv10_26), + // characterCountBitsForVersions27AndHigher_(cbv27), name_(name) { + characterCountBitsForVersions27AndHigher_(cbv27), + bits_(bits), + name_(name) {} + +Mode& Mode::forBits(int bits, ErrorHandler& err_handler) { + switch (bits) { + case 0x0: + return TERMINATOR; + case 0x1: + return NUMERIC; + case 0x2: + return ALPHANUMERIC; + case 0x3: + return STRUCTURED_APPEND; + case 0x4: + return BYTE; + case 0x5: + return FNC1_FIRST_POSITION; + case 0x7: + return ECI; + case 0x8: + return KANJI; + case 0x9: + return FNC1_SECOND_POSITION; + case 0xD: + // 0xD is defined in GBT 18284-2000, may not be supported in foreign + // country + return HANZI; + default: + ostringstream s; + s << "Illegal mode bits: " << bits; + err_handler = zxing::ReaderErrorHandler(s.str().c_str()); + return TERMINATOR; + } +} + +int Mode::getCharacterCountBits(Version* version) const { + int number = version->getVersionNumber(); + if (number <= 9) { + return characterCountBitsForVersions0To9_; + } else if (number <= 26) { + return characterCountBitsForVersions10To26_; + } else { + return characterCountBitsForVersions27AndHigher_; + } +} + +int Mode::getBits() const { return bits_; } + +string zxing::qrcode::Mode::getName() const { return name_; } diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/mode.hpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/mode.hpp new file mode 100644 index 00000000000..2b3b146a1a1 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/mode.hpp @@ -0,0 +1,52 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_QRCODE_DECODER_MODE_HPP__ +#define __ZXING_QRCODE_DECODER_MODE_HPP__ + +#include "../../common/counted.hpp" +#include "../../errorhandler.hpp" +#include "../version.hpp" + +namespace zxing { +namespace qrcode { + +class Mode { +private: + int characterCountBitsForVersions0To9_; + int characterCountBitsForVersions10To26_; + int characterCountBitsForVersions27AndHigher_; + int bits_; + std::string name_; + + Mode(int cbv0_9, int cbv10_26, int cbv27, int bits, char const* name); + +public: + static Mode TERMINATOR; + static Mode NUMERIC; + static Mode ALPHANUMERIC; + static Mode STRUCTURED_APPEND; + static Mode BYTE; + static Mode ECI; + static Mode KANJI; + static Mode FNC1_FIRST_POSITION; + static Mode FNC1_SECOND_POSITION; + static Mode HANZI; + + static Mode& forBits(int bits, ErrorHandler& err_handler); + // int getCharacterCountBits(Version *version); + int getCharacterCountBits(Version* version) const; + int getBits() const; + string getName() const; +}; +} // namespace qrcode +} // namespace zxing + +#endif // __ZXING_QRCODE_DECODER_MODE_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/qrcode_decoder_metadata.hpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/qrcode_decoder_metadata.hpp new file mode 100644 index 00000000000..ea274f8a5b6 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/qrcode_decoder_metadata.hpp @@ -0,0 +1,66 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_QRCODE_DECODER_QRCODEDECODERMETADATA_HPP__ +#define __ZXING_QRCODE_DECODER_QRCODEDECODERMETADATA_HPP__ + +#include "../../common/array.hpp" +#include "../../common/counted.hpp" +#include "../../resultpoint.hpp" + +// VC++ +// The main class which implements QR Code decoding -- as opposed to locating +// and extracting the QR Code from an image. +// using zxing::ArrayRef; +// using zxing::BitMatrix; + +namespace zxing { +namespace qrcode { + +/** + * Meta-data container for QR Code decoding. Instances of this class may be used + * to convey information back to the decoding caller. Callers are expected to + * process this. + * + * @see com.google.zxing.common.DecoderResult#getOther() + */ +class QRCodeDecoderMetaData : public Counted { +private: + bool mirrored_; + +public: + explicit QRCodeDecoderMetaData(bool mirrored) : mirrored_(mirrored) {} + +public: + /** + * @return true if the QR Code was mirrored. + */ + bool isMirrored() { return mirrored_; }; + + /** + * Apply the result points' order correction due to mirroring. + * + * @param points Array of points to apply mirror correction to. + */ + void applyMirroredCorrection(ArrayRef >& points) { + if (!mirrored_ || points->size() < 3) { + return; + } + Ref bottomLeft = points[0]; + points[0] = points[2]; + points[2] = bottomLeft; + // No need to 'fix' top-left and alignment pattern. + }; +}; + +} // namespace qrcode +} // namespace zxing + +#endif // __ZXING_QRCODE_DECODER_QRCODEDECODERMETADATA_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern.cpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern.cpp new file mode 100644 index 00000000000..d6ac46d09dd --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern.cpp @@ -0,0 +1,45 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "alignment_pattern.hpp" + +#include +using std::abs; +using zxing::Ref; +using zxing::qrcode::AlignmentPattern; +namespace zxing { +namespace qrcode { +AlignmentPattern::AlignmentPattern(float posX, float posY, float estimatedModuleSize) + : ResultPoint(posX, posY), estimatedModuleSize_(estimatedModuleSize) {} + +// Determines if this alignment pattern "about equals" an alignment pattern at +// the stated position and size -- meaning, it is at nearly the same center with +// nearly the same size. +bool AlignmentPattern::aboutEquals(float moduleSize, float i, float j) const { + if (abs(i - getY()) <= moduleSize && abs(j - getX()) <= moduleSize) { + float moduleSizeDiff = abs(moduleSize - estimatedModuleSize_); + return moduleSizeDiff <= 1.0f || moduleSizeDiff <= estimatedModuleSize_; + } + return false; +} + +// Combines this object's current estimate of a finder pattern position and +// module size with a new estimate. It returns a new {@code FinderPattern} +// containing an average of the two. +Ref AlignmentPattern::combineEstimate(float i, float j, + float newModuleSize) const { + float combinedX = (getX() + j) / 2.0f; + float combinedY = (getY() + i) / 2.0f; + float combinedModuleSize = (estimatedModuleSize_ + newModuleSize) / 2.0f; + Ref result(new AlignmentPattern(combinedX, combinedY, combinedModuleSize)); + return result; +} +} // namespace qrcode +} // namespace zxing \ No newline at end of file diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern.hpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern.hpp new file mode 100644 index 00000000000..7d48d6b5ab0 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern.hpp @@ -0,0 +1,37 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_QRCODE_DETECTOR_ALIGNMENT_PATTERN_HPP_ +#define __ZXING_QRCODE_DETECTOR_ALIGNMENT_PATTERN_HPP_ + +#include "../../common/bitmatrix.hpp" +#include "../../resultpoint.hpp" + +#include + +namespace zxing { +namespace qrcode { + +class AlignmentPattern : public ResultPoint { +private: + float estimatedModuleSize_; + +public: + AlignmentPattern(float posX, float posY, float estimatedModuleSize); + bool aboutEquals(float moduleSize, float i, float j) const; + float getModuleSize() { return estimatedModuleSize_; }; + + Ref combineEstimate(float i, float j, float newModuleSize) const; +}; + +} // namespace qrcode +} // namespace zxing + +#endif // __ZXING_QRCODE_DETECTOR_ALIGNMENT_PATTERN_HPP_ diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern_finder.cpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern_finder.cpp new file mode 100644 index 00000000000..e3f0f3447e9 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern_finder.cpp @@ -0,0 +1,239 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "alignment_pattern_finder.hpp" + +#include +#include +#include +#include + +using std::abs; +using std::map; +using std::vector; +using zxing::ErrorHandler; +using zxing::ReaderErrorHandler; +using zxing::Ref; +using zxing::qrcode::AlignmentPattern; +using zxing::qrcode::AlignmentPatternFinder; +using zxing::qrcode::FinderPattern; + +// VC++ +// This class attempts to find alignment patterns in a QR Code. Alignment +// patterns look like finder patterns but are smaller and appear at regular +// intervals throughout the image. At the moment this only looks for the +// bottom-right alignment pattern. This is mostly a simplified copy of {@link +// FinderPatternFinder}. It is copied, pasted and stripped down here for maximum +// performance but does unfortunately duplicat some code. This class is +// thread-safe but not reentrant. Each thread must allocate its own object. +using zxing::BitMatrix; + +// Creates a finder that will look in a portion of the whole image. +AlignmentPatternFinder::AlignmentPatternFinder(Ref image, int startX, int startY, + int width, int height, float moduleSize) + : image_(image), + possibleCenters_(new vector()), + startX_(startX), + startY_(startY), + width_(width), + height_(height), + moduleSize_(moduleSize) {} + +AlignmentPatternFinder::AlignmentPatternFinder(Ref image, float moduleSize) + : image_(image), + moduleSize_(moduleSize) {} + +// This method attempts to find the bottom-right alignment pattern in the image. +// It is a bit messy since it's pretty performance-critical and so is written to +// be fast foremost. +Ref AlignmentPatternFinder::find(ErrorHandler &err_handler) { + int maxJ = startX_ + width_; + int middleI = startY_ + (height_ >> 1); + // We are looking for black/white/black modules in 1:1:1 ratio; + // this tracks the number of black/white/black modules seen so far + vector stateCount(3, 0); + for (int iGen = 0; iGen < height_; iGen++) { + // Search from middle outwards + int i = middleI + ((iGen & 0x01) == 0 ? ((iGen + 1) >> 1) : -((iGen + 1) >> 1)); + stateCount[0] = 0; + stateCount[1] = 0; + stateCount[2] = 0; + int j = startX_; + // Burn off leading white pixels before anything else; if we start in + // the middle of a white run, it doesn't make sense to count its length, + // since we don't know if the white run continued to the left of the + // start point + while (j < maxJ && !image_->get(j, i)) { + j++; + } + int currentState = 0; + while (j < maxJ) { + if (image_->get(j, i)) { + // Black pixel + if (currentState == 1) { // Counting black pixels + stateCount[currentState]++; + } else { // Counting white pixels + if (currentState == 2) { // A winner? + if (foundPatternCross(stateCount)) { // Yes + Ref confirmed(handlePossibleCenter(stateCount, i, j)); + if (confirmed != 0) { + return confirmed; + } + } + stateCount[0] = stateCount[2]; + stateCount[1] = 1; + stateCount[2] = 0; + currentState = 1; + } else { + stateCount[++currentState]++; + } + } + } else { // White pixel + if (currentState == 1) { // Counting black pixels + currentState++; + } + stateCount[currentState]++; + } + j++; + } + if (foundPatternCross(stateCount)) { + Ref confirmed(handlePossibleCenter(stateCount, i, maxJ)); + if (confirmed != 0) { + return confirmed; + } + } + } + // Nothing we saw was observed and confirmed twice. If we had any guess at + // all, return it. + if (possibleCenters_->size() > 0) { + Ref center((*possibleCenters_)[0]); + return center; + } + err_handler = ReaderErrorHandler("Could not find alignment pattern"); + return Ref(); +} + + +// Given a count of black/white/black pixels just seen and an end position, +// figures the location of the center of this black/white/black run. +float AlignmentPatternFinder::centerFromEnd(vector &stateCount, int end) { + return (float)(end - stateCount[2]) - stateCount[1] / 2.0f; +} + + + +bool AlignmentPatternFinder::foundPatternCross(vector &stateCount) { + float maxVariance = moduleSize_ / 2.0f; + for (int i = 0; i < 3; i++) { + if (abs(moduleSize_ - stateCount[i]) >= maxVariance) { + return false; + } + } + return true; +} + +// After a horizontal scan finds a potential alignment pattern, this method +// "cross-checks" by scanning down vertically through the center of the possible +// alignment pattern to see if the same proportion is detected. return vertical +// center of alignment pattern, or nan() if not found startI: row where an +// alignment pattern was detected centerJ: center of the section that appears to +// cross an alignment pattern +// maxCount: maximum reasonable number of modules that should be observed in any +// reading state, +// based on the results of the horizontal scan +float AlignmentPatternFinder::crossCheckVertical(int startI, int centerJ, int maxCount, + int originalStateCountTotal) { + // This is slightly faster than using the Ref. Efficiency is important here + BitMatrix &matrix = *image_; + + int maxI = matrix.getHeight(); + vector stateCount(3, 0); + // Start counting up from center + int i = startI; + while (i >= 0 && matrix.get(centerJ, i) && stateCount[1] <= maxCount) { + stateCount[1]++; + i--; + } + // If already too many modules in this state or ran off the edge: + if (i < 0 || stateCount[1] > maxCount) { + return nan(); + } + while (i >= 0 && !matrix.get(centerJ, i) && stateCount[0] <= maxCount) { + stateCount[0]++; + i--; + } + if (stateCount[0] > maxCount) { + return nan(); + } + + // Now also count down from center + i = startI + 1; + while (i < maxI && matrix.get(centerJ, i) && stateCount[1] <= maxCount) { + stateCount[1]++; + i++; + } + if (i == maxI || stateCount[1] > maxCount) { + return nan(); + } + while (i < maxI && !matrix.get(centerJ, i) && stateCount[2] <= maxCount) { + stateCount[2]++; + i++; + } + if (stateCount[2] > maxCount) { + return nan(); + } + + int stateCountTotal = stateCount[0] + stateCount[1] + stateCount[2]; + if (5 * abs(stateCountTotal - originalStateCountTotal) >= 2 * originalStateCountTotal) { + return nan(); + } + return foundPatternCross(stateCount) ? centerFromEnd(stateCount, i) : nan(); +} + + +// This is called when a horizontal scan finds a possible alignment pattern. It +// will cross check with a vertical scan, and if successful, will see if this +// pattern had been found on a previous horizontal scan. If so, we consider it +// confirmed and conclude we have found the alignment pattern. return {@link +// AlignmentPattern} if we have found the same pattern twice, or null if not i: +// row where alignment pattern may be found j: end of possible alignment pattern +// in row +Ref AlignmentPatternFinder::handlePossibleCenter(vector &stateCount, int i, + int j) { + int stateCountTotal = stateCount[0] + stateCount[1] + stateCount[2]; + float centerJ = centerFromEnd(stateCount, j); + float centerI = crossCheckVertical(i, (int)centerJ, 2 * stateCount[1], stateCountTotal); + if (!isnan(centerI)) { + float estimatedModuleSize = (float)(stateCount[0] + stateCount[1] + stateCount[2]) / 3.0f; + int max = possibleCenters_->size(); + for (int index = 0; index < max; index++) { + Ref center((*possibleCenters_)[index]); + // Look for about the same center and module size: + if (center->aboutEquals(estimatedModuleSize, centerI, centerJ)) { + return center->combineEstimate(centerI, centerJ, estimatedModuleSize); + } + } + // Hadn't found this before; save it + AlignmentPattern *tmp = new AlignmentPattern(centerJ, centerI, estimatedModuleSize); + tmp->retain(); + possibleCenters_->push_back(tmp); + } + Ref result; + return result; +} + + +AlignmentPatternFinder::~AlignmentPatternFinder() { + for (int i = 0; i < int(possibleCenters_->size()); i++) { + (*possibleCenters_)[i]->release(); + (*possibleCenters_)[i] = 0; + } + delete possibleCenters_; +} diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern_finder.hpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern_finder.hpp new file mode 100644 index 00000000000..90851dc1ff5 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern_finder.hpp @@ -0,0 +1,64 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_QRCODE_DETECTOR_ALIGNMENT_PATTERN_FINDER_HPP_ +#define __ZXING_QRCODE_DETECTOR_ALIGNMENT_PATTERN_FINDER_HPP_ + +#include "../../common/bitmatrix.hpp" +#include "../../common/counted.hpp" +#include "../../errorhandler.hpp" +#include "alignment_pattern.hpp" +#include "finder_pattern.hpp" + +#include "alignment_pattern.hpp" + +#include + +namespace zxing { +namespace qrcode { + +class AlignmentPatternFinder : public Counted { +private: + static int CENTER_QUORUM; + static int MIN_SKIP; + static int MAX_MODULES; + + Ref image_; + std::vector *possibleCenters_; + + int startX_; + int startY_; + int width_; + int height_; + float moduleSize_; + static float centerFromEnd(std::vector &stateCount, int end); + float crossCheckVertical(int startI, int centerJ, int maxCount, int originalStateCountTotal); + + +public: + AlignmentPatternFinder(Ref image, int startX, int startY, int width, int height, + float moduleSize); + AlignmentPatternFinder(Ref image, float moduleSize); + ~AlignmentPatternFinder(); + + Ref find(ErrorHandler &err_handler); + bool foundPatternCross(std::vector &stateCount); + Ref handlePossibleCenter(std::vector &stateCount, int i, int j); + + +private: + AlignmentPatternFinder(const AlignmentPatternFinder &); + AlignmentPatternFinder &operator=(const AlignmentPatternFinder &); + +}; +} // namespace qrcode +} // namespace zxing + +#endif // __ZXING_QRCODE_DETECTOR_ALIGNMENT_PATTERN_FINDER_HPP_ diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/detector.cpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/detector.cpp new file mode 100644 index 00000000000..70404734862 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/detector.cpp @@ -0,0 +1,1071 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "detector.hpp" +#include +#include +#include +#include +#include "../../common/grid_sampler.hpp" +#include "../../common/mathutils.hpp" +#include "../../decodehints.hpp" +#include "../version.hpp" +#include "alignment_pattern.hpp" +#include "alignment_pattern_finder.hpp" +#include "finder_pattern.hpp" +#include "finder_pattern_finder.hpp" + +using std::abs; +using std::max; +using std::min; +using std::ostringstream; +using zxing::BitMatrix; +using zxing::DetectorResult; +using zxing::ErrorHandler; +using zxing::PerspectiveTransform; +using zxing::Ref; +using zxing::common::MathUtils; +using zxing::qrcode::AlignmentPattern; +using zxing::qrcode::Detector; +using zxing::qrcode::FinderPattern; + +// VC++ +using zxing::DecodeHints; +using zxing::ResultPoint; +using zxing::UnicomBlock; +using zxing::qrcode::FinderPatternFinder; +using zxing::qrcode::FinderPatternInfo; +using zxing::qrcode::PatternResult; + +// Encapsulates logic that can detect a QR Code in an image, +// even if the QR Code is rotated or skewed, or partially obscured. +Detector::Detector(Ref image, Ref block) : image_(image), block_(block) { + detectorState_ = START; + possiblePatternResults_.clear(); +} + +Ref Detector::getImage() const { return image_; } + +// Detects a QR Code in an image +void Detector::detect(DecodeHints const &hints, ErrorHandler &err_handler) { + FinderPatternFinder finder(image_, block_); + std::vector > finderInfos = finder.find(hints, err_handler); + if (err_handler.ErrCode()) return; + + // Get all possible results + possiblePatternResults_.clear(); + + for (size_t i = 0; i < finderInfos.size(); i++) { + Ref result(new PatternResult(finderInfos[i])); + result->possibleVersion = 0; + result->possibleFix = 0.0f; + result->possibleModuleSize = 0.0f; + + possiblePatternResults_.push_back(result); + } + detectorState_ = FINDFINDERPATTERN; +} + +int Detector::getPossibleAlignmentCount(int idx) { + if (idx >= int(possiblePatternResults_.size())) { + return -1; + } + + ErrorHandler err_handler; + // If it is first time to get, process it now + if (possiblePatternResults_[idx]->possibleAlignmentPatterns.size() == 0) { + Ref result = + processFinderPatternInfo(possiblePatternResults_[idx]->finderPatternInfo, err_handler); + if (err_handler.ErrCode()) return -1; + + possiblePatternResults_[idx] = result; + } + + return possiblePatternResults_[idx]->possibleAlignmentPatterns.size(); +} + +Ref Detector::getResultViaAlignment(int patternIdx, int alignmentIdx, + int possibleDimension, + ErrorHandler &err_handler) { + if (patternIdx >= int(possiblePatternResults_.size()) || patternIdx < 0) { + return Ref(NULL); + } + + if (alignmentIdx >= + int(possiblePatternResults_[patternIdx]->possibleAlignmentPatterns.size()) || + alignmentIdx < 0) { + return Ref(NULL); + } + + // Default is the dimension + if (possibleDimension <= 0) { + possibleDimension = possiblePatternResults_[patternIdx]->getDimension(); + } + + Ref topLeft( + possiblePatternResults_[patternIdx]->finderPatternInfo->getTopLeft()); + Ref topRight( + possiblePatternResults_[patternIdx]->finderPatternInfo->getTopRight()); + Ref bottomLeft( + possiblePatternResults_[patternIdx]->finderPatternInfo->getBottomLeft()); + + Ref alignment( + possiblePatternResults_[patternIdx]->possibleAlignmentPatterns[alignmentIdx]); + Ref transform = + createTransform(topLeft, topRight, bottomLeft, alignment, possibleDimension); + Ref bits(sampleGrid(image_, possibleDimension, transform, err_handler)); + if (err_handler.ErrCode()) return Ref(); + + ArrayRef > corrners(new Array >(4)); + vector points(8, 0.0f); + points[0] = 0.0f; + points[1] = possibleDimension; // bottomLeft + points[2] = 0.0f; + points[3] = 0.0f; // topLeft + points[4] = possibleDimension; + points[5] = 0.0f; // topRight + points[6] = possibleDimension; + points[7] = possibleDimension; // bottomRight + transform->transformPoints(points); + corrners[0].reset(Ref(new FinderPattern(points[0], points[1], 0))); + corrners[1].reset(Ref(new FinderPattern(points[2], points[3], 0))); + corrners[2].reset(Ref(new FinderPattern(points[4], points[5], 0))); + corrners[3].reset(Ref(new FinderPattern(points[6], points[7], 0))); + + Ref result(new DetectorResult(bits, corrners, possibleDimension)); + return result; +} + +bool Detector::hasSameResult(vector > possibleAlignmentPatterns, + Ref alignmentPattern) { + float moduleSize = alignmentPattern->getModuleSize() / 5.0; + + if (moduleSize < 1.0) { + moduleSize = 1.0; + } + + for (size_t i = 0; i < possibleAlignmentPatterns.size(); i++) { + if (possibleAlignmentPatterns[i]->aboutEquals(moduleSize, alignmentPattern->getY(), + alignmentPattern->getX())) { + return true; + } + } + return false; +} + +Ref Detector::getNearestAlignmentPattern(int tryFindRange, float moduleSize, + int estAlignmentX, int estAlignmentY) { + Ref alignmentPattern; + + ErrorHandler err_handler; + for (int i = 2; i <= tryFindRange; i <<= 1) { + err_handler.Reset(); + alignmentPattern = + findAlignmentInRegion(moduleSize, estAlignmentX, estAlignmentY, (float)i, err_handler); + if (err_handler.ErrCode() == 0) break; + } + + return alignmentPattern; +} + +Ref Detector::processFinderPatternInfo(Ref info, + ErrorHandler &err_handler) { + Ref topLeft(info->getTopLeft()); + Ref topRight(info->getTopRight()); + Ref bottomLeft(info->getBottomLeft()); + + Ref result(new PatternResult(info)); + result->finderPatternInfo = info; + result->possibleAlignmentPatterns.clear(); + + float moduleSizeX_ = calculateModuleSizeOneWay( + topLeft, topRight, topLeft->getHorizontalCheckState(), topRight->getHorizontalCheckState()); + float moduleSizeY_ = calculateModuleSizeOneWay( + topLeft, bottomLeft, topLeft->getVerticalCheckState(), bottomLeft->getVerticalCheckState()); + + if (moduleSizeX_ < 1.0f || moduleSizeY_ < 1.0f) { + err_handler = ReaderErrorHandler("bad midule size"); + return Ref(); + } + + float moduleSize = (moduleSizeX_ + moduleSizeY_) / 2.0f; + + if (moduleSize > topLeft->getEstimatedModuleSize() * 1.05 && + moduleSize > topRight->getEstimatedModuleSize() * 1.05 && + moduleSize > bottomLeft->getEstimatedModuleSize() * 1.05) { + moduleSize = (topLeft->getEstimatedModuleSize() + topRight->getEstimatedModuleSize() + + bottomLeft->getEstimatedModuleSize()) / + 3; + moduleSizeX_ = moduleSize; + moduleSizeY_ = moduleSize; + } + result->possibleModuleSize = moduleSize; + + if (moduleSize < 1.0f) { + err_handler = ReaderErrorHandler("bad midule size"); + return Ref(); + } + int dimension = computeDimension(topLeft, topRight, bottomLeft, moduleSizeX_, moduleSizeY_); + Version *provisionalVersion = NULL; + + // Try demension around if it cannot get a version + int dimensionDiff[5] = {0, 1, -1, 2, -2}; + + int oriDimension = dimension; + + for (int i = 0; i < 5; i++) { + err_handler.Reset(); + dimension = oriDimension + dimensionDiff[i]; + + provisionalVersion = Version::getProvisionalVersionForDimension(dimension, err_handler); + if (err_handler.ErrCode() == 0) break; + } + if (provisionalVersion == NULL) { + err_handler = zxing::ReaderErrorHandler("Cannot get version number"); + return Ref(); + } + + result->possibleDimension = dimension; + + result->possibleVersion = provisionalVersion->getVersionNumber(); + + int modulesBetweenFPCenters = provisionalVersion->getDimensionForVersion(err_handler) - 7; + if (err_handler.ErrCode()) return Ref(); + + Ref alignmentPattern; + + // Guess where a "bottom right" finder pattern would have been + float bottomRightX = topRight->getX() - topLeft->getX() + bottomLeft->getX(); + float bottomRightY = topRight->getY() - topLeft->getY() + bottomLeft->getY(); + // Estimate that alignment pattern is closer by 3 modules from "bottom + // right" to known top left location + float correctionToTopLeft = 1.0f - 3.0f / (float)modulesBetweenFPCenters; + int estAlignmentX = + (int)(topLeft->getX() + correctionToTopLeft * (bottomRightX - topLeft->getX())); + int estAlignmentY = + (int)(topLeft->getY() + correctionToTopLeft * (bottomRightY - topLeft->getY())); + + Ref estimateCenter( + new AlignmentPattern(estAlignmentX, estAlignmentY, moduleSize)); + + bool foundFitLine = false; + Ref fitLineCenter; + + fitLineCenter = + findAlignmentWithFitLine(topLeft, topRight, bottomLeft, moduleSize, err_handler); + if (err_handler.ErrCode() == 0) { + if (fitLineCenter != NULL && + MathUtils::isInRange(fitLineCenter->getX(), fitLineCenter->getY(), image_->getWidth(), + image_->getHeight())) { + foundFitLine = true; + } + } + err_handler.Reset(); + + Ref fitAP, estAP; + + // Anything above version 1 has an alignment pattern + if (provisionalVersion->getAlignmentPatternCenters().size()) { + // if(alignmentPattern!=NULL&&alignmentPattern->getX()>0&&alignmentPattern->getY()>0){ + int tryFindRange = provisionalVersion->getDimensionForVersion(err_handler) / 2; + if (err_handler.ErrCode()) return Ref(); + + if (foundFitLine == true) { + fitAP = getNearestAlignmentPattern(tryFindRange, moduleSize, fitLineCenter->getX(), + fitLineCenter->getY()); + + if (fitAP != NULL && !hasSameResult(result->possibleAlignmentPatterns, fitAP)) + // if (fitAP != NULL && + // !hasSameResult(result->possibleAlignmentPatterns, fitAP) && + // checkConvexQuadrilateral(topLeft, topRight, bottomLeft, fitAP)) + { + result->possibleAlignmentPatterns.push_back(fitAP); + } + } + + estAP = getNearestAlignmentPattern(tryFindRange, moduleSize, estimateCenter->getX(), + estimateCenter->getY()); + + if (estAP != NULL && !hasSameResult(result->possibleAlignmentPatterns, estAP)) + // if (estAP != NULL && + // !hasSameResult(result->possibleAlignmentPatterns, estAP) && + // checkConvexQuadrilateral(topLeft, topRight, bottomLeft, estAP)) + { + result->possibleAlignmentPatterns.push_back(estAP); + } + } + + // Any way use the fit line result + if (foundFitLine == true && !hasSameResult(result->possibleAlignmentPatterns, fitLineCenter)) { + float alignmentX = fitLineCenter->getX(); + float alignmentY = fitLineCenter->getY(); + fixAlignmentPattern(alignmentX, alignmentY, moduleSize); + Ref fitLineCenterFixed = + Ref(new AlignmentPattern(alignmentX, alignmentY, moduleSize)); + if (!hasSameResult(result->possibleAlignmentPatterns, fitLineCenterFixed)) { + result->possibleAlignmentPatterns.push_back(fitLineCenterFixed); + } + + if (!hasSameResult(result->possibleAlignmentPatterns, fitLineCenter)) { + result->possibleAlignmentPatterns.push_back(fitLineCenter); + } + } + + if (!hasSameResult(result->possibleAlignmentPatterns, estimateCenter)) { + float alignmentX = estimateCenter->getX(); + float alignmentY = estimateCenter->getY(); + fixAlignmentPattern(alignmentX, alignmentY, moduleSize); + Ref estimateCenterFixed = + Ref(new AlignmentPattern(alignmentX, alignmentY, moduleSize)); + if (!hasSameResult(result->possibleAlignmentPatterns, estimateCenterFixed)) { + result->possibleAlignmentPatterns.push_back(estimateCenterFixed); + } + + if (!hasSameResult(result->possibleAlignmentPatterns, estimateCenter)) { + result->possibleAlignmentPatterns.push_back(estimateCenter); + } + } + Ref NoneEstimateCenter = + Ref(new AlignmentPattern(0, 0, moduleSize)); + result->possibleAlignmentPatterns.push_back(NoneEstimateCenter); + + if (result->possibleAlignmentPatterns.size() > 0) { + result->confirmedAlignmentPattern = result->possibleAlignmentPatterns[0]; + } + detectorState_ = FINDALIGNPATTERN; + + return result; +} + +// Computes an average estimated module size based on estimated derived from the +// positions of the three finder patterns. +float Detector::calculateModuleSize(Ref topLeft, Ref topRight, + Ref bottomLeft) { + // Take the average + return (calculateModuleSizeOneWay(topLeft, topRight, NORMAL, NORMAL) + + calculateModuleSizeOneWay(topLeft, bottomLeft, NORMAL, NORMAL)) / + 2.0f; +} + +// Estimates module size based on two finder patterns +// it uses sizeOfBlackWhiteBlackRunBothWays() to figure the width of each, +// measuring along the axis between their centers. +float Detector::calculateModuleSizeOneWay(Ref pattern, Ref otherPattern, + int patternState, int otherPatternState) { + float moduleSizeEst1 = sizeOfBlackWhiteBlackRunBothWays( + (int)pattern->getX(), (int)pattern->getY(), (int)otherPattern->getX(), + (int)otherPattern->getY(), patternState, false); + float moduleSizeEst2 = sizeOfBlackWhiteBlackRunBothWays( + (int)otherPattern->getX(), (int)otherPattern->getY(), (int)pattern->getX(), + (int)pattern->getY(), otherPatternState, true); + if (zxing::isnan(moduleSizeEst1)) { + return moduleSizeEst2 / 7.0f; + } + if (zxing::isnan(moduleSizeEst2)) { + return moduleSizeEst1 / 7.0f; + } + // Average them, and divide by 7 since we've counted the width of 3 black + // modules, and 1 white and 1 black module on either side. Ergo, divide sum + // by 14. + return (moduleSizeEst1 + moduleSizeEst2) / 14.0f; +} + +// Computes the total width of a finder pattern by looking for a +// black-white-black run from the center in the direction of another point +// (another finder pattern center), and in the opposite direction too. +float Detector::sizeOfBlackWhiteBlackRunBothWays(int fromX, int fromY, int toX, int toY, + int patternState, bool isReverse) { + float result1 = sizeOfBlackWhiteBlackRun(fromX, fromY, toX, toY); + float result = 0.0; + // Now count other way -- don't run off image though of course + float scale = 1.0f; + int otherToX = fromX - (toX - fromX); + if (otherToX < 0) { + scale = (float)fromX / (float)(fromX - otherToX); + otherToX = 0; + } else if (otherToX >= (int)image_->getWidth()) { + scale = (float)(image_->getWidth() - 1 - fromX) / (float)(otherToX - fromX); + otherToX = image_->getWidth() - 1; + } + int otherToY = (int)(fromY - (toY - fromY) * scale); + + scale = 1.0f; + if (otherToY < 0) { + scale = (float)fromY / (float)(fromY - otherToY); + otherToY = 0; + } else if (otherToY >= (int)image_->getHeight()) { + scale = (float)(image_->getHeight() - 1 - fromY) / (float)(otherToY - fromY); + otherToY = image_->getHeight() - 1; + } + otherToX = (int)(fromX + (otherToX - fromX) * scale); + + float result2 = sizeOfBlackWhiteBlackRun(fromX, fromY, otherToX, otherToY); + + if (patternState == FinderPattern::HORIZONTAL_STATE_LEFT_SPILL || + patternState == FinderPattern::VERTICAL_STATE_UP_SPILL) { + if (!isReverse) + result = result1 * 2; + else + result = result2 * 2; + } else if (patternState == FinderPattern::HORIZONTAL_STATE_RIGHT_SPILL || + patternState == FinderPattern::VERTICAL_STATE_DOWN_SPILL) { + if (!isReverse) + result = result2 * 2; + else + result = result1 * 2; + } else { + result = result1 + result2; + } + // Middle pixel is double-counted this way; subtract 1 + return result - 1.0f; +} + +Ref Detector::sampleGrid(Ref image, int dimension, + Ref transform, + ErrorHandler &err_handler) { + GridSampler &sampler = GridSampler::getInstance(); + // return sampler.sampleGrid(image, dimension, transform); + Ref bits = sampler.sampleGrid(image, dimension, transform, err_handler); + if (err_handler.ErrCode()) return Ref(); + return bits; +} + +// This method traces a line from a point in the image, in the direction towards +// another point. It begins in a black region, and keeps going until it finds +// white, then black, then white again. It reports the distance from the start +// to this point. +float Detector::sizeOfBlackWhiteBlackRun(int fromX, int fromY, int toX, int toY) { + // Mild variant of Bresenham's algorithm; + // see http://en.wikipedia.org/wiki/Bresenham's_line_algorithm + bool steep = abs(toY - fromY) > abs(toX - fromX); + if (steep) { + // swap(fromX,fromY) + int temp = fromX; + fromX = fromY; + fromY = temp; + // swap(toX,toY) + temp = toX; + toX = toY; + toY = temp; + } + + int dx = abs(toX - fromX); + int dy = abs(toY - fromY); + int error = -dx >> 1; + int xstep = fromX < toX ? 1 : -1; + int ystep = fromY < toY ? 1 : -1; + // In black pixels, looking for white, first or second time. + int state = 0; + // Loop up until x == toX, but not beyond + int xLimit = toX + xstep; + for (int x = fromX, y = fromY; x != xLimit; x += xstep) { + int realX = steep ? y : x; + int realY = steep ? x : y; + + // Does current pixel mean we have moved white to black or vice versa? + // Scanning black in state 0,2 and white in state 1, so if we find the + // wrong color, advance to next state or end if we are in state 2 + // already + if (!((state == 1) ^ image_->get(realX, realY))) { + if (state == 2) { + return MathUtils::distance(x, y, fromX, fromY); + } + state++; + } + + error += dy; + if (error > 0) { + if (y == toY) { + break; + } + y += ystep; + error -= dx; + } + } + // Found black-white-black; give the benefit of the doubt that the next + // pixel outside the image is "white" so this last point at (toX+xStep,toY) + // is the right ending. This is really a small approximation; + // (toX+xStep,toY+yStep) might be really correct. Ignore this. + if (state == 2) { + return MathUtils::distance(toX + xstep, toY, fromX, fromY); + } + // else we didn't find even black-white-black; no estimate is really + // possible + return nan(); +} + +// Attempts to locate an alignment pattern in a limited region of the image, +// which is guessed to contain it. +Ref Detector::findAlignmentInRegion(float overallEstModuleSize, int estAlignmentX, + int estAlignmentY, float allowanceFactor, + ErrorHandler &err_handler) { + // Look for an alignment pattern (3 modules in size) around where it should + // be + int allowance = (int)(allowanceFactor * overallEstModuleSize); + int alignmentAreaLeftX = max(0, estAlignmentX - allowance); + int alignmentAreaRightX = min((int)(image_->getWidth() - 1), estAlignmentX + allowance); + if (alignmentAreaRightX - alignmentAreaLeftX < overallEstModuleSize * 3) { + err_handler = ReaderErrorHandler("region too small to hold alignment pattern"); + return Ref(); + } + int alignmentAreaTopY = max(0, estAlignmentY - allowance); + int alignmentAreaBottomY = min((int)(image_->getHeight() - 1), estAlignmentY + allowance); + if (alignmentAreaBottomY - alignmentAreaTopY < overallEstModuleSize * 3) { + err_handler = ReaderErrorHandler("region too small to hold alignment pattern"); + return Ref(); + } + + AlignmentPatternFinder alignmentFinder( + image_, alignmentAreaLeftX, alignmentAreaTopY, alignmentAreaRightX - alignmentAreaLeftX, + alignmentAreaBottomY - alignmentAreaTopY, overallEstModuleSize); + + Ref ap = alignmentFinder.find(err_handler); + if (err_handler.ErrCode()) return Ref(); + return ap; + +} + +Ref Detector::findAlignmentWithFitLine(Ref topLeft, + Ref topRight, + Ref bottomLeft, + float moduleSize, + ErrorHandler &err_handler) { + float alignmentX = 0.0f, alignmentY = 0.0f; + int imgWidth = image_->getWidth(); + int imgHeight = image_->getHeight(); + Rect bottomLeftRect, topRightRect; + double rectSize = moduleSize * 7; + bottomLeftRect.x = + (bottomLeft->getX() - rectSize / 2.0f) > 0 ? (bottomLeft->getX() - rectSize / 2.0f) : 0; + bottomLeftRect.y = + (bottomLeft->getY() - rectSize / 2.0f) > 0 ? (bottomLeft->getY() - rectSize / 2.0f) : 0; + bottomLeftRect.width = (bottomLeft->getX() - bottomLeftRect.x) * 2; + if (bottomLeftRect.x + bottomLeftRect.width > imgWidth) + bottomLeftRect.width = imgWidth - bottomLeftRect.x; + bottomLeftRect.height = (bottomLeft->getY() - bottomLeftRect.y) * 2; + if (bottomLeftRect.y + bottomLeftRect.height > imgHeight) + bottomLeftRect.height = imgHeight - bottomLeftRect.y; + + topRightRect.x = + (topRight->getX() - rectSize / 2.0f) > 0 ? (topRight->getX() - rectSize / 2.0f) : 0; + topRightRect.y = + (topRight->getY() - rectSize / 2.0f) > 0 ? (topRight->getY() - rectSize / 2.0f) : 0; + topRightRect.width = (topRight->getX() - topRightRect.x) * 2; + if (topRightRect.x + topRightRect.width > imgWidth) + topRightRect.width = imgWidth - topRightRect.x; + topRightRect.height = (topRight->getY() - topRightRect.y) * 2; + if (topRightRect.y + topRightRect.height > imgHeight) + topRightRect.height = imgHeight - topRightRect.y; + + vector > topRightPoints; + vector > bottomLeftPoints; + + findPointsForLine(topLeft, topRight, bottomLeft, topRightRect, bottomLeftRect, topRightPoints, + bottomLeftPoints, moduleSize); + + int a1; + float k1, b1; + int fitResult = fitLine(topRightPoints, k1, b1, a1); + if (fitResult < 0) { + err_handler = ReaderErrorHandler("Cannot find a valid divide for line fit"); + return Ref(); + } + + int a2; + float k2, b2; + int fitResult2 = fitLine(bottomLeftPoints, k2, b2, a2); + if (fitResult2 < 0) { + err_handler = ReaderErrorHandler("Cannot find a valid divide for line fit"); + return Ref(); + } + + int hasResult = 1; + if (a1 == 0) { + if (a2 == 0) { + hasResult = 0; + } else { + alignmentX = -b1; + alignmentY = b2 - b1 * k2; + } + } else { + if (a2 == 0) { + alignmentX = -b2; + alignmentY = b1 - b2 * k1; + } else { + if (k1 == k2) { + hasResult = 0; + } else { + alignmentX = (b2 - b1) / (k1 - k2); + alignmentY = k1 * alignmentX + b1; + } + } + } + + // Donot have a valid divide + if (hasResult == 0) { + err_handler = ReaderErrorHandler("Cannot find a valid divide for line fit"); + return Ref(); + } + Ref result(new AlignmentPattern(alignmentX, alignmentY, moduleSize)); + return result; +} + +void Detector::fixAlignmentPattern(float &alignmentX, float &alignmentY, float moduleSize) { + int imgWidth = image_->getWidth(); + int imgHeight = image_->getHeight(); + int maxFixStep = moduleSize * 2; + int fixStep = 0; + while (alignmentX < imgWidth && alignmentY < imgHeight && alignmentX > 0 && alignmentY > 0 && + !image_->get(alignmentX, alignmentY) && fixStep < maxFixStep) { + ++fixStep; + // Newest Version: The fix process is like this: + // 1 2 3 + // 4 0 5 + // 6 7 8 + for (int y = alignmentY - fixStep; y <= alignmentY + fixStep; y++) { + if (y == alignmentY - fixStep || y == alignmentY + fixStep) { + for (int x = alignmentX - fixStep; x <= alignmentX + fixStep; x++) { + if (x < imgWidth && y < imgHeight && x > 0 && y > 0 && image_->get(x, y)) { + alignmentX = x; + alignmentY = y; + return; + } + } + } else { + int x = alignmentX - fixStep; + if (x < imgWidth && y < imgHeight && x > 0 && y > 0 && image_->get(x, y)) { + alignmentX = x; + alignmentY = y; + return; + } + x = alignmentX + fixStep; + if (x < imgWidth && y < imgHeight && x > 0 && y > 0 && image_->get(x, y)) { + alignmentX = x; + alignmentY = y; + return; + } + } + } + } + + return; +} + +int Detector::fitLine(vector > &oldPoints, float &k, float &b, int &a) { + a = 1; + k = 0.0f; + b = 0.0f; + int old_num = oldPoints.size(); + if (old_num < 2) { + return -1; + } + float tolerance = 2.0f; + vector > fitPoints; + float pre_diff = -1; + for (vector >::iterator it = oldPoints.begin() + 1; it != oldPoints.end() - 1; + it++) { + float diff_x = 0.0f, diff_y = 0.0f, diff = 0.0f; + if (pre_diff < 0) { + diff_x = (*(it - 1))->getX() - (*it)->getX(); + diff_y = (*(it - 1))->getY() - (*it)->getY(); + diff = (diff_x * diff_x + diff_y * diff_y); + pre_diff = diff; + } + diff_x = (*(it + 1))->getX() - (*it)->getX(); + diff_y = (*(it + 1))->getY() - (*it)->getY(); + diff = (diff_x * diff_x + diff_y * diff_y); + if (pre_diff <= tolerance && diff <= tolerance) { + fitPoints.push_back(*(it)); + } + pre_diff = diff; + } + + int num = fitPoints.size(); + if (num < 2) return -1; + + double x = 0, y = 0, xx = 0, xy = 0, yy = 0, tem = 0; + for (int i = 0; i < num; i++) { + int point_x = fitPoints[i]->getX(); + int point_y = fitPoints[i]->getY(); + x += point_x; + y += point_y; + xx += point_x * point_x; + xy += point_x * point_y; + yy += point_y * point_y; + } + + tem = xx * num - x * x; + if (abs(tem) < 0.0000001) { + // Set b as average x + b = -x / num; + a = 0; + k = 1; + + return 1; + } + + k = (num * xy - x * y) / tem; + b = (y - k * x) / num; + a = 1; + if (abs(k) < 0.01) k = 0; + return 1; +} + +bool Detector::checkTolerance(Ref &topLeft, Ref &topRight, + Rect &topRightRect, double modelSize, Ref &p, int flag) { + int topLeftX = topLeft->getX(), topLeftY = topLeft->getY(), topRightX = topRight->getX(), + topRightY = topRight->getY(); + double left_right_k = 0.0f, left_right_b = 0.0f, left_right_b_tolerance, tolerance_b1 = 0.0f, + tolerance_b2 = 0.0f; + if (flag < 2) { + double tolerance_y1 = 0.0f, tolerance_y2 = 0.0f; + double tolerance_x = topRightRect.x; + if (flag == 1) tolerance_x = topRightRect.x + topRightRect.width; + if (topRightX != topLeftX) { + left_right_k = (topRightY - topLeftY) / (double)(topRightX - topLeftX); + left_right_b = (topRightY - left_right_k * topRightX); + double tmp_1 = modelSize * 2.5f; + double tmp_2 = tmp_1 * left_right_k; + + left_right_b_tolerance = sqrt(tmp_1 * tmp_1 + tmp_2 * tmp_2); + tolerance_b1 = left_right_b - left_right_b_tolerance; + tolerance_b2 = left_right_b + left_right_b_tolerance; + tolerance_y1 = left_right_k * tolerance_x + tolerance_b1; + tolerance_y2 = left_right_k * tolerance_x + tolerance_b2; + } else { + return false; + } + if (p->getY() < tolerance_y1 || p->getY() > tolerance_y2) return false; + return true; + } else { + double tolerance_x1 = 0.0f, tolerance_x2 = 0.0f; + if (topRightY != topLeftY) { + double tolerance_y = topRightRect.y; + if (flag == 3) tolerance_y = topRightRect.y + topRightRect.height; + left_right_k = (topRightX - topLeftX) / (double)(topRightY - topLeftY); + left_right_b = (topRightX - left_right_k * topRightY); + double tmp_1 = modelSize * 2.5f; + double tmp_2 = tmp_1 / left_right_k; + left_right_b_tolerance = sqrt(tmp_1 * tmp_1 + tmp_2 * tmp_2); + tolerance_b1 = left_right_b - left_right_b_tolerance; + tolerance_b2 = left_right_b + left_right_b_tolerance; + tolerance_x1 = left_right_k * tolerance_y + tolerance_b1; + tolerance_x2 = left_right_k * tolerance_y + tolerance_b2; + if (p->getX() < tolerance_x1 || p->getX() > tolerance_x2) return false; + return true; + } else { + return false; + } + } +} + +void Detector::findPointsForLine(Ref &topLeft, Ref &topRight, + Ref &bottomLeft, Rect topRightRect, + Rect bottomLeftRect, vector > &topRightPoints, + vector > &bottomLeftPoints, float modelSize) { + int topLeftX = topLeft->getX(), topLeftY = topLeft->getY(), topRightX = topRight->getX(), + topRightY = topRight->getY(); + if (!topRightPoints.empty()) topRightPoints.clear(); + if (!bottomLeftPoints.empty()) bottomLeftPoints.clear(); + + int xMin = 0; + int xMax = 0; + int yMin = 0; + int yMax = 0; + + int imgWidth = image_->getWidth(); + int imgHeight = image_->getHeight(); + + // [-45, 45] or [135, 180) or [-180, -45) + if (topLeftY == topRightY || abs((topRightX - topLeftX) / (topRightY - topLeftY)) >= 1) { + if (topLeftX < topRightX) { + xMin = topRightRect.x; + xMax = topRightRect.x + modelSize * 2; + yMin = topRightRect.y + modelSize; + yMax = topRightRect.y - modelSize + topRightRect.height; + // [-45, 45] TopRight: left, black->white points; BottomLeft: top, black->white points + MathUtils::getRangeValues(xMin, xMax, 0, imgWidth - 1); + MathUtils::getRangeValues(yMin, yMax, 0, imgHeight - 1); + + for (int i = yMin; i < yMax; i++) { + for (int j = xMin; j < xMax; j++) { + // left->right, black->white + if (image_->get(j, i) && !image_->get(j + 1, i)) { + Ref topRightPoint(new ResultPoint(j, i)); + if (checkTolerance(topLeft, topRight, topRightRect, modelSize, + topRightPoint, 0)) { + topRightPoints.push_back(topRightPoint); + break; + } + } + } + } + + xMin = bottomLeftRect.x + modelSize; + xMax = bottomLeftRect.x - modelSize + bottomLeftRect.width; + yMin = bottomLeftRect.y; + yMax = bottomLeftRect.y + 2 * modelSize; + + MathUtils::getRangeValues(xMin, xMax, 0, imgWidth - 1); + MathUtils::getRangeValues(yMin, yMax, 0, imgHeight - 1); + + for (int j = xMin; j < xMax; j++) { + for (int i = yMin; i < yMax; i++) { + // top to down, black->white + if (image_->get(j, i) && !image_->get(j, i + 1)) { + Ref bottomLeftPoint(new ResultPoint(j, i)); + if (checkTolerance(topLeft, bottomLeft, bottomLeftRect, modelSize, + bottomLeftPoint, 2)) { + bottomLeftPoints.push_back(bottomLeftPoint); + break; + } + } + } + } + } else { + // white->black points + xMin = topRightRect.x + topRightRect.width - 2 * modelSize; + xMax = topRightRect.x + topRightRect.width; + yMin = topRightRect.y + modelSize; + yMax = topRightRect.y - modelSize + topRightRect.height; + // [135, 180) or [-180, -45) TopRight: right, white->black points; BottomLeft: bottom, + MathUtils::getRangeValues(xMin, xMax, 0, imgWidth - 1); + MathUtils::getRangeValues(yMin, yMax, 0, imgHeight - 1); + + for (int i = yMin; i < yMax; i++) { + for (int j = xMin; j < xMax; j++) { + // left->right, white->black + if (!image_->get(j, i) && image_->get(j + 1, i)) { + Ref topRightPoint(new ResultPoint(j, i)); + if (checkTolerance(topLeft, topRight, topRightRect, modelSize, + topRightPoint, 1)) { + topRightPoints.push_back(topRightPoint); + break; + } + } + } + } + + xMin = bottomLeftRect.x + modelSize; + xMax = bottomLeftRect.x - modelSize + bottomLeftRect.width; + yMin = bottomLeftRect.y + bottomLeftRect.height - 2 * modelSize; + yMax = bottomLeftRect.y + bottomLeftRect.height; + + MathUtils::getRangeValues(xMin, xMax, 0, imgWidth - 1); + MathUtils::getRangeValues(yMin, yMax, 0, imgHeight - 1); + + for (int j = xMin; j < xMax; j++) { + for (int i = yMin; i < yMax; i++) { + // top to down, white->black + if (!image_->get(j, i) && image_->get(j, i + 1)) { + Ref bottomLeftPoint(new ResultPoint(j, i)); + if (checkTolerance(topLeft, bottomLeft, bottomLeftRect, modelSize, + bottomLeftPoint, 3)) { + bottomLeftPoints.push_back(bottomLeftPoint); + break; + } + } + } + } + } + } else { + // (45, 135) or (-45, -135) + // (45, 135) TopRight: top, black->white; BottomRight: right, black->white + if (topLeftY < topRightY) { + xMin = topRightRect.x + modelSize; + xMax = topRightRect.x - modelSize + topRightRect.width; + yMin = topRightRect.y; + yMax = topRightRect.y + 2 * modelSize; + + MathUtils::getRangeValues(xMin, xMax, 0, imgWidth - 1); + MathUtils::getRangeValues(yMin, yMax, 0, imgHeight - 1); + + for (int j = xMin; j < xMax; j++) { + for (int i = yMin; i < yMax; i++) { + // top to down, black->white + if (image_->get(j, i) && !image_->get(j, i + 1)) { + Ref topRightPoint(new ResultPoint(j, i)); + if (checkTolerance(topLeft, topRight, topRightRect, modelSize, + topRightPoint, 2)) { + topRightPoints.push_back(topRightPoint); + break; + } + } + } + } + + xMin = topRightRect.x + topRightRect.width - 2 * modelSize; + xMax = topRightRect.x + topRightRect.width; + yMin = topRightRect.y + modelSize; + yMax = topRightRect.y - modelSize + topRightRect.height; + + MathUtils::getRangeValues(xMin, xMax, 0, imgWidth - 1); + MathUtils::getRangeValues(yMin, yMax, 0, imgHeight - 1); + + for (int i = yMin; i < yMax; i++) { + for (int j = xMin; j < xMax; j++) { + // left to right, white-> black + if (!image_->get(j, i) && image_->get(j + 1, i)) { + Ref bottomLeftPoint(new ResultPoint(j, i)); + if (checkTolerance(topLeft, bottomLeft, bottomLeftRect, modelSize, + bottomLeftPoint, 1)) { + bottomLeftPoints.push_back(bottomLeftPoint); + break; + } + } + } + } + } else { + // (-45, -135) TopRight: bottom, white->black; BottomRight: left, black->white + xMin = topRightRect.x + modelSize; + xMax = topRightRect.x - modelSize + topRightRect.width; + yMin = topRightRect.y + topRightRect.height - 2 * modelSize; + yMax = topRightRect.y + topRightRect.height; + + MathUtils::getRangeValues(xMin, xMax, 0, imgWidth - 1); + MathUtils::getRangeValues(yMin, yMax, 0, imgHeight - 1); + + for (int j = xMin; j < xMax; j++) { + for (int i = yMin; i < yMax; i++) { + // top to down, white->balck + if (!image_->get(j, i) && image_->get(j, i + 1)) { + Ref topRightPoint(new ResultPoint(j, i)); + if (checkTolerance(topLeft, topRight, topRightRect, modelSize, + topRightPoint, 3)) { + topRightPoints.push_back(topRightPoint); + break; + } + } + } + } + + xMin = bottomLeftRect.x; + xMax = bottomLeftRect.x + 2 * modelSize; + yMin = bottomLeftRect.y + modelSize; + yMax = bottomLeftRect.y + bottomLeftRect.height - modelSize; + + MathUtils::getRangeValues(xMin, xMax, 0, imgWidth - 1); + MathUtils::getRangeValues(yMin, yMax, 0, imgHeight - 1); + + for (int i = yMin; i < yMax; i++) { + for (int j = xMin; j < xMax; j++) { + // left to right, black->white + if (image_->get(j, i) && !image_->get(j + 1, i)) { + Ref bottomLeftPoint(new ResultPoint(j, i)); + if (checkTolerance(topLeft, bottomLeft, bottomLeftRect, modelSize, + bottomLeftPoint, 0)) { + bottomLeftPoints.push_back(bottomLeftPoint); + break; + } + } + } + } + } + } +} + +Ref Detector::createTransform(Ref info, + Ref alignmentPattern, + int dimension) { + Ref topLeft(info->getTopLeft()); + Ref topRight(info->getTopRight()); + Ref bottomLeft(info->getBottomLeft()); + Ref transform = + createTransform(topLeft, topRight, bottomLeft, alignmentPattern, dimension); + return transform; +} + +Ref Detector::createTransform(Ref topLeft, + Ref topRight, + Ref bottomLeft, + Ref alignmentPattern, + int dimension) { + float dimMinusThree = (float)dimension - 3.5f; + float bottomRightX; + float bottomRightY; + float sourceBottomRightX; + float sourceBottomRightY; + if (alignmentPattern && alignmentPattern->getX()) { + bottomRightX = alignmentPattern->getX(); + bottomRightY = alignmentPattern->getY(); + sourceBottomRightX = dimMinusThree - 3.0f; + sourceBottomRightY = sourceBottomRightX; + } else { + // Don't have an alignment pattern, just make up the bottom-right point + bottomRightX = (topRight->getX() - topLeft->getX()) + bottomLeft->getX(); + bottomRightY = (topRight->getY() - topLeft->getY()) + bottomLeft->getY(); + float deltaX = topLeft->getX() - bottomLeft->getX(); + float deltaY = topLeft->getY() - bottomLeft->getY(); + if (fabs(deltaX) < fabs(deltaY)) + deltaY = topLeft->getY() - topRight->getY(); + else + deltaX = topLeft->getX() - topRight->getX(); + bottomRightX += 2 * deltaX; + bottomRightY += 2 * deltaY; + sourceBottomRightX = dimMinusThree; + sourceBottomRightY = dimMinusThree; + } + Ref transform(PerspectiveTransform::quadrilateralToQuadrilateral( + 3.5f, 3.5f, dimMinusThree, 3.5f, sourceBottomRightX, sourceBottomRightY, 3.5f, + dimMinusThree, topLeft->getX(), topLeft->getY(), topRight->getX(), topRight->getY(), + bottomRightX, bottomRightY, bottomLeft->getX(), bottomLeft->getY())); + return transform; +} + +// Computes the dimension (number of modules on a size) of the QR code based on +// the position of the finder patterns and estimated module size. +int Detector::computeDimension(Ref topLeft, Ref topRight, + Ref bottomLeft, float moduleSizeX, float moduleSizeY) { + int tltrCentersDimension = ResultPoint::distance(topLeft, topRight) / moduleSizeX; + int tlblCentersDimension = ResultPoint::distance(topLeft, bottomLeft) / moduleSizeY; + + float tmp_dimension = ((tltrCentersDimension + tlblCentersDimension) / 2.0) + 7.0; + int dimension = MathUtils::round(tmp_dimension); + int mod = dimension & 0x03; // mod 4 + + switch (mod) { // mod 4 + case 0: + dimension++; + break; + // 1? do nothing + case 2: + dimension--; + break; + } + return dimension; +} + +bool Detector::checkConvexQuadrilateral(Ref topLeft, Ref topRight, + Ref bottomLeft, Ref bottomRight) { + float v1[2]; + float v2[2]; + float v3[2]; + float v4[2]; + + v1[0] = topLeft->getX() - topRight->getX(); + v1[1] = topLeft->getY() - topRight->getY(); + v2[0] = topRight->getX() - bottomRight->getX(); + v2[1] = topRight->getY() - bottomRight->getY(); + v3[0] = bottomRight->getX() - bottomLeft->getX(); + v3[1] = bottomRight->getY() - bottomLeft->getY(); + v4[0] = bottomLeft->getX() - topLeft->getX(); + v4[1] = bottomLeft->getY() - topLeft->getY(); + + float c1 = MathUtils::VecCross(v1, v2); + float c2 = MathUtils::VecCross(v2, v3); + float c3 = MathUtils::VecCross(v3, v4); + float c4 = MathUtils::VecCross(v4, v1); + + if ((c1 < 0.0 && c2 < 0.0 && c3 < 0.0 && c4 < 0.0) || + (c1 > 0.0 && c2 > 0.0 && c3 > 0.0 && c4 > 0.0)) + return true; + else + return false; +} diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/detector.hpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/detector.hpp new file mode 100644 index 00000000000..cd3020b8f09 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/detector.hpp @@ -0,0 +1,147 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_QRCODE_DETECTOR_DETECTOR_HPP_ +#define __ZXING_QRCODE_DETECTOR_DETECTOR_HPP_ + +#include "../../common/bitmatrix.hpp" +#include "../../common/counted.hpp" +#include "../../common/detector_result.hpp" +#include "../../common/perspective_transform.hpp" +#include "../../common/unicomblock.hpp" +#include "../../errorhandler.hpp" +#include "alignment_pattern.hpp" +#include "finder_pattern.hpp" +#include "finder_pattern_info.hpp" +#include "pattern_result.hpp" + +#include + +namespace zxing { +class DecodeHints; + +namespace qrcode { + +// Possible Detect Result + +class Detector : public Counted { +public: + enum DetectorState { + START = 10, + FINDFINDERPATTERN = 11, + FINDALIGNPATTERN = 12, + }; + + // Fix module size error when LEFT_SPILL or RIGHT_SPILL + enum FinderPatternMode { + NORMAL = 0, + LEFT_SPILL = 1, + RIGHT_SPILL = 2, + UP_SPILL = 3, + DOWN_SPILL = 4, + }; + + typedef struct Rect_ { + int x; + int y; + int width; + int height; + } Rect; + +private: + Ref image_; + Ref block_; + + vector > possiblePatternResults_; + + + DetectorState detectorState_; + +protected: + Ref getImage() const; + static int computeDimension(Ref topLeft, Ref topRight, + Ref bottomLeft, float moduleSizeX, float moduleSizeY); + float calculateModuleSize(Ref topLeft, Ref topRight, + Ref bottomLeft); + float calculateModuleSizeOneWay(Ref pattern, Ref otherPattern, + int patternState, int otherPatternState); + float sizeOfBlackWhiteBlackRunBothWays(int fromX, int fromY, int toX, int toY, int patternState, + bool isReverse); + float sizeOfBlackWhiteBlackRunBothWays(int fromX, int fromY, int toX, int toY); + float sizeOfBlackWhiteBlackRun(int fromX, int fromY, int toX, int toY); + Ref findAlignmentInRegion(float overallEstModuleSize, int estAlignmentX, + int estAlignmentY, float allowanceFactor, + ErrorHandler &err_handler); + Ref findAlignmentWithFitLine(Ref topLeft, + Ref topRight, + Ref bottomLeft, float moduleSize, + ErrorHandler &err_handler); + int fitLine(vector > &oldPoints, float &k, float &b, int &a); + bool checkTolerance(Ref &topLeft, Ref &topRight, Rect &topRightRect, + double modelSize, Ref &p, int flag); + void findPointsForLine(Ref &topLeft, Ref &topRight, + Ref &bottomLeft, Rect topRightRect, Rect bottomLeftRect, + vector > &topRightPoints, + vector > &bottomLeftPoints, float modelSize); + bool checkConvexQuadrilateral(Ref topLeft, Ref topRight, + Ref bottomLeft, Ref bottomRight); + +public: + virtual Ref createTransform(Ref topLeft, + Ref topRight, + Ref bottomLeft, + Ref alignmentPattern, + int dimension); + Ref createTransform(Ref finderPatternInfo, + Ref alignmentPattern, int dimension); + + static Ref sampleGrid(Ref image, int dimension, Ref, + ErrorHandler &err_handler); + + Detector(Ref image, Ref block); + void detect(DecodeHints const &hints, ErrorHandler &err_handler); + Ref getResultViaAlignment(int patternIdx, int alignmentIndex, + int possibleDimension, ErrorHandler &err_handler); + + int getPossiblePatternCount() { return possiblePatternResults_.size(); } + int getPossibleAlignmentCount(int idx); + + Ref getNearestAlignmentPattern(int tryFindRange, float moduleSize, + int estAlignmentX, int estAlignmentY); + bool hasSameResult(vector > possibleAlignmentPatterns, + Ref alignmentPattern); + void fixAlignmentPattern(float &alignmentX, float &alignmentY, float moduleSize); + + Ref processFinderPatternInfo(Ref info, + ErrorHandler &err_handler); + +public: + Ref getFinderPatternInfo(int idx) { + return possiblePatternResults_[idx]->finderPatternInfo; + } + Ref getAlignmentPattern(int patternIdx, int alignmentIdx) { + return possiblePatternResults_[patternIdx]->possibleAlignmentPatterns[alignmentIdx]; + } + + DetectorState getState() { return detectorState_; } + + unsigned int getPossibleVersion(int idx) { + return possiblePatternResults_[idx]->possibleVersion; + } + float getPossibleFix(int idx) { return possiblePatternResults_[idx]->possibleFix; } + float getPossibleModuleSize(int idx) { + return possiblePatternResults_[idx]->possibleModuleSize; + } + int getDimension(int idx) { return possiblePatternResults_[idx]->possibleDimension; } +}; +} // namespace qrcode +} // namespace zxing + +#endif // __ZXING_QRCODE_DETECTOR_DETECTOR_HPP_ diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern.cpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern.cpp new file mode 100644 index 00000000000..143d20df96c --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern.cpp @@ -0,0 +1,94 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "finder_pattern.hpp" + +#include + +using std::abs; +using zxing::Ref; + +namespace zxing { +namespace qrcode { + +FinderPattern::FinderPattern(float posX, float posY, float estimatedModuleSize) + : ResultPoint(posX, posY), + estimatedModuleSize_(estimatedModuleSize), + count_(1), + horizontalState_(FinderPattern::HORIZONTAL_STATE_NORMAL), + verticalState_(FinderPattern::VERTICAL_STATE_NORMAL) { + fix_ = -1.0f; +} + +FinderPattern::FinderPattern(float posX, float posY, float estimatedModuleSize, int count) + : ResultPoint(posX, posY), + estimatedModuleSize_(estimatedModuleSize), + count_(count), + horizontalState_(FinderPattern::HORIZONTAL_STATE_NORMAL), + verticalState_(FinderPattern::VERTICAL_STATE_NORMAL) { + fix_ = -1.0f; +} +int FinderPattern::getCount() const { return count_; } +void FinderPattern::incrementCount() { count_++; } + +bool FinderPattern::aboutEquals(float moduleSize, float i, float j) const { + if (abs(i - getY()) <= moduleSize && abs(j - getX()) <= moduleSize) { + float moduleSizeDiff = abs(moduleSize - estimatedModuleSize_); + return moduleSizeDiff <= 1.0f || moduleSizeDiff <= estimatedModuleSize_; + } + return false; +} + +float FinderPattern::getEstimatedModuleSize() const { return estimatedModuleSize_; } + +Ref FinderPattern::combineEstimate(float i, float j, float newModuleSize) const { + int combinedCount = count_ + 1; + float combinedX = getX(); + float combinedY = getY(); + float combinedModuleSize = getEstimatedModuleSize(); + if (combinedCount <= 3) { + combinedX = (count_ * getX() + j) / combinedCount; + combinedY = (count_ * getY() + i) / combinedCount; + combinedModuleSize = (count_ * getEstimatedModuleSize() + newModuleSize) / combinedCount; + } + return Ref( + new FinderPattern(combinedX, combinedY, combinedModuleSize, combinedCount)); +} + +void FinderPattern::setHorizontalCheckState(int state) { + switch (state) { + case 0: + horizontalState_ = FinderPattern::HORIZONTAL_STATE_NORMAL; + break; + case 1: + horizontalState_ = FinderPattern::HORIZONTAL_STATE_LEFT_SPILL; + break; + case 2: + horizontalState_ = FinderPattern::HORIZONTAL_STATE_RIGHT_SPILL; + break; + } + return; +} +void FinderPattern::setVerticalCheckState(int state) { + switch (state) { + case 0: + verticalState_ = FinderPattern::VERTICAL_STATE_NORMAL; + break; + case 1: + verticalState_ = FinderPattern::VERTICAL_STATE_UP_SPILL; + break; + case 2: + verticalState_ = FinderPattern::VERTICAL_STATE_DOWN_SPILL; + break; + } + return; +} +} // namespace qrcode +} // namespace zxing diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern.hpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern.hpp new file mode 100644 index 00000000000..ef0b0d44b8f --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern.hpp @@ -0,0 +1,61 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_QRCODE_DETECTOR_FINDER_PATTERN_HPP_ +#define __ZXING_QRCODE_DETECTOR_FINDER_PATTERN_HPP_ + +#include "../../common/bitmatrix.hpp" +#include "../../resultpoint.hpp" + +#include + +namespace zxing { +namespace qrcode { + +class FinderPattern : public ResultPoint { +public: + enum CheckState { + HORIZONTAL_STATE_NORMAL = 0, + HORIZONTAL_STATE_LEFT_SPILL = 1, + HORIZONTAL_STATE_RIGHT_SPILL = 2, + VERTICAL_STATE_NORMAL = 3, + VERTICAL_STATE_UP_SPILL = 4, + VERTICAL_STATE_DOWN_SPILL = 5 + }; + +private: + float estimatedModuleSize_; + int count_; + + FinderPattern(float posX, float posY, float estimatedModuleSize, int count); + +public: + FinderPattern(float posX, float posY, float estimatedModuleSize); + int getCount() const; + float getEstimatedModuleSize() const; + void incrementCount(); + bool aboutEquals(float moduleSize, float i, float j) const; + Ref combineEstimate(float i, float j, float newModuleSize) const; + + void setHorizontalCheckState(int state); + void setVerticalCheckState(int state); + + int getHorizontalCheckState() { return horizontalState_; } + int getVerticalCheckState() { return verticalState_; } + +private: + float fix_; + CheckState horizontalState_; + CheckState verticalState_; +}; +} // namespace qrcode +} // namespace zxing + +#endif // __ZXING_QRCODE_DETECTOR_FINDER_PATTERN_HPP_ diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_finder.cpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_finder.cpp new file mode 100644 index 00000000000..fa3f5aaf373 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_finder.cpp @@ -0,0 +1,1515 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "finder_pattern_finder.hpp" +#include "../../common/kmeans.hpp" +#include "../../common/mathutils.hpp" +#include "../../decodehints.hpp" +#include "../../errorhandler.hpp" + +#include +#include + +using std::abs; +using std::max; +using std::sort; +using std::vector; +using zxing::Ref; +using zxing::qrcode::FinderPattern; +using zxing::qrcode::FinderPatternFinder; +using zxing::qrcode::FinderPatternInfo; + +// VC++ + +using zxing::BitMatrix; +using zxing::DecodeHints; +using zxing::ResultPoint; + +namespace zxing { + +namespace qrcode { + + +namespace { +class FurthestFromAverageComparator { +private: + const float averageModuleSize_; + +public: + explicit FurthestFromAverageComparator(float averageModuleSize) + : averageModuleSize_(averageModuleSize) {} + bool operator()(Ref a, Ref b) { + float dA = abs(a->getEstimatedModuleSize() - averageModuleSize_); + float dB = abs(b->getEstimatedModuleSize() - averageModuleSize_); + return dA > dB; + } +}; + +// Orders by furthes from average +class CenterComparator { + const float averageModuleSize_; + +public: + explicit CenterComparator(float averageModuleSize) : averageModuleSize_(averageModuleSize) {} + bool operator()(Ref a, Ref b) { + // N.B.: we want the result in descending order ... + if (a->getCount() != b->getCount()) { + return a->getCount() > b->getCount(); + } else { + float dA = abs(a->getEstimatedModuleSize() - averageModuleSize_); + float dB = abs(b->getEstimatedModuleSize() - averageModuleSize_); + return dA < dB; + } + } +}; + +class CountComparator { +public: + bool operator()(Ref a, Ref b) { + return a->getCount() > b->getCount(); + } +}; + +class ModuleSizeComparator { +public: + bool operator()(Ref a, Ref b) { + return a->getEstimatedModuleSize() > b->getEstimatedModuleSize(); + } +}; + +class BestComparator { +public: + bool operator()(Ref a, Ref b) { + if (a->getCount() != b->getCount()) { + return a->getCount() > b->getCount(); + } else { + return a->getEstimatedModuleSize() > b->getEstimatedModuleSize(); + } + } +}; +class BestComparator2 { +public: + bool operator()(Ref a, Ref b) { + if (a->getCount() != b->getCount()) { + return a->getCount() > b->getCount(); + } else { + int aErr = 0, bErr = 0; + if (a->getHorizontalCheckState() != FinderPattern::HORIZONTAL_STATE_NORMAL) aErr++; + if (a->getVerticalCheckState() != FinderPattern::VERTICAL_STATE_NORMAL) aErr++; + if (b->getHorizontalCheckState() != FinderPattern::HORIZONTAL_STATE_NORMAL) bErr++; + if (b->getVerticalCheckState() != FinderPattern::VERTICAL_STATE_NORMAL) bErr++; + + if (aErr != bErr) { + return aErr < bErr; + } else { + return a->getEstimatedModuleSize() > b->getEstimatedModuleSize(); + } + } + } +}; + +class XComparator { +public: + bool operator()(Ref a, Ref b) { return a->getX() < b->getX(); } +}; + +class YComparator { +public: + bool operator()(Ref a, Ref b) { return a->getY() < b->getY(); } +}; + +} // namespace + +int FinderPatternFinder::CENTER_QUORUM = 2; +int FinderPatternFinder::MIN_SKIP = 1; // 1 pixel/module times MIN_SKIP modules/center +int FinderPatternFinder::MAX_MODULES = 177; // support up to version 40 which has 177 modules +int FinderPatternFinder::INTEGER_MATH_SHIFT = 8; +int FinderPatternFinder::FP_INPUT_CNN_MAX_NUM = 10; +int FinderPatternFinder::FP_IS_SELECT_BEST = 1; +int FinderPatternFinder::FP_IS_SELECT_FILE_BEST = 1; +int FinderPatternFinder::FP_INPUT_MAX_NUM = 100; +int FinderPatternFinder::FP_FILTER_SIZE = 100; +int FinderPatternFinder::FPS_CLUSTER_MAX = 4; +int FinderPatternFinder::FPS_RESULT_MAX = 12; +int FinderPatternFinder::K_FACTOR = 2; + +float FinderPatternFinder::FPS_MS_VAL = 1.0f; +float FinderPatternFinder::FP_COUNT_MIN = 2.0f; +float FinderPatternFinder::FP_MS_MIN = 1.0f; +float FinderPatternFinder::FP_RIGHT_ANGLE = 0.342f; +float FinderPatternFinder::FP_SMALL_ANGLE1 = 0.8191f; +float FinderPatternFinder::FP_SMALL_ANGLE2 = 0.5736f; +float FinderPatternFinder::QR_MIN_FP_AREA_ERR = 3; +float FinderPatternFinder::QR_MIN_FP_MS_ERR = 1; +int FinderPatternFinder::QR_MIN_FP_ACCEPT = 4; + +std::vector> FinderPatternFinder::find(DecodeHints const& hints, + ErrorHandler& err_handler) { + bool tryHarder = true; + + size_t maxI = image_->getHeight(); + size_t maxJ = image_->getWidth(); + // Init pre check result + _horizontalCheckedResult.clear(); + _horizontalCheckedResult.resize(maxJ); + // As this is used often, we use an integer array instead of vector + int stateCount[5]; + + // Let's assume that the maximum version QR Code we support + // (Version 40, 177modules, and finder pattern start at: 0~7) takes up 1/4 + // the height of the image, and then account for the center being 3 + // modules in size. This gives the smallest number of pixels the center + // could be, so skip this often. When trying harder, look for all + // QR versions regardless of how dense they are. + int iSkip = (3 * maxI) / (4 * MAX_MODULES); + if (iSkip < MIN_SKIP || tryHarder) { + iSkip = MIN_SKIP; + } + + // This is slightly faster than using the Ref. Efficiency is important here + BitMatrix& matrix = *image_; + + // If we need to use getRowRecords or getRowCounterOffsetEnd, we should call + // initRowCounters first + matrix.initRowCounters(); + + // scan line algorithm + for (size_t i = iSkip - 1; i < maxI; i += iSkip) { + COUNTER_TYPE* irow_states = matrix.getRowRecords(i); + COUNTER_TYPE* irow_offsets = matrix.getRowRecordsOffset(i); + + size_t rj = matrix.getRowFirstIsWhite(i) ? 1 : 0; + COUNTER_TYPE row_counter_width = matrix.getRowCounterOffsetEnd(i); + // because the rj is black, rj+1 must be white, so we can skip it by +2 + for (; (rj + 4) < size_t(row_counter_width) && (rj + 4) < maxJ; rj += 2) { + stateCount[0] = irow_states[rj]; + stateCount[1] = irow_states[rj + 1]; + stateCount[2] = irow_states[rj + 2]; + stateCount[3] = irow_states[rj + 3]; + stateCount[4] = irow_states[rj + 4]; + + size_t j = irow_offsets[rj + 4] + stateCount[4]; + if (j > maxJ) { + rj = row_counter_width - 1; + continue; + } + if (foundPatternCross(stateCount)) { + if (j == maxJ) { + // check whether it is the "true" central + bool confirmed = handlePossibleCenter(stateCount, i, maxJ); + if (confirmed) { + iSkip = int(possibleCenters_.back()->getEstimatedModuleSize()); + if (iSkip < 1) iSkip = 1; + } + rj = row_counter_width - 1; + continue; + } else { + bool confirmed = handlePossibleCenter(stateCount, i, j); + if (confirmed) { + // Start examining every other line. Checking each line + // turned out to be too expensive and didn't improve + // performance. + iSkip = 2; + if (!hasSkipped_) { + int rowSkip = findRowSkip(); + if (rowSkip > stateCount[2]) { + // Skip rows between row of lower confirmed + // center and top of presumed third confirmed + // center but back up a bit to get a full chance + // of detecting it, entire width of center of + // finder pattern Skip by rowSkip, but back off + // by stateCount[2] (size of last center of + // pattern we saw) to be conservative, and also + // back off by iSkip which is about to be + // re-added + i += rowSkip - stateCount[2] - iSkip; + rj = row_counter_width - 1; + j = maxJ - 1; + } + } + } else { + continue; + } + rj += 4; + } + } + } + } + // use connected cells algorithm + { + for (size_t i = iSkip - 1; i < maxI; i += iSkip) { + COUNTER_TYPE* irow_states = matrix.getRowRecords(i); + COUNTER_TYPE* irow_offsets = matrix.getRowRecordsOffset(i); + COUNTER_TYPE row_counter_width = matrix.getRowCounterOffsetEnd(i); + + for (size_t rj = matrix.getRowFirstIsWhite(i) ? 1 : 0; + (rj + 4) < size_t(row_counter_width); rj += 2) { + if (block_->GetUnicomBlockIndex(i, irow_offsets[rj]) == + block_->GetUnicomBlockIndex(i, irow_offsets[rj + 4]) && + block_->GetUnicomBlockIndex(i, irow_offsets[rj + 1]) == + block_->GetUnicomBlockIndex(i, irow_offsets[rj + 3]) && + block_->GetUnicomBlockIndex(i, irow_offsets[rj]) != + block_->GetUnicomBlockIndex(i, irow_offsets[rj + 2])) { + const int iBlackCir = block_->GetUnicomBlockSize(i, irow_offsets[rj]); + const int iWhiteCir = block_->GetUnicomBlockSize(i, irow_offsets[rj + 1]); + const int iBlackPnt = block_->GetUnicomBlockSize(i, irow_offsets[rj + 2]); + + if (-1 == iBlackCir || -1 == iWhiteCir) continue; + + const float fBlackCir = sqrt(iBlackCir / 24.0); + const float fWhiteCir = sqrt(iWhiteCir / 16.0); + const float fBlackPnt = sqrt(iBlackPnt / 9.0); + + // use center 1:3:1, because the border may be padded. + // a plan for MS + const float fRealMS = sqrt((iWhiteCir + iBlackPnt) / 25.0); + + // b plan for MS + int iTotalCount = 0; + for (int j = 1; j < 4; ++j) iTotalCount += irow_states[rj + j]; + const float fEstRowMS = iTotalCount / 5.0; + + if (fabs(fBlackCir - fWhiteCir) <= QR_MIN_FP_AREA_ERR && + fabs(fWhiteCir - fBlackPnt) <= QR_MIN_FP_AREA_ERR && + fabs(fRealMS - fEstRowMS) < QR_MIN_FP_MS_ERR) { + int centerI = 0; + int centerJ = 0; + if (fRealMS < QR_MIN_FP_ACCEPT) { + centerI = i; + centerJ = irow_offsets[rj + 2] + irow_states[rj + 2] / 2; + } else { + int iMinX = 0, iMinY = 0, iMaxX = 0, iMaxY = 0; + block_->GetMinPoint(i, irow_offsets[rj + 1], iMinY, iMinX); + block_->GetMaxPoint(i, irow_offsets[rj + 3], iMaxY, iMaxX); + centerI = (iMaxY + iMinY) / 2.0; // y + centerJ = (iMaxX + iMinX) / 2.0; // x + } + tryToPushToCenters(centerI, centerJ, fRealMS); + int rowSkip = findRowSkip(); + if (rowSkip > irow_states[rj + 2]) { + // Skip rows between row of lower confirmed center + // and top of presumed third confirmed center but + // back up a bit to get a full chance of detecting + // it, entire width of center of finder pattern Skip + // by rowSkip, but back off by stateCount[2] (size + // of last center of pattern we saw) to be + // conservative, and also back off by iSkip which is + // about to be re-added + i += rowSkip - irow_states[rj + 2] - iSkip; + rj = row_counter_width - 1; + } + rj += 4; + } + } + } + } + } + + vector> patternInfos = getPatternInfosFileMode(hints, err_handler); + if (err_handler.ErrCode()) { + return std::vector>(); + } + // sort with score + sort(patternInfos.begin(), patternInfos.end(), + [](Ref a, Ref b) { + return a->getPossibleFix() > b->getPossibleFix(); + }); + + return patternInfos; +} + +bool FinderPatternFinder::tryToPushToCenters(float centerI, float centerJ, + float estimatedModuleSize, + CrossCheckState horizontalState, + CrossCheckState verticalState) { + for (size_t index = 0; index < possibleCenters_.size(); index++) { + Ref center = possibleCenters_[index]; + // Look for about the same center and module size: + if (center->aboutEquals(estimatedModuleSize, centerI, centerJ)) { + possibleCenters_[index] = + center->combineEstimate(centerI, centerJ, estimatedModuleSize); + possibleCenters_[index]->setHorizontalCheckState( + horizontalState == FinderPatternFinder::NORMAL ? center->getHorizontalCheckState() + : horizontalState); + possibleCenters_[index]->setVerticalCheckState( + verticalState == FinderPatternFinder::NORMAL ? center->getVerticalCheckState() + : verticalState); + return false; + } + } + Ref newPattern(new FinderPattern(centerJ, centerI, estimatedModuleSize)); + newPattern->setHorizontalCheckState(horizontalState); + newPattern->setVerticalCheckState(verticalState); + possibleCenters_.push_back(newPattern); + return true; +} + +bool FinderPatternFinder::isEqualResult(Ref src, Ref dst) { + if (src == NULL) { + return false; + } + + if (dst == NULL) { + return true; + } + + auto topLeft = src->getTopLeft(); + auto bottomLeft = src->getBottomLeft(); + auto topRight = src->getTopRight(); + + return topLeft->aboutEquals(1.0, dst->getTopLeft()->getY(), dst->getTopLeft()->getX()) && + bottomLeft->aboutEquals(1.0, dst->getBottomLeft()->getY(), + dst->getBottomLeft()->getX()) && + topRight->aboutEquals(1.0, dst->getTopRight()->getY(), dst->getTopRight()->getX()); +} + +bool FinderPatternFinder::IsPossibleFindPatterInfo(Ref a, Ref b, + Ref c) { + // check variance + float aMs = a->getEstimatedModuleSize(); + float bMs = b->getEstimatedModuleSize(); + float cMs = c->getEstimatedModuleSize(); + + float avg = (aMs + bMs + cMs) / 3.0; + float val = + sqrt((aMs - avg) * (aMs - avg) + (bMs - avg) * (bMs - avg) + (cMs - avg) * (cMs - avg)); + + if (val >= FPS_MS_VAL) return false; + + float longSize = 0.0; + + return checkIsoscelesRightTriangle(a, b, c, longSize); +} + +void FinderPatternFinder::PushToResult(Ref a, Ref b, + Ref c, + vector>& patternInfos) { + vector> finderPatterns; + finderPatterns.push_back(a); + finderPatterns.push_back(b); + finderPatterns.push_back(c); + vector> finderPattern = orderBestPatterns(finderPatterns); + + Ref patternInfo(new FinderPatternInfo(finderPattern)); + + for (size_t j = 0; j < patternInfos.size(); j++) { + if (isEqualResult(patternInfos[j], patternInfo)) { + return; + } + } + patternInfos.push_back(patternInfo); +} + +vector> FinderPatternFinder::getPatternInfosFileMode( + DecodeHints const& hints, ErrorHandler& err_handler) { + size_t startSize = possibleCenters_.size(); + + if (startSize < 3) { + // Couldn't find enough finder patterns + err_handler = ReaderErrorHandler("Could not find three finder patterns"); + return vector>(); + } + + std::vector> patternInfos; + + if (startSize == 3) { + PushToResult(possibleCenters_[0], possibleCenters_[1], possibleCenters_[2], patternInfos); + return patternInfos; + } + + vector> finderPatterns; + Ref resultBest; + + // select best + if (FP_IS_SELECT_BEST) { + finderPatterns = selectBestPatterns(err_handler); + if (err_handler.ErrCode() == 0) + PushToResult(finderPatterns[0], finderPatterns[1], finderPatterns[2], patternInfos); + } + + if (FP_IS_SELECT_FILE_BEST) { + finderPatterns = selectFileBestPatterns(err_handler); + if (err_handler.ErrCode() == 0) + PushToResult(finderPatterns[0], finderPatterns[1], finderPatterns[2], patternInfos); + } + + // sort and filter + sort(possibleCenters_.begin(), possibleCenters_.end(), ModuleSizeComparator()); + std::vector> standardCenters; + + for (size_t i = 0; i < possibleCenters_.size(); i++) { + if (possibleCenters_[i]->getEstimatedModuleSize() >= FP_MS_MIN && + possibleCenters_[i]->getCount() >= FP_COUNT_MIN) { + standardCenters.push_back(possibleCenters_[i]); + if (standardCenters.size() >= size_t(FP_INPUT_MAX_NUM)) break; + if (hints.getUseNNDetector() && standardCenters.size() >= size_t(FP_INPUT_CNN_MAX_NUM)) + break; + } + } + + if (standardCenters.size() < 3) { + err_handler = ReaderErrorHandler("Could not find three finder patterns"); + return vector>(); + } + + if (standardCenters.size() <= size_t(FP_INPUT_CNN_MAX_NUM)) { + for (size_t x = 0; x < standardCenters.size(); x++) { + for (size_t y = x + 1; y < standardCenters.size(); y++) { + for (size_t z = y + 1; z < standardCenters.size(); z++) { + bool check_result = IsPossibleFindPatterInfo( + standardCenters[x], standardCenters[y], standardCenters[z]); + if (check_result) { + PushToResult(standardCenters[x], standardCenters[y], standardCenters[z], + patternInfos); + } + } + } + } + return patternInfos; + } + + // Kmeans + const int maxepoches = 100; + const int minchanged = 0; + // calculate K + int k = log(float(standardCenters.size())) * K_FACTOR - 1; + if (k < 1) k = 1; + + vector> trainX; + for (size_t i = 0; i < standardCenters.size(); i++) { + vector tmp; + tmp.push_back(standardCenters[i]->getCount()); + tmp.push_back(standardCenters[i]->getEstimatedModuleSize()); + trainX.push_back(tmp); + } + + vector clusters_out = k_means(trainX, k, maxepoches, minchanged); + + for (size_t i = 0; i < clusters_out.size(); i++) { + int cluster_select = 0; + + if (clusters_out[i].samples.size() < 3) { + if (i < clusters_out.size() - 1 && clusters_out[i + 1].samples.size() < 3) { + for (size_t j = 0; j < clusters_out[i].samples.size(); j++) + clusters_out[i + 1].samples.push_back(clusters_out[i].samples[j]); + } + continue; + } + + vector> clusterPatterns; + for (size_t j = 0; j < clusters_out[i].samples.size(); j++) { + clusterPatterns.push_back(standardCenters[clusters_out[i].samples[j]]); + } + + sort(clusterPatterns.begin(), clusterPatterns.end(), BestComparator2()); + + for (size_t x = 0; + x < clusters_out[i].samples.size() && cluster_select <= FPS_CLUSTER_MAX && + patternInfos.size() <= size_t(FPS_RESULT_MAX); + x++) { + for (size_t y = x + 1; + y < clusters_out[i].samples.size() && cluster_select <= FPS_CLUSTER_MAX && + patternInfos.size() <= size_t(FPS_RESULT_MAX); + y++) { + for (size_t z = y + 1; + z < clusters_out[i].samples.size() && cluster_select <= FPS_CLUSTER_MAX && + patternInfos.size() <= size_t(FPS_RESULT_MAX); + z++) { + bool check_result = IsPossibleFindPatterInfo( + clusterPatterns[x], clusterPatterns[y], clusterPatterns[z]); + if (check_result) { + PushToResult(clusterPatterns[x], clusterPatterns[y], clusterPatterns[z], + patternInfos); + cluster_select++; + } + } + } + } + } + return patternInfos; +} + +// Given a count of black/white/black/white/black pixels just seen and an end +// position, figures the location of the center of this run. +float FinderPatternFinder::centerFromEnd(int* stateCount, int end) { + // calculate the center by pattern 1:3:1 is better than pattern 3 + // because the finder pattern is irregular in some case + return (float)(end - stateCount[4]) - (stateCount[3] + stateCount[2] + stateCount[1]) / 2.0f; +} + +// return if the proportions of the counts is close enough to 1/1/3/1/1 ratios +// used by finder patterns to be considered a match +bool FinderPatternFinder::foundPatternCross(int* stateCount) { + int totalModuleSize = 0; + + int stateCountINT[5]; + + int minModuleSizeINT = 3; + minModuleSizeINT <<= INTEGER_MATH_SHIFT; + + for (int i = 0; i < 5; i++) { + if (stateCount[i] <= 0) { + return false; + } + stateCountINT[i] = stateCount[i] << INTEGER_MATH_SHIFT; + totalModuleSize += stateCount[i]; + } + if (totalModuleSize < 7) { + return false; + } + + CURRENT_CHECK_STATE = FinderPatternFinder::NOT_PATTERN; + + totalModuleSize = totalModuleSize << INTEGER_MATH_SHIFT; + + // Newer version to check 1 time, use 3 points + int moduleSize = ((totalModuleSize - stateCountINT[0] - stateCountINT[4])) / 5; + + int maxVariance = moduleSize; + + if (moduleSize > minModuleSizeINT) maxVariance = moduleSize / 2; + + int startCountINT = stateCountINT[0]; + int endCountINT = stateCountINT[4]; + + bool leftFit = (abs(moduleSize - startCountINT) <= maxVariance); + bool rightFit = (abs(moduleSize - endCountINT) <= maxVariance); + + if (leftFit) { + if (rightFit) { + moduleSize = totalModuleSize / 7; + CURRENT_CHECK_STATE = FinderPatternFinder::NORMAL; + } else { + moduleSize = (totalModuleSize - stateCountINT[4]) / 6; + CURRENT_CHECK_STATE = FinderPatternFinder::RIHGT_SPILL; + } + } else { + if (rightFit) { + moduleSize = (totalModuleSize - stateCountINT[0]) / 6; + CURRENT_CHECK_STATE = FinderPatternFinder::LEFT_SPILL; + } else { + // return false; + CURRENT_CHECK_STATE = FinderPatternFinder::LEFT_RIGHT_SPILL; + } + } + + // 1:1:3:1:1 || n:1:3:1:1 || 1:1:3:1:n + if (abs(moduleSize - stateCountINT[1]) <= maxVariance && + abs(3 * moduleSize - stateCountINT[2]) <= 3 * maxVariance && + abs(moduleSize - stateCountINT[3]) <= maxVariance) { + return true; + } + return false; +} + +int FinderPatternFinder::getMinModuleSize() { + int minModuleSize = (3 * min(image_->getWidth(), image_->getHeight())) / (4 * MAX_MODULES); + + if (minModuleSize < MIN_SKIP) { + minModuleSize = MIN_SKIP; + } + + return minModuleSize; +} + +/** + * After a vertical and horizontal scan finds a potential finder pattern, this + * method "cross-cross-cross-checks" by scanning down diagonally through the + * center of the possible finder pattern to see if the same proportion is + * detected. + * + * @param startI row where a finder pattern was detected + * @param centerJ center of the section that appears to cross a finder pattern + * @param maxCount maximum reasonable number of modules that should be + * observed in any reading state, based on the results of the horizontal scan + * @param originalStateCountTotal The original state count total. + * @return true if proportions are withing expected limits + */ +bool FinderPatternFinder::crossCheckDiagonal(int startI, int centerJ, int maxCount, + int originalStateCountTotal) { + int maxI = image_->getHeight(); + int maxJ = image_->getWidth(); + + if ((startI < 0) || (startI > maxI - 1) || (centerJ < 0) || (centerJ > maxJ - 1)) { + return false; + } + + int stateCount[5]; + stateCount[0] = 0; + stateCount[1] = 0; + stateCount[2] = 0; + stateCount[3] = 0; + stateCount[4] = 0; + + if (!image_->get(centerJ, startI)) { + if (startI + 1 < maxI && image_->get(centerJ, startI + 1)) + startI = startI + 1; + else if (0 < startI - 1 && image_->get(centerJ, startI - 1)) + startI = startI - 1; + else + return false; + } + + // This is slightly faster than using the Ref. Efficiency is important here + BitMatrix& matrix = *image_; + + // Start counting up, left from center finding black center mass + int i = 0; + // Fix possible crash 20140418 + // while (startI - i >= 0 && image.get(centerJ - i, startI - i)) { + while ((startI - i >= 0) && (centerJ - i >= 0) && matrix.get(centerJ - i, startI - i)) { + stateCount[2]++; + i++; + } + + if ((startI - i < 0) || (centerJ - i < 0)) { + return false; + } + + // Continue up, left finding white space + while ((startI - i >= 0) && (centerJ - i >= 0) && !matrix.get(centerJ - i, startI - i) && + stateCount[1] <= maxCount) { + stateCount[1]++; + i++; + } + + // If already too many modules in this state or ran off the edge: + if ((startI - i < 0) || (centerJ - i < 0) || stateCount[1] > maxCount) { + return false; + } + + CrossCheckState tmpCheckState = FinderPatternFinder::NORMAL; + + // Continue up, left finding black border + while ((startI - i >= 0) && (centerJ - i >= 0) && matrix.get(centerJ - i, startI - i) && + stateCount[0] <= maxCount) { + stateCount[0]++; + i++; + } + + if (stateCount[0] >= maxCount) { + tmpCheckState = FinderPatternFinder::LEFT_SPILL; + } + + // Now also count down, right from center + i = 1; + while ((startI + i < maxI) && (centerJ + i < maxJ) && matrix.get(centerJ + i, startI + i)) { + stateCount[2]++; + i++; + } + + // Ran off the edge? + if ((startI + i >= maxI) || (centerJ + i >= maxJ)) { + return false; + } + + while ((startI + i < maxI) && (centerJ + i < maxJ) && !matrix.get(centerJ + i, startI + i) && + stateCount[3] < maxCount) { + stateCount[3]++; + i++; + } + + if ((startI + i >= maxI) || (centerJ + i >= maxJ) || stateCount[3] >= maxCount) { + return false; + } + + while ((startI + i < maxI) && (centerJ + i < maxJ) && matrix.get(centerJ + i, startI + i) && + stateCount[4] < maxCount) { + stateCount[4]++; + i++; + } + + if (stateCount[4] >= maxCount) { + tmpCheckState = tmpCheckState == FinderPatternFinder::LEFT_SPILL + ? FinderPatternFinder::LEFT_RIGHT_SPILL + : FinderPatternFinder::RIHGT_SPILL; + } + + bool diagonal_check = foundPatternCross(stateCount); + if (!diagonal_check) return false; + + if (CURRENT_CHECK_STATE == FinderPatternFinder::LEFT_SPILL && + tmpCheckState == FinderPatternFinder::RIHGT_SPILL) + return false; + + if (CURRENT_CHECK_STATE == FinderPatternFinder::RIHGT_SPILL && + tmpCheckState == FinderPatternFinder::LEFT_SPILL) + return false; + + int stateCountTotal = getStateCountTotal(stateCount, CURRENT_CHECK_STATE); + + if (abs(stateCountTotal - originalStateCountTotal) < 2 * originalStateCountTotal) { + return true; + } else { + return false; + } +} + +int FinderPatternFinder::getStateCountTotal(int* stateCount, const CrossCheckState& check_state) { + int stateCountTotal = stateCount[1] + stateCount[2] + stateCount[3]; + if (check_state == FinderPatternFinder::NORMAL) { + stateCountTotal = stateCountTotal + stateCount[0] + stateCount[4]; + } else if (check_state == FinderPatternFinder::LEFT_SPILL) { + stateCountTotal = stateCountTotal + stateCount[1] + stateCount[4]; + } else if (check_state == FinderPatternFinder::RIHGT_SPILL) { + stateCountTotal = stateCountTotal + stateCount[0] + stateCount[3]; + } else if (check_state == FinderPatternFinder::LEFT_RIGHT_SPILL) { + stateCountTotal = stateCountTotal + stateCount[1] + stateCount[3]; + } + return stateCountTotal; +} +// After a horizontal scan finds a potential finder pattern, this method +// "cross-checks" by scanning down vertically through the center of the possible +// finder pattern to see if the same proportion is detected. +float FinderPatternFinder::crossCheckVertical(size_t startI, size_t centerJ, int maxCount, + int originalStateCountTotal, + float& estimatedVerticalModuleSize) { + int maxI = image_->getHeight(); + + int stateCount[5]; + for (int i = 0; i < 5; i++) stateCount[i] = 0; + + if (!image_->get(centerJ, startI)) { + if ((int)startI + 1 < maxI && image_->get(centerJ, startI + 1)) + startI = startI + 1; + else if (0 < (int)startI - 1 && image_->get(centerJ, startI - 1)) + startI = startI - 1; + else + return nan(); + } + + // This is slightly faster than using the Ref. Efficiency is important here + BitMatrix& matrix = *image_; + + bool* imageRow0 = matrix.getRowBoolPtr(0); + bool* p = imageRow0; + int imgWidth = matrix.getWidth(); + + // Start counting up from center + int ii = startI; + + p = imageRow0 + ii * imgWidth + centerJ; + + while (ii >= 0 && *p) { + stateCount[2]++; + ii--; + p -= imgWidth; + } + if (ii < 0) { + return nan(); + } + while (ii >= 0 && !*p && stateCount[1] <= maxCount) { + stateCount[1]++; + ii--; + p -= imgWidth; + } + // If already too many modules in this state or ran off the edge: + if (ii < 0 || stateCount[1] > maxCount) { + return nan(); + } + + CrossCheckState tmpCheckState = FinderPatternFinder::NORMAL; + + while (ii >= 0 && *p /*&& stateCount[0] <= maxCount*/) { // n:1:3:1:1 + stateCount[0]++; + ii--; + p -= imgWidth; + } + + if (stateCount[0] >= maxCount) { + tmpCheckState = FinderPatternFinder::LEFT_SPILL; + } + + // Now also count down from center + ii = startI + 1; + + p = imageRow0 + ii * imgWidth + centerJ; + + while (ii < maxI && *p) { // 1:1:"3":1:1 + // while (ii < maxI && matrix.get(centerJ, ii)) { + stateCount[2]++; + ii++; + + p += imgWidth; + } + if (ii == maxI) { + return nan(); + } + while (ii < maxI && !*p && stateCount[3] < maxCount) { // 1:1:3:"1":1 + stateCount[3]++; + ii++; + + p += imgWidth; + } + if (ii == maxI || stateCount[3] >= maxCount) { + return nan(); + } + + if (tmpCheckState == FinderPatternFinder::LEFT_SPILL) { + while (ii < maxI && *p && stateCount[4] < maxCount) { // 1:1:3:1:"1" + stateCount[4]++; + ii++; + + p += imgWidth; + } + if (stateCount[4] >= maxCount) { + return nan(); + } + } else { // 1:1:3:1:"n" + while (ii < maxI && *p) { + stateCount[4]++; + ii++; + + p += imgWidth; + } + if (stateCount[4] >= maxCount) { + tmpCheckState = FinderPatternFinder::RIHGT_SPILL; + } + } + + bool vertical_check = foundPatternCross(stateCount); + if (!vertical_check) return nan(); + + if ((CURRENT_CHECK_STATE == FinderPatternFinder::LEFT_SPILL && + tmpCheckState == FinderPatternFinder::RIHGT_SPILL) || + (CURRENT_CHECK_STATE == FinderPatternFinder::RIHGT_SPILL && + tmpCheckState == FinderPatternFinder::LEFT_SPILL)) { + return nan(); + } + + int stateCountTotal = getStateCountTotal(stateCount, CURRENT_CHECK_STATE); + + // If we found a finder-pattern-like section, but its size is more than 40% + // different than the original, assume it's a false positive + if (5 * abs(stateCountTotal - originalStateCountTotal) >= 2 * originalStateCountTotal) { + return nan(); + } + + estimatedVerticalModuleSize = (float)stateCountTotal / 7.0f; + + return centerFromEnd(stateCount, ii); +} + +// Like #crossCheckVertical(), and in fact is basically identical, +// except it reads horizontally instead of vertically. This is used to +// cross-cross check a vertical cross check and locate the real center of the +// alignment pattern. +float FinderPatternFinder::crossCheckHorizontal(size_t startJ, size_t centerI, int maxCount, + int originalStateCountTotal, + float& estimatedHorizontalModuleSize) { + int maxJ = image_->getWidth(); + + int stateCount[5]; + for (int i = 0; i < 5; i++) stateCount[i] = 0; + + if (!image_->get(startJ, centerI)) { + if ((int)startJ + 1 < maxJ && image_->get(startJ + 1, centerI)) + startJ = startJ + 1; + else if (0 < (int)startJ - 1 && image_->get(startJ - 1, centerI)) + startJ = startJ - 1; + else + return nan(); + } + + // This is slightly faster than using the Ref. Efficiency is important here + BitMatrix& matrix = *image_; + int j = startJ; + + bool* centerIrow = NULL; + + centerIrow = matrix.getRowBoolPtr(centerI); + + // while (j >= 0 &&matrix.get(j, centerI)) { + while (j >= 0 && centerIrow[j]) { + stateCount[2]++; + j--; + } + if (j < 0) { + return nan(); + } + while (j >= 0 && !centerIrow[j] && stateCount[1] <= maxCount) { + stateCount[1]++; + j--; + } + if (j < 0 || stateCount[1] > maxCount) { + return nan(); + } + CrossCheckState tmpCheckState = FinderPatternFinder::NORMAL; + + while (j >= 0 && centerIrow[j] /* && stateCount[0] <= maxCount*/) { + stateCount[0]++; + j--; + } + if (stateCount[0] >= maxCount) { + tmpCheckState = FinderPatternFinder::LEFT_SPILL; + } + + j = startJ + 1; + while (j < maxJ && centerIrow[j]) { + stateCount[2]++; + j++; + } + if (j == maxJ) { + return nan(); + } + while (j < maxJ && !centerIrow[j] && stateCount[3] < maxCount) { + stateCount[3]++; + j++; + } + if (j == maxJ || stateCount[3] >= maxCount) { + return nan(); + } + + if (tmpCheckState == LEFT_SPILL) { + while (j < maxJ && centerIrow[j] && stateCount[4] <= maxCount) { + stateCount[4]++; + j++; + } + if (stateCount[4] >= maxCount) { + return nan(); + } + } else { + while (j < maxJ && centerIrow[j]) { + stateCount[4]++; + j++; + } + if (stateCount[4] >= maxCount) { + tmpCheckState = RIHGT_SPILL; + } + } + + while (j < maxJ && centerIrow[j] /*&& stateCount[4] < maxCount*/) { + stateCount[4]++; + j++; + } + + // If we found a finder-pattern-like section, but its size is significantly + // different than the original, assume it's a false positive + // int stateCountTotal = stateCount[0] + stateCount[1] + stateCount[2] + + // stateCount[3] + stateCount[4]; + bool horizontal_check = foundPatternCross(stateCount); + if (!horizontal_check) return nan(); + + /* + if(tmpCheckState!=CURRENT_CHECK_STATE) + return nan();*/ + + // Cannot be a LEFT-RIGHT center + if ((CURRENT_CHECK_STATE == FinderPatternFinder::LEFT_SPILL && + tmpCheckState == FinderPatternFinder::RIHGT_SPILL) || + (CURRENT_CHECK_STATE == FinderPatternFinder::RIHGT_SPILL && + tmpCheckState == FinderPatternFinder::LEFT_SPILL)) { + return nan(); + } + + int stateCountTotal = getStateCountTotal(stateCount, CURRENT_CHECK_STATE); + if (5 * abs(stateCountTotal - originalStateCountTotal) >= originalStateCountTotal) { + return nan(); + } + + estimatedHorizontalModuleSize = (float)stateCountTotal / 7.0f; + return centerFromEnd(stateCount, j); +} + +float FinderPatternFinder::hasHorizontalCheckedResult(size_t startJ, size_t centerI) { + for (size_t i = 0; i < _horizontalCheckedResult[startJ].size(); i++) { + if (_horizontalCheckedResult[startJ][i].centerI == centerI) { + return _horizontalCheckedResult[startJ][i].centerJ; + } + } + + return -1.0; +} + +int FinderPatternFinder::addHorizontalCheckedResult(size_t startJ, size_t centerI, float centerJ) { + HorizontalCheckedResult result; + result.centerI = centerI; + result.centerJ = centerJ; + + _horizontalCheckedResult[startJ].push_back(result); + + return 1; +} + +#define CENTER_CHECK_TIME 3 + +/** + *

This is called when a horizontal scan finds a possible alignment pattern. + * It will cross check with a vertical scan, and if successful, will, ah, + * cross-cross-check with another horizontal scan. This is needed primarily to + * locate the real horizontal center of the pattern in cases of extreme skew. + * And then we cross-cross-cross check with another diagonal scan.

+ * + *

If that succeeds the finder pattern location is added to a list that + * tracks the number of times each location has been nearly-matched as a finder + * pattern. Each additional find is more evidence that the location is in fact a + * finder pattern center + * + * @param stateCount reading state module counts from horizontal scan + * @param i row where finder pattern may be found + * @param j end of possible finder pattern in row + * @return true if a finder pattern candidate was found this time + */ +bool FinderPatternFinder::handlePossibleCenter(int* stateCount, size_t i, size_t j) { + CrossCheckState tmpHorizontalState = CURRENT_CHECK_STATE; + float centerJ = centerFromEnd(stateCount, j); + int stateCountTotal = stateCount[1] + stateCount[2] + stateCount[3]; + if (tmpHorizontalState == FinderPatternFinder::NORMAL) { + // 1:1:3:1:1 + stateCountTotal = stateCountTotal + stateCount[0] + stateCount[4]; + } else if (tmpHorizontalState == FinderPatternFinder::LEFT_SPILL) { + // n:1:3:1:1 + stateCountTotal = stateCountTotal + stateCount[1] + stateCount[4]; + } else if (tmpHorizontalState == FinderPatternFinder::RIHGT_SPILL) { + // 1:1:3:1:n + stateCountTotal = stateCountTotal + stateCount[0] + stateCount[3]; + } + float estimatedHorizontalModuleSize = (float)stateCountTotal / 7.0f; + + float estimatedVerticalModuleSize; + + // try different size according to the estimatedHorizontalModuleSize + float tolerateModuleSize = + estimatedHorizontalModuleSize > 4.0 ? estimatedHorizontalModuleSize / 2.0f : 1.0f; + float possbileCenterJs[7] = {centerJ, + centerJ - tolerateModuleSize, + centerJ + tolerateModuleSize, + centerJ - 2 * tolerateModuleSize, + centerJ + 2 * tolerateModuleSize, + centerJ - 3 * tolerateModuleSize, + centerJ + 3 * tolerateModuleSize}; + int image_height = image_->getHeight(); + int image_width = image_->getWidth(); + for (int k = 0; k < CENTER_CHECK_TIME; k++) { + float possibleCenterJ = possbileCenterJs[k]; + if (possibleCenterJ < 0 || possibleCenterJ >= image_width) continue; + float centerI = crossCheckVertical(i, (size_t)possibleCenterJ, stateCount[2], + stateCountTotal, estimatedVerticalModuleSize); + + if (!isnan(centerI) && centerI >= 0.0) { + CrossCheckState tmpVerticalState = CURRENT_CHECK_STATE; + + float moduleSizeDiff = abs(estimatedHorizontalModuleSize - estimatedVerticalModuleSize); + + if (moduleSizeDiff > estimatedHorizontalModuleSize || + moduleSizeDiff > estimatedVerticalModuleSize) + return false; + + tolerateModuleSize = + estimatedVerticalModuleSize > 4.0 ? estimatedVerticalModuleSize / 2.0f : 1.0f; + + float possbileCenterIs[7] = {centerI, + centerI - tolerateModuleSize, + centerI + tolerateModuleSize, + centerI - 2 * tolerateModuleSize, + centerI + 2 * tolerateModuleSize, + centerI - 3 * tolerateModuleSize, + centerI + 3 * tolerateModuleSize}; + + for (int l = 0; l < CENTER_CHECK_TIME; l++) { + float possibleCenterI = possbileCenterIs[l]; + if (possibleCenterI < 0 || possibleCenterI >= image_height) continue; + // Re-cross check + float reEstimatedHorizontalModuleSize; + float cJ = hasHorizontalCheckedResult(centerJ, possibleCenterI); + + if (!isnan(cJ) && cJ >= 0.0) { + centerJ = cJ; + } else { + cJ = centerJ; + + float ccj = + crossCheckHorizontal((size_t)cJ, (size_t)possibleCenterI, stateCount[2], + stateCountTotal, reEstimatedHorizontalModuleSize); + + if (!isnan(ccj)) { + centerJ = ccj; + addHorizontalCheckedResult(cJ, possibleCenterI, ccj); + } + } + if (!isnan(centerJ)) { + tryToPushToCenters( + centerI, centerJ, + (estimatedHorizontalModuleSize + estimatedVerticalModuleSize) / 2.0, + tmpHorizontalState, tmpVerticalState); + return true; + } + } + } + } + + return false; +} + +// return the number of rows we could safely skip during scanning, based on the +// first two finder patterns that have been located. In some cases their +// position will allow us to infer that the third pattern must lie below a +// certain point farther down the image. +int FinderPatternFinder::findRowSkip() { + int max = possibleCenters_.size(); + if (max <= 1) { + return 0; + } + + if (max <= compared_finder_counts) return 0; + + Ref firstConfirmedCenter, secondConfirmedCenter; + + for (int i = 0; i < max - 1; i++) { + firstConfirmedCenter = possibleCenters_[i]; + if (firstConfirmedCenter->getCount() >= CENTER_QUORUM) { + float firstModuleSize = firstConfirmedCenter->getEstimatedModuleSize(); + int j_start = (i < compared_finder_counts) ? compared_finder_counts : i + 1; + for (int j = j_start; j < max; j++) { + secondConfirmedCenter = possibleCenters_[j]; + if (secondConfirmedCenter->getCount() >= CENTER_QUORUM) { + float secondModuleSize = secondConfirmedCenter->getEstimatedModuleSize(); + float moduleSizeDiff = abs(firstModuleSize - secondModuleSize); + if (moduleSizeDiff < 1.0f) { + hasSkipped_ = true; + return (int)(abs(firstConfirmedCenter->getX() - + secondConfirmedCenter->getX()) - + abs(firstConfirmedCenter->getY() - + secondConfirmedCenter->getY())) / + 2; + } + } + } + } + } + + compared_finder_counts = max; + + return 0; +} + +// return the 3 finder patterns from our list of candidates. The "best" are +// those that have been detected at least #CENTER_QUORUM times, and whose module +// size differs from the average among those patterns the least. // +vector> FinderPatternFinder::selectBestPatterns(ErrorHandler& err_handler) { + size_t startSize = possibleCenters_.size(); + + if (startSize < 3) { + // Couldn't find enough finder patterns + err_handler = ReaderErrorHandler("Could not find three finder patterns"); + return vector>(); + } + + vector> result(3); + + if (startSize == 3) { + result[0] = possibleCenters_[0]; + result[1] = possibleCenters_[1]; + result[2] = possibleCenters_[2]; + return result; + } + + sort(possibleCenters_.begin(), possibleCenters_.end(), CountComparator()); + if ((possibleCenters_[2]->getCount() - possibleCenters_[3]->getCount()) > 1 && + possibleCenters_[2]->getCount() > 1) { + result[0] = possibleCenters_[0]; + result[1] = possibleCenters_[1]; + result[2] = possibleCenters_[2]; + return result; + } else if (possibleCenters_[3]->getCount() > 1) { + float totalModuleSize = 0.0f; + for (int i = 0; i < 4; i++) { + totalModuleSize += possibleCenters_[i]->getEstimatedModuleSize(); + } + float everageModuleSize = totalModuleSize / 4.0f; + float maxDiffModuleSize = 0.0f; + int maxID = 0; + for (int i = 0; i < 4; i++) { + float diff = abs(possibleCenters_[i]->getEstimatedModuleSize() - everageModuleSize); + if (diff > maxDiffModuleSize) { + maxDiffModuleSize = diff; + maxID = i; + } + } + switch (maxID) { + case 0: + result[0] = possibleCenters_[1]; + result[1] = possibleCenters_[2]; + result[2] = possibleCenters_[3]; + break; + case 1: + result[0] = possibleCenters_[0]; + result[1] = possibleCenters_[2]; + result[2] = possibleCenters_[3]; + break; + case 2: + result[0] = possibleCenters_[0]; + result[1] = possibleCenters_[1]; + result[2] = possibleCenters_[3]; + break; + default: + result[0] = possibleCenters_[0]; + result[1] = possibleCenters_[1]; + result[2] = possibleCenters_[2]; + break; + } + + return result; + } else if (possibleCenters_[1]->getCount() > 1 && possibleCenters_[2]->getCount() == 1) { + vector> possibleThirdCenter; + float possibleModuleSize = (possibleCenters_[0]->getEstimatedModuleSize() + + possibleCenters_[1]->getEstimatedModuleSize()) / + 2.0f; + for (size_t i = 2; i < startSize; i++) { + if (abs(possibleCenters_[i]->getEstimatedModuleSize() - possibleModuleSize) < + 0.5 * possibleModuleSize) + possibleThirdCenter.push_back(possibleCenters_[i]); + } + float longestSide = 0.0f; + size_t longestId = 0; + for (size_t i = 0; i < possibleThirdCenter.size(); i++) { + float tmpLongSide = 0.0f; + if (checkIsoscelesRightTriangle(possibleCenters_[0], possibleCenters_[1], + possibleThirdCenter[i], tmpLongSide)) { + if (tmpLongSide >= longestSide) { + longestSide = tmpLongSide; + longestId = i; + } + } + } + result[0] = possibleCenters_[0]; + result[1] = possibleCenters_[1]; + + // Error with decoding + if (longestId >= possibleThirdCenter.size()) { + err_handler = ReaderErrorHandler("Not find any available possibleThirdCenter"); + return vector>(); + } else { + result[2] = possibleThirdCenter[longestId]; + } + + return result; + } + + // Filter outlier possibilities whose module size is too different + if (startSize > 3) { + // But we can only afford to do so if we have at least 4 possibilities + // to choose from + float totalModuleSize = 0.0f; + float square = 0.0f; + for (size_t i = 0; i < startSize; i++) { + float size = possibleCenters_[i]->getEstimatedModuleSize(); + totalModuleSize += size; + square += size * size; + } + float average = totalModuleSize / (float)startSize; + float stdDev = (float)sqrt(square / startSize - average * average); + + sort(possibleCenters_.begin(), possibleCenters_.end(), + FurthestFromAverageComparator(average)); + + // float limit = max(0.2f * average, stdDev); + float limit = max(0.5f * average, stdDev); + + for (size_t i = 0; i < possibleCenters_.size() && possibleCenters_.size() > 3; i++) { + if (abs(possibleCenters_[i]->getEstimatedModuleSize() - average) > limit) { + possibleCenters_.erase(possibleCenters_.begin() + i); + i--; + } + } + } + + int tryHardPossibleCenterSize = 15; + int possibleCenterSize = 12; + + if (possibleCenters_.size() > size_t(tryHardPossibleCenterSize)) { + sort(possibleCenters_.begin(), possibleCenters_.end(), CountComparator()); + possibleCenters_.erase(possibleCenters_.begin() + tryHardPossibleCenterSize, + possibleCenters_.end()); + } else if (possibleCenters_.size() > size_t(possibleCenterSize)) { + sort(possibleCenters_.begin(), possibleCenters_.end(), CountComparator()); + possibleCenters_.erase(possibleCenters_.begin() + possibleCenterSize, + possibleCenters_.end()); + } + + if (possibleCenters_.size() >= 6) { + sort(possibleCenters_.begin(), possibleCenters_.end(), XComparator()); + possibleCenters_.erase(possibleCenters_.begin() + 4, possibleCenters_.end() - 2); + sort(possibleCenters_.begin(), possibleCenters_.begin() + 4, YComparator()); + possibleCenters_.erase(possibleCenters_.begin() + 1, possibleCenters_.begin() + 3); + sort(possibleCenters_.end() - 2, possibleCenters_.end(), YComparator()); + possibleCenters_.erase(possibleCenters_.end() - 1, possibleCenters_.end()); + } else if (possibleCenters_.size() > 3) { + // Throw away all but those first size candidate points we found. + float totalModuleSize = 0.0f; + for (size_t i = 0; i < possibleCenters_.size(); i++) { + float size = possibleCenters_[i]->getEstimatedModuleSize(); + totalModuleSize += size; + } + float average = totalModuleSize / (float)possibleCenters_.size(); + sort(possibleCenters_.begin(), possibleCenters_.end(), CenterComparator(average)); + possibleCenters_.erase(possibleCenters_.begin() + 3, possibleCenters_.end()); + } + + result[0] = possibleCenters_[0]; + result[1] = possibleCenters_[1]; + result[2] = possibleCenters_[2]; + + return result; +} + +vector> FinderPatternFinder::selectFileBestPatterns(ErrorHandler& err_handler) { + size_t startSize = possibleCenters_.size(); + + if (startSize < 3) { + // Couldn't find enough finder patterns + err_handler = ReaderErrorHandler("Could not find three finder patterns"); + return vector>(); + } + + vector> result(3); + + if (startSize == 3) { + result[0] = possibleCenters_[0]; + result[1] = possibleCenters_[1]; + result[2] = possibleCenters_[2]; + return result; + } + + sort(possibleCenters_.begin(), possibleCenters_.end(), BestComparator()); + + result[0] = possibleCenters_[0]; + result[1] = possibleCenters_[1]; + result[2] = possibleCenters_[2]; + + for (size_t i = 0; i < possibleCenters_.size() - 2; ++i) { + float tmpLongSide = 0; + + int iCountDiff = 0; + float fModuleSizeDiff = 0; + for (size_t j = 0; j < 3; ++j) { + iCountDiff += abs(possibleCenters_[i + j]->getCount() - + possibleCenters_[i + ((j + 1) % 3)]->getCount()); + fModuleSizeDiff += fabs(possibleCenters_[i + j]->getEstimatedModuleSize() - + possibleCenters_[i + ((j + 1) % 3)]->getEstimatedModuleSize()); + } + + if (iCountDiff > 2) continue; + if (fModuleSizeDiff > 5) continue; + + if (checkIsoscelesRightTriangle(possibleCenters_[i], possibleCenters_[i + 1], + possibleCenters_[i + 2], tmpLongSide)) { + result[0] = possibleCenters_[i]; + result[1] = possibleCenters_[i + 1]; + result[2] = possibleCenters_[i + 2]; + + break; + } + } + + return result; +} + +// Orders an array of three patterns in an order [A,B,C] such that +// AB> FinderPatternFinder::orderBestPatterns( + vector> patterns) { + // Find distances between pattern centers + float abDistance = distance(patterns[0], patterns[1]); + float bcDistance = distance(patterns[1], patterns[2]); + float acDistance = distance(patterns[0], patterns[2]); + + Ref topLeft; + Ref topRight; + Ref bottomLeft; + // Assume one closest to other two is top left; + // topRight and bottomLeft will just be guesses below at first + if (bcDistance >= abDistance && bcDistance >= acDistance) { + topLeft = patterns[0]; + topRight = patterns[1]; + bottomLeft = patterns[2]; + } else if (acDistance >= bcDistance && acDistance >= abDistance) { + topLeft = patterns[1]; + topRight = patterns[0]; + bottomLeft = patterns[2]; + } else { + topLeft = patterns[2]; + topRight = patterns[0]; + bottomLeft = patterns[1]; + } + + // Use cross product to figure out which of other1/2 is the bottom left + // pattern. The vector "top_left -> bottom_left" x "top_left -> top_right" + // should yield a vector with positive z component + if ((bottomLeft->getY() - topLeft->getY()) * (topRight->getX() - topLeft->getX()) < + (bottomLeft->getX() - topLeft->getX()) * (topRight->getY() - topLeft->getY())) { + Ref temp = topRight; + topRight = bottomLeft; + bottomLeft = temp; + } + + vector> results(3); + results[0] = bottomLeft; + results[1] = topLeft; + results[2] = topRight; + + return results; +} + +bool FinderPatternFinder::checkIsoscelesRightTriangle(Ref centerA, + Ref centerB, + Ref centerC, float& longSide) { + float shortSide1, shortSide2; + FinderPatternInfo::calculateSides(centerA, centerB, centerC, longSide, shortSide1, shortSide2); + + auto minAmongThree = [](float a, float b, float c) { return min(min(a, b), c); }; + auto maxAmongThree = [](float a, float b, float c) { return max(max(a, b), c); }; + + float shortSideSqrt1 = sqrt(shortSide1); + float shortSideSqrt2 = sqrt(shortSide2); + float longSideSqrt = sqrt(longSide); + auto minSide = minAmongThree(shortSideSqrt1, shortSideSqrt2, longSideSqrt); + auto maxModuleSize = + maxAmongThree(centerA->getEstimatedModuleSize(), centerB->getEstimatedModuleSize(), + centerC->getEstimatedModuleSize()); + // if edge length smaller than 14 * module size + if (minSide <= maxModuleSize * 14) return false; + + float CosLong = (shortSide1 + shortSide2 - longSide) / (2 * shortSideSqrt1 * shortSideSqrt2); + float CosShort1 = (longSide + shortSide1 - shortSide2) / (2 * longSideSqrt * shortSideSqrt1); + float CosShort2 = (longSide + shortSide2 - shortSide1) / (2 * longSideSqrt * shortSideSqrt2); + + if (abs(CosLong) > FP_RIGHT_ANGLE || + (CosShort1 < FP_SMALL_ANGLE2 || CosShort1 > FP_SMALL_ANGLE1) || + (CosShort2 < FP_SMALL_ANGLE2 || CosShort2 > FP_SMALL_ANGLE1)) { + return false; + } + + return true; +} + +// return distance between two points +float FinderPatternFinder::distance(Ref p1, Ref p2) { + float dx = p1->getX() - p2->getX(); + float dy = p1->getY() - p2->getY(); + return (float)sqrt(dx * dx + dy * dy); +} + +FinderPatternFinder::FinderPatternFinder(Ref image, Ref block) + : finder_time(0), + compared_finder_counts(0), + image_(image), + possibleCenters_(), + hasSkipped_(false), + block_(block) { + CURRENT_CHECK_STATE = FinderPatternFinder::NORMAL; +} + +Ref FinderPatternFinder::getImage() { return image_; } + +vector>& FinderPatternFinder::getPossibleCenters() { return possibleCenters_; } + +} // namespace qrcode +} // namespace zxing \ No newline at end of file diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_finder.hpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_finder.hpp new file mode 100644 index 00000000000..f7f2268cc4c --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_finder.hpp @@ -0,0 +1,139 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_QRCODE_DETECTOR_FINDER_PATTERN_FINDER_HPP_ +#define __ZXING_QRCODE_DETECTOR_FINDER_PATTERN_FINDER_HPP_ + +#include "../../common/bitmatrix.hpp" +#include "../../common/counted.hpp" +#include "../../common/unicomblock.hpp" +#include "../../errorhandler.hpp" +#include "finder_pattern.hpp" +#include "finder_pattern_info.hpp" + +#include + +using zxing::ErrorHandler; +using zxing::ReaderErrorHandler; + +namespace zxing { + +class DecodeHints; + +namespace qrcode { + +class FinderPatternFinder { +public: + enum CrossCheckState { + NORMAL = 0, + LEFT_SPILL = 1, + RIHGT_SPILL = 2, + LEFT_RIGHT_SPILL = 3, + NOT_PATTERN = 4, + }; + +private: + static int CENTER_QUORUM; + static int MIN_SKIP; + static int MAX_MODULES; + static int INTEGER_MATH_SHIFT; + static int FP_INPUT_CNN_MAX_NUM; + static int FP_IS_SELECT_BEST; + static int FP_IS_SELECT_FILE_BEST; + static int FP_INPUT_MAX_NUM; + static int FP_FILTER_SIZE; + static int FPS_CLUSTER_MAX; + static int FPS_RESULT_MAX; + static int K_FACTOR; + + static float FPS_MS_VAL; + static float FP_COUNT_MIN; + static float FP_MS_MIN; + static float FP_RIGHT_ANGLE; + static float FP_SMALL_ANGLE1; + static float FP_SMALL_ANGLE2; + static float QR_MIN_FP_AREA_ERR; + static float QR_MIN_FP_MS_ERR; + static int QR_MIN_FP_ACCEPT; + + int finder_time; + CrossCheckState CURRENT_CHECK_STATE; + int compared_finder_counts; + + struct HorizontalCheckedResult { + size_t centerI; + float centerJ; + }; + + vector > _horizontalCheckedResult; + + // INI CONFIG + +protected: + Ref image_; + std::vector > possibleCenters_; + + bool hasSkipped_; + Ref block_; + + /** stateCount must be int[5] */ + float centerFromEnd(int* stateCount, int end); + // check if satisfies finder pattern + bool foundPatternCross(int* stateCount); + + // try to insert to possibleCenters_ + int getStateCountTotal(int* stateCount, const CrossCheckState& check_state); + bool tryToPushToCenters(float posX, float posY, float estimatedModuleSize, + CrossCheckState horizontalState = FinderPatternFinder::NORMAL, + CrossCheckState verticalState = FinderPatternFinder::NORMAL); + bool crossCheckDiagonal(int startI, int centerJ, int maxCount, int originalStateCountTotal); + float crossCheckVertical(size_t startI, size_t centerJ, int maxCount, + int originalStateCountTota, float& estimatedVerticalModuleSize); + float crossCheckHorizontal(size_t startJ, size_t centerI, int maxCount, + int originalStateCountTotal, float& estimatedHorizontalModuleSize); + + float hasHorizontalCheckedResult(size_t startJ, size_t centerI); + int addHorizontalCheckedResult(size_t startJ, size_t centerI, float centerJ); + int getMinModuleSize(); + + bool isEqualResult(Ref src, Ref dst); + + /** stateCount must be int[5] */ + bool handlePossibleCenter(int* stateCount, size_t i, size_t j); + int findRowSkip(); + + std::vector > selectBestPatterns(ErrorHandler& err_handler); + std::vector > selectFileBestPatterns(ErrorHandler& err_handler); + std::vector > orderBestPatterns(std::vector > patterns); + + vector > getPatternInfosFileMode(DecodeHints const& hints, + ErrorHandler& err_handler); + + bool IsPossibleFindPatterInfo(Ref a, Ref b, Ref c); + void PushToResult(Ref a, Ref b, Ref c, + vector >& patternInfos); + + Ref getImage(); + std::vector >& getPossibleCenters(); + +public: + void InitConfig(); + float distance(Ref p1, Ref p2); + FinderPatternFinder(Ref image, Ref block); + + std::vector > find(DecodeHints const& hints, ErrorHandler& err_handler); + + bool checkIsoscelesRightTriangle(Ref centerA, Ref centerB, + Ref centerC, float& longSide); +}; +} // namespace qrcode +} // namespace zxing + +#endif // __ZXING_QRCODE_DETECTOR_FINDER_PATTERN_FINDER_HPP_ diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_info.cpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_info.cpp new file mode 100644 index 00000000000..ad365ba3bcf --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_info.cpp @@ -0,0 +1,99 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "finder_pattern_info.hpp" +#include + +#include + +namespace zxing { +namespace qrcode { +FinderPatternInfo::FinderPatternInfo(std::vector > patternCenters) + : bottomLeft_(patternCenters[0]), + topLeft_(patternCenters[1]), + topRight_(patternCenters[2]), + possibleFix_(0) { + estimateFinderPatternInfo(); +} + +Ref FinderPatternInfo::getBottomLeft() { return bottomLeft_; } + +Ref FinderPatternInfo::getTopLeft() { return topLeft_; } + +Ref FinderPatternInfo::getTopRight() { return topRight_; } + + +float FinderPatternInfo::getPossibleFix() { return possibleFix_; } + +float FinderPatternInfo::getAnglePossibleFix() { return anglePossibleFix_; } + +// bottomLeft_ => centerA +void FinderPatternInfo::calculateSides(Ref centerA, Ref centerB, + Ref centerC, float &longSide, + float &shortSide1, float &shortSide2) { + float a_m_b_x = centerA->getX() - centerB->getX(); + float a_m_b_y = centerA->getY() - centerB->getY(); + float ab_s = a_m_b_x * a_m_b_x + a_m_b_y * a_m_b_y; + float a_m_c_x = centerA->getX() - centerC->getX(); + float a_m_c_y = centerA->getY() - centerC->getY(); + float ac_s = a_m_c_x * a_m_c_x + a_m_c_y * a_m_c_y; + float b_m_c_x = centerB->getX() - centerC->getX(); + float b_m_c_y = centerB->getY() - centerC->getY(); + float bc_s = b_m_c_x * b_m_c_x + b_m_c_y * b_m_c_y; + + if (ab_s > bc_s && ab_s > ac_s) { + longSide = ab_s; + shortSide1 = ac_s; + shortSide2 = bc_s; + + } else if (bc_s > ab_s && bc_s > ac_s) { + longSide = bc_s; + shortSide1 = ab_s; + shortSide2 = ac_s; + } else { + longSide = ac_s; + shortSide1 = ab_s; + shortSide2 = bc_s; + } +} +void FinderPatternInfo::estimateFinderPatternInfo() { + float longSide, shortSide1, shortSide2; + calculateSides(bottomLeft_, topLeft_, topRight_, longSide, shortSide1, shortSide2); + + float CosLong = + (shortSide1 + shortSide2 - longSide) / (2 * sqrt(shortSide1) * sqrt(shortSide2)); + float CosShort1 = + (longSide + shortSide1 - shortSide2) / (2 * sqrt(longSide) * sqrt(shortSide1)); + float CosShort2 = + (longSide + shortSide2 - shortSide1) / (2 * sqrt(longSide) * sqrt(shortSide2)); + + float fAngleLong = acos(CosLong) * 180 / acos(-1.0); + float fAngleShort1 = acos(CosShort1) * 180 / acos(-1.0); + float fAngleShort2 = acos(CosShort2) * 180 / acos(-1.0); + if (fAngleShort1 < fAngleShort2) swap(fAngleShort1, fAngleShort2); + + float fLongDiff = fabs(fAngleLong - 90); + float fLongScore = 100.0 - fLongDiff; + + float fShortDiff = std::max(fabs(fAngleShort1 - 45), fabs(fAngleShort2 - 45)); + float fShortScore = 100.0 - 2 * fShortDiff; + + float fFinalScore = std::min(fShortScore, fLongScore); + + anglePossibleFix_ = fFinalScore / 100.0; + + int totalCount = (bottomLeft_->getCount() + topLeft_->getCount() + topRight_->getCount()); + + float fCountScore = (max(3, min(totalCount, 10)) - 3) / (10.0 - 3.0); + + possibleFix_ = anglePossibleFix_ * 0.5 + fCountScore * 0.5; +} +} // namespace qrcode +} // namespace zxing diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_info.hpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_info.hpp new file mode 100644 index 00000000000..a016060bd95 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_info.hpp @@ -0,0 +1,48 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_QRCODE_DETECTOR_FINDER_PATTERN_INFO_HPP_ +#define __ZXING_QRCODE_DETECTOR_FINDER_PATTERN_INFO_HPP_ + +#include "../../common/array.hpp" +#include "../../common/counted.hpp" +#include "finder_pattern.hpp" + +#include + +namespace zxing { +namespace qrcode { + +class FinderPatternInfo : public Counted { +private: + Ref bottomLeft_; + Ref topLeft_; + Ref topRight_; + float possibleFix_; + float anglePossibleFix_; + +public: + explicit FinderPatternInfo(std::vector > patternCenters); + + Ref getBottomLeft(); + Ref getTopLeft(); + Ref getTopRight(); + void estimateFinderPatternInfo(); + float getPossibleFix(); + float getAnglePossibleFix(); + // to void code duplicated + static void calculateSides(Ref centerA, Ref centerB, + Ref centerC, float &longSide, float &shortSide1, + float &shortSide2); +}; +} // namespace qrcode +} // namespace zxing + +#endif // __ZXING_QRCODE_DETECTOR_FINDER_PATTERN_INFO_HPP_ diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/pattern_result.cpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/pattern_result.cpp new file mode 100644 index 00000000000..e4a2abbb901 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/pattern_result.cpp @@ -0,0 +1,32 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). +#include "pattern_result.hpp" + +// VC++ +using namespace std; +using namespace zxing; +using namespace qrcode; + +using std::abs; +using zxing::Ref; +using zxing::ResultPoint; +using zxing::qrcode::FinderPattern; +using zxing::qrcode::FinderPatternInfo; +using zxing::qrcode::PatternResult; + +PatternResult::PatternResult(Ref info) { + finderPatternInfo = info; + possibleAlignmentPatterns.clear(); +} + +void PatternResult::setConfirmedAlignmentPattern(int index) { + if (index >= int(possibleAlignmentPatterns.size())) return; + confirmedAlignmentPattern = possibleAlignmentPatterns[index]; +} diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/pattern_result.hpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/pattern_result.hpp new file mode 100644 index 00000000000..fce734dc822 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/pattern_result.hpp @@ -0,0 +1,51 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_QRCODE_DETECTOR_PATTERN_RESULT_HPP_ +#define __ZXING_QRCODE_DETECTOR_PATTERN_RESULT_HPP_ + +#include "../../common/array.hpp" +#include "../../common/bitmatrix.hpp" +#include "../../common/counted.hpp" +#include "../../common/detector_result.hpp" +#include "../../resultpoint.hpp" +#include "alignment_pattern.hpp" +#include "finder_pattern.hpp" +#include "finder_pattern_info.hpp" + +#include + +namespace zxing { +namespace qrcode { +class PatternResult : public Counted { +public: + Ref finderPatternInfo; + vector > possibleAlignmentPatterns; + Ref confirmedAlignmentPattern; + int possibleDimension; + // vector possibleDimensions; + unsigned int possibleVersion; + float possibleFix; + float possibleModuleSize; + + explicit PatternResult(Ref info); + void setConfirmedAlignmentPattern(int index); + int getPossibleAlignmentCount() { return possibleAlignmentPatterns.size(); } + // int getPossibleDimensionCount(); +public: + unsigned int getPossibleVersion() { return possibleVersion; } + float getPossibleFix() { return possibleFix; } + float getPossibleModuleSize() { return possibleModuleSize; } + int getDimension() { return possibleDimension; } +}; +} // namespace qrcode +} // namespace zxing + +#endif // __ZXING_QRCODE_DETECTOR_PATTERN_RESULT_HPP_ diff --git a/modules/wechat_qrcode/src/zxing/qrcode/error_correction_level.cpp b/modules/wechat_qrcode/src/zxing/qrcode/error_correction_level.cpp new file mode 100644 index 00000000000..a9e376ef153 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/error_correction_level.cpp @@ -0,0 +1,46 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "error_correction_level.hpp" + +using std::string; +using zxing::ErrorHandler; + +namespace zxing { +namespace qrcode { + +ErrorCorrectionLevel::ErrorCorrectionLevel(int inOrdinal, int bits, char const* name) + : ordinal_(inOrdinal), bits_(bits), name_(name) {} + +int ErrorCorrectionLevel::ordinal() const { return ordinal_; } + +int ErrorCorrectionLevel::bits() const { return bits_; } + +string const& ErrorCorrectionLevel::name() const { return name_; } + +ErrorCorrectionLevel::operator string const &() const { return name_; } + +ErrorCorrectionLevel& ErrorCorrectionLevel::forBits(int bits, ErrorHandler& err_handler) { + if (bits < 0 || bits >= N_LEVELS) { + err_handler = zxing::ReaderErrorHandler("Ellegal error correction level bits"); + return *FOR_BITS[0]; + } + return *FOR_BITS[bits]; +} + +ErrorCorrectionLevel ErrorCorrectionLevel::L(0, 0x01, "L"); +ErrorCorrectionLevel ErrorCorrectionLevel::M(1, 0x00, "M"); +ErrorCorrectionLevel ErrorCorrectionLevel::Q(2, 0x03, "Q"); +ErrorCorrectionLevel ErrorCorrectionLevel::H(3, 0x02, "H"); +ErrorCorrectionLevel* ErrorCorrectionLevel::FOR_BITS[] = {&M, &L, &H, &Q}; +int ErrorCorrectionLevel::N_LEVELS = 4; + +} // namespace qrcode +} // namespace zxing diff --git a/modules/wechat_qrcode/src/zxing/qrcode/error_correction_level.hpp b/modules/wechat_qrcode/src/zxing/qrcode/error_correction_level.hpp new file mode 100644 index 00000000000..67344a83b8e --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/error_correction_level.hpp @@ -0,0 +1,44 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_QRCODE_ERROR_CORRECTION_LEVEL_HPP__ +#define __ZXING_QRCODE_ERROR_CORRECTION_LEVEL_HPP__ + +#include "../errorhandler.hpp" + +namespace zxing { +namespace qrcode { + +class ErrorCorrectionLevel { +private: + int ordinal_; + int bits_; + std::string name_; + ErrorCorrectionLevel(int inOrdinal, int bits, char const* name); + static ErrorCorrectionLevel* FOR_BITS[]; + static int N_LEVELS; + +public: + static ErrorCorrectionLevel L; + static ErrorCorrectionLevel M; + static ErrorCorrectionLevel Q; + static ErrorCorrectionLevel H; + + int ordinal() const; + int bits() const; + std::string const& name() const; + operator std::string const &() const; + + static ErrorCorrectionLevel& forBits(int bits, ErrorHandler& err_handler); +}; +} // namespace qrcode +} // namespace zxing + +#endif // __ZXING_QRCODE_ERROR_CORRECTION_LEVEL_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/qrcode/format_information.cpp b/modules/wechat_qrcode/src/zxing/qrcode/format_information.cpp new file mode 100644 index 00000000000..abd97040a92 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/format_information.cpp @@ -0,0 +1,114 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "format_information.hpp" + +#include + +using zxing::ErrorHandler; + +namespace zxing { +namespace qrcode { + +int FormatInformation::FORMAT_INFO_MASK_QR = 0x5412; +int FormatInformation::FORMAT_INFO_DECODE_LOOKUP[][2] = { + {0x5412, 0x00}, {0x5125, 0x01}, {0x5E7C, 0x02}, {0x5B4B, 0x03}, {0x45F9, 0x04}, {0x40CE, 0x05}, + {0x4F97, 0x06}, {0x4AA0, 0x07}, {0x77C4, 0x08}, {0x72F3, 0x09}, {0x7DAA, 0x0A}, {0x789D, 0x0B}, + {0x662F, 0x0C}, {0x6318, 0x0D}, {0x6C41, 0x0E}, {0x6976, 0x0F}, {0x1689, 0x10}, {0x13BE, 0x11}, + {0x1CE7, 0x12}, {0x19D0, 0x13}, {0x0762, 0x14}, {0x0255, 0x15}, {0x0D0C, 0x16}, {0x083B, 0x17}, + {0x355F, 0x18}, {0x3068, 0x19}, {0x3F31, 0x1A}, {0x3A06, 0x1B}, {0x24B4, 0x1C}, {0x2183, 0x1D}, + {0x2EDA, 0x1E}, {0x2BED, 0x1F}, +}; +int FormatInformation::N_FORMAT_INFO_DECODE_LOOKUPS = 32; + +int FormatInformation::BITS_SET_IN_HALF_BYTE[] = {0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4}; + +FormatInformation::FormatInformation(int formatInfo, float possiableFix, ErrorHandler& err_handler) + : errorCorrectionLevel_(ErrorCorrectionLevel::forBits((formatInfo >> 3) & 0x03, err_handler)), + dataMask_((char)(formatInfo & 0x07)) { + possiableFix_ = possiableFix; + if (err_handler.ErrCode()) return; +} + +ErrorCorrectionLevel& FormatInformation::getErrorCorrectionLevel() { return errorCorrectionLevel_; } + +char FormatInformation::getDataMask() { return dataMask_; } + +float FormatInformation::getPossiableFix() { return possiableFix_; } + +int FormatInformation::numBitsDiffering(int a, int b) { + a ^= b; + return BITS_SET_IN_HALF_BYTE[a & 0x0F] + BITS_SET_IN_HALF_BYTE[(a >> 4 & 0x0F)] + + BITS_SET_IN_HALF_BYTE[(a >> 8 & 0x0F)] + BITS_SET_IN_HALF_BYTE[(a >> 12 & 0x0F)] + + BITS_SET_IN_HALF_BYTE[(a >> 16 & 0x0F)] + BITS_SET_IN_HALF_BYTE[(a >> 20 & 0x0F)] + + BITS_SET_IN_HALF_BYTE[(a >> 24 & 0x0F)] + BITS_SET_IN_HALF_BYTE[(a >> 28 & 0x0F)]; +} + +Ref FormatInformation::decodeFormatInformation(int maskedFormatInfo1, + int maskedFormatInfo2) { + Ref result(doDecodeFormatInformation(maskedFormatInfo1, maskedFormatInfo2)); + if (result != 0) { + return result; + } + // Should return null, but, some QR codes apparently + // do not mask this info. Try again by actually masking the pattern + // first + return doDecodeFormatInformation(maskedFormatInfo1 ^ FORMAT_INFO_MASK_QR, + maskedFormatInfo2 ^ FORMAT_INFO_MASK_QR); +} +Ref FormatInformation::doDecodeFormatInformation(int maskedFormatInfo1, + int maskedFormatInfo2) { + ErrorHandler err_handler; + int distance = numBitsDiffering(maskedFormatInfo1, maskedFormatInfo2); + float possiableFix_ = (16.0 - (distance > 16 ? 16 : distance)) / 16.0; + + // Find the int in FORMAT_INFO_DECODE_LOOKUP with fewest bits differing + int bestDifference = std::numeric_limits::max(); + int bestFormatInfo = 0; + for (int i = 0; i < N_FORMAT_INFO_DECODE_LOOKUPS; i++) { + int* decodeInfo = FORMAT_INFO_DECODE_LOOKUP[i]; + int targetInfo = decodeInfo[0]; + if (targetInfo == maskedFormatInfo1 || targetInfo == maskedFormatInfo2) { + // Found an exact match + Ref result( + new FormatInformation(decodeInfo[1], possiableFix_, err_handler)); + if (err_handler.ErrCode()) return Ref(); + return result; + } + int bitsDifference = numBitsDiffering(maskedFormatInfo1, targetInfo); + if (bitsDifference < bestDifference) { + bestFormatInfo = decodeInfo[1]; + bestDifference = bitsDifference; + } + if (maskedFormatInfo1 != maskedFormatInfo2) { + // also try the other option + bitsDifference = numBitsDiffering(maskedFormatInfo2, targetInfo); + if (bitsDifference < bestDifference) { + bestFormatInfo = decodeInfo[1]; + bestDifference = bitsDifference; + } + } + } + if (bestDifference <= 3) { + Ref result( + new FormatInformation(bestFormatInfo, possiableFix_, err_handler)); + if (err_handler.ErrCode()) return Ref(); + return result; + } + Ref result; + return result; +} + +bool operator==(const FormatInformation& a, const FormatInformation& b) { + return &(a.errorCorrectionLevel_) == &(b.errorCorrectionLevel_) && a.dataMask_ == b.dataMask_; +} + +} // namespace qrcode +} // namespace zxing diff --git a/modules/wechat_qrcode/src/zxing/qrcode/format_information.hpp b/modules/wechat_qrcode/src/zxing/qrcode/format_information.hpp new file mode 100644 index 00000000000..249c35b74bb --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/format_information.hpp @@ -0,0 +1,48 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_QRCODE_FORMAT_INFORMATION_HPP__ +#define __ZXING_QRCODE_FORMAT_INFORMATION_HPP__ + +#include "../common/counted.hpp" +#include "../errorhandler.hpp" +#include "error_correction_level.hpp" + +namespace zxing { +namespace qrcode { + +class FormatInformation : public Counted { +private: + static int FORMAT_INFO_MASK_QR; + static int FORMAT_INFO_DECODE_LOOKUP[][2]; + static int N_FORMAT_INFO_DECODE_LOOKUPS; + static int BITS_SET_IN_HALF_BYTE[]; + + ErrorCorrectionLevel &errorCorrectionLevel_; + char dataMask_; + float possiableFix_; + + FormatInformation(int formatInfo, float possiableFix, ErrorHandler &err_handler); + +public: + static int numBitsDiffering(int a, int b); + static Ref decodeFormatInformation(int maskedFormatInfo1, + int maskedFormatInfo2); + static Ref doDecodeFormatInformation(int maskedFormatInfo1, + int maskedFormatInfo2); + ErrorCorrectionLevel &getErrorCorrectionLevel(); + char getDataMask(); + float getPossiableFix(); + friend bool operator==(const FormatInformation &a, const FormatInformation &b); +}; +} // namespace qrcode +} // namespace zxing + +#endif // __ZXING_QRCODE_FORMAT_INFORMATION_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/qrcode/qrcode_reader.cpp b/modules/wechat_qrcode/src/zxing/qrcode/qrcode_reader.cpp new file mode 100644 index 00000000000..318816f0e51 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/qrcode_reader.cpp @@ -0,0 +1,491 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "qrcode_reader.hpp" +#include +#include "../common/bitarray.hpp" +#include "detector/detector.hpp" + + +using zxing::ErrorHandler; + +namespace zxing { +namespace qrcode { + +QRCodeReader::QRCodeReader() : decoder_() { + readerState_ = QRCodeReader::READER_START; + detectedDimension_ = -1; + lastDecodeTime_ = 0; + lastDecodeID_ = 0; + decodeID_ = 0; + lastPossibleAPCount_ = 0; + possibleAPCount_ = 0; + lastSamePossibleAPCountTimes_ = 0; + samePossibleAPCountTimes_ = 0; + lastRecommendedImageSizeType_ = 0; + recommendedImageSizeType_ = 0; + smoothMaxMultiple_ = 40; +} + +Ref QRCodeReader::decode(Ref image) { return decode(image, DecodeHints()); } + +Ref QRCodeReader::decode(Ref image, DecodeHints hints) { + // Binarize image using the Histogram Binarized method and be binarized + ErrorHandler err_handler; + Ref imageBitMatrix = image->getBlackMatrix(err_handler); + if (err_handler.ErrCode() || imageBitMatrix == NULL) return Ref(); + + Ref rst = decodeMore(image, imageBitMatrix, hints, err_handler); + if (err_handler.ErrCode() || rst == NULL) { + // black white mirro!!! + Ref invertedMatrix = image->getInvertedMatrix(err_handler); + if (err_handler.ErrCode() || invertedMatrix == NULL) return Ref(); + Ref tmp_rst = decodeMore(image, invertedMatrix, hints, err_handler); + if (err_handler.ErrCode() || tmp_rst == NULL) return Ref(); + return tmp_rst; + } + + return rst; +} + +Ref QRCodeReader::decodeMore(Ref image, Ref imageBitMatrix, + DecodeHints hints, ErrorHandler &err_handler) { + nowHints_ = hints; + std::string ept; + + if (imageBitMatrix == NULL) return Ref(); + image->m_poUnicomBlock->Init(); + image->m_poUnicomBlock->Reset(imageBitMatrix); + + for (int tryTimes = 0; tryTimes < 1; tryTimes++) { + Ref detector(new Detector(imageBitMatrix, image->m_poUnicomBlock)); + err_handler.Reset(); + + detector->detect(hints, err_handler); + if (err_handler.ErrCode()) { + err_handler = zxing::ReaderErrorHandler("error detect"); + setReaderState(detector->getState()); + ept = err_handler.ErrMsg(); + continue; + } + + setReaderState(detector->getState()); + + int possiblePatternCount = detector->getPossiblePatternCount(); + + if (possiblePatternCount <= 0) { + continue; + } + for (int i = 0; i < possiblePatternCount; i++) { + // filter and perserve the highest score. + Ref patternInfo = detector->getFinderPatternInfo(i); + setPatternFix(patternInfo->getPossibleFix()); + if (patternInfo->getAnglePossibleFix() < 0.6 && i) continue; + + int possibleAlignmentCount = 0; + possibleAlignmentCount = detector->getPossibleAlignmentCount(i); + if (possibleAlignmentCount < 0) continue; + + detectedDimension_ = detector->getDimension(i); + possibleModuleSize_ = detector->getPossibleModuleSize(i); + setPossibleAPCountByVersion(detector->getPossibleVersion(i)); + + vector needTryVariousDeimensions(possibleAlignmentCount, false); + for (int j = 0; j < possibleAlignmentCount; j++) { + ArrayRef > points; + err_handler.Reset(); + Ref detectorResult = + detector->getResultViaAlignment(i, j, detectedDimension_, err_handler); + if (err_handler.ErrCode()) { + ept = err_handler.ErrCode(); + setDecoderFix(decoder_.getPossibleFix(), points); + setReaderState(decoder_.getState()); + + if ((patternInfo->getPossibleFix() > 0.9 && decoder_.getPossibleFix() < 0.1)) { + needTryVariousDeimensions[j] = true; + } + continue; + } + + points = detectorResult->getPoints(); + Ref decoderResult( + decoder_.decode(detectorResult->getBits(), err_handler)); + if (err_handler.ErrCode()) { + ept = err_handler.ErrCode(); + setDecoderFix(decoder_.getPossibleFix(), points); + setReaderState(decoder_.getState()); + + if ((patternInfo->getPossibleFix() > 0.9 && decoder_.getPossibleFix() < 0.1)) { + needTryVariousDeimensions[j] = true; + } + continue; + } + + // If the code was mirrored: swap the bottom-left and the + // top-right points. + if (decoderResult->getOtherClassName() == "QRCodeDecoderMetaData") { + decoderResult->getOther()->applyMirroredCorrection(points); + } + + setDecoderFix(decoder_.getPossibleFix(), points); + setReaderState(decoder_.getState()); + + Ref result( + new Result(decoderResult->getText(), decoderResult->getRawBytes(), points, + decoderResult->getCharset(), decoderResult->getQRCodeVersion(), + decoderResult->getEcLevel(), decoderResult->getCharsetMode())); + setSuccFix(points); + + return result; + } + // try different dimentions + for (int j = 0; j < possibleAlignmentCount; j++) { + err_handler.Reset(); + ArrayRef > points; + if (needTryVariousDeimensions[j]) { + vector possibleDimensions = getPossibleDimentions(detectedDimension_); + for (size_t k = 1; k < possibleDimensions.size(); k++) { + err_handler.Reset(); + int dimension = possibleDimensions[k]; + + Ref detectorResult = + detector->getResultViaAlignment(i, j, dimension, err_handler); + if (err_handler.ErrCode() || detectorResult == NULL) { + ept = err_handler.ErrMsg(); + setDecoderFix(decoder_.getPossibleFix(), points); + setReaderState(decoder_.getState()); + continue; + } + + points = detectorResult->getPoints(); + Ref decoderResult( + decoder_.decode(detectorResult->getBits(), err_handler)); + if (err_handler.ErrCode() || decoderResult == NULL) { + ept = err_handler.ErrMsg(); + setDecoderFix(decoder_.getPossibleFix(), points); + setReaderState(decoder_.getState()); + continue; + } + + if (decoderResult->getOtherClassName() == "QRCodeDecoderMetaData") { + decoderResult->getOther()->applyMirroredCorrection(points); + } + + setDecoderFix(decoder_.getPossibleFix(), points); + setReaderState(decoder_.getState()); + + detectedDimension_ = possibleDimensions[k]; + Ref result(new Result( + decoderResult->getText(), decoderResult->getRawBytes(), points, + decoderResult->getCharset(), decoderResult->getQRCodeVersion(), + decoderResult->getEcLevel(), decoderResult->getCharsetMode())); + + setSuccFix(points); + return result; + } + } + } + } + } + return Ref(); +} + +vector QRCodeReader::getPossibleDimentions(int detectDimension) { + vector possibleDimentions; + possibleDimentions.clear(); + + if (detectDimension < 0) { + return possibleDimentions; + } + + possibleDimentions.push_back(detectDimension); + + if (detectDimension <= 169 && detectDimension >= 73) { + possibleDimentions.push_back(detectDimension + 4); + possibleDimentions.push_back(detectDimension - 4); + possibleDimentions.push_back(detectDimension - 8); + possibleDimentions.push_back(detectDimension + 8); + } else if (detectDimension <= 69 && detectDimension >= 45) { + possibleDimentions.push_back(detectDimension + 4); + possibleDimentions.push_back(detectDimension - 4); + } + + if (detectDimension == 19) { + possibleDimentions.push_back(21); + } + + return possibleDimentions; +} + +void QRCodeReader::setPossibleAPCountByVersion(unsigned int version) { + // cout<<"setPossibleAPCountByVersion"< input, Ref output, + int window) { + BitMatrix &imatrix = *input; + BitMatrix &omatrix = *output; + window >>= 1; + int count = 0; + int width = input->getWidth(); + int height = input->getHeight(); + int bitsize = imatrix.getRowBitsSize(); + + bool *jrowtoset = new bool[bitsize]; + + bool *jrow = NULL; + + jrow = NULL; + + unsigned int size = window * window; + + for (int j = (window + 1); j < (height - 1 - window); ++j) { + int y1 = j - window - 1; + int y2 = j + window; + + int offset1 = y1 * width; + int offset2 = y2 * width; + + jrow = imatrix.getRowBoolPtr(j); + + memcpy(jrowtoset, jrow, bitsize * sizeof(bool)); + + for (int i = (window + 1); i < (width - 1 - window); ++i) { + int x1 = i - window - 1; + int x2 = i + window; + unsigned int sum = integral[offset2 + x2] - integral[offset2 + x1] + + integral[offset1 + x2] - integral[offset1 + x1]; + bool b = jrow[i]; + bool result; + // the middle 1/3 contains informations of corner, these + // informations is useful for finding the finder pattern + int sum3 = 3 * sum; + if ((unsigned int)sum3 <= size) { + result = false; + } else if ((unsigned int)sum3 >= size * 2) { + result = true; + } else { + result = b; + } + + if (result) { + jrowtoset[i] = true; + } + count += (result ^ b) == 1 ? 1U : 0U; + } + omatrix.setRowBool(j, jrowtoset); + } + + delete[] jrowtoset; + return count; +} + +void QRCodeReader::initIntegralOld(unsigned int *integral, Ref input) { + BitMatrix &matrix = *input; + int width = input->getWidth(); + int height = input->getHeight(); + + bool *therow = NULL; + + therow = matrix.getRowBoolPtr(0); + + integral[0] = therow[0]; + + int *s = new int[width]; + + memset(s, 0, width * sizeof(int)); + + integral[0] = therow[0]; + for (int j = 1; j < width; j++) { + integral[j] = integral[j - 1] + therow[j]; + s[j] += therow[j]; + } + + int offset = width; + unsigned int prevSum = 0; + + for (int i = 1; i < height; i++) { + offset = i * width; + therow = matrix.getRowBoolPtr(i); + + integral[offset] = integral[offset - width] + therow[0]; + offset++; + + for (int j = 1; j < width; j++) { + s[j] += therow[j]; + integral[offset] = prevSum + s[j]; + prevSum = integral[offset]; + offset++; + } + } + + delete[] s; + + return; +} + +void QRCodeReader::initIntegral(unsigned int *integral, Ref input) { + BitMatrix &matrix = *input; + int width = input->getWidth(); + int height = input->getHeight(); + + bool *therow = NULL; + + therow = matrix.getRowBoolPtr(0); + + // first row only + int rs = 0; + for (int j = 0; j < width; j++) { + rs += therow[j]; + integral[j] = rs; + } + + // remaining cells are sum above and to the left + int offset = 0; + + for (int i = 1; i < height; ++i) { + therow = matrix.getRowBoolPtr(i); + + rs = 0; + + offset += width; + + for (int j = 0; j < width; ++j) { + rs += therow[j]; + integral[offset + j] = rs + integral[offset - width + j]; + } + } + + return; +} + +int QRCodeReader::getRecommendedImageSizeTypeInteral() { + if (time(0) - lastDecodeTime_ > 30) recommendedImageSizeType_ = 0; + return recommendedImageSizeType_; +} + +unsigned int QRCodeReader::getDecodeID() { return decodeID_; } + +void QRCodeReader::setDecodeID(unsigned int id) { + lastDecodeTime_ = time(0); + + decodeID_ = id; + if (decodeID_ != lastDecodeID_) { + lastDecodeID_ = decodeID_; + lastPossibleAPCount_ = possibleAPCount_; + lastSamePossibleAPCountTimes_ = samePossibleAPCountTimes_; + lastRecommendedImageSizeType_ = getRecommendedImageSizeTypeInteral(); + possibleAPCount_ = 0; + recommendedImageSizeType_ = 0; + } +} + +QRCodeReader::~QRCodeReader() {} +Decoder &QRCodeReader::getDecoder() { return decoder_; } + +unsigned int QRCodeReader::getPossibleAPType() { + int version = (detectedDimension_ - 21) / 4 + 1; + setPossibleAPCountByVersion(version); + return possibleAPCount_; +} +int QRCodeReader::getPossibleFixType() { return possibleQrcodeInfo_.possibleFix > 0.0 ? 1 : 0; } + +void QRCodeReader::setPatternFix(float possibleFix) { + possibleQrcodeInfo_.patternPossibleFix = possibleFix; +} + +void QRCodeReader::setDecoderFix(float possibleFix, ArrayRef > border) { + float realFix = possibleFix; + if (possibleQrcodeInfo_.possibleFix < realFix) { + possibleQrcodeInfo_.possibleFix = realFix; + possibleQrcodeInfo_.qrcodeBorder.clear(); + possibleQrcodeInfo_.possibleModuleSize = possibleModuleSize_; + if (border) { + for (int i = 0; i < 4; ++i) { + possibleQrcodeInfo_.qrcodeBorder.push_back(border[i]); + } + } + } +} +void QRCodeReader::setSuccFix(ArrayRef > border) { + possibleQrcodeInfo_.qrcodeBorder.clear(); + possibleQrcodeInfo_.possibleModuleSize = possibleModuleSize_; + if (border) { + for (int i = 0; i < 4; ++i) { + possibleQrcodeInfo_.qrcodeBorder.push_back(border[i]); + } + } +} + +void QRCodeReader::setReaderState(Detector::DetectorState state) { + switch (state) { + case Detector::START: + this->readerState_ = QRCodeReader::DETECT_START; + break; + case Detector::FINDFINDERPATTERN: + this->readerState_ = QRCodeReader::DETECT_FINDFINDERPATTERN; + break; + case Detector::FINDALIGNPATTERN: + this->readerState_ = QRCodeReader::DETECT_FINDALIGNPATTERN; + break; + } + return; +} +void QRCodeReader::setReaderState(Decoder::DecoderState state) { + switch (state) { + case Decoder::NOTSTART: + this->readerState_ = QRCodeReader::DETECT_FAILD; + break; + case Decoder::START: + if (this->readerState_ < QRCodeReader::DECODE_START) { + this->readerState_ = QRCodeReader::DECODE_START; + } + break; + case Decoder::READVERSION: + if (this->readerState_ < QRCodeReader::DECODE_READVERSION) { + this->readerState_ = QRCodeReader::DECODE_READVERSION; + } + break; + case Decoder::READERRORCORRECTIONLEVEL: + if (this->readerState_ < QRCodeReader::DECODE_READERRORCORRECTIONLEVEL) { + this->readerState_ = QRCodeReader::DECODE_READERRORCORRECTIONLEVEL; + } + break; + case Decoder::READCODEWORDSORRECTIONLEVEL: + if (this->readerState_ < QRCodeReader::DECODE_READCODEWORDSORRECTIONLEVEL) { + this->readerState_ = QRCodeReader::DECODE_READCODEWORDSORRECTIONLEVEL; + } + break; + case Decoder::FINISH: + if (this->readerState_ < QRCodeReader::DECODE_FINISH) { + this->readerState_ = QRCodeReader::DECODE_FINISH; + } + break; + } + return; +} +} // namespace qrcode +} // namespace zxing diff --git a/modules/wechat_qrcode/src/zxing/qrcode/qrcode_reader.hpp b/modules/wechat_qrcode/src/zxing/qrcode/qrcode_reader.hpp new file mode 100644 index 00000000000..bb0ccf61e7d --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/qrcode_reader.hpp @@ -0,0 +1,131 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_QRCODE_QRCODE_READER_HPP__ +#define __ZXING_QRCODE_QRCODE_READER_HPP__ + +#include "../decodehints.hpp" +#include "../errorhandler.hpp" +#include "../reader.hpp" +#include "decoder/decoder.hpp" +#include "decoder/qrcode_decoder_metadata.hpp" +#include "detector/detector.hpp" + +namespace zxing { +namespace qrcode { + +struct QBAR_QRCODE_DETECT_INFO { + int possibleFixIndex; + unsigned int possibleAPType; + + // QRCodeReader Info + float possibleFix; + float patternPossibleFix; + int pyramidLev; + float possibleModuleSize; + std::vector > qrcodeBorder; + + QBAR_QRCODE_DETECT_INFO() { clear(); } + + void clear() { + possibleFixIndex = -1; + possibleAPType = 0; + possibleModuleSize = 0; + + possibleFix = 0; + patternPossibleFix = 0; + pyramidLev = 0; + qrcodeBorder.clear(); + } +}; + +class QRCodeReader : public Reader { +public: + enum ReaderState { + READER_START = -1, + DETECT_START = 0, + DETECT_FINDFINDERPATTERN = 1, + DETECT_FINDALIGNPATTERN = 2, + DETECT_FAILD = 3, + DECODE_START = 4, + DECODE_READVERSION = 5, + DECODE_READERRORCORRECTIONLEVEL = 6, + DECODE_READCODEWORDSORRECTIONLEVEL = 7, + DECODE_FINISH = 8 + }; + +private: + Decoder decoder_; + int detectedDimension_; + ReaderState readerState_; + DecodeHints nowHints_; + +protected: + Decoder& getDecoder(); + +public: + QRCodeReader(); + virtual ~QRCodeReader(); + string name() override { return "qrcode"; } + + Ref decode(Ref image) override; + Ref decode(Ref image, DecodeHints hints) override; + + Ref decodeMore(Ref image, Ref imageBitMatrix, + DecodeHints hints, ErrorHandler& err_handler); + +private: + QBAR_QRCODE_DETECT_INFO possibleQrcodeInfo_; + +protected: + void setPossibleAPCountByVersion(unsigned int version); + int getRecommendedImageSizeTypeInteral(); + static void initIntegralOld(unsigned int* integral, Ref input); + static void initIntegral(unsigned int* integral, Ref input); + static int smooth(unsigned int* integral, Ref input, Ref output, + int window); + unsigned int lastDecodeTime_; + unsigned int lastDecodeID_; + unsigned int decodeID_; + int lastPossibleAPCount_; + int possibleAPCount_; + int possibleModuleSize_; + unsigned int lastSamePossibleAPCountTimes_; + unsigned int samePossibleAPCountTimes_; + unsigned int lastRecommendedImageSizeType_; + unsigned int recommendedImageSizeType_; + unsigned int smoothMaxMultiple_; + +public: + virtual unsigned int getDecodeID() override; + virtual void setDecodeID(unsigned int id) override; + virtual float getPossibleFix() override; + virtual unsigned int getPossibleAPType(); + virtual int getPossibleFixType(); + + void setReaderState(Detector::DetectorState state); + void setReaderState(Decoder::DecoderState state); + + void setPatternFix(float possibleFix); + void setDecoderFix(float possibleFix, ArrayRef > border); + void setSuccFix(ArrayRef > border); + + ReaderState getReaderState() { return this->readerState_; } + float calQrcodeArea(Ref detectorResult); + float calTriangleArea(Ref centerA, Ref centerB, + Ref centerC); + + vector getPossibleDimentions(int detectDimension); +}; + +} // namespace qrcode +} // namespace zxing + +#endif // __ZXING_QRCODE_QRCODE_READER_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/qrcode/version.cpp b/modules/wechat_qrcode/src/zxing/qrcode/version.cpp new file mode 100644 index 00000000000..0c1a4f92820 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/version.cpp @@ -0,0 +1,504 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "version.hpp" +#include "format_information.hpp" + +#include +#include +#include + +using std::numeric_limits; +using std::vector; +using zxing::ErrorHandler; + +namespace zxing { +namespace qrcode { + +ECB::ECB(int count, int dataCodewords) : count_(count), dataCodewords_(dataCodewords) {} + +int ECB::getCount() { return count_; } + +int ECB::getDataCodewords() { return dataCodewords_; } + +ECBlocks::ECBlocks(int ecCodewords, ECB *ecBlocks) + : ecCodewords_(ecCodewords), ecBlocks_(1, ecBlocks) {} + +ECBlocks::ECBlocks(int ecCodewords, ECB *ecBlocks1, ECB *ecBlocks2) + : ecCodewords_(ecCodewords), ecBlocks_(1, ecBlocks1) { + ecBlocks_.push_back(ecBlocks2); +} + +int ECBlocks::getECCodewords() { return ecCodewords_; } + +std::vector &ECBlocks::getECBlocks() { return ecBlocks_; } + +ECBlocks::~ECBlocks() { + for (size_t i = 0; i < ecBlocks_.size(); i++) { + delete ecBlocks_[i]; + } +} + +unsigned int Version::VERSION_DECODE_INFO[] = { + 0x07C94, 0x085BC, 0x09A99, 0x0A4D3, 0x0BBF6, 0x0C762, 0x0D847, 0x0E60D, 0x0F928, + 0x10B78, 0x1145D, 0x12A17, 0x13532, 0x149A6, 0x15683, 0x168C9, 0x177EC, 0x18EC4, + 0x191E1, 0x1AFAB, 0x1B08E, 0x1CC1A, 0x1D33F, 0x1ED75, 0x1F250, 0x209D5, 0x216F0, + 0x228BA, 0x2379F, 0x24B0B, 0x2542E, 0x26A64, 0x27541, 0x28C69}; +int Version::N_VERSION_DECODE_INFOS = 34; + +vector > Version::VERSIONS; +static int N_VERSIONS = Version::buildVersions(); + +int Version::getVersionNumber() { return versionNumber_; } + +vector &Version::getAlignmentPatternCenters() { return alignmentPatternCenters_; } + +int Version::getTotalCodewords() { return totalCodewords_; } + +int Version::getDimensionForVersion(ErrorHandler &err_handler) { + if (versionNumber_ < 1 || versionNumber_ > N_VERSIONS) { + err_handler = zxing::ReaderErrorHandler("versionNumber must be between 1 and 40"); + return -1; + } + return 17 + 4 * versionNumber_; +} + +ECBlocks &Version::getECBlocksForLevel(ErrorCorrectionLevel &ecLevel) { + return *ecBlocks_[ecLevel.ordinal()]; +} + +static vector *intArray(size_t n...) { + va_list ap; + va_start(ap, n); + vector *result = new vector(n); + for (size_t i = 0; i < n; i++) { + (*result)[i] = va_arg(ap, int); + } + va_end(ap); + return result; +} + +Version *Version::getProvisionalVersionForDimension(int dimension, ErrorHandler &err_handler) { + if (dimension % 4 != 1) { + err_handler = zxing::FormatErrorHandler("dimension % 4 != 1"); + return NULL; + } + + // return Version::getVersionForNumber((dimension - 17) >> 2); + Version *version = Version::getVersionForNumber((dimension - 17) >> 2, err_handler); + if (err_handler.ErrCode()) { + err_handler = zxing::FormatErrorHandler("err format"); + return NULL; + } + return version; +} + +Version *Version::getVersionForNumber(int versionNumber, ErrorHandler &err_handler) { + if (versionNumber < 1 || versionNumber > N_VERSIONS) { + err_handler = zxing::ReaderErrorHandler("versionNumber must be between 1 and 40"); + return NULL; + } + return VERSIONS[versionNumber - 1]; +} + +Version::Version(int versionNumber, vector *alignmentPatternCenters, ECBlocks *ecBlocks1, + ECBlocks *ecBlocks2, ECBlocks *ecBlocks3, ECBlocks *ecBlocks4) + : versionNumber_(versionNumber), + alignmentPatternCenters_(*alignmentPatternCenters), + ecBlocks_(4), + totalCodewords_(0) { + ecBlocks_[0] = ecBlocks1; + ecBlocks_[1] = ecBlocks2; + ecBlocks_[2] = ecBlocks3; + ecBlocks_[3] = ecBlocks4; + + int total = 0; + int ecCodewords = ecBlocks1->getECCodewords(); + vector &ecbArray = ecBlocks1->getECBlocks(); + for (size_t i = 0; i < ecbArray.size(); i++) { + ECB *ecBlock = ecbArray[i]; + total += ecBlock->getCount() * (ecBlock->getDataCodewords() + ecCodewords); + } + totalCodewords_ = total; +} + +Version::~Version() { + delete &alignmentPatternCenters_; + for (size_t i = 0; i < ecBlocks_.size(); i++) { + delete ecBlocks_[i]; + } +} + +Version *Version::decodeVersionInformation(unsigned int versionBits) { + int bestDifference = numeric_limits::max(); + size_t bestVersion = 0; + ErrorHandler err_handler; + for (int i = 0; i < N_VERSION_DECODE_INFOS; i++) { + unsigned targetVersion = VERSION_DECODE_INFO[i]; + // Do the version info bits match exactly? done. + if (targetVersion == versionBits) { + // return getVersionForNumber(i + 7 ); + Version *version = getVersionForNumber(i + 7, err_handler); + if (err_handler.ErrCode()) return 0; + return version; + } + // Otherwise see if this is the closest to a real version info bit + // string we have seen so far + int bitsDifference = FormatInformation::numBitsDiffering(versionBits, targetVersion); + if (bitsDifference < bestDifference) { + bestVersion = i + 7; + bestDifference = bitsDifference; + } + } + // We can tolerate up to 3 bits of error since no two version info codewords + // will differ in less than 4 bits. + if (bestDifference <= 3) { + // return getVersionForNumber(bestVersion); + Version *version = getVersionForNumber(bestVersion, err_handler); + if (err_handler.ErrCode()) return 0; + return version; + } + // If we didn't find a close enough match, fail + return 0; +} + +Ref Version::buildFixedPatternValue(ErrorHandler &err_handler) { + int dimension = getDimensionForVersion(err_handler); + if (err_handler.ErrCode()) return Ref(); + + Ref fixedInfo(new BitMatrix(dimension, err_handler)); + if (err_handler.ErrCode()) return Ref(); + + // first timming patterns + for (int i = 0; i < dimension; i += 2) fixedInfo->set(i, 6); + for (int i = 0; i < dimension; i += 2) fixedInfo->set(6, i); + + // FP top left + fixedInfo->setRegion(0, 0, 8, 8, err_handler); + fixedInfo->flipRegion(0, 0, 8, 8, err_handler); + fixedInfo->flipRegion(0, 0, 7, 7, err_handler); + fixedInfo->flipRegion(1, 1, 5, 5, err_handler); + fixedInfo->flipRegion(2, 2, 3, 3, err_handler); + + // FP top right + fixedInfo->setRegion(dimension - 8, 0, 8, 8, err_handler); + fixedInfo->flipRegion(dimension - 8, 0, 8, 8, err_handler); + fixedInfo->flipRegion(dimension - 7, 0, 7, 7, err_handler); + fixedInfo->flipRegion(dimension - 6, 1, 5, 5, err_handler); + fixedInfo->flipRegion(dimension - 5, 2, 3, 3, err_handler); + + // FP bottom left + fixedInfo->setRegion(0, dimension - 8, 8, 8, err_handler); + fixedInfo->flipRegion(0, dimension - 8, 8, 8, err_handler); + fixedInfo->flipRegion(0, dimension - 7, 7, 7, err_handler); + fixedInfo->flipRegion(1, dimension - 6, 5, 5, err_handler); + fixedInfo->flipRegion(2, dimension - 5, 3, 3, err_handler); + if (err_handler.ErrCode()) return Ref(); + + // Alignment patterns + size_t max = alignmentPatternCenters_.size(); + for (size_t x = 0; x < max; x++) { + int i = alignmentPatternCenters_[x] - 2; + for (size_t y = 0; y < max; y++) { + if ((x == 0 && (y == 0 || y == max - 1)) || (x == max - 1 && y == 0)) { + // No alignment patterns near the three finder patterns + continue; + } + fixedInfo->setRegion(alignmentPatternCenters_[y] - 2, i, 5, 5, err_handler); + // fixedInfo->flipRegion(alignmentPatternCenters_[y] - 2, i, 5, 5); + fixedInfo->flipRegion(alignmentPatternCenters_[y] - 1, i + 1, 3, 3, err_handler); + fixedInfo->flipRegion(alignmentPatternCenters_[y], i + 2, 1, 1, err_handler); + if (err_handler.ErrCode()) return Ref(); + } + } + return fixedInfo; +} + +Ref Version::buildFixedPatternTemplate(ErrorHandler &err_handler) { + int dimension = getDimensionForVersion(err_handler); + Ref functionPattern(new BitMatrix(dimension, err_handler)); + if (err_handler.ErrCode()) return Ref(); + + // Top left finder pattern + separator + format + functionPattern->setRegion(0, 0, 8, 8, err_handler); + // Top right finder pattern + separator + format + functionPattern->setRegion(dimension - 8, 0, 8, 8, err_handler); + // Bottom left finder pattern + separator + format + functionPattern->setRegion(0, dimension - 8, 8, 8, err_handler); + if (err_handler.ErrCode()) return Ref(); + + // Alignment patterns + size_t max = alignmentPatternCenters_.size(); + for (size_t x = 0; x < max; x++) { + int i = alignmentPatternCenters_[x] - 2; + for (size_t y = 0; y < max; y++) { + if ((x == 0 && (y == 0 || y == max - 1)) || (x == max - 1 && y == 0)) { + // No alignment patterns near the three finder patterns + continue; + } + functionPattern->setRegion(alignmentPatternCenters_[y] - 2, i, 5, 5, err_handler); + } + } + + // Vertical timing pattern + functionPattern->setRegion(6, 8, 1, dimension - 16, err_handler); + // Horizontal timing pattern + functionPattern->setRegion(8, 6, dimension - 16, 1, err_handler); + if (err_handler.ErrCode()) return Ref(); + + return functionPattern; +} + +Ref Version::buildFunctionPattern(ErrorHandler &err_handler) { + int dimension = getDimensionForVersion(err_handler); + Ref functionPattern(new BitMatrix(dimension, err_handler)); + if (err_handler.ErrCode()) return Ref(); + + // Top left finder pattern + separator + format + functionPattern->setRegion(0, 0, 9, 9, err_handler); + // Top right finder pattern + separator + format + functionPattern->setRegion(dimension - 8, 0, 8, 9, err_handler); + // Bottom left finder pattern + separator + format + functionPattern->setRegion(0, dimension - 8, 9, 8, err_handler); + + // Alignment patterns + size_t max = alignmentPatternCenters_.size(); + for (size_t x = 0; x < max; x++) { + int i = alignmentPatternCenters_[x] - 2; + for (size_t y = 0; y < max; y++) { + if ((x == 0 && (y == 0 || y == max - 1)) || (x == max - 1 && y == 0)) { + // No alignment patterns near the three finder patterns + continue; + } + functionPattern->setRegion(alignmentPatternCenters_[y] - 2, i, 5, 5, err_handler); + } + } + + // Vertical timing pattern + functionPattern->setRegion(6, 9, 1, dimension - 17, err_handler); + // Horizontal timing pattern + functionPattern->setRegion(9, 6, dimension - 17, 1, err_handler); + if (err_handler.ErrCode()) return Ref(); + + if (versionNumber_ > 6) { + // Version info, top right + functionPattern->setRegion(dimension - 11, 0, 3, 6, err_handler); + // Version info, bottom left + functionPattern->setRegion(0, dimension - 11, 6, 3, err_handler); + if (err_handler.ErrCode()) return Ref(); + } + + return functionPattern; +} + +int Version::buildVersions() { + VERSIONS.push_back(Ref(new Version( + 1, intArray(0), new ECBlocks(7, new ECB(1, 19)), new ECBlocks(10, new ECB(1, 16)), + new ECBlocks(13, new ECB(1, 13)), new ECBlocks(17, new ECB(1, 9))))); + VERSIONS.push_back(Ref(new Version( + 2, intArray(2, 6, 18), new ECBlocks(10, new ECB(1, 34)), new ECBlocks(16, new ECB(1, 28)), + new ECBlocks(22, new ECB(1, 22)), new ECBlocks(28, new ECB(1, 16))))); + VERSIONS.push_back(Ref(new Version( + 3, intArray(2, 6, 22), new ECBlocks(15, new ECB(1, 55)), new ECBlocks(26, new ECB(1, 44)), + new ECBlocks(18, new ECB(2, 17)), new ECBlocks(22, new ECB(2, 13))))); + VERSIONS.push_back(Ref(new Version( + 4, intArray(2, 6, 26), new ECBlocks(20, new ECB(1, 80)), new ECBlocks(18, new ECB(2, 32)), + new ECBlocks(26, new ECB(2, 24)), new ECBlocks(16, new ECB(4, 9))))); + VERSIONS.push_back(Ref(new Version( + 5, intArray(2, 6, 30), new ECBlocks(26, new ECB(1, 108)), new ECBlocks(24, new ECB(2, 43)), + new ECBlocks(18, new ECB(2, 15), new ECB(2, 16)), + new ECBlocks(22, new ECB(2, 11), new ECB(2, 12))))); + VERSIONS.push_back(Ref(new Version( + 6, intArray(2, 6, 34), new ECBlocks(18, new ECB(2, 68)), new ECBlocks(16, new ECB(4, 27)), + new ECBlocks(24, new ECB(4, 19)), new ECBlocks(28, new ECB(4, 15))))); + VERSIONS.push_back(Ref(new Version( + 7, intArray(3, 6, 22, 38), new ECBlocks(20, new ECB(2, 78)), + new ECBlocks(18, new ECB(4, 31)), new ECBlocks(18, new ECB(2, 14), new ECB(4, 15)), + new ECBlocks(26, new ECB(4, 13), new ECB(1, 14))))); + VERSIONS.push_back( + Ref(new Version(8, intArray(3, 6, 24, 42), new ECBlocks(24, new ECB(2, 97)), + new ECBlocks(22, new ECB(2, 38), new ECB(2, 39)), + new ECBlocks(22, new ECB(4, 18), new ECB(2, 19)), + new ECBlocks(26, new ECB(4, 14), new ECB(2, 15))))); + VERSIONS.push_back( + Ref(new Version(9, intArray(3, 6, 26, 46), new ECBlocks(30, new ECB(2, 116)), + new ECBlocks(22, new ECB(3, 36), new ECB(2, 37)), + new ECBlocks(20, new ECB(4, 16), new ECB(4, 17)), + new ECBlocks(24, new ECB(4, 12), new ECB(4, 13))))); + VERSIONS.push_back(Ref(new Version(10, intArray(3, 6, 28, 50), + new ECBlocks(18, new ECB(2, 68), new ECB(2, 69)), + new ECBlocks(26, new ECB(4, 43), new ECB(1, 44)), + new ECBlocks(24, new ECB(6, 19), new ECB(2, 20)), + new ECBlocks(28, new ECB(6, 15), new ECB(2, 16))))); + VERSIONS.push_back( + Ref(new Version(11, intArray(3, 6, 30, 54), new ECBlocks(20, new ECB(4, 81)), + new ECBlocks(30, new ECB(1, 50), new ECB(4, 51)), + new ECBlocks(28, new ECB(4, 22), new ECB(4, 23)), + new ECBlocks(24, new ECB(3, 12), new ECB(8, 13))))); + VERSIONS.push_back(Ref(new Version(12, intArray(3, 6, 32, 58), + new ECBlocks(24, new ECB(2, 92), new ECB(2, 93)), + new ECBlocks(22, new ECB(6, 36), new ECB(2, 37)), + new ECBlocks(26, new ECB(4, 20), new ECB(6, 21)), + new ECBlocks(28, new ECB(7, 14), new ECB(4, 15))))); + VERSIONS.push_back( + Ref(new Version(13, intArray(3, 6, 34, 62), new ECBlocks(26, new ECB(4, 107)), + new ECBlocks(22, new ECB(8, 37), new ECB(1, 38)), + new ECBlocks(24, new ECB(8, 20), new ECB(4, 21)), + new ECBlocks(22, new ECB(12, 11), new ECB(4, 12))))); + VERSIONS.push_back(Ref(new Version( + 14, intArray(4, 6, 26, 46, 66), new ECBlocks(30, new ECB(3, 115), new ECB(1, 116)), + new ECBlocks(24, new ECB(4, 40), new ECB(5, 41)), + new ECBlocks(20, new ECB(11, 16), new ECB(5, 17)), + new ECBlocks(24, new ECB(11, 12), new ECB(5, 13))))); + VERSIONS.push_back(Ref(new Version( + 15, intArray(4, 6, 26, 48, 70), new ECBlocks(22, new ECB(5, 87), new ECB(1, 88)), + new ECBlocks(24, new ECB(5, 41), new ECB(5, 42)), + new ECBlocks(30, new ECB(5, 24), new ECB(7, 25)), + new ECBlocks(24, new ECB(11, 12), new ECB(7, 13))))); + VERSIONS.push_back(Ref(new Version( + 16, intArray(4, 6, 26, 50, 74), new ECBlocks(24, new ECB(5, 98), new ECB(1, 99)), + new ECBlocks(28, new ECB(7, 45), new ECB(3, 46)), + new ECBlocks(24, new ECB(15, 19), new ECB(2, 20)), + new ECBlocks(30, new ECB(3, 15), new ECB(13, 16))))); + VERSIONS.push_back(Ref(new Version( + 17, intArray(4, 6, 30, 54, 78), new ECBlocks(28, new ECB(1, 107), new ECB(5, 108)), + new ECBlocks(28, new ECB(10, 46), new ECB(1, 47)), + new ECBlocks(28, new ECB(1, 22), new ECB(15, 23)), + new ECBlocks(28, new ECB(2, 14), new ECB(17, 15))))); + VERSIONS.push_back(Ref(new Version( + 18, intArray(4, 6, 30, 56, 82), new ECBlocks(30, new ECB(5, 120), new ECB(1, 121)), + new ECBlocks(26, new ECB(9, 43), new ECB(4, 44)), + new ECBlocks(28, new ECB(17, 22), new ECB(1, 23)), + new ECBlocks(28, new ECB(2, 14), new ECB(19, 15))))); + VERSIONS.push_back(Ref(new Version( + 19, intArray(4, 6, 30, 58, 86), new ECBlocks(28, new ECB(3, 113), new ECB(4, 114)), + new ECBlocks(26, new ECB(3, 44), new ECB(11, 45)), + new ECBlocks(26, new ECB(17, 21), new ECB(4, 22)), + new ECBlocks(26, new ECB(9, 13), new ECB(16, 14))))); + VERSIONS.push_back(Ref(new Version( + 20, intArray(4, 6, 34, 62, 90), new ECBlocks(28, new ECB(3, 107), new ECB(5, 108)), + new ECBlocks(26, new ECB(3, 41), new ECB(13, 42)), + new ECBlocks(30, new ECB(15, 24), new ECB(5, 25)), + new ECBlocks(28, new ECB(15, 15), new ECB(10, 16))))); + VERSIONS.push_back(Ref(new Version( + 21, intArray(5, 6, 28, 50, 72, 94), new ECBlocks(28, new ECB(4, 116), new ECB(4, 117)), + new ECBlocks(26, new ECB(17, 42)), new ECBlocks(28, new ECB(17, 22), new ECB(6, 23)), + new ECBlocks(30, new ECB(19, 16), new ECB(6, 17))))); + VERSIONS.push_back(Ref(new Version( + 22, intArray(5, 6, 26, 50, 74, 98), new ECBlocks(28, new ECB(2, 111), new ECB(7, 112)), + new ECBlocks(28, new ECB(17, 46)), new ECBlocks(30, new ECB(7, 24), new ECB(16, 25)), + new ECBlocks(24, new ECB(34, 13))))); + VERSIONS.push_back(Ref(new Version( + 23, intArray(5, 6, 30, 54, 78, 102), new ECBlocks(30, new ECB(4, 121), new ECB(5, 122)), + new ECBlocks(28, new ECB(4, 47), new ECB(14, 48)), + new ECBlocks(30, new ECB(11, 24), new ECB(14, 25)), + new ECBlocks(30, new ECB(16, 15), new ECB(14, 16))))); + VERSIONS.push_back(Ref(new Version( + 24, intArray(5, 6, 28, 54, 80, 106), new ECBlocks(30, new ECB(6, 117), new ECB(4, 118)), + new ECBlocks(28, new ECB(6, 45), new ECB(14, 46)), + new ECBlocks(30, new ECB(11, 24), new ECB(16, 25)), + new ECBlocks(30, new ECB(30, 16), new ECB(2, 17))))); + VERSIONS.push_back(Ref(new Version( + 25, intArray(5, 6, 32, 58, 84, 110), new ECBlocks(26, new ECB(8, 106), new ECB(4, 107)), + new ECBlocks(28, new ECB(8, 47), new ECB(13, 48)), + new ECBlocks(30, new ECB(7, 24), new ECB(22, 25)), + new ECBlocks(30, new ECB(22, 15), new ECB(13, 16))))); + VERSIONS.push_back(Ref(new Version( + 26, intArray(5, 6, 30, 58, 86, 114), new ECBlocks(28, new ECB(10, 114), new ECB(2, 115)), + new ECBlocks(28, new ECB(19, 46), new ECB(4, 47)), + new ECBlocks(28, new ECB(28, 22), new ECB(6, 23)), + new ECBlocks(30, new ECB(33, 16), new ECB(4, 17))))); + VERSIONS.push_back(Ref(new Version( + 27, intArray(5, 6, 34, 62, 90, 118), new ECBlocks(30, new ECB(8, 122), new ECB(4, 123)), + new ECBlocks(28, new ECB(22, 45), new ECB(3, 46)), + new ECBlocks(30, new ECB(8, 23), new ECB(26, 24)), + new ECBlocks(30, new ECB(12, 15), new ECB(28, 16))))); + VERSIONS.push_back( + Ref(new Version(28, intArray(6, 6, 26, 50, 74, 98, 122), + new ECBlocks(30, new ECB(3, 117), new ECB(10, 118)), + new ECBlocks(28, new ECB(3, 45), new ECB(23, 46)), + new ECBlocks(30, new ECB(4, 24), new ECB(31, 25)), + new ECBlocks(30, new ECB(11, 15), new ECB(31, 16))))); + VERSIONS.push_back( + Ref(new Version(29, intArray(6, 6, 30, 54, 78, 102, 126), + new ECBlocks(30, new ECB(7, 116), new ECB(7, 117)), + new ECBlocks(28, new ECB(21, 45), new ECB(7, 46)), + new ECBlocks(30, new ECB(1, 23), new ECB(37, 24)), + new ECBlocks(30, new ECB(19, 15), new ECB(26, 16))))); + VERSIONS.push_back( + Ref(new Version(30, intArray(6, 6, 26, 52, 78, 104, 130), + new ECBlocks(30, new ECB(5, 115), new ECB(10, 116)), + new ECBlocks(28, new ECB(19, 47), new ECB(10, 48)), + new ECBlocks(30, new ECB(15, 24), new ECB(25, 25)), + new ECBlocks(30, new ECB(23, 15), new ECB(25, 16))))); + VERSIONS.push_back( + Ref(new Version(31, intArray(6, 6, 30, 56, 82, 108, 134), + new ECBlocks(30, new ECB(13, 115), new ECB(3, 116)), + new ECBlocks(28, new ECB(2, 46), new ECB(29, 47)), + new ECBlocks(30, new ECB(42, 24), new ECB(1, 25)), + new ECBlocks(30, new ECB(23, 15), new ECB(28, 16))))); + VERSIONS.push_back(Ref( + new Version(32, intArray(6, 6, 34, 60, 86, 112, 138), new ECBlocks(30, new ECB(17, 115)), + new ECBlocks(28, new ECB(10, 46), new ECB(23, 47)), + new ECBlocks(30, new ECB(10, 24), new ECB(35, 25)), + new ECBlocks(30, new ECB(19, 15), new ECB(35, 16))))); + VERSIONS.push_back( + Ref(new Version(33, intArray(6, 6, 30, 58, 86, 114, 142), + new ECBlocks(30, new ECB(17, 115), new ECB(1, 116)), + new ECBlocks(28, new ECB(14, 46), new ECB(21, 47)), + new ECBlocks(30, new ECB(29, 24), new ECB(19, 25)), + new ECBlocks(30, new ECB(11, 15), new ECB(46, 16))))); + VERSIONS.push_back( + Ref(new Version(34, intArray(6, 6, 34, 62, 90, 118, 146), + new ECBlocks(30, new ECB(13, 115), new ECB(6, 116)), + new ECBlocks(28, new ECB(14, 46), new ECB(23, 47)), + new ECBlocks(30, new ECB(44, 24), new ECB(7, 25)), + new ECBlocks(30, new ECB(59, 16), new ECB(1, 17))))); + VERSIONS.push_back( + Ref(new Version(35, intArray(7, 6, 30, 54, 78, 102, 126, 150), + new ECBlocks(30, new ECB(12, 121), new ECB(7, 122)), + new ECBlocks(28, new ECB(12, 47), new ECB(26, 48)), + new ECBlocks(30, new ECB(39, 24), new ECB(14, 25)), + new ECBlocks(30, new ECB(22, 15), new ECB(41, 16))))); + VERSIONS.push_back( + Ref(new Version(36, intArray(7, 6, 24, 50, 76, 102, 128, 154), + new ECBlocks(30, new ECB(6, 121), new ECB(14, 122)), + new ECBlocks(28, new ECB(6, 47), new ECB(34, 48)), + new ECBlocks(30, new ECB(46, 24), new ECB(10, 25)), + new ECBlocks(30, new ECB(2, 15), new ECB(64, 16))))); + VERSIONS.push_back( + Ref(new Version(37, intArray(7, 6, 28, 54, 80, 106, 132, 158), + new ECBlocks(30, new ECB(17, 122), new ECB(4, 123)), + new ECBlocks(28, new ECB(29, 46), new ECB(14, 47)), + new ECBlocks(30, new ECB(49, 24), new ECB(10, 25)), + new ECBlocks(30, new ECB(24, 15), new ECB(46, 16))))); + VERSIONS.push_back( + Ref(new Version(38, intArray(7, 6, 32, 58, 84, 110, 136, 162), + new ECBlocks(30, new ECB(4, 122), new ECB(18, 123)), + new ECBlocks(28, new ECB(13, 46), new ECB(32, 47)), + new ECBlocks(30, new ECB(48, 24), new ECB(14, 25)), + new ECBlocks(30, new ECB(42, 15), new ECB(32, 16))))); + VERSIONS.push_back( + Ref(new Version(39, intArray(7, 6, 26, 54, 82, 110, 138, 166), + new ECBlocks(30, new ECB(20, 117), new ECB(4, 118)), + new ECBlocks(28, new ECB(40, 47), new ECB(7, 48)), + new ECBlocks(30, new ECB(43, 24), new ECB(22, 25)), + new ECBlocks(30, new ECB(10, 15), new ECB(67, 16))))); + VERSIONS.push_back( + Ref(new Version(40, intArray(7, 6, 30, 58, 86, 114, 142, 170), + new ECBlocks(30, new ECB(19, 118), new ECB(6, 119)), + new ECBlocks(28, new ECB(18, 47), new ECB(31, 48)), + new ECBlocks(30, new ECB(34, 24), new ECB(34, 25)), + new ECBlocks(30, new ECB(20, 15), new ECB(61, 16))))); + return VERSIONS.size(); +} + +} // namespace qrcode +} // namespace zxing diff --git a/modules/wechat_qrcode/src/zxing/qrcode/version.hpp b/modules/wechat_qrcode/src/zxing/qrcode/version.hpp new file mode 100644 index 00000000000..c6fc6fe81fe --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/qrcode/version.hpp @@ -0,0 +1,88 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_QRCODE_VERSION_HPP__ +#define __ZXING_QRCODE_VERSION_HPP__ + +#include "../common/bitmatrix.hpp" +#include "../common/counted.hpp" +#include "../errorhandler.hpp" +#include "error_correction_level.hpp" + +#include + +namespace zxing { +namespace qrcode { + +// Encapsualtes the parameters for one error-correction block in one symbol +// version. This includes the number of data codewords, and the number of times +// a block with these parameters is used consecutively in the QR code version's +// format. +class ECB { +private: + int count_; + int dataCodewords_; + +public: + ECB(int count, int dataCodewords); + int getCount(); + int getDataCodewords(); +}; + +// Encapsulates a set of error-correction blocks in one symbol version. Most +// versions will use blocks of differing sizes within one version, so, this +// encapsulates the parameters for each set of blocks. It also holds the number +// of error-correction codewords per block since it will be the same across all +// blocks within one version.

+class ECBlocks { +private: + int ecCodewords_; + std::vector ecBlocks_; + +public: + ECBlocks(int ecCodewords, ECB *ecBlocks); + ECBlocks(int ecCodewords, ECB *ecBlocks1, ECB *ecBlocks2); + int getECCodewords(); + std::vector &getECBlocks(); + ~ECBlocks(); +}; + +class Version : public Counted { +private: + int versionNumber_; + std::vector &alignmentPatternCenters_; + std::vector ecBlocks_; + int totalCodewords_; + Version(int versionNumber, std::vector *alignmentPatternCenters, ECBlocks *ecBlocks1, + ECBlocks *ecBlocks2, ECBlocks *ecBlocks3, ECBlocks *ecBlocks4); + +public: + static unsigned int VERSION_DECODE_INFO[]; + static int N_VERSION_DECODE_INFOS; + static std::vector > VERSIONS; + + ~Version(); + int getVersionNumber(); + std::vector &getAlignmentPatternCenters(); + int getTotalCodewords(); + int getDimensionForVersion(ErrorHandler &err_handler); + ECBlocks &getECBlocksForLevel(ErrorCorrectionLevel &ecLevel); + static Version *getProvisionalVersionForDimension(int dimension, ErrorHandler &err_handler); + static Version *getVersionForNumber(int versionNumber, ErrorHandler &err_handler); + static Version *decodeVersionInformation(unsigned int versionBits); + Ref buildFunctionPattern(ErrorHandler &err_handler); + Ref buildFixedPatternValue(ErrorHandler &err_handler); + Ref buildFixedPatternTemplate(ErrorHandler &err_handler); + static int buildVersions(); +}; +} // namespace qrcode +} // namespace zxing + +#endif // __ZXING_QRCODE_VERSION_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/reader.cpp b/modules/wechat_qrcode/src/zxing/reader.cpp new file mode 100644 index 00000000000..adb445d7bcb --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/reader.cpp @@ -0,0 +1,28 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "reader.hpp" + +namespace zxing { + +Reader::~Reader() {} + +Ref Reader::decode(Ref image) { return decode(image, DecodeHints()); } + +unsigned int Reader::getDecodeID() { return 0; } + +void Reader::setDecodeID(unsigned int) {} + +float Reader::getPossibleFix() { return 0.0; } + + +string Reader::name() { return "unknow"; } + +} // namespace zxing diff --git a/modules/wechat_qrcode/src/zxing/reader.hpp b/modules/wechat_qrcode/src/zxing/reader.hpp new file mode 100644 index 00000000000..cb3e7eaf3f0 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/reader.hpp @@ -0,0 +1,39 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_READER_HPP__ +#define __ZXING_READER_HPP__ + +#include "binarybitmap.hpp" +#include "decodehints.hpp" +#include "errorhandler.hpp" +#include "result.hpp" + +namespace zxing { + +class Reader : public Counted { +protected: + Reader() {} + +public: + virtual Ref decode(Ref image); + virtual Ref decode(Ref image, DecodeHints hints) = 0; + + virtual ~Reader(); + virtual string name(); + virtual unsigned int getDecodeID(); + virtual void setDecodeID(unsigned int id); + + virtual float getPossibleFix(); +}; + +} // namespace zxing + +#endif // __ZXING_READER_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/result.cpp b/modules/wechat_qrcode/src/zxing/result.cpp new file mode 100644 index 00000000000..cd8c2a102fd --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/result.cpp @@ -0,0 +1,71 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "result.hpp" + +using zxing::ArrayRef; +using zxing::Ref; +using zxing::Result; +using zxing::ResultPoint; +using zxing::String; + +Result::Result(Ref text, ArrayRef rawBytes, ArrayRef > resultPoints) + : text_(text), rawBytes_(rawBytes), resultPoints_(resultPoints) { + charset_ = "UTF-8"; + + qrcodeVersion_ = -1; + pyramidLv_ = -1; + binaryMethod_ = -1; + ecLevel_ = '0'; +} + +Result::Result(Ref text, ArrayRef rawBytes, ArrayRef > resultPoints, + std::string charset) + : text_(text), rawBytes_(rawBytes), resultPoints_(resultPoints), charset_(charset) { + qrcodeVersion_ = -1; + pyramidLv_ = -1; + binaryMethod_ = -1; + ecLevel_ = '0'; +} + +Result::Result(Ref text, ArrayRef rawBytes, ArrayRef > resultPoints, + std::string charset, int QRCodeVersion, string ecLevel, string charsetMode) + : text_(text), + rawBytes_(rawBytes), + resultPoints_(resultPoints), + charset_(charset), + qrcodeVersion_(QRCodeVersion), + ecLevel_(ecLevel), + charsetMode_(charsetMode) { + pyramidLv_ = -1; + binaryMethod_ = -1; +} + +Result::~Result() {} + +Ref Result::getText() { return text_; } + +ArrayRef Result::getRawBytes() { return rawBytes_; } + +ArrayRef > const& Result::getResultPoints() const { return resultPoints_; } + +ArrayRef >& Result::getResultPoints() { return resultPoints_; } + +void Result::enlargeResultPoints(int scale) { + for (int i = 0; i < resultPoints_->size(); i++) { + resultPoints_[i] = Ref(new ResultPoint( + resultPoints_[i]->getX() * (float)scale, resultPoints_[i]->getY() * (float)scale)); + } + return; +} + +std::string Result::getCharset() const { return charset_; } + +std::string zxing::Result::getChartsetMode() const { return charsetMode_; } diff --git a/modules/wechat_qrcode/src/zxing/result.hpp b/modules/wechat_qrcode/src/zxing/result.hpp new file mode 100644 index 00000000000..6bf053c0f19 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/result.hpp @@ -0,0 +1,78 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_RESULT_HPP__ +#define __ZXING_RESULT_HPP__ + +#include +#include "common/array.hpp" +#include "common/counted.hpp" +#include "common/str.hpp" +#include "resultpoint.hpp" + +#include + +namespace zxing { + +class Result : public Counted { +private: + Ref text_; + ArrayRef rawBytes_; + ArrayRef > resultPoints_; + std::string charset_; + int qrcodeVersion_; + int pyramidLv_; + int binaryMethod_; + string ecLevel_; + string charsetMode_; + string scale_list_; + float decode_scale_; + uint32_t detect_time_; + uint32_t sr_time_; + +public: + Result(Ref text, ArrayRef rawBytes, ArrayRef > resultPoints); + + Result(Ref text, ArrayRef rawBytes, ArrayRef > resultPoints, + std::string charset); + + Result(Ref text, ArrayRef rawBytes, ArrayRef > resultPoints, + std::string charset, int QRCodeVersion, string ecLevel, string charsetMode); + + ~Result(); + + Ref getText(); + ArrayRef getRawBytes(); + ArrayRef > const& getResultPoints() const; + ArrayRef >& getResultPoints(); + std::string getCharset() const; + std::string getChartsetMode() const; + void enlargeResultPoints(int scale); + + int getQRCodeVersion() const { return qrcodeVersion_; }; + void setQRCodeVersion(int QRCodeVersion) { qrcodeVersion_ = QRCodeVersion; }; + int getPyramidLv() const { return pyramidLv_; }; + void setPyramidLv(int pyramidLv) { pyramidLv_ = pyramidLv; }; + int getBinaryMethod() const { return binaryMethod_; }; + void setBinaryMethod(int binaryMethod) { binaryMethod_ = binaryMethod; }; + string getEcLevel() const { return ecLevel_; } + void setEcLevel(char ecLevel) { ecLevel_ = ecLevel; } + std::string getScaleList() { return scale_list_; }; + void setScaleList(const std::string& scale_list) { scale_list_ = scale_list; }; + float getDecodeScale() { return decode_scale_; }; + void setDecodeScale(float decode_scale) { decode_scale_ = decode_scale; }; + uint32_t getDetectTime() { return detect_time_; }; + void setDetectTime(uint32_t detect_time) { detect_time_ = detect_time; }; + uint32_t getSrTime() { return sr_time_; }; + void setSrTime(uint32_t sr_time) { sr_time_ = sr_time; }; +}; + +} // namespace zxing +#endif // __ZXING_RESULT_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/resultpoint.cpp b/modules/wechat_qrcode/src/zxing/resultpoint.cpp new file mode 100644 index 00000000000..4cd5290a6d4 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/resultpoint.cpp @@ -0,0 +1,101 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#include "resultpoint.hpp" +#include "common/mathutils.hpp" + +using zxing::common::MathUtils; + +namespace zxing { + +ResultPoint::ResultPoint() : posX_(0), posY_(0) {} + +ResultPoint::ResultPoint(float x, float y) : posX_(x), posY_(y) {} + +ResultPoint::ResultPoint(int x, int y) : posX_(float(x)), posY_(float(y)) {} + +ResultPoint::~ResultPoint() {} + +float ResultPoint::getX() const { return posX_; } + +float ResultPoint::getY() const { return posY_; } + +void ResultPoint::SetX(float fX) { posX_ = fX; } + +void ResultPoint::SetY(float fY) { posY_ = fY; } + +bool ResultPoint::equals(Ref other) { + return (fabs(posX_ - other->getX()) <= 1e-6) && (fabs(posY_ - other->getY()) <= 1e-6); +} + +/** + *

Orders an array of three ResultPoints in an order [A,B,C] such that AB < + * AC and BC < AC and the angle between BC and BA is less than 180 degrees. + */ +void ResultPoint::orderBestPatterns(std::vector > &patterns) { + // Find distances between pattern centers + float zeroOneDistance = distance(patterns[0]->getX(), patterns[1]->getX(), patterns[0]->getY(), + patterns[1]->getY()); + float oneTwoDistance = distance(patterns[1]->getX(), patterns[2]->getX(), patterns[1]->getY(), + patterns[2]->getY()); + float zeroTwoDistance = distance(patterns[0]->getX(), patterns[2]->getX(), patterns[0]->getY(), + patterns[2]->getY()); + + Ref pointA, pointB, pointC; + // Assume one closest to other two is B; A and C will just be guesses at + // first + if (oneTwoDistance >= zeroOneDistance && oneTwoDistance >= zeroTwoDistance) { + pointB = patterns[0]; + pointA = patterns[1]; + pointC = patterns[2]; + } else if (zeroTwoDistance >= oneTwoDistance && zeroTwoDistance >= zeroOneDistance) { + pointB = patterns[1]; + pointA = patterns[0]; + pointC = patterns[2]; + } else { + pointB = patterns[2]; + pointA = patterns[0]; + pointC = patterns[1]; + } + + // Use cross product to figure out whether A and C are correct or flipped. + // This asks whether BC x BA has a positive z component, which is the + // arrangement we want for A, B, C. If it's negative, then we've got it + // flipped around and should swap A and C. + if (crossProductZ(pointA, pointB, pointC) < 0.0f) { + Ref temp = pointA; + pointA = pointC; + pointC = temp; + } + + patterns[0] = pointA; + patterns[1] = pointB; + patterns[2] = pointC; +} + +float ResultPoint::distance(Ref pattern1, Ref pattern2) { + return MathUtils::distance(pattern1->posX_, pattern1->posY_, pattern2->posX_, pattern2->posY_); +} + +float ResultPoint::distance(float x1, float x2, float y1, float y2) { + float xDiff = x1 - x2; + float yDiff = y1 - y2; + return (float)sqrt((double)(xDiff * xDiff + yDiff * yDiff)); +} + +float ResultPoint::crossProductZ(Ref pointA, Ref pointB, + Ref pointC) { + float bX = pointB->getX(); + float bY = pointB->getY(); + return ((pointC->getX() - bX) * (pointA->getY() - bY)) - + ((pointC->getY() - bY) * (pointA->getX() - bX)); +} +} // namespace zxing diff --git a/modules/wechat_qrcode/src/zxing/resultpoint.hpp b/modules/wechat_qrcode/src/zxing/resultpoint.hpp new file mode 100644 index 00000000000..3237c36260f --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/resultpoint.hpp @@ -0,0 +1,49 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_RESULTPOINT_HPP__ +#define __ZXING_RESULTPOINT_HPP__ + +#include "common/counted.hpp" + +#include + +namespace zxing { + +class ResultPoint : public Counted { +protected: + float posX_; + float posY_; + +public: + ResultPoint(); + ResultPoint(float x, float y); + ResultPoint(int x, int y); + virtual ~ResultPoint(); + + virtual float getX() const; + virtual float getY() const; + virtual void SetX(float fX); + virtual void SetY(float fY); + + bool equals(Ref other); + + static void orderBestPatterns(std::vector > &patterns); + static float distance(Ref point1, Ref point2); + static float distance(float x1, float x2, float y1, float y2); + +private: + static float crossProductZ(Ref pointA, Ref pointB, + Ref pointC); +}; + +} // namespace zxing + +#endif // __ZXING_RESULTPOINT_HPP__ diff --git a/modules/wechat_qrcode/src/zxing/zxing.hpp b/modules/wechat_qrcode/src/zxing/zxing.hpp new file mode 100644 index 00000000000..83936500ee1 --- /dev/null +++ b/modules/wechat_qrcode/src/zxing/zxing.hpp @@ -0,0 +1,91 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. +// +// Modified from ZXing. Copyright ZXing authors. +// Licensed under the Apache License, Version 2.0 (the "License"). + +#ifndef __ZXING_ZXING_HPP__ +#define __ZXING_ZXING_HPP__ + + +#define COUNTER_TYPE short + + +#define ZXING_ARRAY_LEN(v) ((int)(sizeof(v) / sizeof(v[0]))) +#define ZX_LOG_DIGITS(digits) \ + ((digits == 8) \ + ? 3 \ + : ((digits == 16) \ + ? 4 \ + : ((digits == 32) ? 5 : ((digits == 64) ? 6 : ((digits == 128) ? 7 : (-1)))))) + +#ifndef USE_QRCODE_ONLY +#define USE_ONED_WRITER 1 +#endif + +#if defined(__ANDROID_API__) + +#ifndef NO_ICONV +#define NO_ICONV +#endif + +#endif + + + +#ifndef NO_ICONV_INSIDE +#define NO_ICONV_INSIDE +#endif + +#define ZXING_MAX_WIDTH 2048 +#define ZXING_MAX_HEIGHT 2048 + +namespace zxing { +typedef char byte; +typedef unsigned char boolean; +// typedef unsigned short ushort; +} // namespace zxing + +#include + +#if defined(_WIN32) || defined(_WIN64) + +#ifndef NO_ICONV +#define NO_ICONV +#endif + +#include + +namespace zxing { +inline bool isnan(float v) { return _isnan(v) != 0; } +inline bool isnan(double v) { return _isnan(v) != 0; } +inline float nan() { return std::numeric_limits::quiet_NaN(); } +} // namespace zxing + +#else + +#include + +#undef isnan +namespace zxing { +inline bool isnan(float v) { return std::isnan(v); } +inline bool isnan(double v) { return std::isnan(v); } +inline float nan() { return std::numeric_limits::quiet_NaN(); } +} // namespace zxing + +//#endif + +#endif + +#ifndef ZXING_TIME +#define ZXING_TIME(string) (void)0 +#endif +#ifndef ZXING_TIME_MARK +#define ZXING_TIME_MARK(string) (void)0 +#endif + +#endif // __ZXING_ZXING_HPP__ diff --git a/modules/wechat_qrcode/test/test_main.cpp b/modules/wechat_qrcode/test/test_main.cpp new file mode 100644 index 00000000000..47f2ebac939 --- /dev/null +++ b/modules/wechat_qrcode/test/test_main.cpp @@ -0,0 +1,14 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#include "test_precomp.hpp" + +#if defined(HAVE_HPX) +#include +#endif + +CV_TEST_MAIN("cv") diff --git a/modules/wechat_qrcode/test/test_precomp.hpp b/modules/wechat_qrcode/test/test_precomp.hpp new file mode 100644 index 00000000000..0f5ee5ba4e8 --- /dev/null +++ b/modules/wechat_qrcode/test/test_precomp.hpp @@ -0,0 +1,14 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#ifndef __OPENCV_TEST_PRECOMP_HPP__ +#define __OPENCV_TEST_PRECOMP_HPP__ + +#include "opencv2/ts.hpp" +#include "opencv2/wechat_qrcode.hpp" + +#endif diff --git a/modules/wechat_qrcode/test/test_qrcode.cpp b/modules/wechat_qrcode/test/test_qrcode.cpp new file mode 100644 index 00000000000..5de65338f0c --- /dev/null +++ b/modules/wechat_qrcode/test/test_qrcode.cpp @@ -0,0 +1,293 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Tencent is pleased to support the open source community by making WeChat QRCode available. +// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + +#include "test_precomp.hpp" + +namespace opencv_test { +namespace { +std::string qrcode_images_name[] = { + "version_1_down.jpg", /*"version_1_left.jpg", "version_1_right.jpg", "version_1_up.jpg",*/ + "version_1_top.jpg", + /*"version_2_down.jpg",*/ "version_2_left.jpg", /*"version_2_right.jpg",*/ + "version_2_up.jpg", + "version_2_top.jpg", + "version_3_down.jpg", + "version_3_left.jpg", + /*"version_3_right.jpg",*/ "version_3_up.jpg", + "version_3_top.jpg", + "version_4_down.jpg", + "version_4_left.jpg", + /*"version_4_right.jpg",*/ "version_4_up.jpg", + "version_4_top.jpg", + "version_5_down.jpg", + "version_5_left.jpg", + /*"version_5_right.jpg",*/ "version_5_up.jpg", + "version_5_top.jpg", + "russian.jpg", + "kanji.jpg", /*"link_github_ocv.jpg",*/ + "link_ocv.jpg", + "link_wiki_cv.jpg"}; + +std::string qrcode_images_close[] = {/*"close_1.png",*/ "close_2.png", "close_3.png", "close_4.png", + "close_5.png"}; +std::string qrcode_images_monitor[] = {"monitor_1.png", "monitor_2.png", "monitor_3.png", + "monitor_4.png", "monitor_5.png"}; +std::string qrcode_images_curved[] = {"curved_1.jpg", /*"curved_2.jpg", "curved_3.jpg", + "curved_4.jpg",*/ + "curved_5.jpg", "curved_6.jpg", + /*"curved_7.jpg", "curved_8.jpg"*/}; +// std::string qrcode_images_multiple[] = {"2_qrcodes.png", "3_close_qrcodes.png", "3_qrcodes.png", +// "4_qrcodes.png", "5_qrcodes.png", "6_qrcodes.png", +// "7_qrcodes.png", "8_close_qrcodes.png"}; + +typedef testing::TestWithParam Objdetect_QRCode; +TEST_P(Objdetect_QRCode, regression) { + const std::string name_current_image = GetParam(); + const std::string root = "qrcode/"; + + std::string image_path = findDataFile(root + name_current_image); + Mat src = imread(image_path, IMREAD_GRAYSCALE); + ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path; + + vector points; + // can not find the model file + // so we temporarily comment it out + // auto detector = wechat_qrcode::WeChatQRCode( + // findDataFile("detect.prototxt", false), findDataFile("detect.caffemodel", false), + // findDataFile("sr.prototxt", false), findDataFile("sr.caffemodel", false)); + auto detector = wechat_qrcode::WeChatQRCode(); + auto decoded_info = detector.detectAndDecode(src, points); + + const std::string dataset_config = findDataFile(root + "dataset_config.json"); + FileStorage file_config(dataset_config, FileStorage::READ); + ASSERT_TRUE(file_config.isOpened()) << "Can't read validation data: " << dataset_config; + { + FileNode images_list = file_config["test_images"]; + size_t images_count = static_cast(images_list.size()); + ASSERT_GT(images_count, 0u) + << "Can't find validation data entries in 'test_images': " << dataset_config; + + for (size_t index = 0; index < images_count; index++) { + FileNode config = images_list[(int)index]; + std::string name_test_image = config["image_name"]; + if (name_test_image == name_current_image) { + std::string original_info = config["info"]; + string decoded_str; + if (decoded_info.size()) { + decoded_str = decoded_info[0]; + } + EXPECT_EQ(decoded_str, original_info); + return; // done + } + } + std::cerr << "Not found results for '" << name_current_image + << "' image in config file:" << dataset_config << std::endl + << "Re-run tests with enabled UPDATE_QRCODE_TEST_DATA macro to update test data." + << std::endl; + } +} + +typedef testing::TestWithParam Objdetect_QRCode_Close; +TEST_P(Objdetect_QRCode_Close, regression) { + const std::string name_current_image = GetParam(); + const std::string root = "qrcode/close/"; + + std::string image_path = findDataFile(root + name_current_image); + Mat src = imread(image_path, IMREAD_GRAYSCALE); + ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path; + + vector points; + // can not find the model file + // so we temporarily comment it out + // auto detector = wechat_qrcode::WeChatQRCode( + // findDataFile("detect.prototxt", false), findDataFile("detect.caffemodel", false), + // findDataFile("sr.prototxt", false), findDataFile("sr.caffemodel", false)); + auto detector = wechat_qrcode::WeChatQRCode(); + auto decoded_info = detector.detectAndDecode(src, points); + + const std::string dataset_config = findDataFile(root + "dataset_config.json"); + FileStorage file_config(dataset_config, FileStorage::READ); + ASSERT_TRUE(file_config.isOpened()) << "Can't read validation data: " << dataset_config; + { + FileNode images_list = file_config["close_images"]; + size_t images_count = static_cast(images_list.size()); + ASSERT_GT(images_count, 0u) + << "Can't find validation data entries in 'close_images': " << dataset_config; + + for (size_t index = 0; index < images_count; index++) { + FileNode config = images_list[(int)index]; + std::string name_test_image = config["image_name"]; + if (name_test_image == name_current_image) { + std::string original_info = config["info"]; + string decoded_str; + if (decoded_info.size()) { + decoded_str = decoded_info[0]; + } + EXPECT_EQ(decoded_str, original_info); + return; // done + } + } + std::cerr << "Not found results for '" << name_current_image + << "' image in config file:" << dataset_config << std::endl + << "Re-run tests with enabled UPDATE_QRCODE_TEST_DATA macro to update test data." + << std::endl; + } +} + +typedef testing::TestWithParam Objdetect_QRCode_Monitor; +TEST_P(Objdetect_QRCode_Monitor, regression) { + const std::string name_current_image = GetParam(); + const std::string root = "qrcode/monitor/"; + + std::string image_path = findDataFile(root + name_current_image); + Mat src = imread(image_path, IMREAD_GRAYSCALE); + ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path; + + vector points; + // can not find the model file + // so we temporarily comment it out + // auto detector = wechat_qrcode::WeChatQRCode( + // findDataFile("detect.prototxt", false), findDataFile("detect.caffemodel", false), + // findDataFile("sr.prototxt", false), findDataFile("sr.caffemodel", false)); + auto detector = wechat_qrcode::WeChatQRCode(); + auto decoded_info = detector.detectAndDecode(src, points); + + const std::string dataset_config = findDataFile(root + "dataset_config.json"); + FileStorage file_config(dataset_config, FileStorage::READ); + ASSERT_TRUE(file_config.isOpened()) << "Can't read validation data: " << dataset_config; + { + FileNode images_list = file_config["monitor_images"]; + size_t images_count = static_cast(images_list.size()); + ASSERT_GT(images_count, 0u) + << "Can't find validation data entries in 'monitor_images': " << dataset_config; + + for (size_t index = 0; index < images_count; index++) { + FileNode config = images_list[(int)index]; + std::string name_test_image = config["image_name"]; + if (name_test_image == name_current_image) { + std::string original_info = config["info"]; + string decoded_str; + if (decoded_info.size()) { + decoded_str = decoded_info[0]; + } + EXPECT_EQ(decoded_str, original_info); + return; // done + } + } + std::cerr << "Not found results for '" << name_current_image + << "' image in config file:" << dataset_config << std::endl + << "Re-run tests with enabled UPDATE_QRCODE_TEST_DATA macro to update test data." + << std::endl; + } +} + +typedef testing::TestWithParam Objdetect_QRCode_Curved; +TEST_P(Objdetect_QRCode_Curved, regression) { + const std::string name_current_image = GetParam(); + const std::string root = "qrcode/curved/"; + + std::string image_path = findDataFile(root + name_current_image); + Mat src = imread(image_path, IMREAD_GRAYSCALE); + ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path; + + vector points; + // can not find the model file + // so we temporarily comment it out + // auto detector = wechat_qrcode::WeChatQRCode( + // findDataFile("detect.prototxt", false), findDataFile("detect.caffemodel", false), + // findDataFile("sr.prototxt", false), findDataFile("sr.caffemodel", false)); + auto detector = wechat_qrcode::WeChatQRCode(); + auto decoded_info = detector.detectAndDecode(src, points); + + const std::string dataset_config = findDataFile(root + "dataset_config.json"); + FileStorage file_config(dataset_config, FileStorage::READ); + ASSERT_TRUE(file_config.isOpened()) << "Can't read validation data: " << dataset_config; + { + FileNode images_list = file_config["test_images"]; + size_t images_count = static_cast(images_list.size()); + ASSERT_GT(images_count, 0u) + << "Can't find validation data entries in 'test_images': " << dataset_config; + + for (size_t index = 0; index < images_count; index++) { + FileNode config = images_list[(int)index]; + std::string name_test_image = config["image_name"]; + if (name_test_image == name_current_image) { + std::string original_info = config["info"]; + string decoded_str; + if (decoded_info.size()) { + decoded_str = decoded_info[0]; + } + EXPECT_EQ(decoded_str, original_info); + return; // done + } + } + std::cerr << "Not found results for '" << name_current_image + << "' image in config file:" << dataset_config << std::endl + << "Re-run tests with enabled UPDATE_QRCODE_TEST_DATA macro to update test data." + << std::endl; + } +} + +typedef testing::TestWithParam Objdetect_QRCode_Multi; +TEST_P(Objdetect_QRCode_Multi, regression) { + const std::string name_current_image = GetParam(); + const std::string root = "qrcode/multiple/"; + + std::string image_path = findDataFile(root + name_current_image); + Mat src = imread(image_path); + ASSERT_FALSE(src.empty()) << "Can't read image: " << image_path; + + vector points; + // can not find the model file + // so we temporarily comment it out + // auto detector = wechat_qrcode::WeChatQRCode( + // findDataFile("detect.prototxt", false), findDataFile("detect.caffemodel", false), + // findDataFile("sr.prototxt", false), findDataFile("sr.caffemodel", false)); + auto detector = wechat_qrcode::WeChatQRCode(); + vector decoded_info = detector.detectAndDecode(src, points); + + const std::string dataset_config = findDataFile(root + "dataset_config.json"); + FileStorage file_config(dataset_config, FileStorage::READ); + ASSERT_TRUE(file_config.isOpened()) << "Can't read validation data: " << dataset_config; + { + FileNode images_list = file_config["multiple_images"]; + size_t images_count = static_cast(images_list.size()); + ASSERT_GT(images_count, 0u) + << "Can't find validation data entries in 'test_images': " << dataset_config; + for (size_t index = 0; index < images_count; index++) { + FileNode config = images_list[(int)index]; + std::string name_test_image = config["image_name"]; + if (name_test_image == name_current_image) { + size_t count_eq_info = 0; + for (int i = 0; i < int(decoded_info.size()); i++) { + for (int j = 0; j < int(config["info"].size()); j++) { + std::string original_info = config["info"][j]; + if (original_info == decoded_info[i]) { + count_eq_info++; + break; + } + } + } + EXPECT_EQ(config["info"].size(), count_eq_info); + return; // done + } + } + std::cerr << "Not found results for '" << name_current_image + << "' image in config file:" << dataset_config << std::endl + << "Re-run tests with enabled UPDATE_QRCODE_TEST_DATA macro to update test data." + << std::endl; + } +} + +INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode, testing::ValuesIn(qrcode_images_name)); +INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode_Close, testing::ValuesIn(qrcode_images_close)); +INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode_Monitor, testing::ValuesIn(qrcode_images_monitor)); +INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode_Curved, testing::ValuesIn(qrcode_images_curved)); +// INSTANTIATE_TEST_CASE_P(/**/, Objdetect_QRCode_Multi, testing::ValuesIn(qrcode_images_multiple)); + +} // namespace +} // namespace opencv_test From af72ddd320634cedca4e27b70cc75d87433664bc Mon Sep 17 00:00:00 2001 From: dddzg Date: Wed, 20 Jan 2021 20:48:21 +0800 Subject: [PATCH 20/70] remove useless license --- modules/wechat_qrcode/LICENSE | 47 +---------------------------------- 1 file changed, 1 insertion(+), 46 deletions(-) diff --git a/modules/wechat_qrcode/LICENSE b/modules/wechat_qrcode/LICENSE index ce2fbff9fec..33358269b03 100644 --- a/modules/wechat_qrcode/LICENSE +++ b/modules/wechat_qrcode/LICENSE @@ -250,49 +250,4 @@ You may add Your own copyright statement to Your modifications and may provide a 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. -END OF TERMS AND CONDITIONS - - -======================================================================== -jai-imageio -======================================================================== - -Copyright (c) 2005 Sun Microsystems, Inc. -Copyright © 2010-2014 University of Manchester -Copyright © 2010-2015 Stian Soiland-Reyes -Copyright © 2015 Peter Hull -All Rights Reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - -- Redistribution of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -- Redistribution in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - -Neither the name of Sun Microsystems, Inc. or the names of -contributors may be used to endorse or promote products derived -from this software without specific prior written permission. - -This software is provided "AS IS," without a warranty of any -kind. ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND -WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT, ARE HEREBY -EXCLUDED. SUN MIDROSYSTEMS, INC. ("SUN") AND ITS LICENSORS SHALL -NOT BE LIABLE FOR ANY DAMAGES SUFFERED BY LICENSEE AS A RESULT OF -USING, MODIFYING OR DISTRIBUTING THIS SOFTWARE OR ITS -DERIVATIVES. IN NO EVENT WILL SUN OR ITS LICENSORS BE LIABLE FOR -ANY LOST REVENUE, PROFIT OR DATA, OR FOR DIRECT, INDIRECT, SPECIAL, -CONSEQUENTIAL, INCIDENTAL OR PUNITIVE DAMAGES, HOWEVER CAUSED AND -REGARDLESS OF THE THEORY OF LIABILITY, ARISING OUT OF THE USE OF OR -INABILITY TO USE THIS SOFTWARE, EVEN IF SUN HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - -You acknowledge that this software is not designed or intended for -use in the design, construction, operation or maintenance of any -nuclear facility. \ No newline at end of file +END OF TERMS AND CONDITIONS \ No newline at end of file From 06f690fc829630a491a544b0bff476289343b0df Mon Sep 17 00:00:00 2001 From: dddzg Date: Wed, 20 Jan 2021 20:50:53 +0800 Subject: [PATCH 21/70] remove licenses in samples --- modules/wechat_qrcode/samples/qrcode.py | 7 ------- modules/wechat_qrcode/samples/qrcode_example.cpp | 7 ------- 2 files changed, 14 deletions(-) diff --git a/modules/wechat_qrcode/samples/qrcode.py b/modules/wechat_qrcode/samples/qrcode.py index e3cc003c110..ff95f2b051e 100644 --- a/modules/wechat_qrcode/samples/qrcode.py +++ b/modules/wechat_qrcode/samples/qrcode.py @@ -1,10 +1,3 @@ -# This file is part of OpenCV project. -# It is subject to the license terms in the LICENSE file found in the top-level directory -# of this distribution and at http://opencv.org/license.html. -# -# Tencent is pleased to support the open source community by making WeChat QRCode available. -# Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - import cv2 import sys diff --git a/modules/wechat_qrcode/samples/qrcode_example.cpp b/modules/wechat_qrcode/samples/qrcode_example.cpp index 62217b258f1..046525b90f2 100644 --- a/modules/wechat_qrcode/samples/qrcode_example.cpp +++ b/modules/wechat_qrcode/samples/qrcode_example.cpp @@ -1,10 +1,3 @@ -// This file is part of OpenCV project. -// It is subject to the license terms in the LICENSE file found in the top-level directory -// of this distribution and at http://opencv.org/license.html. -// -// Tencent is pleased to support the open source community by making WeChat QRCode available. -// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - #include #include #include From 7b872c993dbf19d656b23e26a3706117a6dad5e7 Mon Sep 17 00:00:00 2001 From: dddzg Date: Wed, 20 Jan 2021 21:29:26 +0800 Subject: [PATCH 22/70] add precomp --- modules/wechat_qrcode/src/binarizermgr.cpp | 4 ++-- modules/wechat_qrcode/src/decodermgr.cpp | 4 ++-- modules/wechat_qrcode/src/detector/align.cpp | 2 +- modules/wechat_qrcode/src/detector/align.hpp | 1 - modules/wechat_qrcode/src/detector/ssd_detector.cpp | 3 +-- modules/wechat_qrcode/src/imgsource.cpp | 4 ++-- modules/wechat_qrcode/src/scale/super_scale.cpp | 4 ++-- modules/wechat_qrcode/src/wechat_qrcode.cpp | 3 +-- 8 files changed, 11 insertions(+), 14 deletions(-) diff --git a/modules/wechat_qrcode/src/binarizermgr.cpp b/modules/wechat_qrcode/src/binarizermgr.cpp index d651ff07be5..db019cd50e9 100644 --- a/modules/wechat_qrcode/src/binarizermgr.cpp +++ b/modules/wechat_qrcode/src/binarizermgr.cpp @@ -4,10 +4,10 @@ // // Tencent is pleased to support the open source community by making WeChat QRCode available. // Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - +#include "precomp.hpp" #include "binarizermgr.hpp" #include "imgsource.hpp" -#include "precomp.hpp" + using zxing::Binarizer; using zxing::LuminanceSource; diff --git a/modules/wechat_qrcode/src/decodermgr.cpp b/modules/wechat_qrcode/src/decodermgr.cpp index cf40b4fffbc..06706eed2f7 100644 --- a/modules/wechat_qrcode/src/decodermgr.cpp +++ b/modules/wechat_qrcode/src/decodermgr.cpp @@ -4,9 +4,9 @@ // // Tencent is pleased to support the open source community by making WeChat QRCode available. // Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - -#include "decodermgr.hpp" #include "precomp.hpp" +#include "decodermgr.hpp" + using zxing::ArrayRef; using zxing::BinaryBitmap; diff --git a/modules/wechat_qrcode/src/detector/align.cpp b/modules/wechat_qrcode/src/detector/align.cpp index 06fe6cf8fa1..9a34f001292 100644 --- a/modules/wechat_qrcode/src/detector/align.cpp +++ b/modules/wechat_qrcode/src/detector/align.cpp @@ -4,7 +4,7 @@ // // Tencent is pleased to support the open source community by making WeChat QRCode available. // Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - +#include "../precomp.hpp" #include "align.hpp" using std::max; diff --git a/modules/wechat_qrcode/src/detector/align.hpp b/modules/wechat_qrcode/src/detector/align.hpp index c8cbc17849e..2ad88a5b6a9 100644 --- a/modules/wechat_qrcode/src/detector/align.hpp +++ b/modules/wechat_qrcode/src/detector/align.hpp @@ -10,7 +10,6 @@ #include #include -#include "../precomp.hpp" #include "opencv2/core.hpp" #include "opencv2/imgproc.hpp" diff --git a/modules/wechat_qrcode/src/detector/ssd_detector.cpp b/modules/wechat_qrcode/src/detector/ssd_detector.cpp index f0455534ffd..cba7f602758 100644 --- a/modules/wechat_qrcode/src/detector/ssd_detector.cpp +++ b/modules/wechat_qrcode/src/detector/ssd_detector.cpp @@ -4,9 +4,8 @@ // // Tencent is pleased to support the open source community by making WeChat QRCode available. // Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - -#include "ssd_detector.hpp" #include "../precomp.hpp" +#include "ssd_detector.hpp" #define CLIP(x, x1, x2) max(x1, min(x, x2)) namespace cv { namespace wechat_qrcode { diff --git a/modules/wechat_qrcode/src/imgsource.cpp b/modules/wechat_qrcode/src/imgsource.cpp index 7bec2989b6c..bd49de4ea89 100644 --- a/modules/wechat_qrcode/src/imgsource.cpp +++ b/modules/wechat_qrcode/src/imgsource.cpp @@ -7,9 +7,9 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - -#include "imgsource.hpp" #include "precomp.hpp" +#include "imgsource.hpp" + using zxing::ArrayRef; using zxing::ByteMatrix; using zxing::ErrorHandler; diff --git a/modules/wechat_qrcode/src/scale/super_scale.cpp b/modules/wechat_qrcode/src/scale/super_scale.cpp index 5a70975ec87..8b3b11383a1 100644 --- a/modules/wechat_qrcode/src/scale/super_scale.cpp +++ b/modules/wechat_qrcode/src/scale/super_scale.cpp @@ -4,9 +4,9 @@ // // Tencent is pleased to support the open source community by making WeChat QRCode available. // Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - -#include "super_scale.hpp" #include "../precomp.hpp" +#include "super_scale.hpp" + #define CLIP(x, x1, x2) max(x1, min(x, x2)) namespace cv { diff --git a/modules/wechat_qrcode/src/wechat_qrcode.cpp b/modules/wechat_qrcode/src/wechat_qrcode.cpp index 270f6920b8c..bfaa5afc4c7 100644 --- a/modules/wechat_qrcode/src/wechat_qrcode.cpp +++ b/modules/wechat_qrcode/src/wechat_qrcode.cpp @@ -4,14 +4,13 @@ // // Tencent is pleased to support the open source community by making WeChat QRCode available. // Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - +#include "precomp.hpp" #include "opencv2/wechat_qrcode.hpp" #include "decodermgr.hpp" #include "detector/align.hpp" #include "detector/ssd_detector.hpp" #include "opencv2/core.hpp" #include "opencv2/core/utils/filesystem.hpp" -#include "precomp.hpp" #include "scale/super_scale.hpp" #include "zxing/result.hpp" using cv::InputArray; From 9750e14937da12e6b079b9a0ce8b8b225921eb4e Mon Sep 17 00:00:00 2001 From: dddzg Date: Thu, 21 Jan 2021 11:46:36 +0800 Subject: [PATCH 23/70] use CV_Assert --- modules/wechat_qrcode/src/wechat_qrcode.cpp | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/modules/wechat_qrcode/src/wechat_qrcode.cpp b/modules/wechat_qrcode/src/wechat_qrcode.cpp index bfaa5afc4c7..754bcfb6c67 100644 --- a/modules/wechat_qrcode/src/wechat_qrcode.cpp +++ b/modules/wechat_qrcode/src/wechat_qrcode.cpp @@ -54,13 +54,11 @@ WeChatQRCode::WeChatQRCode(const String& detector_prototxt_path, if (!detector_caffe_model_path.empty() && !detector_prototxt_path.empty()) { // initialize detector model (caffe) p->use_nn_detector_ = true; - CV_CheckEQ(utils::fs::exists(detector_prototxt_path), true, - "fail to find detector caffe prototxt file"); - CV_CheckEQ(utils::fs::exists(detector_caffe_model_path), true, - "fail to find detector caffe model file"); + CV_Assert(utils::fs::exists(detector_prototxt_path)); + CV_Assert(utils::fs::exists(detector_caffe_model_path)); p->detector_ = make_shared(); auto ret = p->detector_->init(detector_prototxt_path, detector_caffe_model_path); - CV_CheckEQ(ret, 0, "fail to load the detector model."); + CV_Assert(ret == 0); } else { p->use_nn_detector_ = false; p->detector_ = NULL; @@ -71,14 +69,12 @@ WeChatQRCode::WeChatQRCode(const String& detector_prototxt_path, p->super_resolution_model_ = make_shared(); if (!super_resolution_prototxt_path.empty() && !super_resolution_caffe_model_path.empty()) { p->use_nn_sr_ = true; - // initialize dnn model (onnx format) - CV_CheckEQ(utils::fs::exists(super_resolution_prototxt_path), true, - "fail to find super resolution prototxt model file"); - CV_CheckEQ(utils::fs::exists(super_resolution_caffe_model_path), true, - "fail to find super resolution caffe model file"); + // initialize dnn model (caffe format) + CV_Assert(utils::fs::exists(super_resolution_prototxt_path)); + CV_Assert(utils::fs::exists(super_resolution_caffe_model_path)); auto ret = p->super_resolution_model_->init(super_resolution_prototxt_path, super_resolution_caffe_model_path); - CV_CheckEQ(ret, 0, "fail to load the super resolution model."); + CV_Assert(ret == 0); } else { p->use_nn_sr_ = false; } @@ -159,7 +155,7 @@ vector WeChatQRCode::Impl::detect(const Mat& img) { if (use_nn_detector_) { // use cnn detector auto ret = applyDetector(img, points); - CV_CheckEQ(ret, 0, "fail to apply detector."); + CV_Assert(ret == 0); } else { auto width = img.cols, height = img.rows; // if there is no detector, use the full image as input From be6718e52ff7160da999ea4e36b675df1b531ce5 Mon Sep 17 00:00:00 2001 From: dddzg Date: Thu, 21 Jan 2021 11:48:08 +0800 Subject: [PATCH 24/70] remove useless cv:: --- modules/wechat_qrcode/src/wechat_qrcode.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/modules/wechat_qrcode/src/wechat_qrcode.cpp b/modules/wechat_qrcode/src/wechat_qrcode.cpp index 754bcfb6c67..d4856ba22f0 100644 --- a/modules/wechat_qrcode/src/wechat_qrcode.cpp +++ b/modules/wechat_qrcode/src/wechat_qrcode.cpp @@ -13,7 +13,6 @@ #include "opencv2/core/utils/filesystem.hpp" #include "scale/super_scale.hpp" #include "zxing/result.hpp" -using cv::InputArray; namespace cv { namespace wechat_qrcode { class WeChatQRCode::Impl { @@ -91,7 +90,7 @@ vector WeChatQRCode::detectAndDecode(InputArray img, OutputArrayOfArrays int incn = img.channels(); CV_Check(incn, incn == 1 || incn == 3 || incn == 4, ""); if (incn == 3 || incn == 4) { - cv::cvtColor(img, input_img, cv::COLOR_BGR2GRAY); + cvtColor(img, input_img, COLOR_BGR2GRAY); } else { input_img = img.getMat(); } @@ -122,7 +121,7 @@ vector WeChatQRCode::Impl::decode(const Mat& img, vector& candidate } vector decode_results; for (auto& point : candidate_points) { - cv::Mat cropped_img; + Mat cropped_img; if (use_nn_detector_) { Align aligner; cropped_img = cropObj(img, point, aligner); @@ -132,7 +131,7 @@ vector WeChatQRCode::Impl::decode(const Mat& img, vector& candidate // scale_list contains different scale ratios auto scale_list = getScaleList(cropped_img.cols, cropped_img.rows); for (auto cur_scale : scale_list) { - cv::Mat scaled_img = + Mat scaled_img = super_resolution_model_->processImageScale(cropped_img, cur_scale, use_nn_sr_); string result; DecoderMgr decodemgr; @@ -173,7 +172,7 @@ vector WeChatQRCode::Impl::detect(const Mat& img) { return points; } -int WeChatQRCode::Impl::applyDetector(const cv::Mat& img, vector& points) { +int WeChatQRCode::Impl::applyDetector(const Mat& img, vector& points) { int img_w = img.cols; int img_h = img.rows; @@ -188,7 +187,7 @@ int WeChatQRCode::Impl::applyDetector(const cv::Mat& img, vector& points) { return 0; } -cv::Mat WeChatQRCode::Impl::cropObj(const cv::Mat& img, const Mat& point, Align& aligner) { +Mat WeChatQRCode::Impl::cropObj(const Mat& img, const Mat& point, Align& aligner) { // make some padding to boost the qrcode details recall. float padding_w = 0.1f, padding_h = 0.1f; auto min_padding = 15; From edc5bb71e5cadc00f1ae1beef8a919b5a2943399 Mon Sep 17 00:00:00 2001 From: Vadim Pisarevsky Date: Thu, 21 Jan 2021 15:38:19 +0800 Subject: [PATCH 25/70] Merge pull request #2806 from vpisarev:kinfu_demo_orbbec * added orbbec camera intrinsic parameters for kinfu demo * removed trailing whitespace --- modules/rgbd/samples/io_utils.hpp | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/modules/rgbd/samples/io_utils.hpp b/modules/rgbd/samples/io_utils.hpp index c96d6c5345d..f0a3a1e7372 100644 --- a/modules/rgbd/samples/io_utils.hpp +++ b/modules/rgbd/samples/io_utils.hpp @@ -89,6 +89,19 @@ static const float k2 = -0.34f; static const float k3 = 0.12f; }; // namespace Kinect2Params +namespace AstraParams +{ +static const Size frameSize = Size(640, 480); +// approximate values, no guarantee to be correct +static const float fx = 535.4f; +static const float fy = 539.2f; +static const float cx = 320.1f; +static const float cy = 247.6f; +static const float k1 = 0.0f; +static const float k2 = 0.0f; +static const float k3 = 0.0f; +}; // namespace Kinect2Params + struct DepthSource { public: @@ -97,7 +110,8 @@ struct DepthSource DEPTH_LIST, DEPTH_KINECT2_LIST, DEPTH_KINECT2, - DEPTH_REALSENSE + DEPTH_REALSENSE, + DEPTH_ASTRA }; DepthSource(int cam) : DepthSource("", cam) {} @@ -116,7 +130,10 @@ struct DepthSource vc = VideoCapture(VideoCaptureAPIs::CAP_OPENNI2 + cam); if (vc.isOpened()) { - sourceType = Type::DEPTH_KINECT2; + if(cam == 20) + sourceType = Type::DEPTH_ASTRA; + else + sourceType = Type::DEPTH_KINECT2; } else { @@ -201,6 +218,15 @@ struct DepthSource frameSize = Kinect2Params::frameSize; } + else if (sourceType == Type::DEPTH_ASTRA) + { + fx = AstraParams::fx; + fy = AstraParams::fy; + cx = AstraParams::cx; + cy = AstraParams::cy; + + frameSize = AstraParams::frameSize; + } else { if (sourceType == Type::DEPTH_REALSENSE) From 962fd354b5ec56f6909951b843dc65d7b510d099 Mon Sep 17 00:00:00 2001 From: Vadim Pisarevsky Date: Thu, 21 Jan 2021 18:30:40 +0800 Subject: [PATCH 26/70] hopefully fixed compile error on Windows --- modules/wechat_qrcode/src/zxing/common/mathutils.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/wechat_qrcode/src/zxing/common/mathutils.hpp b/modules/wechat_qrcode/src/zxing/common/mathutils.hpp index 95ac189c4c2..11470184072 100644 --- a/modules/wechat_qrcode/src/zxing/common/mathutils.hpp +++ b/modules/wechat_qrcode/src/zxing/common/mathutils.hpp @@ -15,6 +15,8 @@ #if (defined __GNUC__ && defined __x86_64__ && defined __SSE2__ && !defined __APPLE__ && \ !defined __GXX_WEAK__) #include +#elif defined _MSC_VER && (defined _M_X64 || defined _M_IX86) +#include #endif #include From d733a80100bcc2145c2011ade96c53e7e2140767 Mon Sep 17 00:00:00 2001 From: LaurentBerger Date: Fri, 22 Jan 2021 09:09:36 +0100 Subject: [PATCH 27/70] Merge pull request #2819 from LaurentBerger:pyb_qd * solved issue 2813 * review --- modules/stereo/CMakeLists.txt | 2 +- .../opencv2/stereo/quasi_dense_stereo.hpp | 66 ++++++++++--------- .../stereo/misc/python/pyopencv_stereo.hpp | 17 +++++ .../stereo/misc/python/test/test_stereo.py | 20 ++++++ modules/stereo/samples/dense_disparity.cpp | 2 +- modules/stereo/samples/sample_quasi_dense.py | 21 ++++++ modules/stereo/src/quasi_dense_stereo.cpp | 20 +++--- 7 files changed, 104 insertions(+), 44 deletions(-) create mode 100644 modules/stereo/misc/python/pyopencv_stereo.hpp create mode 100644 modules/stereo/misc/python/test/test_stereo.py create mode 100644 modules/stereo/samples/sample_quasi_dense.py diff --git a/modules/stereo/CMakeLists.txt b/modules/stereo/CMakeLists.txt index 7863cbf8d30..827331b81e9 100644 --- a/modules/stereo/CMakeLists.txt +++ b/modules/stereo/CMakeLists.txt @@ -1,2 +1,2 @@ set(the_description "Stereo Correspondence") -ocv_define_module(stereo opencv_imgproc opencv_features2d opencv_core opencv_tracking) +ocv_define_module(stereo opencv_imgproc opencv_features2d opencv_core opencv_tracking WRAP python) diff --git a/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp b/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp index b302c133464..197d762b6c7 100644 --- a/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp +++ b/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp @@ -24,42 +24,44 @@ namespace stereo // A basic match structure -struct CV_EXPORTS Match +struct CV_EXPORTS_W_SIMPLE MatchQuasiDense { - cv::Point2i p0; - cv::Point2i p1; - float corr; + CV_PROP_RW cv::Point2i p0; + CV_PROP_RW cv::Point2i p1; + CV_PROP_RW float corr; - bool operator < (const Match & rhs) const//fixme may be used uninitialized in this function + CV_WRAP MatchQuasiDense() { corr = 0; } + + CV_WRAP_AS(apply) bool operator < (const MatchQuasiDense & rhs) const//fixme may be used uninitialized in this function { return this->corr < rhs.corr; } }; -struct CV_EXPORTS PropagationParameters +struct CV_EXPORTS_W_SIMPLE PropagationParameters { - int corrWinSizeX; // similarity window - int corrWinSizeY; + CV_PROP_RW int corrWinSizeX; // similarity window + CV_PROP_RW int corrWinSizeY; - int borderX; // border to ignore - int borderY; + CV_PROP_RW int borderX; // border to ignore + CV_PROP_RW int borderY; //matching - float correlationThreshold; // correlation threshold - float textrureThreshold; // texture threshold + CV_PROP_RW float correlationThreshold; // correlation threshold + CV_PROP_RW float textrureThreshold; // texture threshold - int neighborhoodSize; // neighborhood size - int disparityGradient; // disparity gradient threshold + CV_PROP_RW int neighborhoodSize; // neighborhood size + CV_PROP_RW int disparityGradient; // disparity gradient threshold // Parameters for LK flow algorithm - int lkTemplateSize; - int lkPyrLvl; - int lkTermParam1; - float lkTermParam2; + CV_PROP_RW int lkTemplateSize; + CV_PROP_RW int lkPyrLvl; + CV_PROP_RW int lkTermParam1; + CV_PROP_RW float lkTermParam2; // Parameters for GFT algorithm. - float gftQualityThres; - int gftMinSeperationDist; - int gftMaxNumFeatures; + CV_PROP_RW float gftQualityThres; + CV_PROP_RW int gftMinSeperationDist; + CV_PROP_RW int gftMaxNumFeatures; }; @@ -90,14 +92,14 @@ struct CV_EXPORTS PropagationParameters * */ -class CV_EXPORTS QuasiDenseStereo +class CV_EXPORTS_W QuasiDenseStereo { public: /** * @brief destructor * Method to free all the memory allocated by matrices and vectors in this class. */ - virtual ~QuasiDenseStereo() = 0; + CV_WRAP virtual ~QuasiDenseStereo() = 0; /** @@ -113,7 +115,7 @@ class CV_EXPORTS QuasiDenseStereo * in case of video processing. * @sa loadParameters */ - virtual int loadParameters(cv::String filepath) = 0; + CV_WRAP virtual int loadParameters(cv::String filepath) = 0; /** @@ -124,7 +126,7 @@ class CV_EXPORTS QuasiDenseStereo * @note This method can be used to generate a template file for tuning the class. * @sa loadParameters */ - virtual int saveParameters(cv::String filepath) = 0; + CV_WRAP virtual int saveParameters(cv::String filepath) = 0; /** @@ -133,7 +135,7 @@ class CV_EXPORTS QuasiDenseStereo * @note The method clears the sMatches vector. * @note The returned Match elements inside the sMatches vector, do not use corr member. */ - virtual void getSparseMatches(std::vector &sMatches) = 0; + CV_WRAP virtual void getSparseMatches(CV_OUT std::vector &sMatches) = 0; /** @@ -142,7 +144,7 @@ class CV_EXPORTS QuasiDenseStereo * @note The method clears the denseMatches vector. * @note The returned Match elements inside the sMatches vector, do not use corr member. */ - virtual void getDenseMatches(std::vector &denseMatches) = 0; + CV_WRAP virtual void getDenseMatches(CV_OUT std::vector &denseMatches) = 0; @@ -158,7 +160,7 @@ class CV_EXPORTS QuasiDenseStereo * @sa sparseMatching * @sa quasiDenseMatching */ - virtual void process(const cv::Mat &imgLeft ,const cv::Mat &imgRight) = 0; + CV_WRAP virtual void process(const cv::Mat &imgLeft ,const cv::Mat &imgRight) = 0; /** @@ -169,7 +171,7 @@ class CV_EXPORTS QuasiDenseStereo * @retval cv::Point(0, 0) (NO_MATCH) if no match is found in the right image for the specified pixel location in the left image. * @note This method should be always called after process, otherwise the matches will not be correct. */ - virtual cv::Point2f getMatch(const int x, const int y) = 0; + CV_WRAP virtual cv::Point2f getMatch(const int x, const int y) = 0; /** @@ -180,13 +182,13 @@ class CV_EXPORTS QuasiDenseStereo * @sa computeDisparity * @sa quantizeDisparity */ - virtual cv::Mat getDisparity(uint8_t disparityLvls=50) = 0; + CV_WRAP virtual cv::Mat getDisparity(uint8_t disparityLvls=50) = 0; - static cv::Ptr create(cv::Size monoImgSize, cv::String paramFilepath = cv::String()); + CV_WRAP static cv::Ptr create(cv::Size monoImgSize, cv::String paramFilepath = cv::String()); - PropagationParameters Param; + CV_PROP_RW PropagationParameters Param; }; } //namespace cv diff --git a/modules/stereo/misc/python/pyopencv_stereo.hpp b/modules/stereo/misc/python/pyopencv_stereo.hpp new file mode 100644 index 00000000000..6fba7e05341 --- /dev/null +++ b/modules/stereo/misc/python/pyopencv_stereo.hpp @@ -0,0 +1,17 @@ +#ifdef HAVE_OPENCV_STEREO +typedef std::vector vector_MatchQuasiDense; + +template<> struct pyopencvVecConverter +{ + static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) + { + return pyopencv_to_generic_vec(obj, value, info); + } + + static PyObject* from(const std::vector& value) + { + return pyopencv_from_generic_vec(value); + } +}; + +#endif diff --git a/modules/stereo/misc/python/test/test_stereo.py b/modules/stereo/misc/python/test/test_stereo.py new file mode 100644 index 00000000000..2023123eb93 --- /dev/null +++ b/modules/stereo/misc/python/test/test_stereo.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +import cv2 as cv + +from tests_common import NewOpenCVTests + +class quasi_dense_stereo_test(NewOpenCVTests): + + def test_simple(self): + + stereo = cv.stereo.QuasiDenseStereo_create((100, 100)) + self.assertIsNotNone(stereo) + + dense_matches = cv.stereo_MatchQuasiDense() + self.assertIsNotNone(dense_matches) + + parameters = cv.stereo_PropagationParameters() + self.assertIsNotNone(parameters) + +if __name__ == '__main__': + NewOpenCVTests.bootstrap() diff --git a/modules/stereo/samples/dense_disparity.cpp b/modules/stereo/samples/dense_disparity.cpp index 0e257def230..e7822366623 100644 --- a/modules/stereo/samples/dense_disparity.cpp +++ b/modules/stereo/samples/dense_disparity.cpp @@ -46,7 +46,7 @@ int main() //! [export] - vector matches; + vector matches; stereo->getDenseMatches(matches); std::ofstream dense("./dense.txt", std::ios::out); for (uint i=0; i< matches.size(); i++) diff --git a/modules/stereo/samples/sample_quasi_dense.py b/modules/stereo/samples/sample_quasi_dense.py new file mode 100644 index 00000000000..a6059e70179 --- /dev/null +++ b/modules/stereo/samples/sample_quasi_dense.py @@ -0,0 +1,21 @@ +import numpy as np +import cv2 as cv + +left_img = cv.imread(cv.samples.findFile("aloeL.jpg"), cv.IMREAD_COLOR) +right_img = cv.imread(cv.samples.findFile("aloeR.jpg"), cv.IMREAD_COLOR) + +frame_size = leftImg.shape[0:2]; + +stereo = cv.stereo.QuasiDenseStereo_create(frame_size[::-1]) +stereo.process(left_img, right_img) +disp = stereo.getDisparity(80) +cv.imshow("disparity", disp) +cv.waitKey() +dense_matches = stereo.getDenseMatches() +try: + with open("dense.txt", "wt") as f: + # if you want all matches use for idx in len(dense_matches): It can be a big file + for idx in range(0, min(10, len(dense_matches))): + nb = f.write(str(dense_matches[idx].p0) + "\t" + str(dense_matches[idx].p1) + "\t" + str(dense_matches[idx].corr) + "\n") +except: + print("Cannot open file") diff --git a/modules/stereo/src/quasi_dense_stereo.cpp b/modules/stereo/src/quasi_dense_stereo.cpp index 7a6513cb843..a4d196ae11b 100644 --- a/modules/stereo/src/quasi_dense_stereo.cpp +++ b/modules/stereo/src/quasi_dense_stereo.cpp @@ -13,7 +13,7 @@ namespace stereo { #define NO_MATCH cv::Point(0,0) -typedef std::priority_queue, std::less > t_matchPriorityQueue; +typedef std::priority_queue, std::less > t_matchPriorityQueue; class QuasiDenseStereoImpl : public QuasiDenseStereo @@ -165,7 +165,7 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo t_matchPriorityQueue Local; // Get the best seed at the moment - Match m = seeds.top(); + MatchQuasiDense m = seeds.top(); seeds.pop(); // Ignore the border @@ -209,7 +209,7 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo // push back if this is valid match if( corr > Param.correlationThreshold ) { - Match nm; + MatchQuasiDense nm; nm.p0 = p0; nm.p1 = p1; nm.corr = corr; @@ -223,7 +223,7 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo // Get seeds from the local while( !Local.empty() ) { - Match lm = Local.top(); + MatchQuasiDense lm = Local.top(); Local.pop(); // Check if its unique in both ref and dst. if(refMap.at(lm.p0.y, lm.p0.x) != NO_MATCH) @@ -410,7 +410,7 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo for(uint i=0; i < featuresLeft.size(); i++) { // Calculate correlation and store match in Seeds. - Match m; + MatchQuasiDense m; m.p0 = cv::Point2i(featuresLeft[i]); m.p1 = cv::Point2i(featuresRight[i]); m.corr = 0; @@ -442,7 +442,7 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo * @retval true If the feature is in the border of the image. * @retval false If the feature is not in the border of image. */ - bool CheckBorder(Match m, int bx, int by, int w, int h) + bool CheckBorder(MatchQuasiDense m, int bx, int by, int w, int h) { if(m.p0.xw-bx || m.p0.yh-by || m.p1.xw-bx || m.p1.yh-by) @@ -492,9 +492,9 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo //------------------------------------------------------------------------- - void getSparseMatches(std::vector &sMatches) override + void getSparseMatches(std::vector &sMatches) override { - Match tmpMatch; + MatchQuasiDense tmpMatch; sMatches.clear(); sMatches.reserve(leftFeatures.size()); for (uint i=0; i &denseMatches) override + void getDenseMatches(std::vector &denseMatches) override { - Match tmpMatch; + MatchQuasiDense tmpMatch; denseMatches.clear(); denseMatches.reserve(dMatchesLen); for (int row=0; row Date: Sun, 24 Jan 2021 20:29:52 +0900 Subject: [PATCH 28/70] [moved from opencv] fix wrong index original commit: https://github.com/opencv/opencv/commit/3cfe7b9af99aaeb4ca7d387f78134896c7b3456d --- modules/cudalegacy/src/calib3d.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/cudalegacy/src/calib3d.cpp b/modules/cudalegacy/src/calib3d.cpp index b58ca3a98d3..03d05be0455 100644 --- a/modules/cudalegacy/src/calib3d.cpp +++ b/modules/cudalegacy/src/calib3d.cpp @@ -283,7 +283,7 @@ void cv::cuda::solvePnPRansac(const Mat& object, const Mat& image, const Mat& ca p_transf.z = rot[6] * p.x + rot[7] * p.y + rot[8] * p.z + transl[2]; p_proj.x = p_transf.x / p_transf.z; p_proj.y = p_transf.y / p_transf.z; - if (norm(p_proj - image_normalized.at(0, i)) < max_dist) + if (norm(p_proj - image_normalized.at(i)) < max_dist) inliers->push_back(i); } } From 6e52be36a5560f2f4242171f71a5904ddf597e59 Mon Sep 17 00:00:00 2001 From: Tomoaki Teshima Date: Mon, 25 Jan 2021 08:40:01 +0900 Subject: [PATCH 29/70] disable NVIDIA Optical Flow SDK when CUDA < 10.0 * follow the review comment * add missing constructors when not covered * add definition in cu file --- modules/cudaoptflow/CMakeLists.txt | 34 ++++++++++--------- .../cudaoptflow/src/cuda/nvidiaOpticalFlow.cu | 2 ++ modules/cudaoptflow/src/nvidiaOpticalFlow.cpp | 25 ++++++++++---- 3 files changed, 39 insertions(+), 22 deletions(-) diff --git a/modules/cudaoptflow/CMakeLists.txt b/modules/cudaoptflow/CMakeLists.txt index 7d2d3e74eef..d6ae902637d 100644 --- a/modules/cudaoptflow/CMakeLists.txt +++ b/modules/cudaoptflow/CMakeLists.txt @@ -8,21 +8,23 @@ ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4127 /wd4324 /wd4512 -Wundef -Wmissing-d ocv_define_module(cudaoptflow opencv_video opencv_optflow opencv_cudaarithm opencv_cudawarping opencv_cudaimgproc OPTIONAL opencv_cudalegacy WRAP python) -set(NVIDIA_OPTICAL_FLOW_2_0_HEADERS_COMMIT "edb50da3cf849840d680249aa6dbef248ebce2ca") -set(NVIDIA_OPTICAL_FLOW_2_0_HEADERS_MD5 "a73cd48b18dcc0cc8933b30796074191") -set(NVIDIA_OPTICAL_FLOW_2_0_HEADERS_PATH "${OpenCV_BINARY_DIR}/3rdparty/NVIDIAOpticalFlowSDK_2_0_Headers") -ocv_download(FILENAME "${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_COMMIT}.zip" - HASH ${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_MD5} - URL - "https://github.com/NVIDIA/NVIDIAOpticalFlowSDK/archive/" - DESTINATION_DIR "${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_PATH}" - STATUS NVIDIA_OPTICAL_FLOW_2_0_HEADERS_DOWNLOAD_SUCCESS - ID "NVIDIA_OPTICAL_FLOW" - UNPACK RELATIVE_URL) +if(NOT CUDA_VERSION VERSION_LESS "10.0") + set(NVIDIA_OPTICAL_FLOW_2_0_HEADERS_COMMIT "edb50da3cf849840d680249aa6dbef248ebce2ca") + set(NVIDIA_OPTICAL_FLOW_2_0_HEADERS_MD5 "a73cd48b18dcc0cc8933b30796074191") + set(NVIDIA_OPTICAL_FLOW_2_0_HEADERS_PATH "${OpenCV_BINARY_DIR}/3rdparty/NVIDIAOpticalFlowSDK_2_0_Headers") + ocv_download(FILENAME "${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_COMMIT}.zip" + HASH ${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_MD5} + URL "https://github.com/NVIDIA/NVIDIAOpticalFlowSDK/archive/" + DESTINATION_DIR "${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_PATH}" + STATUS NVIDIA_OPTICAL_FLOW_2_0_HEADERS_DOWNLOAD_SUCCESS + ID "NVIDIA_OPTICAL_FLOW" + UNPACK RELATIVE_URL) -if(NOT NVIDIA_OPTICAL_FLOW_2_0_HEADERS_DOWNLOAD_SUCCESS) - message(STATUS "Failed to download NVIDIA_Optical_Flow_2_0 Headers") -else() - add_definitions(-DHAVE_NVIDIA_OPTFLOW=1) - ocv_include_directories(SYSTEM "${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_PATH}/NVIDIAOpticalFlowSDK-${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_COMMIT}") + if(NOT NVIDIA_OPTICAL_FLOW_2_0_HEADERS_DOWNLOAD_SUCCESS) + message(STATUS "Failed to download NVIDIA_Optical_Flow_2_0 Headers") + else() + message(STATUS "Building with NVIDIA Optical Flow API 2.0") + add_definitions(-DHAVE_NVIDIA_OPTFLOW=2) + ocv_include_directories(SYSTEM "${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_PATH}/NVIDIAOpticalFlowSDK-${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_COMMIT}") + endif() endif() \ No newline at end of file diff --git a/modules/cudaoptflow/src/cuda/nvidiaOpticalFlow.cu b/modules/cudaoptflow/src/cuda/nvidiaOpticalFlow.cu index 1ad21143e70..f308710d54e 100644 --- a/modules/cudaoptflow/src/cuda/nvidiaOpticalFlow.cu +++ b/modules/cudaoptflow/src/cuda/nvidiaOpticalFlow.cu @@ -22,6 +22,7 @@ typedef signed int int32_t; #define SMEM_COLS ((BLOCKDIM_X)/2) #define SMEM_ROWS ((BLOCKDIM_Y)/2) +#ifdef HAVE_NVIDIA_OPTFLOW namespace cv { namespace cuda { namespace device { namespace optflow_nvidia { static const char *_cudaGetErrorEnum(cudaError_t error) { return cudaGetErrorName(error); } @@ -95,5 +96,6 @@ void FlowUpsample(void* srcDevPtr, uint32_t nSrcWidth, uint32_t nSrcPitch, uint3 checkCudaErrors(cudaGetLastError()); }}}}} +#endif //HAVE_NVIDIA_OPTFLOW #endif \ No newline at end of file diff --git a/modules/cudaoptflow/src/nvidiaOpticalFlow.cpp b/modules/cudaoptflow/src/nvidiaOpticalFlow.cpp index df715698933..b937c77621a 100644 --- a/modules/cudaoptflow/src/nvidiaOpticalFlow.cpp +++ b/modules/cudaoptflow/src/nvidiaOpticalFlow.cpp @@ -9,26 +9,39 @@ #if !defined HAVE_CUDA || defined(CUDA_DISABLER) cv::Ptr cv::cuda::NvidiaOpticalFlow_1_0::create - (int, int, NVIDIA_OF_PERF_LEVEL, bool, bool, bool, int, Stream&, Stream&) { + (cv::Size, NVIDIA_OF_PERF_LEVEL, bool, bool, bool, int, Stream&, Stream&) { throw_no_cuda(); return cv::Ptr(); } cv::Ptr cv::cuda::NvidiaOpticalFlow_2_0::create( - int, int, NVIDIA_OF_PERF_LEVEL, NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE, NVIDIA_OF_HINT_VECTOR_GRID_SIZE, - bool, int, cv::Rect*, bool, bool, bool, int, Stream&, Stream&) { + cv::Size, NVIDIA_OF_PERF_LEVEL, NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE, NVIDIA_OF_HINT_VECTOR_GRID_SIZE, + bool, bool, bool, int, Stream&, Stream&) { + throw_no_cuda(); return cv::Ptr(); +} + +cv::Ptr cv::cuda::NvidiaOpticalFlow_2_0::create( + cv::Size, std::vector, NVIDIA_OF_PERF_LEVEL, NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE, NVIDIA_OF_HINT_VECTOR_GRID_SIZE, + bool, bool, bool, int, Stream&, Stream&) { throw_no_cuda(); return cv::Ptr(); } #elif !defined HAVE_NVIDIA_OPTFLOW cv::Ptr cv::cuda::NvidiaOpticalFlow_1_0::create( - int, int, NVIDIA_OF_PERF_LEVEL, bool, bool, bool, int, Stream&, Stream&) + cv::Size, NVIDIA_OF_PERF_LEVEL, bool, bool, bool, int, Stream&, Stream&) +{ + CV_Error(cv::Error::HeaderIsNull, "OpenCV was build without NVIDIA OpticalFlow support"); +} + +cv::Ptr cv::cuda::NvidiaOpticalFlow_2_0::create( + cv::Size, NVIDIA_OF_PERF_LEVEL, NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE, NVIDIA_OF_HINT_VECTOR_GRID_SIZE, + bool, bool, bool, int, Stream&, Stream&) { CV_Error(cv::Error::HeaderIsNull, "OpenCV was build without NVIDIA OpticalFlow support"); } cv::Ptr cv::cuda::NvidiaOpticalFlow_2_0::create( - int, int, NVIDIA_OF_PERF_LEVEL, NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE, NVIDIA_OF_HINT_VECTOR_GRID_SIZE, - bool, int, cv::Rect*, bool, bool, bool, int, , Stream&, Stream&) + cv::Size, std::vector, NVIDIA_OF_PERF_LEVEL, NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE, NVIDIA_OF_HINT_VECTOR_GRID_SIZE, + bool, bool, bool, int, Stream&, Stream&) { CV_Error(cv::Error::HeaderIsNull, "OpenCV was build without NVIDIA OpticalFlow support"); } From 3778fa373af4dbd6772cf373a04721d47987d413 Mon Sep 17 00:00:00 2001 From: Tomoaki Teshima Date: Mon, 25 Jan 2021 17:14:37 +0900 Subject: [PATCH 30/70] [moved from opencv] fix peaky test failure * follow the review comment original commit: https://github.com/opencv/opencv/commit/96e0902f39bccaecf6e950de528d1eb853f47f32 --- modules/cudalegacy/test/test_calib3d.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/cudalegacy/test/test_calib3d.cpp b/modules/cudalegacy/test/test_calib3d.cpp index e21432a8a4c..0d8522e287d 100644 --- a/modules/cudalegacy/test/test_calib3d.cpp +++ b/modules/cudalegacy/test/test_calib3d.cpp @@ -163,7 +163,7 @@ struct SolvePnPRansac : testing::TestWithParam CUDA_TEST_P(SolvePnPRansac, Accuracy) { - cv::Mat object = randomMat(cv::Size(5000, 1), CV_32FC3, 0, 100); + cv::Mat object = randomMat(cv::Size(5000, 1), CV_32FC3, -2000, 2000); cv::Mat camera_mat = randomMat(cv::Size(3, 3), CV_32F, 0.5, 1); camera_mat.at(0, 1) = 0.f; camera_mat.at(1, 0) = 0.f; @@ -174,7 +174,7 @@ CUDA_TEST_P(SolvePnPRansac, Accuracy) cv::Mat rvec_gold; cv::Mat tvec_gold; rvec_gold = randomMat(cv::Size(3, 1), CV_32F, 0, 1); - tvec_gold = randomMat(cv::Size(3, 1), CV_32F, 0, 1); + tvec_gold = randomMat(cv::Size(3, 1), CV_32F, 0, 1000); cv::projectPoints(object, rvec_gold, tvec_gold, camera_mat, cv::Mat(1, 8, CV_32F, cv::Scalar::all(0)), image_vec); cv::Mat rvec, tvec; @@ -184,7 +184,7 @@ CUDA_TEST_P(SolvePnPRansac, Accuracy) rvec, tvec, false, 200, 2.f, 100, &inliers); ASSERT_LE(cv::norm(rvec - rvec_gold), 1e-3); - ASSERT_LE(cv::norm(tvec - tvec_gold), 1e-3); + ASSERT_LE(cv::norm(tvec, tvec_gold, NORM_L2 | NORM_RELATIVE), 1e-3); } INSTANTIATE_TEST_CASE_P(CUDA_Calib3D, SolvePnPRansac, ALL_DEVICES); From 58ab920d041083afefb281af7884c187f14b50dc Mon Sep 17 00:00:00 2001 From: Tomoaki Teshima Date: Tue, 26 Jan 2021 12:45:08 +0900 Subject: [PATCH 31/70] fix build failure when CUDA >= 10.0 --- modules/cudaoptflow/src/cuda/nvidiaOpticalFlow.cu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/cudaoptflow/src/cuda/nvidiaOpticalFlow.cu b/modules/cudaoptflow/src/cuda/nvidiaOpticalFlow.cu index f308710d54e..e73eb430081 100644 --- a/modules/cudaoptflow/src/cuda/nvidiaOpticalFlow.cu +++ b/modules/cudaoptflow/src/cuda/nvidiaOpticalFlow.cu @@ -22,7 +22,7 @@ typedef signed int int32_t; #define SMEM_COLS ((BLOCKDIM_X)/2) #define SMEM_ROWS ((BLOCKDIM_Y)/2) -#ifdef HAVE_NVIDIA_OPTFLOW +#if defined(__CUDACC_VER_MAJOR__) && (10 <= __CUDACC_VER_MAJOR__) namespace cv { namespace cuda { namespace device { namespace optflow_nvidia { static const char *_cudaGetErrorEnum(cudaError_t error) { return cudaGetErrorName(error); } @@ -96,6 +96,6 @@ void FlowUpsample(void* srcDevPtr, uint32_t nSrcWidth, uint32_t nSrcPitch, uint3 checkCudaErrors(cudaGetLastError()); }}}}} -#endif //HAVE_NVIDIA_OPTFLOW +#endif //defined(__CUDACC_VER_MAJOR__) && (10 <= __CUDACC_VER_MAJOR__) #endif \ No newline at end of file From 4e606621e2d2b0ee7ad58c5ad3e0cc30ad3e7ba8 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 26 Jan 2021 15:21:12 +0000 Subject: [PATCH 32/70] valid point issue 2818 fix --- modules/rgbd/src/tsdf.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/rgbd/src/tsdf.cpp b/modules/rgbd/src/tsdf.cpp index 1e8704170f4..a35899d9ab7 100644 --- a/modules/rgbd/src/tsdf.cpp +++ b/modules/rgbd/src/tsdf.cpp @@ -419,8 +419,11 @@ struct RaycastInvoker : ParallelLoopBody // near clipping plane const float clip = 0.f; - float tmin = max(v_reduce_max(minAx), clip); - float tmax = v_reduce_min(maxAx); + float _minAx[4], _maxAx[4]; + v_store(_minAx, minAx); + v_store(_maxAx, maxAx); + float tmin = max( {_minAx[0], _minAx[1], _minAx[2], clip} ); + float tmax = min( {_maxAx[0], _maxAx[1], _maxAx[2]} ); // precautions against getting coordinates out of bounds tmin = tmin + tstep; From 714f07dace5a84f1ec1aaef9d6b58b86ba7132d0 Mon Sep 17 00:00:00 2001 From: aaarthurliu Date: Wed, 27 Jan 2021 10:37:37 +0800 Subject: [PATCH 33/70] fix readme typo inside modules --- modules/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/README.md b/modules/README.md index 0a6d55b5345..b156c02b555 100644 --- a/modules/README.md +++ b/modules/README.md @@ -36,7 +36,7 @@ $ cmake -D OPENCV_EXTRA_MODULES_PATH=/modules -D BUILD_opencv_ Date: Sun, 31 Jan 2021 07:31:52 +0900 Subject: [PATCH 34/70] get rid of kernel build failure * prevent out-of-bounds access * prevent deallocate before return * fix build warning --- modules/rgbd/src/opencl/tsdf.cl | 14 ++++++++------ modules/rgbd/src/tsdf.cpp | 13 ++++++------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/modules/rgbd/src/opencl/tsdf.cl b/modules/rgbd/src/opencl/tsdf.cl index e57bbdbabae..b9186826236 100644 --- a/modules/rgbd/src/opencl/tsdf.cl +++ b/modules/rgbd/src/opencl/tsdf.cl @@ -4,8 +4,7 @@ // This code is also subject to the license terms in the LICENSE_KinectFusion.md file found in this module's directory -typedef __INT8_TYPE__ int8_t; - +typedef char int8_t; typedef int8_t TsdfType; typedef uchar WeightType; @@ -28,14 +27,17 @@ static inline float tsdfToFloat(TsdfType num) } __kernel void preCalculationPixNorm (__global float * pixNorms, + int pix_step, int pix_offset, + int pix_rows, int pix_cols, const __global float * xx, const __global float * yy, - int width) + int width, int height) { int i = get_global_id(0); int j = get_global_id(1); int idx = i*width + j; - pixNorms[idx] = sqrt(xx[j] * xx[j] + yy[i] * yy[i] + 1.0f); + if(i < height && j < width && idx < pix_cols) + pixNorms[idx] = sqrt(xx[j] * xx[j] + yy[i] * yy[i] + 1.0f); } __kernel void integrate(__global const char * depthptr, @@ -85,7 +87,7 @@ __kernel void integrate(__global const char * depthptr, int volYidx = x*volDims.x + y*volDims.y; int startZ, endZ; - if(fabs(zStep.z) > 1e-5) + if(fabs(zStep.z) > 1e-5f) { int baseZ = convert_int(-basePt.z / zStep.z); if(zStep.z > 0) @@ -162,7 +164,7 @@ __kernel void integrate(__global const char * depthptr, if(v == 0) continue; - int idx = projected.y * depth_rows + projected.x; + int idx = projected.y * depth_cols + projected.x; float pixNorm = pixNorms[idx]; //float pixNorm = length(camPixVec); diff --git a/modules/rgbd/src/tsdf.cpp b/modules/rgbd/src/tsdf.cpp index a35899d9ab7..f80ba986a7a 100644 --- a/modules/rgbd/src/tsdf.cpp +++ b/modules/rgbd/src/tsdf.cpp @@ -836,11 +836,11 @@ void TSDFVolumeGPU::reset() volume.setTo(Scalar(0, 0)); } -static cv::UMat preCalculationPixNormGPU(int depth_rows, int depth_cols, Vec2f fxy, Vec2f cxy) +static void preCalculationPixNormGPU(int depth_rows, int depth_cols, Vec2f fxy, Vec2f cxy, UMat& pixNorm) { Mat x(1, depth_cols, CV_32F); Mat y(1, depth_rows, CV_32F); - Mat _pixNorm(1, depth_rows * depth_cols, CV_32F); + pixNorm.create(1, depth_rows * depth_cols, CV_32F); for (int i = 0; i < depth_cols; i++) x.at(0, i) = (i - cxy[0]) / fxy[0]; @@ -859,14 +859,13 @@ static cv::UMat preCalculationPixNormGPU(int depth_rows, int depth_cols, Vec2f f throw std::runtime_error("Failed to create kernel: " + errorStr); AccessFlag af = ACCESS_READ; - UMat pixNorm = _pixNorm.getUMat(af); UMat xx = x.getUMat(af); UMat yy = y.getUMat(af); - kk.args(ocl::KernelArg::PtrReadWrite(pixNorm), + kk.args(ocl::KernelArg::ReadWrite(pixNorm), ocl::KernelArg::PtrReadOnly(xx), ocl::KernelArg::PtrReadOnly(yy), - depth_cols); + depth_cols, depth_rows); size_t globalSize[2]; globalSize[0] = depth_rows; @@ -875,7 +874,7 @@ static cv::UMat preCalculationPixNormGPU(int depth_rows, int depth_cols, Vec2f f if (!kk.run(2, globalSize, NULL, true)) throw std::runtime_error("Failed to run kernel"); - return pixNorm; + return; } // use depth instead of distance (optimization) @@ -910,7 +909,7 @@ void TSDFVolumeGPU::integrate(InputArray _depth, float depthFactor, frameParams[2] = intrinsics.fx; frameParams[3] = intrinsics.fy; frameParams[4] = intrinsics.cx; frameParams[5] = intrinsics.cy; - pixNorms = preCalculationPixNormGPU(depth.rows, depth.cols, fxy, cxy); + preCalculationPixNormGPU(depth.rows, depth.cols, fxy, cxy, pixNorms); } // TODO: optimization possible From 3d8ec32805b8b85aca2313bb37d67f0dfd5a6450 Mon Sep 17 00:00:00 2001 From: Tomoaki Teshima Date: Sun, 31 Jan 2021 08:23:57 +0900 Subject: [PATCH 35/70] adjust threshold for Jetson TX1/TX2 --- modules/xfeatures2d/test/test_surf.cuda.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/xfeatures2d/test/test_surf.cuda.cpp b/modules/xfeatures2d/test/test_surf.cuda.cpp index b93c6f4bbd1..48ff01822a3 100644 --- a/modules/xfeatures2d/test/test_surf.cuda.cpp +++ b/modules/xfeatures2d/test/test_surf.cuda.cpp @@ -181,7 +181,7 @@ testing::internal::ValueArray3 thresholdValues = testing::Values( - SURF_HessianThreshold(813.0), + SURF_HessianThreshold(830.0), SURF_HessianThreshold(1000.0)); #endif From 8c385d58f683163fd5c56c5e43ce571bb9584564 Mon Sep 17 00:00:00 2001 From: Pavel Rojtberg Date: Mon, 1 Feb 2021 18:01:15 +0100 Subject: [PATCH 36/70] ovis: also take down SceneManager when Window is deleted --- modules/ovis/src/ovis.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/ovis/src/ovis.cpp b/modules/ovis/src/ovis.cpp index 2f9f69a6f73..5901f54e749 100644 --- a/modules/ovis/src/ovis.cpp +++ b/modules/ovis/src/ovis.cpp @@ -410,6 +410,8 @@ class WindowSceneImpl : public WindowScene { texMgr.remove(texName, RESOURCEGROUP_NAME); } + + root->destroySceneManager(sceneMgr); } if(_app->mainWin == this && (flags & SCENE_SEPARATE)) From efbf1b8f4f83feb03db5d3659b88cd19638a539c Mon Sep 17 00:00:00 2001 From: Tomoaki Teshima Date: Fri, 5 Feb 2021 22:35:15 +0900 Subject: [PATCH 37/70] fix segmentation fault on Arm 32bit platform --- modules/rgbd/src/tsdf.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/rgbd/src/tsdf.cpp b/modules/rgbd/src/tsdf.cpp index f80ba986a7a..412c2fe789e 100644 --- a/modules/rgbd/src/tsdf.cpp +++ b/modules/rgbd/src/tsdf.cpp @@ -231,11 +231,11 @@ inline Point3f TSDFVolumeCPU::getNormalVoxel(const Point3f& _p) const inline v_float32x4 TSDFVolumeCPU::getNormalVoxel(const v_float32x4& p) const { - if(v_check_any((p < v_float32x4(1.f, 1.f, 1.f, 0.f)) + - (p >= v_float32x4((float)(volResolution.x-2), + if(v_check_any (p < v_float32x4(1.f, 1.f, 1.f, 0.f)) || + v_check_any (p >= v_float32x4((float)(volResolution.x-2), (float)(volResolution.y-2), (float)(volResolution.z-2), 1.f)) - )) + ) return nanv; v_int32x4 ip = v_floor(p); From 38f46874aed044d2d7f0b4ae093b41636c7a7a45 Mon Sep 17 00:00:00 2001 From: batters21 Date: Mon, 1 Feb 2021 15:31:04 +0300 Subject: [PATCH 38/70] issue 2817 fixed --- modules/rgbd/src/kinfu_frame.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/rgbd/src/kinfu_frame.cpp b/modules/rgbd/src/kinfu_frame.cpp index ce160dd5f3c..76d80902b7b 100644 --- a/modules/rgbd/src/kinfu_frame.cpp +++ b/modules/rgbd/src/kinfu_frame.cpp @@ -623,8 +623,9 @@ void makeFrameFromDepth(InputArray _depth, // looks like OpenCV's bilateral filter works the same as KinFu's Depth smooth; - - bilateralFilter(depth, smooth, kernelSize, sigmaDepth*depthFactor, sigmaSpatial); + Depth depthNoNans = depth.clone(); + patchNaNs(depthNoNans); + bilateralFilter(depthNoNans, smooth, kernelSize, sigmaDepth*depthFactor, sigmaSpatial); // depth truncation can be used in some scenes Depth depthThreshold; From 461b56f602f7c0a4189a4759af5c219f06d56e78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Andrieux?= Date: Fri, 5 Feb 2021 21:47:52 +0100 Subject: [PATCH 39/70] Merge pull request #2845 from kevineor:DeltaECIEDE2000-fix deltaCIEDE2000_ fix into the MCC module * deltaCIEDE2000_ fix, did not pass the test case 17 and 19 of Gaurav Sharma's test data * Update ccm test case --- modules/mcc/src/distance.cpp | 3 ++- modules/mcc/test/test_ccm.cpp | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/mcc/src/distance.cpp b/modules/mcc/src/distance.cpp index 7996379e939..c5eb1b59514 100644 --- a/modules/mcc/src/distance.cpp +++ b/modules/mcc/src/distance.cpp @@ -139,7 +139,8 @@ double deltaCIEDE2000_(const Vec3d& lab1, const Vec3d& lab2, const double& kL, double sC = 1.0 + 0.045 * C_bar_apo; double sH = 1.0 + 0.015 * C_bar_apo * T; double sL = 1.0 + ((0.015 * pow(l_bar_apo - 50.0, 2.0)) / sqrt(20.0 + pow(l_bar_apo - 50.0, 2.0))); - double RT = -2.0 * G * sin(toRad(60.0) * exp(-pow((H_bar_apo - toRad(275.0)) / toRad(25.0), 2.0))); + double R_C = 2.0 * sqrt(pow(C_bar_apo, 7.0) / (pow(C_bar_apo, 7.0) + pow(25, 7))); + double RT = -sin(toRad(60.0) * exp(-pow((H_bar_apo - toRad(275.0)) / toRad(25.0), 2.0))) * R_C; double res = (pow(delta_L_apo / (kL * sL), 2.0) + pow(delta_C_apo / (kC * sC), 2.0) + pow(delta_H_apo / (kH * sH), 2.0) + RT * (delta_C_apo / (kC * sC)) * (delta_H_apo / (kH * sH))); return res > 0 ? sqrt(res) : 0; } diff --git a/modules/mcc/test/test_ccm.cpp b/modules/mcc/test/test_ccm.cpp index 56ac51db410..cd6498f1b6e 100644 --- a/modules/mcc/test/test_ccm.cpp +++ b/modules/mcc/test/test_ccm.cpp @@ -99,9 +99,9 @@ TEST(CV_ccmRunColorCorrection, test_model) Mat ccm = (Mat_(3, 3) << - 0.37408717, 0.02066172, 0.05796725, - 0.12684056, 0.77364991, -0.01566532, - -0.27464866, 0.00652140, 2.74593262); + 0.37406520, 0.02066507, 0.05804047, + 0.12719672, 0.77389268, -0.01569404, + -0.27627010, 0.00603427, 2.74272981); ASSERT_MAT_NEAR(model.getCCM(), ccm, 1e-4); } TEST(CV_ccmRunColorCorrection, test_masks_weights_1) From c918b081756cae2712cc42ed889e633a80ce0d62 Mon Sep 17 00:00:00 2001 From: Namgoo Lee Date: Wed, 3 Feb 2021 13:55:52 +0900 Subject: [PATCH 40/70] Test code - GpuMatND & Mat interoperability --- modules/cudev/test/test_nd.cu | 250 ++++++++++++++++++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 modules/cudev/test/test_nd.cu diff --git a/modules/cudev/test/test_nd.cu b/modules/cudev/test/test_nd.cu new file mode 100644 index 00000000000..2fc3f396b95 --- /dev/null +++ b/modules/cudev/test/test_nd.cu @@ -0,0 +1,250 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +#include "test_precomp.hpp" + +namespace opencv_test { namespace { + +template +class GpuMatNDTest : public ::testing::Test +{ +public: + using MatType = Mat_; + using CnType = typename Mat_::channel_type; + static constexpr int cn = DataType::channels; + using SizeArray = GpuMatND::SizeArray; + + static MatType RandomMat(const SizeArray& size) + { + const auto dims = static_cast(size.size()); + + MatType ret(dims, size.data()); + + for (ElemType& elem : ret) + for (int i = 0; i < cn; ++i) + elem[i] = cv::randu(); + + return ret; + } + + static std::vector RandomRange(const SizeArray& size) + { + const auto dims = static_cast(size.size()); + + std::vector ret; + + const auto margin = cv::randu() & 0x1 + 1; // 1 or 2 + + for (int s : size) + if (s > margin * 2) + ret.emplace_back(margin, s-margin); + else + ret.push_back(Range::all()); + + if (dims == 1) + { + // Mat expects two ranges even in this case + ret.push_back(Range::all()); + } + + return ret; + } + + static std::vector RandomRange2D(const SizeArray& size) + { + const auto dims = static_cast(size.size()); + + std::vector ret = RandomRange(size); + + for (int i = 0; i < dims - 2; ++i) + { + const auto start = cv::randu() % size[i]; + ret[i] = Range(static_cast(start), static_cast(start) + 1); + } + + return ret; + } + + static void doTest1(const SizeArray& size) + { + const MatType gold = RandomMat(size); + + MatType dst; + GpuMatND gmat; + + // simple upload, download test for GpuMatND + gmat.upload(gold); + gmat.download(dst); + EXPECT_TRUE(std::equal(gold.begin(), gold.end(), dst.begin())); + } + + static void doTest2(const SizeArray& size) + { + const MatType gold = RandomMat(size); + const std::vector ranges = RandomRange(size); + const MatType goldSub = gold(ranges); + + MatType dst; + GpuMatND gmat; + + // upload partial mat, download it, and compare + gmat.upload(goldSub); + gmat.download(dst); + EXPECT_TRUE(std::equal(goldSub.begin(), goldSub.end(), dst.begin())); + + // upload full mat, extract partial mat from it, download it, and compare + gmat.upload(gold); + gmat = gmat(ranges); + gmat.download(dst); + EXPECT_TRUE(std::equal(goldSub.begin(), goldSub.end(), dst.begin())); + } + + static void doTest3(const SizeArray& size) + { + if (std::is_same::value) // GpuMat::convertTo is not implemented for CV_16F + return; + + const MatType gold = RandomMat(size); + const std::vector ranges = RandomRange2D(size); + + MatType dst; + GpuMatND gmat; + + // Test GpuMatND to GpuMat conversion: + // extract a 2D-plane and set its elements in the extracted region to 1 + // compare the values of the full mat between Mat and GpuMatND + + gmat.upload(gold); + GpuMat plane = gmat(ranges).createGpuMatHeader(); + EXPECT_TRUE(!plane.refcount); // plane points to externally allocated memory(a part of gmat) + + const GpuMat dummy = plane.clone(); + EXPECT_TRUE(dummy.refcount); // dummy is clone()-ed from plane, so it manages its memory + + // currently, plane(GpuMat) points to a sub-matrix of gmat(GpuMatND) + // in this case, dummy and plane have same size and type, + // so plane does not get reallocated inside convertTo, + // so this convertTo sets a sub-matrix region of gmat to 1 + dummy.convertTo(plane, -1, 0, 1); + EXPECT_TRUE(!plane.refcount); // plane still points to externally allocated memory(a part of gmat) + + gmat.download(dst); + + // set a sub-matrix region of gold to 1 + Mat plane_ = gold(ranges); + const Mat dummy_ = plane_.clone(); + dummy_.convertTo(plane_, -1, 0, 1); + + EXPECT_TRUE(std::equal(gold.begin(), gold.end(), dst.begin())); + } + + static void doTest4(const SizeArray& size) + { + if (std::is_same::value) // GpuMat::convertTo is not implemented for CV_16F + return; + + const MatType gold = RandomMat(size); + const std::vector ranges = RandomRange2D(size); + + MatType dst; + GpuMatND gmat; + + // Test handling external memory + gmat.upload(gold); + const GpuMatND external(gmat.size, gmat.type(), gmat.getDevicePtr(), {gmat.step.begin(), gmat.step.end() - 1}); + + // set a sub-matrix region of external to 2 + GpuMat plane = external(ranges).createGpuMatHeader(); + const GpuMat dummy = plane.clone(); + dummy.convertTo(plane, -1, 0, 2); + external.download(dst); + + // set a sub-matrix region of gold to 2 + Mat plane_ = gold(ranges); + const Mat dummy_ = plane_.clone(); + dummy_.convertTo(plane_, -1, 0, 2); + + EXPECT_TRUE(std::equal(gold.begin(), gold.end(), dst.begin())); + } + + static void doTest5(const SizeArray& size) + { + if (std::is_same::value) // GpuMat::convertTo is not implemented for CV_16F + return; + + const MatType gold = RandomMat(size); + const std::vector ranges = RandomRange(size); + MatType goldSub = gold(ranges); + + MatType dst; + GpuMatND gmat; + + // Upload a sub-mat, set a sub-region of the sub-mat to 3, download, and compare + gmat.upload(goldSub); + const std::vector rangesInRanges = RandomRange2D(gmat.size); + + GpuMat plane = gmat(rangesInRanges).createGpuMatHeader(); + const GpuMat dummy = plane.clone(); + dummy.convertTo(plane, -1, 0, 3); + gmat.download(dst); + + Mat plane_ = goldSub(rangesInRanges); + const Mat dummy_ = plane_.clone(); + dummy_.convertTo(plane_, -1, 0, 3); + + EXPECT_TRUE(std::equal(goldSub.begin(), goldSub.end(), dst.begin())); + } +}; + +using ElemTypes = ::testing::Types< + Vec, Vec, Vec, Vec, // CV_8U + Vec, Vec, Vec, Vec, // CV_8S + Vec, Vec, Vec, Vec, // CV_16U + Vec, Vec, Vec, Vec, // CV_16S + Vec, Vec, Vec, Vec, // CV_32S + Vec, Vec, Vec, Vec, // CV_32F + Vec, Vec, Vec, Vec, //CV_64F + Vec, Vec, Vec, Vec // CV_16F +>; + +using SizeArray = GpuMatND::SizeArray; + +#define DIFFERENT_SIZES_ND std::vector{ \ + SizeArray{2, 1}, SizeArray{3, 2, 1}, SizeArray{1, 3, 2, 1}, SizeArray{2, 1, 3, 2, 1}, SizeArray{3, 2, 1, 3, 2, 1}, \ + SizeArray{1}, SizeArray{1, 1}, SizeArray{1, 1, 1}, SizeArray{1, 1, 1, 1}, \ + SizeArray{4}, SizeArray{4, 4}, SizeArray{4, 4, 4}, SizeArray{4, 4, 4, 4}, \ + SizeArray{11}, SizeArray{13, 11}, SizeArray{17, 13, 11}, SizeArray{19, 17, 13, 11}} + +TYPED_TEST_CASE(GpuMatNDTest, ElemTypes); + +TYPED_TEST(GpuMatNDTest, Test1) +{ + for (auto& size : DIFFERENT_SIZES_ND) + GpuMatNDTest::doTest1(size); +} + +TYPED_TEST(GpuMatNDTest, Test2) +{ + for (auto& size : DIFFERENT_SIZES_ND) + GpuMatNDTest::doTest2(size); +} + +TYPED_TEST(GpuMatNDTest, Test3) +{ + for (auto& size : DIFFERENT_SIZES_ND) + GpuMatNDTest::doTest3(size); +} + +TYPED_TEST(GpuMatNDTest, Test4) +{ + for (auto& size : DIFFERENT_SIZES_ND) + GpuMatNDTest::doTest4(size); +} + +TYPED_TEST(GpuMatNDTest, Test5) +{ + for (auto& size : DIFFERENT_SIZES_ND) + GpuMatNDTest::doTest5(size); +} + +}} // namespace From c56f8185502082573577be01745d88e87910e816 Mon Sep 17 00:00:00 2001 From: Tobias Senst Date: Sat, 6 Feb 2021 14:13:09 +0100 Subject: [PATCH 41/70] ximgproc(ric): resolve issue 2796 --- .../ximgproc/sparse_match_interpolator.hpp | 4 +- .../test/test_sparse_match_interpolator.py | 71 +++++++++++++++++++ .../src/sparse_match_interpolators.cpp | 58 ++++++++++----- .../test/test_sparse_match_interpolator.cpp | 15 ++++ 4 files changed, 130 insertions(+), 18 deletions(-) create mode 100644 modules/ximgproc/misc/python/test/test_sparse_match_interpolator.py diff --git a/modules/ximgproc/include/opencv2/ximgproc/sparse_match_interpolator.hpp b/modules/ximgproc/include/opencv2/ximgproc/sparse_match_interpolator.hpp index 80e20571477..feae7261ef6 100644 --- a/modules/ximgproc/include/opencv2/ximgproc/sparse_match_interpolator.hpp +++ b/modules/ximgproc/include/opencv2/ximgproc/sparse_match_interpolator.hpp @@ -57,12 +57,12 @@ class CV_EXPORTS_W SparseMatchInterpolator : public Algorithm @param from_image first of the two matched images, 8-bit single-channel or three-channel. @param from_points points of the from_image for which there are correspondences in the - to_image (Point2f vector, size shouldn't exceed 32767) + to_image (Point2f vector or Mat of depth CV_32F) @param to_image second of the two matched images, 8-bit single-channel or three-channel. @param to_points points in the to_image corresponding to from_points - (Point2f vector, size shouldn't exceed 32767) + (Point2f vector or Mat of depth CV_32F) @param dense_flow output dense matching (two-channel CV_32F image) */ diff --git a/modules/ximgproc/misc/python/test/test_sparse_match_interpolator.py b/modules/ximgproc/misc/python/test/test_sparse_match_interpolator.py new file mode 100644 index 00000000000..0b8b977abe6 --- /dev/null +++ b/modules/ximgproc/misc/python/test/test_sparse_match_interpolator.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +import numpy as np +import cv2 as cv + +from tests_common import NewOpenCVTests + +class Interpolator_test(NewOpenCVTests): + def test_edgeaware_interpolator(self): + # readGT + MAX_DIF = 1.0 + MAX_MEAN_DIF = 1.0 / 256.0 + + src = cv.imread(self.find_file("cv/optflow/RubberWhale1.png"), cv.IMREAD_COLOR) + self.assertFalse(src is None) + + ref_flow = cv.readOpticalFlow(self.find_file("cv/sparse_match_interpolator/RubberWhale_reference_result.flo")) + self.assertFalse(ref_flow is None) + + matches = np.genfromtxt(self.find_file("cv/sparse_match_interpolator/RubberWhale_sparse_matches.txt")).astype(np.float32) + from_points = matches[:,0:2] + to_points = matches[:,2:4] + interpolator = cv.ximgproc.createEdgeAwareInterpolator() + interpolator.setK(128) + interpolator.setSigma(0.05) + interpolator.setUsePostProcessing(True) + interpolator.setFGSLambda(500.0) + interpolator.setFGSSigma(1.5) + + dense_flow = interpolator.interpolate(src, from_points, src, to_points) + + self.assertTrue(cv.norm(dense_flow, ref_flow, cv.NORM_INF) <= MAX_DIF) + self.assertTrue(cv.norm(dense_flow, ref_flow, cv.NORM_L1) <= (MAX_MEAN_DIF * dense_flow.shape[0] * dense_flow.shape[1])) + + def test_ric_interpolator(self): + # readGT + MAX_DIF = 6.0 + MAX_MEAN_DIF = 60.0 / 256.0 + + src0 = cv.imread(self.find_file("cv/optflow/RubberWhale1.png"), cv.IMREAD_COLOR) + self.assertFalse(src0 is None) + + src1 = cv.imread(self.find_file("cv/optflow/RubberWhale2.png"), cv.IMREAD_COLOR) + self.assertFalse(src1 is None) + + ref_flow = cv.readOpticalFlow(self.find_file("cv/sparse_match_interpolator/RubberWhale_reference_result.flo")) + self.assertFalse(ref_flow is None) + + matches = np.genfromtxt(self.find_file("cv/sparse_match_interpolator/RubberWhale_sparse_matches.txt")).astype(np.float32) + from_points = matches[:,0:2] + to_points = matches[:,2:4] + + interpolator = cv.ximgproc.createRICInterpolator() + interpolator.setK(32) + interpolator.setSuperpixelSize(15) + interpolator.setSuperpixelNNCnt(150) + interpolator.setSuperpixelRuler(15.0) + interpolator.setSuperpixelMode(cv.ximgproc.SLIC) + interpolator.setAlpha(0.7) + interpolator.setModelIter(4) + interpolator.setRefineModels(True) + interpolator.setMaxFlow(250) + interpolator.setUseVariationalRefinement(True) + interpolator.setUseGlobalSmootherFilter(True) + interpolator.setFGSLambda(500.0) + interpolator.setFGSSigma(1.5) + dense_flow = interpolator.interpolate(src0, from_points, src1, to_points) + self.assertTrue(cv.norm(dense_flow, ref_flow, cv.NORM_INF) <= MAX_DIF) + self.assertTrue(cv.norm(dense_flow, ref_flow, cv.NORM_L1) <= (MAX_MEAN_DIF * dense_flow.shape[0] * dense_flow.shape[1])) + +if __name__ == '__main__': + NewOpenCVTests.bootstrap() \ No newline at end of file diff --git a/modules/ximgproc/src/sparse_match_interpolators.cpp b/modules/ximgproc/src/sparse_match_interpolators.cpp index caec6983c3d..e60ee98f1e9 100644 --- a/modules/ximgproc/src/sparse_match_interpolators.cpp +++ b/modules/ximgproc/src/sparse_match_interpolators.cpp @@ -173,19 +173,33 @@ Ptr EdgeAwareInterpolatorImpl::create() void EdgeAwareInterpolatorImpl::interpolate(InputArray from_image, InputArray from_points, InputArray, InputArray to_points, OutputArray dense_flow) { CV_Assert( !from_image.empty() && (from_image.depth() == CV_8U) && (from_image.channels() == 3 || from_image.channels() == 1) ); - CV_Assert( !from_points.empty() && from_points.isVector() && - !to_points .empty() && to_points .isVector() && - from_points.sameSize(to_points) ); + CV_Assert( !from_points.empty() && !to_points.empty() && from_points.sameSize(to_points) ); + CV_Assert((from_points.isVector() || from_points.isMat()) && from_points.depth() == CV_32F); + CV_Assert((to_points.isVector() || to_points.isMat()) && to_points.depth() == CV_32F); + CV_Assert(from_points.sameSize(to_points)); w = from_image.cols(); h = from_image.rows(); - vector from_vector = *(const vector*)from_points.getObj(); - vector to_vector = *(const vector*)to_points .getObj(); - vector matches_vector(from_vector.size()); - for(unsigned int i=0;i matches_vector(npoints); + for(unsigned int i=0;i(i), to_mat.at(i)); + + sort(matches_vector.begin(),matches_vector.end()); + match_num = (int)matches_vector.size(); CV_Assert(match_num from_vector = *(const vector*)from_points.getObj(); - vector to_vector = *(const vector*)to_points.getObj(); - vector matches_vector(from_vector.size()); - for (unsigned int i = 0; i < from_vector.size(); i++) - matches_vector[i] = SparseMatch(from_vector[i], to_vector[i]); + Mat from_mat = from_points.getMat(); + Mat to_mat = to_points.getMat(); + int npoints = from_mat.checkVector(2, CV_32F, false); + + if (from_mat.channels() != 2) + from_mat = from_mat.reshape(2, npoints); + + if (to_mat.channels() != 2 ){ + to_mat = to_mat.reshape(2, npoints); + npoints = from_mat.checkVector(2, CV_32F, false); + } + + vector matches_vector(npoints); + for(unsigned int i=0;i(i),to_mat.at(i)); - match_num = static_cast(from_vector.size()); + match_num = static_cast(matches_vector.size()); Mat src = from_image.getMat(); Size src_size = src.size(); diff --git a/modules/ximgproc/test/test_sparse_match_interpolator.cpp b/modules/ximgproc/test/test_sparse_match_interpolator.cpp index e5f0f31f7f7..261d6109bf9 100644 --- a/modules/ximgproc/test/test_sparse_match_interpolator.cpp +++ b/modules/ximgproc/test/test_sparse_match_interpolator.cpp @@ -93,6 +93,14 @@ TEST(InterpolatorTest, ReferenceAccuracy) EXPECT_LE(cv::norm(res_flow, ref_flow, NORM_INF), MAX_DIF); EXPECT_LE(cv::norm(res_flow, ref_flow, NORM_L1) , MAX_MEAN_DIF*res_flow.total()); + + Mat from_point_mat(from_points); + Mat to_points_mat(to_points); + interpolator->interpolate(src,from_point_mat,Mat(),to_points_mat,res_flow); + + EXPECT_LE(cv::norm(res_flow, ref_flow, NORM_INF), MAX_DIF); + EXPECT_LE(cv::norm(res_flow, ref_flow, NORM_L1) , MAX_MEAN_DIF*res_flow.total()); + } TEST(InterpolatorTest, RICReferenceAccuracy) @@ -141,6 +149,13 @@ TEST(InterpolatorTest, RICReferenceAccuracy) EXPECT_LE(cv::norm(res_flow, ref_flow, NORM_INF), MAX_DIF); EXPECT_LE(cv::norm(res_flow, ref_flow, NORM_L1), MAX_MEAN_DIF*res_flow.total()); + + Mat from_point_mat(from_points); + Mat to_points_mat(to_points); + interpolator->interpolate(src, from_point_mat, src1, to_points_mat, res_flow); + + EXPECT_LE(cv::norm(res_flow, ref_flow, NORM_INF), MAX_DIF); + EXPECT_LE(cv::norm(res_flow, ref_flow, NORM_L1) , MAX_MEAN_DIF*res_flow.total()); } TEST_P(InterpolatorTest, MultiThreadReproducibility) From 483b39684d4aaabc82a08cf648db540d46768cb1 Mon Sep 17 00:00:00 2001 From: Stefan Brechtken Date: Sun, 7 Feb 2021 13:31:56 +0100 Subject: [PATCH 42/70] fixes #2662 --- modules/alphamat/src/cm.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/alphamat/src/cm.cpp b/modules/alphamat/src/cm.cpp index 19dcb961e4a..660962eba39 100644 --- a/modules/alphamat/src/cm.cpp +++ b/modules/alphamat/src/cm.cpp @@ -90,7 +90,7 @@ void lle(my_vector_of_vectors_t& indm, my_vector_of_vectors_t& samples, float ep Mat ptDotN(20, 1, DataType::type), imd(20, 1, DataType::type); Mat Cones(20, 1, DataType::type), Cinv(20, 1, DataType::type); float alpha, beta, lagrangeMult; - Cones += 1; + Cones = 1; C = 0; rhs = 1; From 6c422507639c0592acaa2f67c7bcd6116a6a887a Mon Sep 17 00:00:00 2001 From: dddzg Date: Mon, 8 Feb 2021 19:41:40 +0800 Subject: [PATCH 43/70] Merge pull request #2849 from WeChatCV:wechat_qrcode Wechat qrcode module refinement * remove useles fixedType * refine wechat qrcode module * fix _isnan error --- .../include/opencv2/wechat_qrcode.hpp | 8 +-- modules/wechat_qrcode/samples/qrcode.py | 2 +- modules/wechat_qrcode/src/detector/align.cpp | 2 +- .../src/detector/ssd_detector.cpp | 4 ++ modules/wechat_qrcode/src/precomp.hpp | 2 +- modules/wechat_qrcode/src/wechat_qrcode.cpp | 5 +- modules/wechat_qrcode/src/zxing/binarizer.cpp | 4 +- .../wechat_qrcode/src/zxing/binarybitmap.cpp | 2 +- .../wechat_qrcode/src/zxing/common/array.hpp | 2 - .../adaptive_threshold_mean_binarizer.cpp | 5 +- .../adaptive_threshold_mean_binarizer.hpp | 1 - .../binarizer/fast_window_binarizer.cpp | 5 +- .../binarizer/global_histogram_binarizer.cpp | 35 +----------- .../binarizer/global_histogram_binarizer.hpp | 11 +++- .../common/binarizer/hybrid_binarizer.cpp | 13 ++--- .../binarizer/simple_adaptive_binarizer.cpp | 8 +-- .../src/zxing/common/bitarray.cpp | 4 +- .../src/zxing/common/bitarray.hpp | 3 -- .../src/zxing/common/bitmatrix.cpp | 8 +-- .../src/zxing/common/bitmatrix.hpp | 3 -- .../src/zxing/common/bitsource.cpp | 2 +- .../src/zxing/common/bytematrix.cpp | 9 +--- .../src/zxing/common/bytematrix.hpp | 2 - .../src/zxing/common/character.hpp | 54 ------------------- .../src/zxing/common/characterseteci.cpp | 4 +- .../src/zxing/common/decoder_result.cpp | 9 ++-- .../src/zxing/common/decoder_result.hpp | 2 - .../src/zxing/common/detector_result.cpp | 2 +- .../common/greyscale_luminance_source.cpp | 2 +- .../greyscale_rotated_luminance_source.cpp | 7 +-- .../src/zxing/common/grid_sampler.cpp | 7 +-- .../src/zxing/common/imagecut.cpp | 3 +- .../src/zxing/common/imagecut.hpp | 3 -- .../wechat_qrcode/src/zxing/common/kmeans.cpp | 9 +--- .../src/zxing/common/mathutils.hpp | 38 ------------- .../zxing/common/perspective_transform.cpp | 4 +- .../zxing/common/reedsolomon/genericgf.cpp | 4 +- .../zxing/common/reedsolomon/genericgf.hpp | 2 - .../common/reedsolomon/genericgfpoly.cpp | 4 +- .../common/reedsolomon/genericgfpoly.hpp | 2 - .../reedsolomon/reed_solomon_decoder.cpp | 7 +-- .../reedsolomon/reed_solomon_decoder.hpp | 3 -- .../wechat_qrcode/src/zxing/common/str.cpp | 10 +--- .../wechat_qrcode/src/zxing/common/str.hpp | 7 +-- .../src/zxing/common/stringutils.cpp | 6 +-- .../src/zxing/common/unicomblock.cpp | 5 +- .../src/zxing/common/unicomblock.hpp | 3 -- .../wechat_qrcode/src/zxing/errorhandler.cpp | 3 +- .../src/zxing/luminance_source.cpp | 2 +- .../zxing/qrcode/decoder/bitmatrixparser.cpp | 3 +- .../src/zxing/qrcode/decoder/datablock.cpp | 2 +- .../src/zxing/qrcode/decoder/datablock.hpp | 1 - .../src/zxing/qrcode/decoder/datamask.cpp | 2 +- .../src/zxing/qrcode/decoder/datamask.hpp | 3 -- .../decoder/decoded_bit_stream_parser.cpp | 4 +- .../decoder/decoded_bit_stream_parser.hpp | 3 -- .../src/zxing/qrcode/decoder/decoder.cpp | 6 +-- .../src/zxing/qrcode/decoder/mode.cpp | 2 +- .../decoder/qrcode_decoder_metadata.hpp | 2 - .../qrcode/detector/alignment_pattern.cpp | 5 +- .../qrcode/detector/alignment_pattern.hpp | 3 -- .../detector/alignment_pattern_finder.cpp | 10 +--- .../detector/alignment_pattern_finder.hpp | 4 -- .../src/zxing/qrcode/detector/detector.cpp | 12 ++--- .../src/zxing/qrcode/detector/detector.hpp | 2 - .../zxing/qrcode/detector/finder_pattern.cpp | 5 +- .../zxing/qrcode/detector/finder_pattern.hpp | 2 - .../qrcode/detector/finder_pattern_finder.cpp | 9 +--- .../qrcode/detector/finder_pattern_finder.hpp | 3 -- .../qrcode/detector/finder_pattern_info.cpp | 5 +- .../qrcode/detector/finder_pattern_info.hpp | 2 - .../zxing/qrcode/detector/pattern_result.cpp | 6 +-- .../zxing/qrcode/detector/pattern_result.hpp | 3 -- .../zxing/qrcode/error_correction_level.cpp | 4 +- .../src/zxing/qrcode/format_information.cpp | 2 +- .../src/zxing/qrcode/qrcode_reader.cpp | 2 +- .../src/zxing/qrcode/version.cpp | 6 +-- .../src/zxing/qrcode/version.hpp | 2 - modules/wechat_qrcode/src/zxing/reader.cpp | 2 +- modules/wechat_qrcode/src/zxing/result.cpp | 2 +- .../wechat_qrcode/src/zxing/resultpoint.cpp | 2 +- .../wechat_qrcode/src/zxing/resultpoint.hpp | 2 - modules/wechat_qrcode/src/zxing/zxing.hpp | 17 +----- 83 files changed, 86 insertions(+), 396 deletions(-) delete mode 100644 modules/wechat_qrcode/src/zxing/common/character.hpp diff --git a/modules/wechat_qrcode/include/opencv2/wechat_qrcode.hpp b/modules/wechat_qrcode/include/opencv2/wechat_qrcode.hpp index 69ac40b7bd7..7400d2b0889 100644 --- a/modules/wechat_qrcode/include/opencv2/wechat_qrcode.hpp +++ b/modules/wechat_qrcode/include/opencv2/wechat_qrcode.hpp @@ -15,7 +15,7 @@ namespace wechat_qrcode { //! @addtogroup wechat_qrcode //! @{ /** - * @brief QRCode includes two CNN-based models: + * @brief WeChat QRCode includes two CNN-based models: * A object detection model and a super resolution model. * Object detection model is applied to detect QRCode with the bounding box. * super resolution model is applied to zoom in QRCode when it is small. @@ -24,9 +24,9 @@ namespace wechat_qrcode { class CV_EXPORTS_W WeChatQRCode { public: /** - * @brief Initialize the QRCode. - * Two models are packaged with caffe format. - * Therefore, there are prototxt and caffe model two files. + * @brief Initialize the WeChatQRCode. + * It includes two models, which are packaged with caffe format. + * Therefore, there are prototxt and caffe models (In total, four paramenters). * * @param detector_prototxt_path prototxt file path for the detector * @param detector_caffe_model_path caffe model file path for the detector diff --git a/modules/wechat_qrcode/samples/qrcode.py b/modules/wechat_qrcode/samples/qrcode.py index ff95f2b051e..fd79607efcf 100644 --- a/modules/wechat_qrcode/samples/qrcode.py +++ b/modules/wechat_qrcode/samples/qrcode.py @@ -32,7 +32,7 @@ if camIdx < 0: res, points = detector.detectAndDecode(img) - print(res) + print(res,points) else: cap = cv2.VideoCapture(camIdx) while True: diff --git a/modules/wechat_qrcode/src/detector/align.cpp b/modules/wechat_qrcode/src/detector/align.cpp index 9a34f001292..05ab53352fc 100644 --- a/modules/wechat_qrcode/src/detector/align.cpp +++ b/modules/wechat_qrcode/src/detector/align.cpp @@ -58,7 +58,7 @@ Mat Align::crop(const Mat &inputImg, const Mat &srcPts, const float paddingW, co Rect crop_roi(crop_x_, crop_y_, end_x - crop_x_ + 1, end_y - crop_y_ + 1); Mat dst = inputImg(crop_roi).clone(); - if (rotate90_) dst = dst.t(); // really is just transpose + if (rotate90_) dst = dst.t(); // transpose return dst; } diff --git a/modules/wechat_qrcode/src/detector/ssd_detector.cpp b/modules/wechat_qrcode/src/detector/ssd_detector.cpp index cba7f602758..e55e6ef1f29 100644 --- a/modules/wechat_qrcode/src/detector/ssd_detector.cpp +++ b/modules/wechat_qrcode/src/detector/ssd_detector.cpp @@ -26,9 +26,13 @@ vector SSDDetector::forward(Mat img, const int target_width, const int targ auto prob = net_.forward("detection_output"); vector point_list; + // the shape is (1,1,100,7)=>(batch,channel,count,dim) for (int row = 0; row < prob.size[2]; row++) { const float* prob_score = prob.ptr(0, 0, row); + // prob_score[0] is not used. + // prob_score[1]==1 stands for qrcode if (prob_score[1] == 1) { + // prob_score[2] is the probability of the qrcode, which is not used. auto point = Mat(4, 2, CV_32FC1); float x0 = CLIP(prob_score[3] * img_w, 0.0f, img_w - 1.0f); float y0 = CLIP(prob_score[4] * img_h, 0.0f, img_h - 1.0f); diff --git a/modules/wechat_qrcode/src/precomp.hpp b/modules/wechat_qrcode/src/precomp.hpp index 46c09c35c32..1fc21c0f454 100644 --- a/modules/wechat_qrcode/src/precomp.hpp +++ b/modules/wechat_qrcode/src/precomp.hpp @@ -21,7 +21,7 @@ #include #include #include - +#include #include "imgsource.hpp" using std::ostringstream; using std::string; diff --git a/modules/wechat_qrcode/src/wechat_qrcode.cpp b/modules/wechat_qrcode/src/wechat_qrcode.cpp index d4856ba22f0..7a4037c9c61 100644 --- a/modules/wechat_qrcode/src/wechat_qrcode.cpp +++ b/modules/wechat_qrcode/src/wechat_qrcode.cpp @@ -103,10 +103,7 @@ vector WeChatQRCode::detectAndDecode(InputArray img, OutputArrayOfArrays for (size_t i = 0; i < res_points.size(); i++) { Mat tmp_point; tmp_points.push_back(tmp_point); - res_points[i].convertTo(((OutputArray)tmp_points[i]), - ((OutputArray)tmp_points[i]).fixedType() - ? ((OutputArray)tmp_points[i]).type() - : CV_32FC2); + res_points[i].convertTo(((OutputArray)tmp_points[i]), CV_32FC2); } points.createSameSize(tmp_points, CV_32FC2); points.assign(tmp_points); diff --git a/modules/wechat_qrcode/src/zxing/binarizer.cpp b/modules/wechat_qrcode/src/zxing/binarizer.cpp index 758629fab13..685082870bb 100644 --- a/modules/wechat_qrcode/src/zxing/binarizer.cpp +++ b/modules/wechat_qrcode/src/zxing/binarizer.cpp @@ -7,11 +7,9 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../precomp.hpp" #include "binarizer.hpp" -#include - namespace zxing { Binarizer::Binarizer(Ref source) : source_(source) { diff --git a/modules/wechat_qrcode/src/zxing/binarybitmap.cpp b/modules/wechat_qrcode/src/zxing/binarybitmap.cpp index d617ac9ecee..28352ca8f5b 100644 --- a/modules/wechat_qrcode/src/zxing/binarybitmap.cpp +++ b/modules/wechat_qrcode/src/zxing/binarybitmap.cpp @@ -7,7 +7,7 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../precomp.hpp" #include "binarybitmap.hpp" using zxing::BinaryBitmap; diff --git a/modules/wechat_qrcode/src/zxing/common/array.hpp b/modules/wechat_qrcode/src/zxing/common/array.hpp index 2edb946a23b..9187937e884 100644 --- a/modules/wechat_qrcode/src/zxing/common/array.hpp +++ b/modules/wechat_qrcode/src/zxing/common/array.hpp @@ -13,8 +13,6 @@ #include "counted.hpp" -#include - namespace zxing { template diff --git a/modules/wechat_qrcode/src/zxing/common/binarizer/adaptive_threshold_mean_binarizer.cpp b/modules/wechat_qrcode/src/zxing/common/binarizer/adaptive_threshold_mean_binarizer.cpp index 57570941b02..38a79b378e1 100644 --- a/modules/wechat_qrcode/src/zxing/common/binarizer/adaptive_threshold_mean_binarizer.cpp +++ b/modules/wechat_qrcode/src/zxing/common/binarizer/adaptive_threshold_mean_binarizer.cpp @@ -4,10 +4,9 @@ // // Tencent is pleased to support the open source community by making WeChat QRCode available. // Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - +#include "../../../precomp.hpp" #include "adaptive_threshold_mean_binarizer.hpp" -using namespace std; -using namespace zxing; +using zxing::AdaptiveThresholdMeanBinarizer; namespace { const int BLOCK_SIZE = 25; diff --git a/modules/wechat_qrcode/src/zxing/common/binarizer/adaptive_threshold_mean_binarizer.hpp b/modules/wechat_qrcode/src/zxing/common/binarizer/adaptive_threshold_mean_binarizer.hpp index aba64ea88c9..bd51dde8cef 100644 --- a/modules/wechat_qrcode/src/zxing/common/binarizer/adaptive_threshold_mean_binarizer.hpp +++ b/modules/wechat_qrcode/src/zxing/common/binarizer/adaptive_threshold_mean_binarizer.hpp @@ -9,7 +9,6 @@ #define __ZXING_COMMON_ADAPTIVE_THRESHOLD_MEAN_BINARIZER_HPP__ #include #include -#include #include "../../binarizer.hpp" #include "../../errorhandler.hpp" #include "../bitarray.hpp" diff --git a/modules/wechat_qrcode/src/zxing/common/binarizer/fast_window_binarizer.cpp b/modules/wechat_qrcode/src/zxing/common/binarizer/fast_window_binarizer.cpp index d9a5e1edc36..b0d9f657ef9 100644 --- a/modules/wechat_qrcode/src/zxing/common/binarizer/fast_window_binarizer.cpp +++ b/modules/wechat_qrcode/src/zxing/common/binarizer/fast_window_binarizer.cpp @@ -7,10 +7,9 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../../precomp.hpp" #include "fast_window_binarizer.hpp" -using namespace std; -using namespace zxing; +using zxing::FastWindowBinarizer; namespace { diff --git a/modules/wechat_qrcode/src/zxing/common/binarizer/global_histogram_binarizer.cpp b/modules/wechat_qrcode/src/zxing/common/binarizer/global_histogram_binarizer.cpp index 8c3bc60fa3d..0f1aa0567c0 100644 --- a/modules/wechat_qrcode/src/zxing/common/binarizer/global_histogram_binarizer.cpp +++ b/modules/wechat_qrcode/src/zxing/common/binarizer/global_histogram_binarizer.cpp @@ -7,19 +7,9 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../../precomp.hpp" #include "global_histogram_binarizer.hpp" - -using zxing::ArrayRef; -using zxing::Binarizer; -using zxing::BitArray; -using zxing::BitMatrix; -using zxing::ErrorHandler; using zxing::GlobalHistogramBinarizer; -using zxing::Ref; - -// VC++ -using zxing::LuminanceSource; namespace { const int LUMINANCE_BITS = 5; @@ -76,12 +66,6 @@ int GlobalHistogramBinarizer::estimateBlackPoint(ArrayRef const& _buckets, int maxBucketCount = 0; int firstPeak = 0; int firstPeakSize = 0; - if (false) { - for (int x = 0; x < numBuckets; x++) { - cerr << _buckets[x] << " "; - } - cerr << endl; - } for (int x = 0; x < numBuckets; x++) { if (_buckets[x] > firstPeakSize) { firstPeak = x; @@ -282,23 +266,6 @@ int GlobalHistogramBinarizer::binarizeImage0(ErrorHandler& err_handler) { LuminanceSource& source = *getLuminanceSource(); Ref matrix(new BitMatrix(width, height, err_handler)); if (err_handler.ErrCode()) return -1; - -#ifdef INPUT_BINARIZED - { - ArrayRef localLuminances = source.getMatrix(); - for (int y = 0; y < height; y++) { - int offset = y * width; - for (int x = 0; x < width; x++) { - int pixel = localLuminances[offset + x] & 0xff; - if (pixel < 128) { - matrix->set(x, y); - } - } - } - } - matrix0_ = matrix; - return 0; -#endif // Quickly calculates the histogram by sampling four rows from the image. // This proved to be more robust on the blackbox tests than sampling a // diagonal as we used to do. diff --git a/modules/wechat_qrcode/src/zxing/common/binarizer/global_histogram_binarizer.hpp b/modules/wechat_qrcode/src/zxing/common/binarizer/global_histogram_binarizer.hpp index 4f090225663..f61b3cfebef 100644 --- a/modules/wechat_qrcode/src/zxing/common/binarizer/global_histogram_binarizer.hpp +++ b/modules/wechat_qrcode/src/zxing/common/binarizer/global_histogram_binarizer.hpp @@ -16,8 +16,17 @@ #include "../array.hpp" #include "../bitarray.hpp" #include "../bitmatrix.hpp" +#include "../bytematrix.hpp" + +using zxing::ArrayRef; +using zxing::Binarizer; +using zxing::BitArray; +using zxing::BitMatrix; +using zxing::ByteMatrix; +using zxing::ErrorHandler; +using zxing::LuminanceSource; +using zxing::Ref; -//#define INPUT_BINARIZED namespace zxing { class GlobalHistogramBinarizer : public Binarizer { diff --git a/modules/wechat_qrcode/src/zxing/common/binarizer/hybrid_binarizer.cpp b/modules/wechat_qrcode/src/zxing/common/binarizer/hybrid_binarizer.cpp index 527f8b4e4ce..6787b24642a 100644 --- a/modules/wechat_qrcode/src/zxing/common/binarizer/hybrid_binarizer.cpp +++ b/modules/wechat_qrcode/src/zxing/common/binarizer/hybrid_binarizer.cpp @@ -7,15 +7,11 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../../precomp.hpp" #include "hybrid_binarizer.hpp" -#include -#include -#include -#include -using namespace std; -using namespace zxing; +using zxing::HybridBinarizer; +using zxing::BINARIZER_BLOCK; // This class uses 5*5 blocks to compute local luminance, where each block is // 8*8 pixels So this is the smallest dimension in each axis we can accept. @@ -52,9 +48,6 @@ HybridBinarizer::HybridBinarizer(Ref source) : GlobalHistogramB } HybridBinarizer::~HybridBinarizer() { - // delete [] _bitCached; - // delete [] subSumPoints; - // delete [] subSumColumn; } Ref HybridBinarizer::createBinarizer(Ref source) { diff --git a/modules/wechat_qrcode/src/zxing/common/binarizer/simple_adaptive_binarizer.cpp b/modules/wechat_qrcode/src/zxing/common/binarizer/simple_adaptive_binarizer.cpp index 8f610ca31d5..2364bc39a3c 100644 --- a/modules/wechat_qrcode/src/zxing/common/binarizer/simple_adaptive_binarizer.cpp +++ b/modules/wechat_qrcode/src/zxing/common/binarizer/simple_adaptive_binarizer.cpp @@ -7,15 +7,11 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../../precomp.hpp" #include "simple_adaptive_binarizer.hpp" -#include -using namespace std; -using namespace zxing; +using zxing::SimpleAdaptiveBinarizer; -// VC++ -using zxing::LuminanceSource; SimpleAdaptiveBinarizer::SimpleAdaptiveBinarizer(Ref source) : GlobalHistogramBinarizer(source) { diff --git a/modules/wechat_qrcode/src/zxing/common/bitarray.cpp b/modules/wechat_qrcode/src/zxing/common/bitarray.cpp index 87e3aa3d0f6..768f6256d72 100644 --- a/modules/wechat_qrcode/src/zxing/common/bitarray.cpp +++ b/modules/wechat_qrcode/src/zxing/common/bitarray.cpp @@ -7,14 +7,12 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../precomp.hpp" #include "bitarray.hpp" -using std::vector; using zxing::ArrayRef; using zxing::BitArray; using zxing::ErrorHandler; -// VC++ using zxing::Ref; #if __WORDSIZE == 64 diff --git a/modules/wechat_qrcode/src/zxing/common/bitarray.hpp b/modules/wechat_qrcode/src/zxing/common/bitarray.hpp index 79c5280c60b..b6f1f56000e 100644 --- a/modules/wechat_qrcode/src/zxing/common/bitarray.hpp +++ b/modules/wechat_qrcode/src/zxing/common/bitarray.hpp @@ -16,9 +16,6 @@ #include "array.hpp" #include "counted.hpp" #include -#include -#include -#include namespace zxing { diff --git a/modules/wechat_qrcode/src/zxing/common/bitmatrix.cpp b/modules/wechat_qrcode/src/zxing/common/bitmatrix.cpp index 103140756a2..53df192a7b6 100644 --- a/modules/wechat_qrcode/src/zxing/common/bitmatrix.cpp +++ b/modules/wechat_qrcode/src/zxing/common/bitmatrix.cpp @@ -7,14 +7,8 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../precomp.hpp" #include "bitmatrix.hpp" -#include -#include -#include - -using std::ostream; -using std::ostringstream; using zxing::ArrayRef; using zxing::BitArray; diff --git a/modules/wechat_qrcode/src/zxing/common/bitmatrix.hpp b/modules/wechat_qrcode/src/zxing/common/bitmatrix.hpp index 624db9c5a0a..3cf7ec3c367 100644 --- a/modules/wechat_qrcode/src/zxing/common/bitmatrix.hpp +++ b/modules/wechat_qrcode/src/zxing/common/bitmatrix.hpp @@ -15,9 +15,6 @@ #include "array.hpp" #include "bitarray.hpp" #include "counted.hpp" - -#include -#include using namespace std; namespace zxing { diff --git a/modules/wechat_qrcode/src/zxing/common/bitsource.cpp b/modules/wechat_qrcode/src/zxing/common/bitsource.cpp index 6ff5feaba3e..52187e1d2ce 100644 --- a/modules/wechat_qrcode/src/zxing/common/bitsource.cpp +++ b/modules/wechat_qrcode/src/zxing/common/bitsource.cpp @@ -7,7 +7,7 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../precomp.hpp" #include "bitsource.hpp" #include diff --git a/modules/wechat_qrcode/src/zxing/common/bytematrix.cpp b/modules/wechat_qrcode/src/zxing/common/bytematrix.cpp index b5370671db3..61cf2517c2d 100644 --- a/modules/wechat_qrcode/src/zxing/common/bytematrix.cpp +++ b/modules/wechat_qrcode/src/zxing/common/bytematrix.cpp @@ -4,15 +4,8 @@ // // Tencent is pleased to support the open source community by making WeChat QRCode available. // Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - +#include "../../precomp.hpp" #include "bytematrix.hpp" -#include -#include -#include -#include - -using std::ostream; -using std::ostringstream; using zxing::ArrayRef; using zxing::ByteMatrix; diff --git a/modules/wechat_qrcode/src/zxing/common/bytematrix.hpp b/modules/wechat_qrcode/src/zxing/common/bytematrix.hpp index d3bd0038103..c3ee35ea587 100644 --- a/modules/wechat_qrcode/src/zxing/common/bytematrix.hpp +++ b/modules/wechat_qrcode/src/zxing/common/bytematrix.hpp @@ -13,8 +13,6 @@ #include "bitarray.hpp" #include "counted.hpp" -#include - namespace zxing { class ByteMatrix : public Counted { diff --git a/modules/wechat_qrcode/src/zxing/common/character.hpp b/modules/wechat_qrcode/src/zxing/common/character.hpp deleted file mode 100644 index 4059d8dcf5c..00000000000 --- a/modules/wechat_qrcode/src/zxing/common/character.hpp +++ /dev/null @@ -1,54 +0,0 @@ -// This file is part of OpenCV project. -// It is subject to the license terms in the LICENSE file found in the top-level directory -// of this distribution and at http://opencv.org/license.html. -// -// Tencent is pleased to support the open source community by making WeChat QRCode available. -// Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - -#ifndef __ZXING_COMMON_CHARACTER_HPP___ -#define __ZXING_COMMON_CHARACTER_HPP___ - -#include -#include - -#include - -using namespace std; - -namespace zxing { - -class Character { -public: - static char toUpperCase(char c) { return toupper(c); }; - - static bool isDigit(char c) { - if (c < '0' || c > '9') { - return false; - } - - return true; - - // return isdigit(c); - }; - - static int digit(char c, int radix) { - // return digit(c, radix); - - if (c >= '0' && c <= '9') { - return (int)(c - '0'); - } - - if (c >= 'a' && c <= 'z' && c < (radix + 'a' - 10)) { - return (int)(c - 'a' + 10); - } - - if (c >= 'A' && c <= 'Z' && c < (radix + 'A' - 10)) { - return (int)(c - 'A' + 10); - } - - return -1; - } -}; -} // namespace zxing - -#endif // __ZXING_COMMON_CHARACTER_HPP___ diff --git a/modules/wechat_qrcode/src/zxing/common/characterseteci.cpp b/modules/wechat_qrcode/src/zxing/common/characterseteci.cpp index 5ba94e22990..acef2044eb0 100644 --- a/modules/wechat_qrcode/src/zxing/common/characterseteci.cpp +++ b/modules/wechat_qrcode/src/zxing/common/characterseteci.cpp @@ -7,10 +7,8 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../precomp.hpp" #include "characterseteci.hpp" -using std::string; - using zxing::common::CharacterSetECI; // Fix memory leak diff --git a/modules/wechat_qrcode/src/zxing/common/decoder_result.cpp b/modules/wechat_qrcode/src/zxing/common/decoder_result.cpp index 1340084abcc..3de6656fc15 100644 --- a/modules/wechat_qrcode/src/zxing/common/decoder_result.cpp +++ b/modules/wechat_qrcode/src/zxing/common/decoder_result.cpp @@ -7,12 +7,13 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../precomp.hpp" #include "decoder_result.hpp" -using namespace std; -using namespace zxing; - +using zxing::DecoderResult; +using zxing::Ref; +using zxing::ArrayRef; +using zxing::String; DecoderResult::DecoderResult(ArrayRef rawBytes, Ref text, ArrayRef >& byteSegments, string const& ecLevel) : rawBytes_(rawBytes), text_(text), byteSegments_(byteSegments), ecLevel_(ecLevel) { diff --git a/modules/wechat_qrcode/src/zxing/common/decoder_result.hpp b/modules/wechat_qrcode/src/zxing/common/decoder_result.hpp index 730b166baad..7ffd002c03f 100644 --- a/modules/wechat_qrcode/src/zxing/common/decoder_result.hpp +++ b/modules/wechat_qrcode/src/zxing/common/decoder_result.hpp @@ -16,8 +16,6 @@ #include "counted.hpp" #include "str.hpp" -#include - namespace zxing { class DecoderResult : public Counted { diff --git a/modules/wechat_qrcode/src/zxing/common/detector_result.cpp b/modules/wechat_qrcode/src/zxing/common/detector_result.cpp index 408ad2c3945..2e50a5dc405 100644 --- a/modules/wechat_qrcode/src/zxing/common/detector_result.cpp +++ b/modules/wechat_qrcode/src/zxing/common/detector_result.cpp @@ -7,7 +7,7 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../precomp.hpp" #include "detector_result.hpp" namespace zxing { diff --git a/modules/wechat_qrcode/src/zxing/common/greyscale_luminance_source.cpp b/modules/wechat_qrcode/src/zxing/common/greyscale_luminance_source.cpp index 0efad7560e5..cf35ccbcb17 100644 --- a/modules/wechat_qrcode/src/zxing/common/greyscale_luminance_source.cpp +++ b/modules/wechat_qrcode/src/zxing/common/greyscale_luminance_source.cpp @@ -7,7 +7,7 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../precomp.hpp" #include "greyscale_luminance_source.hpp" #include "bytematrix.hpp" #include "greyscale_rotated_luminance_source.hpp" diff --git a/modules/wechat_qrcode/src/zxing/common/greyscale_rotated_luminance_source.cpp b/modules/wechat_qrcode/src/zxing/common/greyscale_rotated_luminance_source.cpp index 6e131d346f1..4f9e6261909 100644 --- a/modules/wechat_qrcode/src/zxing/common/greyscale_rotated_luminance_source.cpp +++ b/modules/wechat_qrcode/src/zxing/common/greyscale_rotated_luminance_source.cpp @@ -7,7 +7,7 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../precomp.hpp" #include "greyscale_rotated_luminance_source.hpp" #include "bytematrix.hpp" using zxing::ArrayRef; @@ -42,11 +42,6 @@ ArrayRef GreyscaleRotatedLuminanceSource::getRow(int y, ArrayRef row row = ArrayRef(getWidth()); } int offset = (left_ * dataWidth_) + (dataWidth_ - 1 - (y + top_)); - using namespace std; - if (false) { - cerr << offset << " = " << top_ << " " << left_ << " " << getHeight() << " " << getWidth() - << " " << y << endl; - } for (int x = 0; x < getWidth(); x++) { row[x] = greyData_[offset]; offset += dataWidth_; diff --git a/modules/wechat_qrcode/src/zxing/common/grid_sampler.cpp b/modules/wechat_qrcode/src/zxing/common/grid_sampler.cpp index c99a99e3fb1..52564f2f323 100644 --- a/modules/wechat_qrcode/src/zxing/common/grid_sampler.cpp +++ b/modules/wechat_qrcode/src/zxing/common/grid_sampler.cpp @@ -7,15 +7,12 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../precomp.hpp" #include "grid_sampler.hpp" #include "perspective_transform.hpp" - -#include #include namespace zxing { -using namespace std; GridSampler GridSampler::gridSampler; @@ -28,7 +25,7 @@ Ref GridSampler::sampleGrid(Ref image, int dimension, Ref bits(new BitMatrix(dimension, err_handler)); if (err_handler.ErrCode()) return Ref(); - vector points(dimension << 1, (const float)0.0f); + vector points(dimension << 1, 0.0f); int outlier = 0; int maxOutlier = dimension * dimension * 3 / 10 - 1; diff --git a/modules/wechat_qrcode/src/zxing/common/imagecut.cpp b/modules/wechat_qrcode/src/zxing/common/imagecut.cpp index 07300cc112b..86fdf7f3c45 100644 --- a/modules/wechat_qrcode/src/zxing/common/imagecut.cpp +++ b/modules/wechat_qrcode/src/zxing/common/imagecut.cpp @@ -4,10 +4,9 @@ // // Tencent is pleased to support the open source community by making WeChat QRCode available. // Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - +#include "../../precomp.hpp" #include "imagecut.hpp" -#include namespace zxing { diff --git a/modules/wechat_qrcode/src/zxing/common/imagecut.hpp b/modules/wechat_qrcode/src/zxing/common/imagecut.hpp index 2b85d77e933..377191ff1fe 100644 --- a/modules/wechat_qrcode/src/zxing/common/imagecut.hpp +++ b/modules/wechat_qrcode/src/zxing/common/imagecut.hpp @@ -7,12 +7,9 @@ #ifndef __ZXING_COMMON_IMAGECUT_HPP__ #define __ZXING_COMMON_IMAGECUT_HPP__ -#include #include "bytematrix.hpp" #include "counted.hpp" -#include - namespace zxing { typedef struct _ImageCutResult { diff --git a/modules/wechat_qrcode/src/zxing/common/kmeans.cpp b/modules/wechat_qrcode/src/zxing/common/kmeans.cpp index f7e2446f11c..2c55edc3564 100644 --- a/modules/wechat_qrcode/src/zxing/common/kmeans.cpp +++ b/modules/wechat_qrcode/src/zxing/common/kmeans.cpp @@ -4,14 +4,9 @@ // // Tencent is pleased to support the open source community by making WeChat QRCode available. // Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - +#include "../../precomp.hpp" #include "kmeans.hpp" -#include -#include -#include - -using namespace std; typedef unsigned int uint; namespace zxing { @@ -21,8 +16,6 @@ double cal_distance(vector a, vector b) { const float KMEANS_MS_FACTOR = 1; uint da = a.size(); - uint db = b.size(); - if (da != db) cerr << "Dimensions of two vectors must be same!!\n"; double val = 0.0; for (uint i = 0; i < da; i++) { if (i == 1) diff --git a/modules/wechat_qrcode/src/zxing/common/mathutils.hpp b/modules/wechat_qrcode/src/zxing/common/mathutils.hpp index 11470184072..bd46622466a 100644 --- a/modules/wechat_qrcode/src/zxing/common/mathutils.hpp +++ b/modules/wechat_qrcode/src/zxing/common/mathutils.hpp @@ -32,44 +32,6 @@ class MathUtils { ~MathUtils(); public: - /** - * Ends up being a bit faster than {@link Math#round(float)}. This merely - * rounds its argument to the nearest int, where x.5 rounds up to x+1. - */ - static inline int round(double value) { - // return (int) (d + 0.5f); - -#if (defined _MSC_VER && defined _M_X64) || \ - (defined __GNUC__ && defined __x86_64__ && defined __SSE2__ && !defined __APPLE__ && \ - !defined __GXX_WEAK__) - __m128d t = _mm_set_sd(value); - return _mm_cvtsd_si32(t); -#elif defined _MSC_VER && defined _M_IX86 - int t; - __asm - { - fld value; - fistp t; - } - return t; -#elif defined _MSC_VER && defined _M_ARM && defined HAVE_TEGRA_OPTIMIZATION - TEGRA_ROUND(value); -#elif defined HAVE_LRINT || defined CV_ICC || defined __GNUC__ -#ifdef HAVE_TEGRA_OPTIMIZATION - TEGRA_ROUND(value); -#else - return (int)lrint(value); -#endif -#else - double intpart, fractpart; - fractpart = modf(value, &intpart); - if ((fabs(fractpart) != 0.5) || ((((int)intpart) % 2) != 0)) - return (int)(value + (value >= 0 ? 0.5 : -0.5)); - else - return (int)intpart; -#endif - } - static inline float distance(float aX, float aY, float bX, float bY) { float xDiff = aX - bX; float yDiff = aY - bY; diff --git a/modules/wechat_qrcode/src/zxing/common/perspective_transform.cpp b/modules/wechat_qrcode/src/zxing/common/perspective_transform.cpp index 7779694f9e9..74ac2cd7f02 100644 --- a/modules/wechat_qrcode/src/zxing/common/perspective_transform.cpp +++ b/modules/wechat_qrcode/src/zxing/common/perspective_transform.cpp @@ -7,13 +7,11 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../precomp.hpp" #include "perspective_transform.hpp" -#include namespace zxing { -using namespace std; // This class implements a perspective transform in two dimensions. Given four // source and four destination points, it will compute the transformation diff --git a/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgf.cpp b/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgf.cpp index 51e1d99eac5..8e3ad184501 100644 --- a/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgf.cpp +++ b/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgf.cpp @@ -7,12 +7,10 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../../precomp.hpp" #include "genericgf.hpp" #include "genericgfpoly.hpp" -#include - using zxing::ErrorHandler; using zxing::GenericGF; using zxing::GenericGFPoly; diff --git a/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgf.hpp b/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgf.hpp index 7029af2b4ee..aabbeaf0833 100644 --- a/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgf.hpp +++ b/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgf.hpp @@ -14,8 +14,6 @@ #include "../../errorhandler.hpp" #include "../counted.hpp" -#include - namespace zxing { class GenericGFPoly; diff --git a/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgfpoly.cpp b/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgfpoly.cpp index 96f9fece18c..4098a379fdf 100644 --- a/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgfpoly.cpp +++ b/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgfpoly.cpp @@ -7,12 +7,10 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../../precomp.hpp" #include "genericgfpoly.hpp" #include "genericgf.hpp" -#include - using zxing::ArrayRef; using zxing::ErrorHandler; using zxing::GenericGFPoly; diff --git a/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgfpoly.hpp b/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgfpoly.hpp index c6c17bcb804..031c0b7b65c 100644 --- a/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgfpoly.hpp +++ b/modules/wechat_qrcode/src/zxing/common/reedsolomon/genericgfpoly.hpp @@ -15,8 +15,6 @@ #include "../array.hpp" #include "../counted.hpp" -#include - namespace zxing { class GenericGF; diff --git a/modules/wechat_qrcode/src/zxing/common/reedsolomon/reed_solomon_decoder.cpp b/modules/wechat_qrcode/src/zxing/common/reedsolomon/reed_solomon_decoder.cpp index dfaed4f4b14..fec9c82880b 100644 --- a/modules/wechat_qrcode/src/zxing/common/reedsolomon/reed_solomon_decoder.cpp +++ b/modules/wechat_qrcode/src/zxing/common/reedsolomon/reed_solomon_decoder.cpp @@ -7,19 +7,14 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../../precomp.hpp" #include "reed_solomon_decoder.hpp" -#include -#include -using std::vector; using zxing::ArrayRef; using zxing::ErrorHandler; using zxing::GenericGFPoly; using zxing::ReedSolomonDecoder; using zxing::Ref; - -// VC++ using zxing::GenericGF; ReedSolomonDecoder::ReedSolomonDecoder(Ref field_) : field(field_) {} diff --git a/modules/wechat_qrcode/src/zxing/common/reedsolomon/reed_solomon_decoder.hpp b/modules/wechat_qrcode/src/zxing/common/reedsolomon/reed_solomon_decoder.hpp index 26daa79d2b5..15271dd0207 100644 --- a/modules/wechat_qrcode/src/zxing/common/reedsolomon/reed_solomon_decoder.hpp +++ b/modules/wechat_qrcode/src/zxing/common/reedsolomon/reed_solomon_decoder.hpp @@ -17,9 +17,6 @@ #include "genericgf.hpp" #include "genericgfpoly.hpp" -#include -#include - namespace zxing { class GenericGFPoly; class GenericGF; diff --git a/modules/wechat_qrcode/src/zxing/common/str.cpp b/modules/wechat_qrcode/src/zxing/common/str.cpp index 82e1155ff1e..1de3dd028ea 100644 --- a/modules/wechat_qrcode/src/zxing/common/str.cpp +++ b/modules/wechat_qrcode/src/zxing/common/str.cpp @@ -7,18 +7,12 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../precomp.hpp" #include "str.hpp" -#include - -using std::string; using zxing::Ref; using zxing::String; using zxing::StrUtil; - -using namespace std; - String::String(const std::string& text) : text_(text) {} String::String(int capacity) { text_.reserve(capacity); } @@ -84,7 +78,7 @@ string StrUtil::numberToString(T Number) { template T StrUtil::stringToNumber(const string& Text) { - istringstream ss(Text); + std::istringstream ss(Text); T result; return ss >> result ? result : 0; } diff --git a/modules/wechat_qrcode/src/zxing/common/str.hpp b/modules/wechat_qrcode/src/zxing/common/str.hpp index 46cb9dcd712..3d20910f3fe 100644 --- a/modules/wechat_qrcode/src/zxing/common/str.hpp +++ b/modules/wechat_qrcode/src/zxing/common/str.hpp @@ -11,16 +11,11 @@ #ifndef __ZXING_COMMON_STR_HPP__ #define __ZXING_COMMON_STR_HPP__ -#include -#include #include "counted.hpp" -#include #include #include - -using namespace std; - +using std::string; namespace zxing { class String : public Counted { diff --git a/modules/wechat_qrcode/src/zxing/common/stringutils.cpp b/modules/wechat_qrcode/src/zxing/common/stringutils.cpp index 2a4a01868f4..ed9fe3e9d3c 100644 --- a/modules/wechat_qrcode/src/zxing/common/stringutils.cpp +++ b/modules/wechat_qrcode/src/zxing/common/stringutils.cpp @@ -7,14 +7,10 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../precomp.hpp" #include "../common/stringutils.hpp" #include "../decodehints.hpp" -#include - -using namespace std; -using namespace zxing; using namespace zxing::common; // N.B.: these are the iconv strings for at least some versions of iconv diff --git a/modules/wechat_qrcode/src/zxing/common/unicomblock.cpp b/modules/wechat_qrcode/src/zxing/common/unicomblock.cpp index ae51cb742b3..5c10a2d0e38 100644 --- a/modules/wechat_qrcode/src/zxing/common/unicomblock.cpp +++ b/modules/wechat_qrcode/src/zxing/common/unicomblock.cpp @@ -4,11 +4,8 @@ // // Tencent is pleased to support the open source community by making WeChat QRCode available. // Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - +#include "../../precomp.hpp" #include "unicomblock.hpp" -#include - -#include namespace zxing { short UnicomBlock::SEARCH_POS[4][2] = {{1, 0}, {-1, 0}, {0, 1}, {0, -1}}; diff --git a/modules/wechat_qrcode/src/zxing/common/unicomblock.hpp b/modules/wechat_qrcode/src/zxing/common/unicomblock.hpp index 858e03f018f..9b6871b2f44 100644 --- a/modules/wechat_qrcode/src/zxing/common/unicomblock.hpp +++ b/modules/wechat_qrcode/src/zxing/common/unicomblock.hpp @@ -10,9 +10,6 @@ #include "bitmatrix.hpp" #include "counted.hpp" -#include -#include - namespace zxing { class UnicomBlock : public Counted { public: diff --git a/modules/wechat_qrcode/src/zxing/errorhandler.cpp b/modules/wechat_qrcode/src/zxing/errorhandler.cpp index 466dd346af9..c8604ab6dc7 100644 --- a/modules/wechat_qrcode/src/zxing/errorhandler.cpp +++ b/modules/wechat_qrcode/src/zxing/errorhandler.cpp @@ -4,10 +4,9 @@ // // Tencent is pleased to support the open source community by making WeChat QRCode available. // Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. - +#include "../precomp.hpp" #include "errorhandler.hpp" -#include namespace zxing { ErrorHandler::ErrorHandler() : err_code_(0), err_msg_("") { Init(); } diff --git a/modules/wechat_qrcode/src/zxing/luminance_source.cpp b/modules/wechat_qrcode/src/zxing/luminance_source.cpp index e67052409f2..bb0b41b56b8 100644 --- a/modules/wechat_qrcode/src/zxing/luminance_source.cpp +++ b/modules/wechat_qrcode/src/zxing/luminance_source.cpp @@ -7,7 +7,7 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../precomp.hpp" #include "luminance_source.hpp" #include diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/bitmatrixparser.cpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/bitmatrixparser.cpp index 98189c005a0..6d426d076d3 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/decoder/bitmatrixparser.cpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/bitmatrixparser.cpp @@ -7,9 +7,8 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../../precomp.hpp" #include "bitmatrixparser.hpp" -#include #include "datamask.hpp" using zxing::ErrorHandler; diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/datablock.cpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/datablock.cpp index 1b329cf579c..da6afaa5054 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/decoder/datablock.cpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/datablock.cpp @@ -7,7 +7,7 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../../precomp.hpp" #include "datablock.hpp" namespace zxing { namespace qrcode { diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/datablock.hpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/datablock.hpp index 8b23e6e6c61..f95a29e45c0 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/decoder/datablock.hpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/datablock.hpp @@ -17,7 +17,6 @@ #include "../error_correction_level.hpp" #include "../version.hpp" -#include namespace zxing { namespace qrcode { diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/datamask.cpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/datamask.cpp index 56c42b5bc24..68ccb6ff503 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/decoder/datamask.cpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/datamask.cpp @@ -7,7 +7,7 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../../precomp.hpp" #include "datamask.hpp" namespace zxing { namespace qrcode { diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/datamask.hpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/datamask.hpp index 98b7f9e8569..c8f276fda01 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/decoder/datamask.hpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/datamask.hpp @@ -15,9 +15,6 @@ #include "../../common/bitmatrix.hpp" #include "../../common/counted.hpp" #include "../../errorhandler.hpp" - -#include - namespace zxing { namespace qrcode { diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoded_bit_stream_parser.cpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoded_bit_stream_parser.cpp index 86c8c43b207..05de793ced6 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoded_bit_stream_parser.cpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoded_bit_stream_parser.cpp @@ -7,12 +7,10 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../../precomp.hpp" #include "decoded_bit_stream_parser.hpp" #include "../../common/stringutils.hpp" #include "../../zxing.hpp" - -#include #ifndef NO_ICONV_INSIDE #include #endif diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoded_bit_stream_parser.hpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoded_bit_stream_parser.hpp index 3c047545fce..e8887ef654b 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoded_bit_stream_parser.hpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoded_bit_stream_parser.hpp @@ -20,9 +20,6 @@ #include "../../errorhandler.hpp" #include "mode.hpp" -#include -#include -#include namespace zxing { namespace qrcode { diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoder.cpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoder.cpp index d0e68a8f888..4ebe4754319 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoder.cpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/decoder.cpp @@ -7,7 +7,7 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../../precomp.hpp" #include "decoder.hpp" #include "../error_correction_level.hpp" #include "../version.hpp" @@ -15,10 +15,6 @@ #include "decoded_bit_stream_parser.hpp" #include "qrcode_decoder_metadata.hpp" -#include - -using std::cout; -using std::endl; using zxing::DecoderResult; using zxing::Ref; using zxing::qrcode::Decoder; diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/mode.cpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/mode.cpp index 5ba6ab91d7c..f96b63b38c5 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/decoder/mode.cpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/mode.cpp @@ -7,7 +7,7 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../../precomp.hpp" #include "mode.hpp" #include "../../common/counted.hpp" #include "../../zxing.hpp" diff --git a/modules/wechat_qrcode/src/zxing/qrcode/decoder/qrcode_decoder_metadata.hpp b/modules/wechat_qrcode/src/zxing/qrcode/decoder/qrcode_decoder_metadata.hpp index ea274f8a5b6..e7718931c76 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/decoder/qrcode_decoder_metadata.hpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/decoder/qrcode_decoder_metadata.hpp @@ -18,8 +18,6 @@ // VC++ // The main class which implements QR Code decoding -- as opposed to locating // and extracting the QR Code from an image. -// using zxing::ArrayRef; -// using zxing::BitMatrix; namespace zxing { namespace qrcode { diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern.cpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern.cpp index d6ac46d09dd..deefeeb6619 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern.cpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern.cpp @@ -7,11 +7,8 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../../precomp.hpp" #include "alignment_pattern.hpp" - -#include -using std::abs; using zxing::Ref; using zxing::qrcode::AlignmentPattern; namespace zxing { diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern.hpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern.hpp index 7d48d6b5ab0..74a15e32f79 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern.hpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern.hpp @@ -13,9 +13,6 @@ #include "../../common/bitmatrix.hpp" #include "../../resultpoint.hpp" - -#include - namespace zxing { namespace qrcode { diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern_finder.cpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern_finder.cpp index e3f0f3447e9..072371dad7b 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern_finder.cpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern_finder.cpp @@ -7,17 +7,9 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../../precomp.hpp" #include "alignment_pattern_finder.hpp" -#include -#include -#include -#include - -using std::abs; -using std::map; -using std::vector; using zxing::ErrorHandler; using zxing::ReaderErrorHandler; using zxing::Ref; diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern_finder.hpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern_finder.hpp index 90851dc1ff5..ace3064a5eb 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern_finder.hpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/alignment_pattern_finder.hpp @@ -17,10 +17,6 @@ #include "alignment_pattern.hpp" #include "finder_pattern.hpp" -#include "alignment_pattern.hpp" - -#include - namespace zxing { namespace qrcode { diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/detector.cpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/detector.cpp index 70404734862..e22a852f9c3 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/detector/detector.cpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/detector.cpp @@ -7,11 +7,8 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../../precomp.hpp" #include "detector.hpp" -#include -#include -#include #include #include "../../common/grid_sampler.hpp" #include "../../common/mathutils.hpp" @@ -21,11 +18,8 @@ #include "alignment_pattern_finder.hpp" #include "finder_pattern.hpp" #include "finder_pattern_finder.hpp" +#include "opencv2/core.hpp" -using std::abs; -using std::max; -using std::min; -using std::ostringstream; using zxing::BitMatrix; using zxing::DetectorResult; using zxing::ErrorHandler; @@ -1027,7 +1021,7 @@ int Detector::computeDimension(Ref topLeft, Ref topRig int tlblCentersDimension = ResultPoint::distance(topLeft, bottomLeft) / moduleSizeY; float tmp_dimension = ((tltrCentersDimension + tlblCentersDimension) / 2.0) + 7.0; - int dimension = MathUtils::round(tmp_dimension); + int dimension = cvRound(tmp_dimension); int mod = dimension & 0x03; // mod 4 switch (mod) { // mod 4 diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/detector.hpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/detector.hpp index cd3020b8f09..fbb8147dc91 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/detector/detector.hpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/detector.hpp @@ -22,8 +22,6 @@ #include "finder_pattern_info.hpp" #include "pattern_result.hpp" -#include - namespace zxing { class DecodeHints; diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern.cpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern.cpp index 143d20df96c..c357f312df5 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern.cpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern.cpp @@ -7,12 +7,9 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../../precomp.hpp" #include "finder_pattern.hpp" -#include - -using std::abs; using zxing::Ref; namespace zxing { diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern.hpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern.hpp index ef0b0d44b8f..d7733e0c6f1 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern.hpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern.hpp @@ -14,8 +14,6 @@ #include "../../common/bitmatrix.hpp" #include "../../resultpoint.hpp" -#include - namespace zxing { namespace qrcode { diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_finder.cpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_finder.cpp index fa3f5aaf373..438928c093d 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_finder.cpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_finder.cpp @@ -7,20 +7,13 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../../precomp.hpp" #include "finder_pattern_finder.hpp" #include "../../common/kmeans.hpp" #include "../../common/mathutils.hpp" #include "../../decodehints.hpp" #include "../../errorhandler.hpp" -#include -#include - -using std::abs; -using std::max; -using std::sort; -using std::vector; using zxing::Ref; using zxing::qrcode::FinderPattern; using zxing::qrcode::FinderPatternFinder; diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_finder.hpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_finder.hpp index f7f2268cc4c..442198bfd80 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_finder.hpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_finder.hpp @@ -17,9 +17,6 @@ #include "../../errorhandler.hpp" #include "finder_pattern.hpp" #include "finder_pattern_info.hpp" - -#include - using zxing::ErrorHandler; using zxing::ReaderErrorHandler; diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_info.cpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_info.cpp index ad365ba3bcf..2e121f97ed5 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_info.cpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_info.cpp @@ -7,11 +7,8 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../../precomp.hpp" #include "finder_pattern_info.hpp" -#include - -#include namespace zxing { namespace qrcode { diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_info.hpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_info.hpp index a016060bd95..223524e2242 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_info.hpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/finder_pattern_info.hpp @@ -15,8 +15,6 @@ #include "../../common/counted.hpp" #include "finder_pattern.hpp" -#include - namespace zxing { namespace qrcode { diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/pattern_result.cpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/pattern_result.cpp index e4a2abbb901..b44349ee99c 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/detector/pattern_result.cpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/pattern_result.cpp @@ -7,14 +7,10 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). +#include "../../../precomp.hpp" #include "pattern_result.hpp" -// VC++ -using namespace std; -using namespace zxing; -using namespace qrcode; -using std::abs; using zxing::Ref; using zxing::ResultPoint; using zxing::qrcode::FinderPattern; diff --git a/modules/wechat_qrcode/src/zxing/qrcode/detector/pattern_result.hpp b/modules/wechat_qrcode/src/zxing/qrcode/detector/pattern_result.hpp index fce734dc822..93a4f2ba3af 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/detector/pattern_result.hpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/detector/pattern_result.hpp @@ -19,9 +19,6 @@ #include "alignment_pattern.hpp" #include "finder_pattern.hpp" #include "finder_pattern_info.hpp" - -#include - namespace zxing { namespace qrcode { class PatternResult : public Counted { diff --git a/modules/wechat_qrcode/src/zxing/qrcode/error_correction_level.cpp b/modules/wechat_qrcode/src/zxing/qrcode/error_correction_level.cpp index a9e376ef153..1b2c63dee25 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/error_correction_level.cpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/error_correction_level.cpp @@ -7,10 +7,8 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../precomp.hpp" #include "error_correction_level.hpp" - -using std::string; using zxing::ErrorHandler; namespace zxing { diff --git a/modules/wechat_qrcode/src/zxing/qrcode/format_information.cpp b/modules/wechat_qrcode/src/zxing/qrcode/format_information.cpp index abd97040a92..5395f791fcb 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/format_information.cpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/format_information.cpp @@ -7,7 +7,7 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../precomp.hpp" #include "format_information.hpp" #include diff --git a/modules/wechat_qrcode/src/zxing/qrcode/qrcode_reader.cpp b/modules/wechat_qrcode/src/zxing/qrcode/qrcode_reader.cpp index 318816f0e51..64a6c08c9f3 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/qrcode_reader.cpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/qrcode_reader.cpp @@ -7,7 +7,7 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../precomp.hpp" #include "qrcode_reader.hpp" #include #include "../common/bitarray.hpp" diff --git a/modules/wechat_qrcode/src/zxing/qrcode/version.cpp b/modules/wechat_qrcode/src/zxing/qrcode/version.cpp index 0c1a4f92820..1f08a6b6e2b 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/version.cpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/version.cpp @@ -7,16 +7,12 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../../precomp.hpp" #include "version.hpp" #include "format_information.hpp" #include -#include -#include - using std::numeric_limits; -using std::vector; using zxing::ErrorHandler; namespace zxing { diff --git a/modules/wechat_qrcode/src/zxing/qrcode/version.hpp b/modules/wechat_qrcode/src/zxing/qrcode/version.hpp index c6fc6fe81fe..68bb76740bf 100644 --- a/modules/wechat_qrcode/src/zxing/qrcode/version.hpp +++ b/modules/wechat_qrcode/src/zxing/qrcode/version.hpp @@ -16,8 +16,6 @@ #include "../errorhandler.hpp" #include "error_correction_level.hpp" -#include - namespace zxing { namespace qrcode { diff --git a/modules/wechat_qrcode/src/zxing/reader.cpp b/modules/wechat_qrcode/src/zxing/reader.cpp index adb445d7bcb..da54ccb6b86 100644 --- a/modules/wechat_qrcode/src/zxing/reader.cpp +++ b/modules/wechat_qrcode/src/zxing/reader.cpp @@ -7,7 +7,7 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../precomp.hpp" #include "reader.hpp" namespace zxing { diff --git a/modules/wechat_qrcode/src/zxing/result.cpp b/modules/wechat_qrcode/src/zxing/result.cpp index cd8c2a102fd..e33c2c5188c 100644 --- a/modules/wechat_qrcode/src/zxing/result.cpp +++ b/modules/wechat_qrcode/src/zxing/result.cpp @@ -7,7 +7,7 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../precomp.hpp" #include "result.hpp" using zxing::ArrayRef; diff --git a/modules/wechat_qrcode/src/zxing/resultpoint.cpp b/modules/wechat_qrcode/src/zxing/resultpoint.cpp index 4cd5290a6d4..0fab518d2ef 100644 --- a/modules/wechat_qrcode/src/zxing/resultpoint.cpp +++ b/modules/wechat_qrcode/src/zxing/resultpoint.cpp @@ -8,7 +8,7 @@ // // Modified from ZXing. Copyright ZXing authors. // Licensed under the Apache License, Version 2.0 (the "License"). - +#include "../precomp.hpp" #include "resultpoint.hpp" #include "common/mathutils.hpp" diff --git a/modules/wechat_qrcode/src/zxing/resultpoint.hpp b/modules/wechat_qrcode/src/zxing/resultpoint.hpp index 3237c36260f..cb5d05d0d6b 100644 --- a/modules/wechat_qrcode/src/zxing/resultpoint.hpp +++ b/modules/wechat_qrcode/src/zxing/resultpoint.hpp @@ -13,8 +13,6 @@ #include "common/counted.hpp" -#include - namespace zxing { class ResultPoint : public Counted { diff --git a/modules/wechat_qrcode/src/zxing/zxing.hpp b/modules/wechat_qrcode/src/zxing/zxing.hpp index 83936500ee1..cae7e7dbac2 100644 --- a/modules/wechat_qrcode/src/zxing/zxing.hpp +++ b/modules/wechat_qrcode/src/zxing/zxing.hpp @@ -52,35 +52,22 @@ typedef unsigned char boolean; #include -#if defined(_WIN32) || defined(_WIN64) +#if defined(_MSC_VER) #ifndef NO_ICONV #define NO_ICONV #endif -#include - -namespace zxing { -inline bool isnan(float v) { return _isnan(v) != 0; } -inline bool isnan(double v) { return _isnan(v) != 0; } -inline float nan() { return std::numeric_limits::quiet_NaN(); } -} // namespace zxing - -#else +#endif #include -#undef isnan namespace zxing { inline bool isnan(float v) { return std::isnan(v); } inline bool isnan(double v) { return std::isnan(v); } inline float nan() { return std::numeric_limits::quiet_NaN(); } } // namespace zxing -//#endif - -#endif - #ifndef ZXING_TIME #define ZXING_TIME(string) (void)0 #endif From 1a5e6ed911b96c113409ac8f67cd2f8916e14ec9 Mon Sep 17 00:00:00 2001 From: Pavel Rojtberg Date: Mon, 8 Feb 2021 18:41:46 +0100 Subject: [PATCH 44/70] ovis: also unregister SceneManager when Window is deleted 8c385d58f683163fd5c56c5e43ce571bb9584564 was incomplete --- modules/ovis/src/ovis.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/ovis/src/ovis.cpp b/modules/ovis/src/ovis.cpp index 5901f54e749..257e2315cda 100644 --- a/modules/ovis/src/ovis.cpp +++ b/modules/ovis/src/ovis.cpp @@ -411,6 +411,7 @@ class WindowSceneImpl : public WindowScene texMgr.remove(texName, RESOURCEGROUP_NAME); } + RTShader::ShaderGenerator::getSingleton().removeSceneManager(sceneMgr); root->destroySceneManager(sceneMgr); } From 5f8f4b5b86566872f130544169126ae4e514e8ca Mon Sep 17 00:00:00 2001 From: Pavel Rojtberg Date: Mon, 8 Feb 2021 18:57:40 +0100 Subject: [PATCH 45/70] ovis: do not leak camera when !SCENE_SEPARATE --- modules/ovis/src/ovis.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/ovis/src/ovis.cpp b/modules/ovis/src/ovis.cpp index 257e2315cda..b7db2aad54e 100644 --- a/modules/ovis/src/ovis.cpp +++ b/modules/ovis/src/ovis.cpp @@ -414,6 +414,11 @@ class WindowSceneImpl : public WindowScene RTShader::ShaderGenerator::getSingleton().removeSceneManager(sceneMgr); root->destroySceneManager(sceneMgr); } + else + { + // we share everything, but the camera + sceneMgr->destroyCamera(title); + } if(_app->mainWin == this && (flags & SCENE_SEPARATE)) { From d1099dc872da0a70b9018b1968c51b3ff8878101 Mon Sep 17 00:00:00 2001 From: Yahui Wang Date: Fri, 5 Feb 2021 11:17:40 +0800 Subject: [PATCH 46/70] Fix CUDA GoodFeaturesToTrackDetector not threadsafe bug --- modules/cudaimgproc/src/cuda/gftt.cu | 58 +++++++++++++--------------- modules/cudaimgproc/src/gftt.cpp | 22 +++++++++-- 2 files changed, 45 insertions(+), 35 deletions(-) diff --git a/modules/cudaimgproc/src/cuda/gftt.cu b/modules/cudaimgproc/src/cuda/gftt.cu index ab8713f868a..66bd6e0dbc2 100644 --- a/modules/cudaimgproc/src/cuda/gftt.cu +++ b/modules/cudaimgproc/src/cuda/gftt.cu @@ -52,37 +52,33 @@ namespace cv { namespace cuda { namespace device { namespace gfft { - texture eigTex(0, cudaFilterModePoint, cudaAddressModeClamp); - - __device__ int g_counter = 0; - - template __global__ void findCorners(float threshold, const Mask mask, float2* corners, int max_count, int rows, int cols) + template __global__ void findCorners(float threshold, const Mask mask, float2* corners, int max_count, int rows, int cols, cudaTextureObject_t eigTex, int *g_counter) { const int j = blockIdx.x * blockDim.x + threadIdx.x; const int i = blockIdx.y * blockDim.y + threadIdx.y; if (i > 0 && i < rows - 1 && j > 0 && j < cols - 1 && mask(i, j)) { - float val = tex2D(eigTex, j, i); + float val = tex2D(eigTex, j, i); if (val > threshold) { float maxVal = val; - maxVal = ::fmax(tex2D(eigTex, j - 1, i - 1), maxVal); - maxVal = ::fmax(tex2D(eigTex, j , i - 1), maxVal); - maxVal = ::fmax(tex2D(eigTex, j + 1, i - 1), maxVal); + maxVal = ::fmax(tex2D(eigTex, j - 1, i - 1), maxVal); + maxVal = ::fmax(tex2D(eigTex, j , i - 1), maxVal); + maxVal = ::fmax(tex2D(eigTex, j + 1, i - 1), maxVal); - maxVal = ::fmax(tex2D(eigTex, j - 1, i), maxVal); - maxVal = ::fmax(tex2D(eigTex, j + 1, i), maxVal); + maxVal = ::fmax(tex2D(eigTex, j - 1, i), maxVal); + maxVal = ::fmax(tex2D(eigTex, j + 1, i), maxVal); - maxVal = ::fmax(tex2D(eigTex, j - 1, i + 1), maxVal); - maxVal = ::fmax(tex2D(eigTex, j , i + 1), maxVal); - maxVal = ::fmax(tex2D(eigTex, j + 1, i + 1), maxVal); + maxVal = ::fmax(tex2D(eigTex, j - 1, i + 1), maxVal); + maxVal = ::fmax(tex2D(eigTex, j , i + 1), maxVal); + maxVal = ::fmax(tex2D(eigTex, j + 1, i + 1), maxVal); if (val == maxVal) { - const int ind = ::atomicAdd(&g_counter, 1); + const int ind = ::atomicAdd(g_counter, 1); if (ind < max_count) corners[ind] = make_float2(j, i); @@ -91,22 +87,20 @@ namespace cv { namespace cuda { namespace device } } - int findCorners_gpu(PtrStepSzf eig, float threshold, PtrStepSzb mask, float2* corners, int max_count, cudaStream_t stream) + int findCorners_gpu(const cudaTextureObject_t &eigTex, const int &rows, const int &cols, float threshold, PtrStepSzb mask, float2* corners, int max_count, cudaStream_t stream) { - void* counter_ptr; - cudaSafeCall( cudaGetSymbolAddress(&counter_ptr, g_counter) ); + int* counter_ptr; + cudaSafeCall( cudaMalloc(&counter_ptr, sizeof(int)) ); cudaSafeCall( cudaMemsetAsync(counter_ptr, 0, sizeof(int), stream) ); - bindTexture(&eigTex, eig); - dim3 block(16, 16); - dim3 grid(divUp(eig.cols, block.x), divUp(eig.rows, block.y)); + dim3 grid(divUp(cols, block.x), divUp(rows, block.y)); if (mask.data) - findCorners<<>>(threshold, SingleMask(mask), corners, max_count, eig.rows, eig.cols); + findCorners<<>>(threshold, SingleMask(mask), corners, max_count, rows, cols, eigTex, counter_ptr); else - findCorners<<>>(threshold, WithOutMask(), corners, max_count, eig.rows, eig.cols); + findCorners<<>>(threshold, WithOutMask(), corners, max_count, rows, cols, eigTex, counter_ptr); cudaSafeCall( cudaGetLastError() ); @@ -122,25 +116,27 @@ namespace cv { namespace cuda { namespace device class EigGreater { public: + EigGreater(const cudaTextureObject_t &eigTex_) : eigTex(eigTex_) + { + } __device__ __forceinline__ bool operator()(float2 a, float2 b) const { - return tex2D(eigTex, a.x, a.y) > tex2D(eigTex, b.x, b.y); + return tex2D(eigTex, a.x, a.y) > tex2D(eigTex, b.x, b.y); } - }; + cudaTextureObject_t eigTex; + }; - void sortCorners_gpu(PtrStepSzf eig, float2* corners, int count, cudaStream_t stream) + void sortCorners_gpu(const cudaTextureObject_t &eigTex, float2* corners, int count, cudaStream_t stream) { - bindTexture(&eigTex, eig); - thrust::device_ptr ptr(corners); #if THRUST_VERSION >= 100802 if (stream) - thrust::sort(thrust::cuda::par(ThrustAllocator::getAllocator()).on(stream), ptr, ptr + count, EigGreater()); + thrust::sort(thrust::cuda::par(ThrustAllocator::getAllocator()).on(stream), ptr, ptr + count, EigGreater(eigTex)); else - thrust::sort(thrust::cuda::par(ThrustAllocator::getAllocator()), ptr, ptr + count, EigGreater()); + thrust::sort(thrust::cuda::par(ThrustAllocator::getAllocator()), ptr, ptr + count, EigGreater(eigTex)); #else - thrust::sort(ptr, ptr + count, EigGreater()); + thrust::sort(ptr, ptr + count, EigGreater(eigTex)); #endif } } // namespace optical_flow diff --git a/modules/cudaimgproc/src/gftt.cpp b/modules/cudaimgproc/src/gftt.cpp index bf5d01b1174..f25158a68d8 100644 --- a/modules/cudaimgproc/src/gftt.cpp +++ b/modules/cudaimgproc/src/gftt.cpp @@ -55,8 +55,8 @@ namespace cv { namespace cuda { namespace device { namespace gfft { - int findCorners_gpu(PtrStepSzf eig, float threshold, PtrStepSzb mask, float2* corners, int max_count, cudaStream_t stream); - void sortCorners_gpu(PtrStepSzf eig, float2* corners, int count, cudaStream_t stream); + int findCorners_gpu(const cudaTextureObject_t &eigTex_, const int &rows, const int &cols, float threshold, PtrStepSzb mask, float2* corners, int max_count, cudaStream_t stream); + void sortCorners_gpu(const cudaTextureObject_t &eigTex_, float2* corners, int count, cudaStream_t stream); } }}} @@ -112,7 +112,21 @@ namespace cudaStream_t stream_ = StreamAccessor::getStream(stream); ensureSizeIsEnough(1, std::max(1000, static_cast(image.size().area() * 0.05)), CV_32FC2, tmpCorners_); - int total = findCorners_gpu(eig_, static_cast(maxVal * qualityLevel_), mask, tmpCorners_.ptr(), tmpCorners_.cols, stream_); + //create texture object for findCorners_gpu and sortCorners_gpu + cudaTextureDesc texDesc; + memset(&texDesc, 0, sizeof(texDesc)); + texDesc.readMode = cudaReadModeElementType; + texDesc.filterMode = cudaFilterModePoint; + texDesc.addressMode[0] = cudaAddressModeClamp; + texDesc.addressMode[1] = cudaAddressModeClamp; + texDesc.addressMode[2] = cudaAddressModeClamp; + + cudaTextureObject_t eigTex_; + PtrStepSzf eig = eig_; + cv::cuda::device::createTextureObjectPitch2D(&eigTex_, eig, texDesc); + + int total = findCorners_gpu(eigTex_, eig_.rows, eig_.cols, static_cast(maxVal * qualityLevel_), mask, tmpCorners_.ptr(), tmpCorners_.cols, stream_); + if (total == 0) { @@ -120,7 +134,7 @@ namespace return; } - sortCorners_gpu(eig_, tmpCorners_.ptr(), total, stream_); + sortCorners_gpu(eigTex_, tmpCorners_.ptr(), total, stream_); if (minDistance_ < 1) { From 24baeaae27919d1ee31490669af2144cd122cd31 Mon Sep 17 00:00:00 2001 From: Roman Golovanov Date: Thu, 18 Feb 2021 19:46:39 +0200 Subject: [PATCH 47/70] Fix omnidir::undistortPoints The relevant bug was reported in https://github.com/opencv/opencv_contrib/issues/1612 The _xi was erroneously applied at points re-projection to camera plane. _xi parameter was already taken in use while projection of points to unit sphere. --- modules/ccalib/src/omnidir.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ccalib/src/omnidir.cpp b/modules/ccalib/src/omnidir.cpp index a67a6bd6c07..83e48725da6 100644 --- a/modules/ccalib/src/omnidir.cpp +++ b/modules/ccalib/src/omnidir.cpp @@ -330,7 +330,7 @@ void cv::omnidir::undistortPoints( InputArray distorted, OutputArray undistorted Vec3d Xs = Xw / cv::norm(Xw); // reproject to camera plane - Vec3d ppu = Vec3d(Xs[0]/(Xs[2]+_xi), Xs[1]/(Xs[2]+_xi), 1.0); + Vec3d ppu = Vec3d(Xs[0]/Xs[2], Xs[1]/Xs[2], 1.0); if (undistorted.depth() == CV_32F) { dstf[i] = Vec2f((float)ppu[0], (float)ppu[1]); From 7ede6585f5d52fd5547fc41b0d531fb7179917fb Mon Sep 17 00:00:00 2001 From: DumDereDum <46279571+DumDereDum@users.noreply.github.com> Date: Sat, 20 Feb 2021 19:33:19 +0300 Subject: [PATCH 48/70] Merge pull request #2734 from DumDereDum:new_HashTSDF_GPU_class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New HashTSDFVolumeGPU class * create new class * rewrite makeHashTSDFVolume * it builds * minor fi * raycast - done * all code builds * Docs fix * –fetch_normals fix * I want it to work * minor fix * valid points checking passed * it works! * minor fixes * warning fix * last replacements * warning fix * create hash_tsdf.cl * add preCalculationPixNormGPU * add integrateVolumeUnitGPU * Docs fix * create simple volume table * add functionality to the table * add new functions * create new hash table implementation * minor fix * create new hash tsdf implementaton * add expand feature * minorfix * error fix * it builds * new hash table work * Docs fix * warnings fix * warning fix 1 * warning fix 2 * warning fix * remove extra coments * create new function integrateAllVolumeUnitsGPU * add hash table function * extend kernel signature * minor fix * extent Volume_NODE signature and add functionality * kernel args fix * smth * minor fix * kernal args fix * try to fix reading bug * hash table fix * add totalVolUnits feature * it builds * minor fix * integrate fix * fix work with table * minor table fi * I am stumped * try to fix * add checking to find error * add copyto * vol2cam fix * try to fix * we are the champions * remove some comments * remove last comments * docs fix * warning fix * warning fix 1 * warning fix 2 * create raycast kernel * extend raycast gpu signature * minor fix * integrate all volUnits - done * docs fix * warning fix * extend raycast signature * raycast work * bug fix * calc point4 fix * minor fix * add _at function * raycast done, but work incorrect * fin dug * bug fix * getNorm works correctly * minor fix * raycast - done * minor fixes * docs fix * warning fix * minor fix * minor fix * warning fix * warning fix 1 * win warning fix * mac fix * win debug fix * mac cl warning fix * allCam2vol fix * rename num with bufferNum * remove nums hardcode * minor fix * change logic of calculating hash * replace float16 by vload16 * replace true with false in zFirstMemOrder * poses fix * add new tests * debug print * debug print 1 * replace [] with .s * minor fix * minor fix 1 * remove print * hashtsdf work fix * hashtable fix * resrite hashtasble update * test_tsdf fix * override warning fix * docs fix * mutex fix * setUseOptimizes replace false with true * minor fix * merge with master * all minor fixes * add CPU code for debug * settings set * move code from hpp to cpp * minor opencl fix * fid bug * coment fix * minor fix * bug fix * integrate fixed * speed up * remove extra code * raycast fix, but still not work * raycast fix * HT refactoring: constants * HT refactoring: calc_hash() * HT refactoring: methods set shortened * no typedef VolumeIndex * fix a bug in CPU version * VolumeIndex * identation * HashTSDF refactoring: a lot of things * less TODOs * lastVisibleIndex dropped from HT * HT refactor: find -> findRow * interpolate & getNormal small hash: no queried * totalVolUnits, indexes -> volumeUnitIndices, better iteration * raycast artifacts fixed * interpolation in getNormal is on * minors * findRow() refactored * volStrides, lastVolIndex, volDims, neighbourCoords * normals, vectorization * interpolation in getNormalVoxel() * no 16*16*16, use volumeUnitResolution instead * resolution, stride... * raycast: cur idx, etc. * CPU debugging code removed * volUnitsData moved to OpenCL memory * raycast: no copy * allocate accelerated a bit * hash_tsdf.cl: pointers fixed * allVol2cam, poses moved to kernel; rowRange for volUnitsData init * TSDF voxel: constructor; row range fixed * rowRange * pixNorms fix + minors * depth is now UMat * markActive is done on GPU, lastVisibleIndices and isActiveFlags moved to GPU * CL: minor * volumeUnitIndices -> GPU * ToyHashMap instead of HT, volumeUnitIndices removed, lastVolIndex removed, + a lot of debugging code * warning about local pix capacity * CPU allocation: find in global HT first * ToyHashMap: -> tsdf_functions.hpp * debugging code + to be removed * ptr -> at * volumeUnitResolution is limited to a power of 2 * rename ToyHashMap to ToyHashSet, hashMap to hashTable * minor * getNormalVoxel .cl: no float conversions * getNormalVoxel does not need voxelSizeInv anymore; orig and dir pre-scaled to get rid of extra multiplication * minor fix * linux critical bug fix * docs fix * mac warning fix * remove extra test code * linux warning fix * remove comments * capacity = 2048 * capacity = 1024 * hashtables fixes * minor code removement * change pixNorm calculation * remove buff_lvl and neighboorCoords; rename degree * move volStrides to HashTSDFVolume * HashTSDFVolumeGPU constructor fix * move preCalculatePixNormsGPU to tsdf_functions * minor code removement * replace copyto() with getUMat() * remove debug code * replace copyTo with getUmat again * replace types in cl * remove NAN_ELEMENT * remove Volume_NODE struct and findRow() * make hash_divisor constant * uncomment camTrans * rename data; remove extra val * fix merge conflict * create empty UMat * cl fix * build error fix * clear HashTSDFVolumeGPU() * use constuctor delegation at HashTSDFVolumeCPU * remove cam2volTransGPU * add minor fix Co-authored-by: Saratovtsev Co-authored-by: Rostislav Vasilikhin --- modules/rgbd/samples/large_kinfu_demo.cpp | 3 +- modules/rgbd/src/hash_tsdf.cpp | 1113 +++++++++++++++++++-- modules/rgbd/src/hash_tsdf.hpp | 102 +- modules/rgbd/src/large_kinfu.cpp | 6 +- modules/rgbd/src/opencl/hash_tsdf.cl | 653 ++++++++++++ modules/rgbd/src/opencl/tsdf.cl | 14 - modules/rgbd/src/opencl/tsdf_functions.cl | 19 + modules/rgbd/src/submap.hpp | 12 +- modules/rgbd/src/tsdf.cpp | 60 +- modules/rgbd/src/tsdf.hpp | 9 +- modules/rgbd/src/tsdf_functions.cpp | 48 +- modules/rgbd/src/tsdf_functions.hpp | 242 +++++ modules/rgbd/src/volume.cpp | 8 +- modules/rgbd/test/test_tsdf.cpp | 14 +- 14 files changed, 2046 insertions(+), 257 deletions(-) create mode 100644 modules/rgbd/src/opencl/hash_tsdf.cl create mode 100644 modules/rgbd/src/opencl/tsdf_functions.cl diff --git a/modules/rgbd/samples/large_kinfu_demo.cpp b/modules/rgbd/samples/large_kinfu_demo.cpp index 40d14ca8cae..e78c3511215 100644 --- a/modules/rgbd/samples/large_kinfu_demo.cpp +++ b/modules/rgbd/samples/large_kinfu_demo.cpp @@ -127,8 +127,7 @@ int main(int argc, char** argv) // These params can be different for each depth sensor ds->updateParams(*params); - // Disabled until there is no OpenCL accelerated HashTSDF is available - cv::setUseOptimized(false); + cv::setUseOptimized(true); if (!idle) largeKinfu = LargeKinfu::create(params); diff --git a/modules/rgbd/src/hash_tsdf.cpp b/modules/rgbd/src/hash_tsdf.cpp index afdeef37063..08338fe83ef 100644 --- a/modules/rgbd/src/hash_tsdf.cpp +++ b/modules/rgbd/src/hash_tsdf.cpp @@ -15,9 +15,10 @@ #include "opencv2/core/utility.hpp" #include "opencv2/core/utils/trace.hpp" #include "utils.hpp" +#include "opencl_kernels_rgbd.hpp" #define USE_INTERPOLATION_IN_GETNORMAL 1 -#define VOLUMES_SIZE 1024 +#define VOLUMES_SIZE 8192 namespace cv { @@ -35,13 +36,17 @@ HashTSDFVolume::HashTSDFVolume(float _voxelSize, cv::Matx44f _pose, float _rayca zFirstMemOrder(_zFirstMemOrder) { truncDist = std::max(_truncDist, 4.0f * voxelSize); -} -HashTSDFVolumeCPU::HashTSDFVolumeCPU(float _voxelSize, const Matx44f& _pose, float _raycastStepFactor, float _truncDist, - int _maxWeight, float _truncateThreshold, int _volumeUnitRes, bool _zFirstMemOrder) - :HashTSDFVolume(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, _truncateThreshold, _volumeUnitRes, - _zFirstMemOrder) -{ + if (!(volumeUnitResolution & (volumeUnitResolution - 1))) + { + // vuRes is a power of 2, let's get this power + volumeUnitDegree = trailingZeros32(volumeUnitResolution); + } + else + { + CV_Error(Error::StsBadArg, "Volume unit resolution should be a power of 2"); + } + int xdim, ydim, zdim; if (zFirstMemOrder) { @@ -56,21 +61,110 @@ HashTSDFVolumeCPU::HashTSDFVolumeCPU(float _voxelSize, const Matx44f& _pose, flo zdim = volumeUnitResolution * volumeUnitResolution; } volStrides = Vec4i(xdim, ydim, zdim); +} + +//! Spatial hashing +struct tsdf_hash +{ + size_t operator()(const Vec3i& x) const noexcept + { + size_t seed = 0; + constexpr uint32_t GOLDEN_RATIO = 0x9e3779b9; + for (uint16_t i = 0; i < 3; i++) + { + seed ^= std::hash()(x[i]) + GOLDEN_RATIO + (seed << 6) + (seed >> 2); + } + return seed; + } +}; + +struct VolumeUnit +{ + cv::Vec3i coord; + int index; + cv::Matx44f pose; + int lastVisibleIndex = 0; + bool isActive; +}; + +typedef std::unordered_set VolumeUnitIndexSet; +typedef std::unordered_map VolumeUnitIndexes; + +class HashTSDFVolumeCPU : public HashTSDFVolume +{ +public: + // dimension in voxels, size in meters + HashTSDFVolumeCPU(float _voxelSize, const Matx44f& _pose, float _raycastStepFactor, float _truncDist, int _maxWeight, + float _truncateThreshold, int _volumeUnitRes, bool zFirstMemOrder = true); + + HashTSDFVolumeCPU(const VolumeParams& _volumeParams, bool zFirstMemOrder = true); + + void integrate(InputArray _depth, float depthFactor, const Matx44f& cameraPose, const kinfu::Intr& intrinsics, + const int frameId = 0) override; + void raycast(const Matx44f& cameraPose, const kinfu::Intr& intrinsics, const Size& frameSize, OutputArray points, + OutputArray normals) const override; + + void fetchNormals(InputArray points, OutputArray _normals) const override; + void fetchPointsNormals(OutputArray points, OutputArray normals) const override; + + void reset() override; + size_t getTotalVolumeUnits() const override { return volumeUnits.size(); } + int getVisibleBlocks(int currFrameId, int frameThreshold) const override; + + //! Return the voxel given the voxel index in the universal volume (1 unit = 1 voxel_length) + TsdfVoxel at(const Vec3i& volumeIdx) const; + + //! Return the voxel given the point in volume coordinate system i.e., (metric scale 1 unit = + //! 1m) + virtual TsdfVoxel at(const cv::Point3f& point) const; + virtual TsdfVoxel _at(const cv::Vec3i& volumeIdx, int indx) const; + + TsdfVoxel atVolumeUnit(const Vec3i& point, const Vec3i& volumeUnitIdx, VolumeUnitIndexes::const_iterator it) const; + + float interpolateVoxelPoint(const Point3f& point) const; + float interpolateVoxel(const cv::Point3f& point) const; + Point3f getNormalVoxel(const cv::Point3f& p) const; + + //! Utility functions for coordinate transformations + Vec3i volumeToVolumeUnitIdx(const Point3f& point) const; + Point3f volumeUnitIdxToVolume(const Vec3i& volumeUnitIdx) const; + + Point3f voxelCoordToVolume(const Vec3i& voxelIdx) const; + Vec3i volumeToVoxelCoord(const Point3f& point) const; + +public: + Vec6f frameParams; + Mat pixNorms; + VolumeUnitIndexes volumeUnits; + cv::Mat volUnitsData; + int lastVolIndex; +}; + + +HashTSDFVolumeCPU::HashTSDFVolumeCPU(float _voxelSize, const Matx44f& _pose, float _raycastStepFactor, float _truncDist, + int _maxWeight, float _truncateThreshold, int _volumeUnitRes, bool _zFirstMemOrder) + :HashTSDFVolume(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, _truncateThreshold, _volumeUnitRes, + _zFirstMemOrder) +{ reset(); } HashTSDFVolumeCPU::HashTSDFVolumeCPU(const VolumeParams& _params, bool _zFirstMemOrder) - : HashTSDFVolume(_params.voxelSize, _params.pose.matrix, _params.raycastStepFactor, _params.tsdfTruncDist, _params.maxWeight, + : HashTSDFVolumeCPU(_params.voxelSize, _params.pose.matrix, _params.raycastStepFactor, _params.tsdfTruncDist, _params.maxWeight, _params.depthTruncThreshold, _params.unitResolution, _zFirstMemOrder) { } + // zero volume, leave rest params the same void HashTSDFVolumeCPU::reset() { CV_TRACE_FUNCTION(); lastVolIndex = 0; volUnitsData = cv::Mat(VOLUMES_SIZE, volumeUnitResolution * volumeUnitResolution * volumeUnitResolution, rawType()); + frameParams = Vec6f(); + pixNorms = Mat(); + volumeUnits = VolumeUnitIndexes(); } void HashTSDFVolumeCPU::integrate(InputArray _depth, float depthFactor, const Matx44f& cameraPose, const Intr& intrinsics, const int frameId) @@ -81,7 +175,7 @@ void HashTSDFVolumeCPU::integrate(InputArray _depth, float depthFactor, const Ma Depth depth = _depth.getMat(); //! Compute volumes to be allocated - const int depthStride = int(log2(volumeUnitResolution)); + const int depthStride = volumeUnitDegree; const float invDepthFactor = 1.f / depthFactor; const Intr::Reprojector reproj(intrinsics.makeReprojector()); const Affine3f cam2vol(pose.inv() * Affine3f(cameraPose)); @@ -111,7 +205,7 @@ void HashTSDFVolumeCPU::integrate(InputArray _depth, float depthFactor, const Ma for (int k = lower_bound[2]; k <= upper_bound[2]; k++) { const Vec3i tsdf_idx = Vec3i(i, j, k); - if (!localAccessVolUnits.count(tsdf_idx)) + if (localAccessVolUnits.count(tsdf_idx) <= 0 && this->volumeUnits.count(tsdf_idx) <= 0) { //! This volume unit will definitely be required for current integration localAccessVolUnits.emplace(tsdf_idx); @@ -124,10 +218,9 @@ void HashTSDFVolumeCPU::integrate(InputArray _depth, float depthFactor, const Ma for (const auto& tsdf_idx : localAccessVolUnits) { //! If the insert into the global set passes - if (!this->volumeUnits.count(tsdf_idx)) + if (!newIndices.count(tsdf_idx)) { // Volume allocation can be performed outside of the lock - this->volumeUnits.emplace(tsdf_idx, VolumeUnit()); newIndices.emplace(tsdf_idx); } } @@ -138,12 +231,13 @@ void HashTSDFVolumeCPU::integrate(InputArray _depth, float depthFactor, const Ma //! Perform the allocation for (auto idx : newIndices) { - VolumeUnit& vu = volumeUnits[idx]; + VolumeUnit& vu = this->volumeUnits.emplace(idx, VolumeUnit()).first->second; + Matx44f subvolumePose = pose.translate(volumeUnitIdxToVolume(idx)).matrix; vu.pose = subvolumePose; vu.index = lastVolIndex; lastVolIndex++; - if (lastVolIndex > VolumeIndex(volUnitsData.size().height)) + if (lastVolIndex > int(volUnitsData.size().height)) { volUnitsData.resize((lastVolIndex - 1) * 2); } @@ -175,14 +269,14 @@ void HashTSDFVolumeCPU::integrate(InputArray _depth, float depthFactor, const Ma Vec3i tsdf_idx = totalVolUnits[i]; VolumeUnitIndexes::iterator it = volumeUnits.find(tsdf_idx); if (it == volumeUnits.end()) - return; + continue; Point3f volumeUnitPos = volumeUnitIdxToVolume(it->first); Point3f volUnitInCamSpace = vol2cam * volumeUnitPos; if (volUnitInCamSpace.z < 0 || volUnitInCamSpace.z > truncateThreshold) { it->second.isActive = false; - return; + continue; } Point2f cameraPoint = proj(volUnitInCamSpace); if (cameraPoint.x >= 0 && cameraPoint.y >= 0 && cameraPoint.x < depth.cols && cameraPoint.y < depth.rows) @@ -250,17 +344,14 @@ cv::Vec3i HashTSDFVolumeCPU::volumeToVoxelCoord(const cv::Point3f& point) const cvFloor(point.z * voxelSizeInv)); } -inline TsdfVoxel HashTSDFVolumeCPU::_at(const cv::Vec3i& volumeIdx, VolumeIndex indx) const +inline TsdfVoxel HashTSDFVolumeCPU::_at(const cv::Vec3i& volumeIdx, int indx) const { //! Out of bounds if ((volumeIdx[0] >= volumeUnitResolution || volumeIdx[0] < 0) || (volumeIdx[1] >= volumeUnitResolution || volumeIdx[1] < 0) || (volumeIdx[2] >= volumeUnitResolution || volumeIdx[2] < 0)) { - TsdfVoxel dummy; - dummy.tsdf = floatToTsdf(1.0f); - dummy.weight = 0; - return dummy; + return TsdfVoxel(floatToTsdf(1.f), 0); } const TsdfVoxel* volData = volUnitsData.ptr(indx); @@ -270,25 +361,21 @@ inline TsdfVoxel HashTSDFVolumeCPU::_at(const cv::Vec3i& volumeIdx, VolumeIndex } inline TsdfVoxel HashTSDFVolumeCPU::at(const cv::Vec3i& volumeIdx) const - { - Vec3i volumeUnitIdx = Vec3i(cvFloor(volumeIdx[0] / volumeUnitResolution), - cvFloor(volumeIdx[1] / volumeUnitResolution), - cvFloor(volumeIdx[2] / volumeUnitResolution)); + Vec3i volumeUnitIdx = Vec3i(volumeIdx[0] >> volumeUnitDegree, + volumeIdx[1] >> volumeUnitDegree, + volumeIdx[2] >> volumeUnitDegree); VolumeUnitIndexes::const_iterator it = volumeUnits.find(volumeUnitIdx); if (it == volumeUnits.end()) { - TsdfVoxel dummy; - dummy.tsdf = floatToTsdf(1.f); - dummy.weight = 0; - return dummy; + return TsdfVoxel(floatToTsdf(1.f), 0); } - cv::Vec3i volUnitLocalIdx = volumeIdx - cv::Vec3i(volumeUnitIdx[0] * volumeUnitResolution, - volumeUnitIdx[1] * volumeUnitResolution, - volumeUnitIdx[2] * volumeUnitResolution); + cv::Vec3i volUnitLocalIdx = volumeIdx - cv::Vec3i(volumeUnitIdx[0] << volumeUnitDegree, + volumeUnitIdx[1] << volumeUnitDegree, + volumeUnitIdx[2] << volumeUnitDegree); volUnitLocalIdx = cv::Vec3i(abs(volUnitLocalIdx[0]), abs(volUnitLocalIdx[1]), abs(volUnitLocalIdx[2])); @@ -298,15 +385,12 @@ inline TsdfVoxel HashTSDFVolumeCPU::at(const cv::Vec3i& volumeIdx) const TsdfVoxel HashTSDFVolumeCPU::at(const Point3f& point) const { - cv::Vec3i volumeUnitIdx = volumeToVolumeUnitIdx(point); + cv::Vec3i volumeUnitIdx = volumeToVolumeUnitIdx(point); VolumeUnitIndexes::const_iterator it = volumeUnits.find(volumeUnitIdx); if (it == volumeUnits.end()) { - TsdfVoxel dummy; - dummy.tsdf = floatToTsdf(1.f); - dummy.weight = 0; - return dummy; + return TsdfVoxel(floatToTsdf(1.f), 0); } cv::Point3f volumeUnitPos = volumeUnitIdxToVolume(volumeUnitIdx); @@ -316,32 +400,15 @@ TsdfVoxel HashTSDFVolumeCPU::at(const Point3f& point) const return _at(volUnitLocalIdx, it->second.index); } -static inline Vec3i voxelToVolumeUnitIdx(const Vec3i& pt, const int vuRes) -{ - if (!(vuRes & (vuRes - 1))) - { - // vuRes is a power of 2, let's get this power - const int p2 = trailingZeros32(vuRes); - return Vec3i(pt[0] >> p2, pt[1] >> p2, pt[2] >> p2); - } - else - { - return Vec3i(cvFloor(float(pt[0]) / vuRes), - cvFloor(float(pt[1]) / vuRes), - cvFloor(float(pt[2]) / vuRes)); - } -} - TsdfVoxel HashTSDFVolumeCPU::atVolumeUnit(const Vec3i& point, const Vec3i& volumeUnitIdx, VolumeUnitIndexes::const_iterator it) const { if (it == volumeUnits.end()) { - TsdfVoxel dummy; - dummy.tsdf = floatToTsdf(1.f); - dummy.weight = 0; - return dummy; + return TsdfVoxel(floatToTsdf(1.f), 0); } - Vec3i volUnitLocalIdx = point - volumeUnitIdx * volumeUnitResolution; + Vec3i volUnitLocalIdx = point - Vec3i(volumeUnitIdx[0] << volumeUnitDegree, + volumeUnitIdx[1] << volumeUnitDegree, + volumeUnitIdx[2] << volumeUnitDegree); // expanding at(), removing bounds check const TsdfVoxel* volData = volUnitsData.ptr(it->second.index); @@ -349,8 +416,6 @@ TsdfVoxel HashTSDFVolumeCPU::atVolumeUnit(const Vec3i& point, const Vec3i& volum return volData[coordBase]; } - - #if USE_INTRINSICS inline float interpolate(float tx, float ty, float tz, float vx[8]) { @@ -413,7 +478,7 @@ float HashTSDFVolumeCPU::interpolateVoxelPoint(const Point3f& point) const { Vec3i pt = iv + neighbourCoords[i]; - Vec3i volumeUnitIdx = voxelToVolumeUnitIdx(pt, volumeUnitResolution); + Vec3i volumeUnitIdx = Vec3i(pt[0] >> volumeUnitDegree, pt[1] >> volumeUnitDegree, pt[2] >> volumeUnitDegree); int dictIdx = (volumeUnitIdx[0] & 1) + (volumeUnitIdx[1] & 1) * 2 + (volumeUnitIdx[2] & 1) * 4; auto it = iterMap[dictIdx]; if (!queried[dictIdx]) @@ -475,7 +540,7 @@ Point3f HashTSDFVolumeCPU::getNormalVoxel(const Point3f &point) const { Vec3i pt = iptVox + offsets[i]; - Vec3i volumeUnitIdx = voxelToVolumeUnitIdx(pt, volumeUnitResolution); + Vec3i volumeUnitIdx = Vec3i(pt[0] >> volumeUnitDegree, pt[1] >> volumeUnitDegree, pt[2] >> volumeUnitDegree); int dictIdx = (volumeUnitIdx[0] & 1) + (volumeUnitIdx[1] & 1) * 2 + (volumeUnitIdx[2] & 1) * 4; auto it = iterMap[dictIdx]; @@ -625,7 +690,6 @@ void HashTSDFVolumeCPU::raycast(const Matx44f& cameraPose, const kinfu::Intr& in cv::Vec3i(std::numeric_limits::min(), std::numeric_limits::min(), std::numeric_limits::min()); - float tprev = tcurr; float prevTsdf = volume.truncDist; Ptr currVolumeUnit; @@ -634,7 +698,6 @@ void HashTSDFVolumeCPU::raycast(const Matx44f& cameraPose, const kinfu::Intr& in Point3f currRayPos = orig + tcurr * rayDirV; cv::Vec3i currVolumeUnitIdx = volume.volumeToVolumeUnitIdx(currRayPos); - VolumeUnitIndexes::const_iterator it = volume.volumeUnits.find(currVolumeUnitIdx); float currTsdf = prevTsdf; @@ -650,7 +713,6 @@ void HashTSDFVolumeCPU::raycast(const Matx44f& cameraPose, const kinfu::Intr& in volume.volumeUnitIdxToVolume(currVolumeUnitIdx); volUnitLocalIdx = volume.volumeToVoxelCoord(currRayPos - currVolUnitPos); - //! TODO: Figure out voxel interpolation TsdfVoxel currVoxel = _at(volUnitLocalIdx, it->second.index); currTsdf = tsdfToFloat(currVoxel.tsdf); @@ -708,17 +770,13 @@ void HashTSDFVolumeCPU::fetchPointsNormals(OutputArray _points, OutputArray _nor bool needNormals(_normals.needed()); Mutex mutex; - auto HashFetchPointsNormalsInvoker = [&](const Range& range) { - - std::vector points, normals; for (int i = range.start; i < range.end; i++) { cv::Vec3i tsdf_idx = totalVolUnits[i]; - VolumeUnitIndexes::const_iterator it = volume.volumeUnits.find(tsdf_idx); Point3f base_point = volume.volumeUnitIdxToVolume(tsdf_idx); if (it != volume.volumeUnits.end()) @@ -753,8 +811,6 @@ void HashTSDFVolumeCPU::fetchPointsNormals(OutputArray _points, OutputArray _nor parallel_for_(fetchRange, HashFetchPointsNormalsInvoker, nstripes); - - std::vector points, normals; for (size_t i = 0; i < pVecs.size(); i++) { @@ -817,5 +873,920 @@ int HashTSDFVolumeCPU::getVisibleBlocks(int currFrameId, int frameThreshold) con return numVisibleBlocks; } + +///////// GPU implementation ///////// + +#ifdef HAVE_OPENCL + +class HashTSDFVolumeGPU : public HashTSDFVolume +{ +public: + HashTSDFVolumeGPU(float _voxelSize, const Matx44f& _pose, float _raycastStepFactor, float _truncDist, int _maxWeight, + float _truncateThreshold, int _volumeUnitRes, bool zFirstMemOrder = false); + + HashTSDFVolumeGPU(const VolumeParams& _volumeParams, bool zFirstMemOrder = false); + + void reset() override; + + void integrateAllVolumeUnitsGPU(const UMat& depth, float depthFactor, const Matx44f& cameraPose, const Intr& intrinsics); + + void allocateVolumeUnits(const UMat& depth, float depthFactor, const Matx44f& cameraPose, const Intr& intrinsics); + + void markActive(const Matx44f& cameraPose, const Intr& intrinsics, const Size frameSz, const int frameId); + + void integrate(InputArray _depth, float depthFactor, const Matx44f& cameraPose, const kinfu::Intr& intrinsics, + const int frameId = 0) override; + void raycast(const Matx44f& cameraPose, const kinfu::Intr& intrinsics, const Size& frameSize, OutputArray points, + OutputArray normals) const override; + + + void fetchNormals(InputArray points, OutputArray _normals) const override; + void fetchPointsNormals(OutputArray points, OutputArray normals) const override; + + size_t getTotalVolumeUnits() const override { return size_t(hashTable.last); } + int getVisibleBlocks(int currFrameId, int frameThreshold) const override; + + + + //! Return the voxel given the point in volume coordinate system i.e., (metric scale 1 unit = + //! 1m) + virtual TsdfVoxel new_at(const cv::Vec3i& volumeIdx, int indx) const; + TsdfVoxel new_atVolumeUnit(const Vec3i& point, const Vec3i& volumeUnitIdx, int indx) const; + + + float interpolateVoxelPoint(const Point3f& point) const; + float interpolateVoxel(const cv::Point3f& point) const; + Point3f getNormalVoxel(const cv::Point3f& p) const; + + //! Utility functions for coordinate transformations + Vec3i volumeToVolumeUnitIdx(const Point3f& point) const; + Point3f volumeUnitIdxToVolume(const Vec3i& volumeUnitIdx) const; + + Point3f voxelCoordToVolume(const Vec3i& voxelIdx) const; + Vec3i volumeToVoxelCoord(const Point3f& point) const; + +public: + Vec6f frameParams; + int bufferSizeDegree; + + // per-volume-unit data + cv::UMat lastVisibleIndices; + + cv::UMat isActiveFlags; + + cv::UMat volUnitsData; + //TODO: remove it when there's no CPU parts + cv::Mat volUnitsDataCopy; + + cv::UMat pixNorms; + + //TODO: move indexes.volumes to GPU + CustomHashSet hashTable; +}; + +HashTSDFVolumeGPU::HashTSDFVolumeGPU(float _voxelSize, const Matx44f& _pose, float _raycastStepFactor, float _truncDist, int _maxWeight, + float _truncateThreshold, int _volumeUnitRes, bool _zFirstMemOrder) + :HashTSDFVolume(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, _truncateThreshold, _volumeUnitRes, _zFirstMemOrder) +{ + if (truncDist >= volumeUnitSize) + { + std::cout << "truncDist exceeds volume unit size, allocation may work incorrectly" << std::endl; + } + + reset(); +} + +HashTSDFVolumeGPU::HashTSDFVolumeGPU(const VolumeParams & _params, bool _zFirstMemOrder) + : HashTSDFVolumeGPU(_params.voxelSize, _params.pose.matrix, _params.raycastStepFactor, _params.tsdfTruncDist, _params.maxWeight, + _params.depthTruncThreshold, _params.unitResolution, _zFirstMemOrder) +{ +} + +// zero volume, leave rest params the same +void HashTSDFVolumeGPU::reset() +{ + CV_TRACE_FUNCTION(); + + bufferSizeDegree = 15; + int buff_lvl = (int) (1 << bufferSizeDegree); + + int volCubed = volumeUnitResolution * volumeUnitResolution * volumeUnitResolution; + volUnitsDataCopy = cv::Mat(buff_lvl, volCubed, rawType()); + + volUnitsData = cv::UMat(buff_lvl, volCubed, CV_8UC2); + + lastVisibleIndices = cv::UMat(buff_lvl, 1, CV_32S); + + isActiveFlags = cv::UMat(buff_lvl, 1, CV_8U); + + hashTable = CustomHashSet(); + + frameParams = Vec6f(); + pixNorms = UMat(); +} + + +void HashTSDFVolumeGPU::integrateAllVolumeUnitsGPU(const UMat& depth, float depthFactor, const Matx44f& cameraPose, const Intr& intrinsics) +{ + CV_TRACE_FUNCTION(); + CV_Assert(!depth.empty()); + + String errorStr; + String name = "integrateAllVolumeUnits"; + ocl::ProgramSource source = ocl::rgbd::hash_tsdf_oclsrc; + String options = "-cl-mad-enable"; + ocl::Kernel k; + k.create(name.c_str(), source, options, &errorStr); + + if (k.empty()) + throw std::runtime_error("Failed to create kernel: " + errorStr); + + float dfac = 1.f / depthFactor; + Vec2f fxy(intrinsics.fx, intrinsics.fy), cxy(intrinsics.cx, intrinsics.cy); + Matx44f vol2camMatrix = (Affine3f(cameraPose).inv() * pose).matrix; + Matx44f camInvMatrix = Affine3f(cameraPose).inv().matrix; + + UMat hashesGpu = Mat(hashTable.hashes, false).getUMat(ACCESS_READ); + UMat hashDataGpu = Mat(hashTable.data, false).getUMat(ACCESS_READ); + + k.args(ocl::KernelArg::ReadOnly(depth), + ocl::KernelArg::PtrReadOnly(hashesGpu), + ocl::KernelArg::PtrReadOnly(hashDataGpu), + ocl::KernelArg::ReadWrite(volUnitsData), + ocl::KernelArg::ReadOnly(pixNorms), + ocl::KernelArg::ReadOnly(isActiveFlags), + vol2camMatrix, + camInvMatrix, + voxelSize, + volumeUnitResolution, + volStrides.val, + fxy.val, + cxy.val, + dfac, + truncDist, + int(maxWeight) + ); + + int resol = volumeUnitResolution; + size_t globalSize[3]; + globalSize[0] = (size_t)resol; + globalSize[1] = (size_t)resol; + globalSize[2] = (size_t)hashTable.last; // num of volume units + + if (!k.run(3, globalSize, NULL, true)) + throw std::runtime_error("Failed to run kernel"); +} + + +void HashTSDFVolumeGPU::allocateVolumeUnits(const UMat& _depth, float depthFactor, const Matx44f& cameraPose, const Intr& intrinsics) +{ + constexpr int pixCapacity = 16; + typedef std::array LocalVolUnits; + + Depth depth = _depth.getMat(ACCESS_READ); + + //! Compute volumes to be allocated + const int depthStride = volumeUnitDegree; + const float invDepthFactor = 1.f / depthFactor; + const Intr::Reprojector reproj(intrinsics.makeReprojector()); + const Affine3f cam2vol(pose.inv() * Affine3f(cameraPose)); + const Point3f truncPt(truncDist, truncDist, truncDist); + Mutex mutex; + + // for new indices + CustomHashSet thm; + + auto fillLocalAcessVolUnits = [&](const Range& xrange, const Range& yrange, CustomHashSet& ghm) + { + for (int y = yrange.start; y < yrange.end; y += depthStride) + { + const depthType* depthRow = depth[y]; + for (int x = xrange.start; x < xrange.end; x += depthStride) + { + depthType z = depthRow[x] * invDepthFactor; + if (z <= 0 || z > this->truncateThreshold) + continue; + Point3f camPoint = reproj(Point3f((float)x, (float)y, z)); + Point3f volPoint = cam2vol * camPoint; + //! Find accessed TSDF volume unit for valid 3D vertex + Vec3i lower_bound = this->volumeToVolumeUnitIdx(volPoint - truncPt); + Vec3i upper_bound = this->volumeToVolumeUnitIdx(volPoint + truncPt); + + int pixLocalCounter = 0; + LocalVolUnits pixLocalVolUnits; + for (int i = lower_bound[0]; i <= upper_bound[0]; i++) + for (int j = lower_bound[1]; j <= upper_bound[1]; j++) + for (int k = lower_bound[2]; k <= upper_bound[2]; k++) + { + const Vec3i tsdf_idx = Vec3i(i, j, k); + + if (hashTable.find(tsdf_idx) < 0) + { + bool found = false; + for (int c = 0; c < pixLocalCounter; c++) + { + if (pixLocalVolUnits[c] == tsdf_idx) + { + found = true; break; + } + } + if (!found) + { + pixLocalVolUnits[pixLocalCounter++] = tsdf_idx; + if (pixLocalCounter >= pixCapacity) + { + return; + } + } + } + } + + // lock localAccessVolUnits somehow + for (int i = 0; i < pixLocalCounter; i++) + { + Vec3i idx = pixLocalVolUnits[i]; + if (!ghm.insert(idx)) + { + //return; + } + } + // unlock + } + } + }; + + Rect dim(0, 0, depth.cols, depth.rows); + Size gsz(32, 32); + Size gg(divUp(dim.width, gsz.width), divUp(dim.height, gsz.height)); + + bool needReallocation = false; + auto allocateLambda = [&](const Range& r) + { + + for (int yg = r.start; yg < r.end; yg++) + { + for (int xg = 0; xg < gg.width; xg++) + { + Rect gr(xg * gsz.width, yg * gsz.height, (xg + 1) * gsz.width, (yg + 1) * gsz.height); + gr = gr & dim; + Range xr(gr.tl().x, gr.br().x), yr(gr.tl().y, gr.br().y); + + CustomHashSet ghm; + + fillLocalAcessVolUnits(xr, yr, ghm); + + if (ghm.last) + { + std::lock_guard al(mutex); + + for (int i = 0; i < ghm.last; i++) + { + Vec4i node = ghm.data[i]; + Vec3i idx(node[0], node[1], node[2]); + + //TODO: 1. add to separate hash map instead, then merge on GPU side + + int result = thm.insert(idx); + if (!result) + { + needReallocation = true; + return; + } + } + } + } + } + + }; + + do + { + if (needReallocation) + { + thm.capacity *= 2; + thm.data.resize(thm.capacity); + + needReallocation = false; + } + + parallel_for_(Range(0, gg.height), allocateLambda); + } while (needReallocation); + + + auto pushToGlobal = [](const CustomHashSet _thm, CustomHashSet& _globalHashMap, + bool& _needReallocation, Mutex& _mutex) + { + for (int i = 0; i < _thm.last; i++) + { + Vec4i node = _thm.data[i]; + Vec3i idx(node[0], node[1], node[2]); + + std::lock_guard al(_mutex); + + int result = _globalHashMap.insert(idx); + if (result == 0) + { + _needReallocation = true; + return; + } + } + }; + + needReallocation = false; + do + { + if (needReallocation) + { + hashTable.capacity *= 2; + hashTable.data.resize(hashTable.capacity); + + needReallocation = false; + } + + pushToGlobal(thm, hashTable, needReallocation, mutex); + } while (needReallocation); +} + + +void HashTSDFVolumeGPU::markActive(const Matx44f& cameraPose, const Intr& intrinsics, const Size frameSz, const int frameId) +{ + //! Mark volumes in the camera frustum as active + String errorStr; + String name = "markActive"; + ocl::ProgramSource source = ocl::rgbd::hash_tsdf_oclsrc; + String options = "-cl-mad-enable"; + ocl::Kernel k; + k.create(name.c_str(), source, options, &errorStr); + + if (k.empty()) + throw std::runtime_error("Failed to create kernel: " + errorStr); + + const Affine3f vol2cam(Affine3f(cameraPose.inv()) * pose); + const Intr::Projector proj(intrinsics.makeProjector()); + Vec2f fxy(proj.fx, proj.fy), cxy(proj.cx, proj.cy); + + UMat hashDataGpu = Mat(hashTable.data, false).getUMat(ACCESS_READ); + + k.args( + ocl::KernelArg::PtrReadOnly(hashDataGpu), + ocl::KernelArg::WriteOnly(isActiveFlags), + ocl::KernelArg::WriteOnly(lastVisibleIndices), + vol2cam.matrix, + fxy, + cxy, + frameSz, + volumeUnitSize, + hashTable.last, + truncateThreshold, + frameId + ); + + size_t globalSize[1] = { (size_t)hashTable.last }; + if (!k.run(1, globalSize, nullptr, true)) + throw std::runtime_error("Failed to run kernel"); +} + + +void HashTSDFVolumeGPU::integrate(InputArray _depth, float depthFactor, const Matx44f& cameraPose, const Intr& intrinsics, const int frameId) +{ + CV_TRACE_FUNCTION(); + + CV_Assert(_depth.type() == DEPTH_TYPE); + UMat depth = _depth.getUMat(); + + // Save length to fill new data in ranges + int sizeBefore = hashTable.last; + allocateVolumeUnits(depth, depthFactor, cameraPose, intrinsics); + int sizeAfter = hashTable.last; + //! Perform the allocation + + // Grow buffers + int buff_lvl = (int)(1 << bufferSizeDegree); + if (sizeAfter >= buff_lvl) + { + bufferSizeDegree = (int)(log2(sizeAfter) + 1); // clz() would be better + int oldBuffSize = buff_lvl; + buff_lvl = (int)pow(2, bufferSizeDegree); + + volUnitsDataCopy.resize(buff_lvl); + + Range oldr(0, oldBuffSize); + int volCubed = volumeUnitResolution * volumeUnitResolution * volumeUnitResolution; + UMat newData(buff_lvl, volCubed, CV_8UC2); + volUnitsData.copyTo(newData.rowRange(oldr)); + volUnitsData = newData; + + UMat newLastVisibleIndices(buff_lvl, 1, CV_32S); + lastVisibleIndices.copyTo(newLastVisibleIndices.rowRange(oldr)); + lastVisibleIndices = newLastVisibleIndices; + + UMat newIsActiveFlags(buff_lvl, 1, CV_8U); + isActiveFlags.copyTo(newIsActiveFlags.rowRange(oldr)); + isActiveFlags = newIsActiveFlags; + } + + // Fill data for new volume units + Range r(sizeBefore, sizeAfter); + if (r.start < r.end) + { + lastVisibleIndices.rowRange(r) = frameId; + isActiveFlags.rowRange(r) = 1; + + TsdfVoxel emptyVoxel(floatToTsdf(0.0f), 0); + volUnitsData.rowRange(r) = Vec2b((uchar)(emptyVoxel.tsdf), (uchar)(emptyVoxel.weight)); + } + + //! Mark volumes in the camera frustum as active + markActive(cameraPose, intrinsics, depth.size(), frameId); + + Vec6f newParams((float)depth.rows, (float)depth.cols, + intrinsics.fx, intrinsics.fy, + intrinsics.cx, intrinsics.cy); + if (!(frameParams == newParams)) + { + frameParams = newParams; + pixNorms = preCalculationPixNormGPU(depth, intrinsics); + } + + //! Integrate the correct volumeUnits + integrateAllVolumeUnitsGPU(depth, depthFactor, cameraPose, intrinsics); +} + + +//TODO: remove these functions when everything is done on GPU +cv::Vec3i HashTSDFVolumeGPU::volumeToVolumeUnitIdx(const cv::Point3f& p) const +{ + return cv::Vec3i(cvFloor(p.x / volumeUnitSize), cvFloor(p.y / volumeUnitSize), + cvFloor(p.z / volumeUnitSize)); +} + +cv::Point3f HashTSDFVolumeGPU::volumeUnitIdxToVolume(const cv::Vec3i& volumeUnitIdx) const +{ + return cv::Point3f(volumeUnitIdx[0] * volumeUnitSize, volumeUnitIdx[1] * volumeUnitSize, + volumeUnitIdx[2] * volumeUnitSize); +} + +cv::Point3f HashTSDFVolumeGPU::voxelCoordToVolume(const cv::Vec3i& voxelIdx) const +{ + return cv::Point3f(voxelIdx[0] * voxelSize, voxelIdx[1] * voxelSize, voxelIdx[2] * voxelSize); +} + +cv::Vec3i HashTSDFVolumeGPU::volumeToVoxelCoord(const cv::Point3f& point) const +{ + return cv::Vec3i(cvFloor(point.x * voxelSizeInv), cvFloor(point.y * voxelSizeInv), + cvFloor(point.z * voxelSizeInv)); +} + +inline TsdfVoxel HashTSDFVolumeGPU::new_at(const cv::Vec3i& volumeIdx, int indx) const +{ + //! Out of bounds + if ((volumeIdx[0] >= volumeUnitResolution || volumeIdx[0] < 0) || + (volumeIdx[1] >= volumeUnitResolution || volumeIdx[1] < 0) || + (volumeIdx[2] >= volumeUnitResolution || volumeIdx[2] < 0)) + { + return TsdfVoxel(floatToTsdf(1.0f), 0); + } + + const TsdfVoxel* volData = volUnitsDataCopy.ptr(indx); + int coordBase = + volumeIdx[0] * volStrides[0] + + volumeIdx[1] * volStrides[1] + + volumeIdx[2] * volStrides[2]; + return volData[coordBase]; +} + +TsdfVoxel HashTSDFVolumeGPU::new_atVolumeUnit(const Vec3i& point, const Vec3i& volumeUnitIdx, int indx) const +{ + if (indx < 0) + { + return TsdfVoxel(floatToTsdf(1.f), 0); + } + Vec3i volUnitLocalIdx = point - Vec3i(volumeUnitIdx[0] << volumeUnitDegree, + volumeUnitIdx[1] << volumeUnitDegree, + volumeUnitIdx[2] << volumeUnitDegree); + + // expanding at(), removing bounds check + const TsdfVoxel* volData = volUnitsDataCopy.ptr(indx); + int coordBase = volUnitLocalIdx[0] * volStrides[0] + + volUnitLocalIdx[1] * volStrides[1] + + volUnitLocalIdx[2] * volStrides[2]; + return volData[coordBase]; +} + +float HashTSDFVolumeGPU::interpolateVoxelPoint(const Point3f& point) const +{ + const Vec3i local_neighbourCoords[] = { {0, 0, 0}, {0, 0, 1}, {0, 1, 0}, {0, 1, 1}, + {1, 0, 0}, {1, 0, 1}, {1, 1, 0}, {1, 1, 1} }; + + // A small hash table to reduce a number of find() calls + // -2 and lower means not queried yet + // -1 means not found + // 0+ means found + int iterMap[8]; + for (int i = 0; i < 8; i++) + { + iterMap[i] = -2; + } + + int ix = cvFloor(point.x); + int iy = cvFloor(point.y); + int iz = cvFloor(point.z); + + float tx = point.x - ix; + float ty = point.y - iy; + float tz = point.z - iz; + + Vec3i iv(ix, iy, iz); + float vx[8]; + for (int i = 0; i < 8; i++) + { + Vec3i pt = iv + local_neighbourCoords[i]; + + Vec3i volumeUnitIdx = Vec3i(pt[0] >> volumeUnitDegree, pt[1] >> volumeUnitDegree, pt[2] >> volumeUnitDegree); + int dictIdx = (volumeUnitIdx[0] & 1) + (volumeUnitIdx[1] & 1) * 2 + (volumeUnitIdx[2] & 1) * 4; + auto it = iterMap[dictIdx]; + if (it < -1) + { + it = hashTable.find(volumeUnitIdx); + iterMap[dictIdx] = it; + } + + vx[i] = new_atVolumeUnit(pt, volumeUnitIdx, it).tsdf; + } + + return interpolate(tx, ty, tz, vx); +} + +inline float HashTSDFVolumeGPU::interpolateVoxel(const cv::Point3f& point) const +{ + return interpolateVoxelPoint(point * voxelSizeInv); +} + +Point3f HashTSDFVolumeGPU::getNormalVoxel(const Point3f& point) const +{ + Vec3f normal = Vec3f(0, 0, 0); + + Point3f ptVox = point * voxelSizeInv; + Vec3i iptVox(cvFloor(ptVox.x), cvFloor(ptVox.y), cvFloor(ptVox.z)); + + // A small hash table to reduce a number of find() calls + // -2 and lower means not queried yet + // -1 means not found + // 0+ means found + int iterMap[8]; + for (int i = 0; i < 8; i++) + { + iterMap[i] = -2; + } + +#if !USE_INTERPOLATION_IN_GETNORMAL + const Vec3i offsets[] = { { 1, 0, 0}, {-1, 0, 0}, { 0, 1, 0}, // 0-3 + { 0, -1, 0}, { 0, 0, 1}, { 0, 0, -1} // 4-7 + }; + const int nVals = 6; + +#else + const Vec3i offsets[] = { { 0, 0, 0}, { 0, 0, 1}, { 0, 1, 0}, { 0, 1, 1}, // 0-3 + { 1, 0, 0}, { 1, 0, 1}, { 1, 1, 0}, { 1, 1, 1}, // 4-7 + {-1, 0, 0}, {-1, 0, 1}, {-1, 1, 0}, {-1, 1, 1}, // 8-11 + { 2, 0, 0}, { 2, 0, 1}, { 2, 1, 0}, { 2, 1, 1}, // 12-15 + { 0, -1, 0}, { 0, -1, 1}, { 1, -1, 0}, { 1, -1, 1}, // 16-19 + { 0, 2, 0}, { 0, 2, 1}, { 1, 2, 0}, { 1, 2, 1}, // 20-23 + { 0, 0, -1}, { 0, 1, -1}, { 1, 0, -1}, { 1, 1, -1}, // 24-27 + { 0, 0, 2}, { 0, 1, 2}, { 1, 0, 2}, { 1, 1, 2}, // 28-31 + }; + const int nVals = 32; +#endif + + float vals[nVals]; + for (int i = 0; i < nVals; i++) + { + Vec3i pt = iptVox + offsets[i]; + + Vec3i volumeUnitIdx = Vec3i(pt[0] >> volumeUnitDegree, pt[1] >> volumeUnitDegree, pt[2] >> volumeUnitDegree); + + int dictIdx = (volumeUnitIdx[0] & 1) + (volumeUnitIdx[1] & 1) * 2 + (volumeUnitIdx[2] & 1) * 4; + auto it = iterMap[dictIdx]; + if (it < -1) + { + it = hashTable.find(volumeUnitIdx); + iterMap[dictIdx] = it; + } + + vals[i] = tsdfToFloat(new_atVolumeUnit(pt, volumeUnitIdx, it).tsdf); + } + +#if !USE_INTERPOLATION_IN_GETNORMAL + for (int c = 0; c < 3; c++) + { + normal[c] = vals[c * 2] - vals[c * 2 + 1]; + } +#else + + float cxv[8], cyv[8], czv[8]; + + // How these numbers were obtained: + // 1. Take the basic interpolation sequence: + // 000, 001, 010, 011, 100, 101, 110, 111 + // where each digit corresponds to shift by x, y, z axis respectively. + // 2. Add +1 for next or -1 for prev to each coordinate to corresponding axis + // 3. Search corresponding values in offsets + const int idxxp[8] = { 8, 9, 10, 11, 0, 1, 2, 3 }; + const int idxxn[8] = { 4, 5, 6, 7, 12, 13, 14, 15 }; + const int idxyp[8] = { 16, 17, 0, 1, 18, 19, 4, 5 }; + const int idxyn[8] = { 2, 3, 20, 21, 6, 7, 22, 23 }; + const int idxzp[8] = { 24, 0, 25, 2, 26, 4, 27, 6 }; + const int idxzn[8] = { 1, 28, 3, 29, 5, 30, 7, 31 }; + +#if !USE_INTRINSICS + for (int i = 0; i < 8; i++) + { + cxv[i] = vals[idxxn[i]] - vals[idxxp[i]]; + cyv[i] = vals[idxyn[i]] - vals[idxyp[i]]; + czv[i] = vals[idxzn[i]] - vals[idxzp[i]]; + } +#else + +# if CV_SIMD >= 32 + v_float32x8 cxp = v_lut(vals, idxxp); + v_float32x8 cxn = v_lut(vals, idxxn); + + v_float32x8 cyp = v_lut(vals, idxyp); + v_float32x8 cyn = v_lut(vals, idxyn); + + v_float32x8 czp = v_lut(vals, idxzp); + v_float32x8 czn = v_lut(vals, idxzn); + + v_float32x8 vcxv = cxn - cxp; + v_float32x8 vcyv = cyn - cyp; + v_float32x8 vczv = czn - czp; + + v_store(cxv, vcxv); + v_store(cyv, vcyv); + v_store(czv, vczv); +# else + v_float32x4 cxp0 = v_lut(vals, idxxp + 0); v_float32x4 cxp1 = v_lut(vals, idxxp + 4); + v_float32x4 cxn0 = v_lut(vals, idxxn + 0); v_float32x4 cxn1 = v_lut(vals, idxxn + 4); + + v_float32x4 cyp0 = v_lut(vals, idxyp + 0); v_float32x4 cyp1 = v_lut(vals, idxyp + 4); + v_float32x4 cyn0 = v_lut(vals, idxyn + 0); v_float32x4 cyn1 = v_lut(vals, idxyn + 4); + + v_float32x4 czp0 = v_lut(vals, idxzp + 0); v_float32x4 czp1 = v_lut(vals, idxzp + 4); + v_float32x4 czn0 = v_lut(vals, idxzn + 0); v_float32x4 czn1 = v_lut(vals, idxzn + 4); + + v_float32x4 cxv0 = cxn0 - cxp0; v_float32x4 cxv1 = cxn1 - cxp1; + v_float32x4 cyv0 = cyn0 - cyp0; v_float32x4 cyv1 = cyn1 - cyp1; + v_float32x4 czv0 = czn0 - czp0; v_float32x4 czv1 = czn1 - czp1; + + v_store(cxv + 0, cxv0); v_store(cxv + 4, cxv1); + v_store(cyv + 0, cyv0); v_store(cyv + 4, cyv1); + v_store(czv + 0, czv0); v_store(czv + 4, czv1); +#endif + +#endif + + float tx = ptVox.x - iptVox[0]; + float ty = ptVox.y - iptVox[1]; + float tz = ptVox.z - iptVox[2]; + + normal[0] = interpolate(tx, ty, tz, cxv); + normal[1] = interpolate(tx, ty, tz, cyv); + normal[2] = interpolate(tx, ty, tz, czv); +#endif + float nv = sqrt(normal[0] * normal[0] + + normal[1] * normal[1] + + normal[2] * normal[2]); + return nv < 0.0001f ? nan3 : normal / nv; +} + + +void HashTSDFVolumeGPU::raycast(const Matx44f& cameraPose, const kinfu::Intr& intrinsics, const Size& frameSize, + OutputArray _points, OutputArray _normals) const +{ + CV_TRACE_FUNCTION(); + CV_Assert(frameSize.area() > 0); + + String errorStr; + String name = "raycast"; + ocl::ProgramSource source = ocl::rgbd::hash_tsdf_oclsrc; + String options = "-cl-mad-enable"; + ocl::Kernel k; + k.create(name.c_str(), source, options, &errorStr); + + if (k.empty()) + throw std::runtime_error("Failed to create kernel: " + errorStr); + + _points.create(frameSize, CV_32FC4); + _normals.create(frameSize, CV_32FC4); + + UMat points = _points.getUMat(); + UMat normals = _normals.getUMat(); + + Intr::Reprojector r = intrinsics.makeReprojector(); + Vec2f finv(r.fxinv, r.fyinv), cxy(r.cx, r.cy); + + Vec4f boxMin, boxMax(volumeUnitSize - voxelSize, + volumeUnitSize - voxelSize, + volumeUnitSize - voxelSize); + + float tstep = truncDist * raycastStepFactor; + + const HashTSDFVolumeGPU& volume(*this); + const Affine3f cam2vol(volume.pose.inv() * Affine3f(cameraPose)); + const Affine3f vol2cam(Affine3f(cameraPose.inv()) * volume.pose); + + Matx44f cam2volRotGPU = cam2vol.matrix; + Matx44f vol2camRotGPU = vol2cam.matrix; + + UMat volPoseGpu = Mat(pose.matrix).getUMat(ACCESS_READ); + UMat invPoseGpu = Mat(pose.inv().matrix).getUMat(ACCESS_READ); + + UMat hashesGpu = Mat(hashTable.hashes, false).getUMat(ACCESS_READ); + UMat hashDataGpu = Mat(hashTable.data, false).getUMat(ACCESS_READ); + + k.args( + ocl::KernelArg::PtrReadOnly(hashesGpu), + ocl::KernelArg::PtrReadOnly(hashDataGpu), + ocl::KernelArg::WriteOnlyNoSize(points), + ocl::KernelArg::WriteOnlyNoSize(normals), + frameSize, + ocl::KernelArg::ReadOnly(volUnitsData), + cam2volRotGPU, + vol2camRotGPU, + float(volume.truncateThreshold), + finv.val, cxy.val, + boxMin.val, boxMax.val, + tstep, + voxelSize, + voxelSizeInv, + volumeUnitSize, + volume.truncDist, + volumeUnitDegree, + volStrides + ); + + size_t globalSize[2]; + globalSize[0] = (size_t)frameSize.width; + globalSize[1] = (size_t)frameSize.height; + + if (!k.run(2, globalSize, NULL, true)) + throw std::runtime_error("Failed to run kernel"); +} + +void HashTSDFVolumeGPU::fetchPointsNormals(OutputArray _points, OutputArray _normals) const +{ + CV_TRACE_FUNCTION(); + + if (_points.needed()) + { + //TODO: remove it when it works w/o CPU code + volUnitsData.copyTo(volUnitsDataCopy); + //TODO: remove it when it works w/o CPU code + //TODO: enable it when it's on GPU + //UMat hashDataGpu(hashMap.capacity, 1, CV_32SC4); + //Mat(hashMap.data, false).copyTo(hashDataGpu); + + std::vector> pVecs, nVecs; + + Range _fetchRange(0, hashTable.last); + + const int nstripes = -1; + + const HashTSDFVolumeGPU& volume(*this); + bool needNormals(_normals.needed()); + Mutex mutex; + + auto _HashFetchPointsNormalsInvoker = [&](const Range& range) + { + std::vector points, normals; + for (int row = range.start; row < range.end; row++) + { + cv::Vec4i idx4 = hashTable.data[row]; + cv::Vec3i idx(idx4[0], idx4[1], idx4[2]); + + Point3f base_point = volume.volumeUnitIdxToVolume(idx); + + std::vector localPoints; + std::vector localNormals; + for (int x = 0; x < volume.volumeUnitResolution; x++) + for (int y = 0; y < volume.volumeUnitResolution; y++) + for (int z = 0; z < volume.volumeUnitResolution; z++) + { + cv::Vec3i voxelIdx(x, y, z); + TsdfVoxel voxel = new_at(voxelIdx, row); + + if (voxel.tsdf != -128 && voxel.weight != 0) + { + Point3f point = base_point + volume.voxelCoordToVolume(voxelIdx); + + localPoints.push_back(toPtype(point)); + if (needNormals) + { + Point3f normal = volume.getNormalVoxel(point); + localNormals.push_back(toPtype(normal)); + } + } + } + + AutoLock al(mutex); + pVecs.push_back(localPoints); + nVecs.push_back(localNormals); + } + }; + + parallel_for_(_fetchRange, _HashFetchPointsNormalsInvoker, nstripes); + + std::vector points, normals; + for (size_t i = 0; i < pVecs.size(); i++) + { + points.insert(points.end(), pVecs[i].begin(), pVecs[i].end()); + normals.insert(normals.end(), nVecs[i].begin(), nVecs[i].end()); + } + + _points.create((int)points.size(), 1, POINT_TYPE); + if (!points.empty()) + Mat((int)points.size(), 1, POINT_TYPE, &points[0]).copyTo(_points.getMat()); + + if (_normals.needed()) + { + _normals.create((int)normals.size(), 1, POINT_TYPE); + if (!normals.empty()) + Mat((int)normals.size(), 1, POINT_TYPE, &normals[0]).copyTo(_normals.getMat()); + } + } +} + +void HashTSDFVolumeGPU::fetchNormals(InputArray _points, OutputArray _normals) const +{ + CV_TRACE_FUNCTION(); + + if (_normals.needed()) + { + //TODO: remove it when it works w/o CPU code + volUnitsData.copyTo(volUnitsDataCopy); + + Points points = _points.getMat(); + CV_Assert(points.type() == POINT_TYPE); + _normals.createSameSize(_points, _points.type()); + Normals normals = _normals.getMat(); + const HashTSDFVolumeGPU& _volume = *this; + auto HashPushNormals = [&](const ptype& point, const int* position) { + const HashTSDFVolumeGPU& volume(_volume); + Affine3f invPose(volume.pose.inv()); + Point3f p = fromPtype(point); + Point3f n = nan3; + if (!isNaN(p)) + { + Point3f voxelPoint = invPose * p; + n = volume.pose.rotation() * volume.getNormalVoxel(voxelPoint); + } + normals(position[0], position[1]) = toPtype(n); + }; + points.forEach(HashPushNormals); + } + +} + +int HashTSDFVolumeGPU::getVisibleBlocks(int currFrameId, int frameThreshold) const +{ + Mat cpuIndices = lastVisibleIndices.getMat(ACCESS_READ); + + int numVisibleBlocks = 0; + //! TODO: Iterate over map parallely? + for (int i = 0; i < hashTable.last; i++) + { + if (cpuIndices.at(i) > (currFrameId - frameThreshold)) + numVisibleBlocks++; + } + return numVisibleBlocks; +} + +#endif + +//template +Ptr makeHashTSDFVolume(const VolumeParams& _params) +{ +#ifdef HAVE_OPENCL + if (ocl::useOpenCL()) + return makePtr(_params.voxelSize, _params.pose.matrix, _params.raycastStepFactor, _params.tsdfTruncDist, _params.maxWeight, + _params.depthTruncThreshold, _params.unitResolution); +#endif + return makePtr(_params.voxelSize, _params.pose.matrix, _params.raycastStepFactor, _params.tsdfTruncDist, _params.maxWeight, + _params.depthTruncThreshold, _params.unitResolution); +} + +//template +Ptr makeHashTSDFVolume(float _voxelSize, Matx44f _pose, float _raycastStepFactor, float _truncDist, + int _maxWeight, float truncateThreshold, int volumeUnitResolution) +{ +#ifdef HAVE_OPENCL + if (ocl::useOpenCL()) + return makePtr(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, truncateThreshold, + volumeUnitResolution); +#endif + return makePtr(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, truncateThreshold, + volumeUnitResolution); +} + } // namespace kinfu } // namespace cv diff --git a/modules/rgbd/src/hash_tsdf.hpp b/modules/rgbd/src/hash_tsdf.hpp index 31bf026785b..34e73f30383 100644 --- a/modules/rgbd/src/hash_tsdf.hpp +++ b/modules/rgbd/src/hash_tsdf.hpp @@ -26,108 +26,26 @@ class HashTSDFVolume : public Volume virtual ~HashTSDFVolume() = default; + virtual int getVisibleBlocks(int currFrameId, int frameThreshold) const = 0; + virtual size_t getTotalVolumeUnits() const = 0; + public: int maxWeight; float truncDist; float truncateThreshold; int volumeUnitResolution; + int volumeUnitDegree; float volumeUnitSize; bool zFirstMemOrder; + Vec4i volStrides; }; -//! Spatial hashing -struct tsdf_hash -{ - size_t operator()(const Vec3i& x) const noexcept - { - size_t seed = 0; - constexpr uint32_t GOLDEN_RATIO = 0x9e3779b9; - for (uint16_t i = 0; i < 3; i++) - { - seed ^= std::hash()(x[i]) + GOLDEN_RATIO + (seed << 6) + (seed >> 2); - } - return seed; - } -}; - -typedef unsigned int VolumeIndex; -struct VolumeUnit -{ - cv::Vec3i coord; - VolumeIndex index; - cv::Matx44f pose; - int lastVisibleIndex = 0; - bool isActive; -}; - -typedef std::unordered_set VolumeUnitIndexSet; -typedef std::unordered_map VolumeUnitIndexes; - -class HashTSDFVolumeCPU : public HashTSDFVolume -{ - public: - // dimension in voxels, size in meters - HashTSDFVolumeCPU(float _voxelSize, const Matx44f& _pose, float _raycastStepFactor, float _truncDist, int _maxWeight, - float _truncateThreshold, int _volumeUnitRes, bool zFirstMemOrder = true); - - HashTSDFVolumeCPU(const VolumeParams& _volumeParams, bool zFirstMemOrder = true); - - void integrate(InputArray _depth, float depthFactor, const Matx44f& cameraPose, const kinfu::Intr& intrinsics, - const int frameId = 0) override; - void raycast(const Matx44f& cameraPose, const kinfu::Intr& intrinsics, const Size& frameSize, OutputArray points, - OutputArray normals) const override; - - void fetchNormals(InputArray points, OutputArray _normals) const override; - void fetchPointsNormals(OutputArray points, OutputArray normals) const override; - - void reset() override; - size_t getTotalVolumeUnits() const { return volumeUnits.size(); } - int getVisibleBlocks(int currFrameId, int frameThreshold) const; - - //! Return the voxel given the voxel index in the universal volume (1 unit = 1 voxel_length) - TsdfVoxel at(const Vec3i& volumeIdx) const; - - //! Return the voxel given the point in volume coordinate system i.e., (metric scale 1 unit = - //! 1m) - virtual TsdfVoxel at(const cv::Point3f& point) const; - virtual TsdfVoxel _at(const cv::Vec3i& volumeIdx, VolumeIndex indx) const; - - TsdfVoxel atVolumeUnit(const Vec3i& point, const Vec3i& volumeUnitIdx, VolumeUnitIndexes::const_iterator it) const; - - - float interpolateVoxelPoint(const Point3f& point) const; - float interpolateVoxel(const cv::Point3f& point) const; - Point3f getNormalVoxel(const cv::Point3f& p) const; - - //! Utility functions for coordinate transformations - Vec3i volumeToVolumeUnitIdx(const Point3f& point) const; - Point3f volumeUnitIdxToVolume(const Vec3i& volumeUnitIdx) const; - - Point3f voxelCoordToVolume(const Vec3i& voxelIdx) const; - Vec3i volumeToVoxelCoord(const Point3f& point) const; - - public: - Vec4i volStrides; - Vec6f frameParams; - Mat pixNorms; - VolumeUnitIndexes volumeUnits; - cv::Mat volUnitsData; - VolumeIndex lastVolIndex; -}; - -template -Ptr makeHashTSDFVolume(const VolumeParams& _volumeParams) -{ - return makePtr(_volumeParams); -} - -template +//template +Ptr makeHashTSDFVolume(const VolumeParams& _volumeParams); +//template Ptr makeHashTSDFVolume(float _voxelSize, Matx44f _pose, float _raycastStepFactor, float _truncDist, - int _maxWeight, float truncateThreshold, int volumeUnitResolution = 16) -{ - return makePtr(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, truncateThreshold, - volumeUnitResolution); -} + int _maxWeight, float truncateThreshold, int volumeUnitResolution = 16); + } // namespace kinfu } // namespace cv #endif diff --git a/modules/rgbd/src/large_kinfu.cpp b/modules/rgbd/src/large_kinfu.cpp index dedeabb82db..16006713c53 100644 --- a/modules/rgbd/src/large_kinfu.cpp +++ b/modules/rgbd/src/large_kinfu.cpp @@ -316,21 +316,21 @@ template void LargeKinfuImpl::getCloud(OutputArray p, OutputArray n) const { auto currSubmap = submapMgr->getCurrentSubmap(); - currSubmap->volume.fetchPointsNormals(p, n); + currSubmap->volume->fetchPointsNormals(p, n); } template void LargeKinfuImpl::getPoints(OutputArray points) const { auto currSubmap = submapMgr->getCurrentSubmap(); - currSubmap->volume.fetchPointsNormals(points, noArray()); + currSubmap->volume->fetchPointsNormals(points, noArray()); } template void LargeKinfuImpl::getNormals(InputArray points, OutputArray normals) const { auto currSubmap = submapMgr->getCurrentSubmap(); - currSubmap->volume.fetchNormals(points, normals); + currSubmap->volume->fetchNormals(points, normals); } // importing class diff --git a/modules/rgbd/src/opencl/hash_tsdf.cl b/modules/rgbd/src/opencl/hash_tsdf.cl new file mode 100644 index 00000000000..0e611c1d3dd --- /dev/null +++ b/modules/rgbd/src/opencl/hash_tsdf.cl @@ -0,0 +1,653 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +// This code is also subject to the license terms in the LICENSE_KinectFusion.md file found in this module's directory + +#define USE_INTERPOLATION_IN_GETNORMAL 1 +#define HASH_DIVISOR 32768 + +typedef char int8_t; +typedef uint int32_t; + +typedef int8_t TsdfType; +typedef uchar WeightType; + +struct TsdfVoxel +{ + TsdfType tsdf; + WeightType weight; +}; + + +static inline TsdfType floatToTsdf(float num) +{ + int8_t res = (int8_t) ( (num * (-128)) ); + res = res ? res : (num < 0 ? 1 : -1); + return res; +} + +static inline float tsdfToFloat(TsdfType num) +{ + return ( (float) num ) / (-128); +} + +static uint calc_hash(int3 x) +{ + unsigned int seed = 0; + unsigned int GOLDEN_RATIO = 0x9e3779b9; + seed ^= x.s0 + GOLDEN_RATIO + (seed << 6) + (seed >> 2); + seed ^= x.s1 + GOLDEN_RATIO + (seed << 6) + (seed >> 2); + seed ^= x.s2 + GOLDEN_RATIO + (seed << 6) + (seed >> 2); + return seed; +} + + +//TODO: make hashDivisor a power of 2 +//TODO: put it to this .cl file as a constant +static int custom_find(int3 idx, const int hashDivisor, __global const int* hashes, + __global const int4* data) +{ + int hash = calc_hash(idx) % hashDivisor; + int place = hashes[hash]; + // search a place + while (place >= 0) + { + if (all(data[place].s012 == idx)) + break; + else + place = data[place].s3; + } + + return place; +} + + + +static void integrateVolumeUnit( + int x, int y, + __global const char * depthptr, + int depth_step, int depth_offset, + int depth_rows, int depth_cols, + __global struct TsdfVoxel * volumeptr, + const __global char * pixNormsPtr, + int pixNormsStep, int pixNormsOffset, + int pixNormsRows, int pixNormsCols, + const float16 vol2camMatrix, + const float voxelSize, + const int4 volResolution4, + const int4 volStrides4, + const float2 fxy, + const float2 cxy, + const float dfac, + const float truncDist, + const int maxWeight + ) +{ + const int3 volResolution = volResolution4.xyz; + + if(x >= volResolution.x || y >= volResolution.y) + return; + + // coord-independent constants + const int3 volStrides = volStrides4.xyz; + const float2 limits = (float2)(depth_cols-1, depth_rows-1); + + const float4 vol2cam0 = vol2camMatrix.s0123; + const float4 vol2cam1 = vol2camMatrix.s4567; + const float4 vol2cam2 = vol2camMatrix.s89ab; + + const float truncDistInv = 1.f/truncDist; + + // optimization of camSpace transformation (vector addition instead of matmul at each z) + float4 inPt = (float4)(x*voxelSize, y*voxelSize, 0, 1); + float3 basePt = (float3)(dot(vol2cam0, inPt), + dot(vol2cam1, inPt), + dot(vol2cam2, inPt)); + + float3 camSpacePt = basePt; + + // zStep == vol2cam*(float3(x, y, 1)*voxelSize) - basePt; + float3 zStep = ((float3)(vol2cam0.z, vol2cam1.z, vol2cam2.z))*voxelSize; + + int volYidx = x*volStrides.x + y*volStrides.y; + + int startZ, endZ; + if(fabs(zStep.z) > 1e-5f) + { + int baseZ = convert_int(-basePt.z / zStep.z); + if(zStep.z > 0) + { + startZ = baseZ; + endZ = volResolution.z; + } + else + { + startZ = 0; + endZ = baseZ; + } + } + else + { + if(basePt.z > 0) + { + startZ = 0; endZ = volResolution.z; + } + else + { + // z loop shouldn't be performed + startZ = endZ = 0; + } + } + + startZ = max(0, startZ); + endZ = min(volResolution.z, endZ); + + for(int z = startZ; z < endZ; z++) + { + // optimization of the following: + //float3 camSpacePt = vol2cam * ((float3)(x, y, z)*voxelSize); + camSpacePt += zStep; + + if(camSpacePt.z <= 0) + continue; + + float3 camPixVec = camSpacePt / camSpacePt.z; + float2 projected = mad(camPixVec.xy, fxy, cxy); // mad(a,b,c) = a * b + c + + float v; + // bilinearly interpolate depth at projected + if(all(projected >= 0) && all(projected < limits)) + { + float2 ip = floor(projected); + int xi = ip.x, yi = ip.y; + + __global const float* row0 = (__global const float*)(depthptr + depth_offset + + (yi+0)*depth_step); + __global const float* row1 = (__global const float*)(depthptr + depth_offset + + (yi+1)*depth_step); + + float v00 = row0[xi+0]; + float v01 = row0[xi+1]; + float v10 = row1[xi+0]; + float v11 = row1[xi+1]; + float4 vv = (float4)(v00, v01, v10, v11); + + // assume correct depth is positive + if(all(vv > 0)) + { + float2 t = projected - ip; + float2 vf = mix(vv.xz, vv.yw, t.x); + v = mix(vf.s0, vf.s1, t.y); + } + else + continue; + } + else + continue; + + if(v == 0) + continue; + + int2 projInt = convert_int2(projected); + float pixNorm = *(__global const float*)(pixNormsPtr + pixNormsOffset + projInt.y*pixNormsStep + projInt.x*sizeof(float)); + //float pixNorm = length(camPixVec); + + // difference between distances of point and of surface to camera + float sdf = pixNorm*(v*dfac - camSpacePt.z); + // possible alternative is: + // float sdf = length(camSpacePt)*(v*dfac/camSpacePt.z - 1.0); + if(sdf >= -truncDist) + { + float tsdf = fmin(1.0f, sdf * truncDistInv); + int volIdx = volYidx + z*volStrides.z; + + struct TsdfVoxel voxel = volumeptr[volIdx]; + float value = tsdfToFloat(voxel.tsdf); + int weight = voxel.weight; + // update TSDF + value = (value*weight + tsdf) / (weight + 1); + weight = min(weight + 1, maxWeight); + + voxel.tsdf = floatToTsdf(value); + voxel.weight = weight; + volumeptr[volIdx] = voxel; + } + } + +} + + +__kernel void integrateAllVolumeUnits( + // depth + __global const char * depthptr, + int depth_step, int depth_offset, + int depth_rows, int depth_cols, + // hashMap + __global const int* hashes, + __global const int4* data, + // volUnitsData + __global struct TsdfVoxel * allVolumePtr, + int table_step, int table_offset, + int table_rows, int table_cols, + // pixNorms + const __global char * pixNormsPtr, + int pixNormsStep, int pixNormsOffset, + int pixNormsRows, int pixNormsCols, + // isActiveFlags + __global const uchar* isActiveFlagsPtr, + int isActiveFlagsStep, int isActiveFlagsOffset, + int isActiveFlagsRows, int isActiveFlagsCols, + // cam matrices: + const float16 vol2cam, + const float16 camInv, + // scalars: + const float voxelSize, + const int volUnitResolution, + const int4 volStrides4, + const float2 fxy, + const float2 cxy, + const float dfac, + const float truncDist, + const int maxWeight + ) +{ + const int hash_divisor = HASH_DIVISOR; + int i = get_global_id(0); + int j = get_global_id(1); + int row = get_global_id(2); + int3 idx = data[row].xyz; + + const int4 volResolution4 = (int4)(volUnitResolution, + volUnitResolution, + volUnitResolution, + volUnitResolution); + + int isActive = *(__global const uchar*)(isActiveFlagsPtr + isActiveFlagsOffset + row); + + if (isActive) + { + int volCubed = volUnitResolution * volUnitResolution * volUnitResolution; + __global struct TsdfVoxel * volumeptr = (__global struct TsdfVoxel*) + (allVolumePtr + table_offset + row * volCubed); + + // volUnit2cam = world2cam * volUnit2world = + // camPoseInv * volUnitPose = camPoseInv * (volPose + idx * volUnitSize) = + // camPoseInv * (volPose + idx * volUnitResolution * voxelSize) = + // camPoseInv * (volPose + mulIdx) = camPoseInv * volPose + camPoseInv * mulIdx = + // vol2cam + camPoseInv * mulIdx + float3 mulIdx = convert_float3(idx * volUnitResolution) * voxelSize; + float16 volUnit2cam = vol2cam; + volUnit2cam.s37b += (float3)(dot(mulIdx, camInv.s012), + dot(mulIdx, camInv.s456), + dot(mulIdx, camInv.s89a)); + + integrateVolumeUnit( + i, j, + depthptr, + depth_step, depth_offset, + depth_rows, depth_cols, + volumeptr, + pixNormsPtr, + pixNormsStep, pixNormsOffset, + pixNormsRows, pixNormsCols, + volUnit2cam, + voxelSize, + volResolution4, + volStrides4, + fxy, + cxy, + dfac, + truncDist, + maxWeight + ); + } +} + + +static struct TsdfVoxel at(int3 volumeIdx, int row, int volumeUnitDegree, + int3 volStrides, __global const struct TsdfVoxel * allVolumePtr, int table_offset) + +{ + //! Out of bounds + if (any(volumeIdx >= (1 << volumeUnitDegree)) || + any(volumeIdx < 0)) + { + struct TsdfVoxel dummy; + dummy.tsdf = floatToTsdf(1.0f); + dummy.weight = 0; + return dummy; + } + + int volCubed = 1 << (volumeUnitDegree*3); + __global struct TsdfVoxel * volData = (__global struct TsdfVoxel*) + (allVolumePtr + table_offset + row * volCubed); + int3 ismul = volumeIdx * volStrides; + int coordBase = ismul.x + ismul.y + ismul.z; + return volData[coordBase]; +} + + +static struct TsdfVoxel atVolumeUnit(int3 volumeIdx, int3 volumeUnitIdx, int row, + int volumeUnitDegree, int3 volStrides, + __global const struct TsdfVoxel * allVolumePtr, int table_offset) + +{ + //! Out of bounds + if (row < 0) + { + struct TsdfVoxel dummy; + dummy.tsdf = floatToTsdf(1.0f); + dummy.weight = 0; + return dummy; + } + + int3 volUnitLocalIdx = volumeIdx - (volumeUnitIdx << volumeUnitDegree); + int volCubed = 1 << (volumeUnitDegree*3); + __global struct TsdfVoxel * volData = (__global struct TsdfVoxel*) + (allVolumePtr + table_offset + row * volCubed); + int3 ismul = volUnitLocalIdx * volStrides; + int coordBase = ismul.x + ismul.y + ismul.z; + return volData[coordBase]; +} + +inline float interpolate(float3 t, float8 vz) +{ + float4 vy = mix(vz.s0246, vz.s1357, t.z); + float2 vx = mix(vy.s02, vy.s13, t.y); + return mix(vx.s0, vx.s1, t.x); +} + +inline float3 getNormalVoxel(float3 ptVox, __global const struct TsdfVoxel* allVolumePtr, + int volumeUnitDegree, + const int hash_divisor, + __global const int* hashes, + __global const int4* data, + + int3 volStrides, int table_offset) +{ + float3 normal = (float3) (0.0f, 0.0f, 0.0f); + float3 fip = floor(ptVox); + int3 iptVox = convert_int3(fip); + + // A small hash table to reduce a number of findRow() calls + // -2 and lower means not queried yet + // -1 means not found + // 0+ means found + int iterMap[8]; + for (int i = 0; i < 8; i++) + { + iterMap[i] = -2; + } + +#if !USE_INTERPOLATION_IN_GETNORMAL + int4 offsets[] = { (int4)( 1, 0, 0, 0), (int4)(-1, 0, 0, 0), (int4)( 0, 1, 0, 0), // 0-3 + (int4)( 0, -1, 0, 0), (int4)( 0, 0, 1, 0), (int4)( 0, 0, -1, 0) // 4-7 + }; + + const int nVals = 6; + float vals[6]; +#else + int4 offsets[]={(int4)( 0, 0, 0, 0), (int4)( 0, 0, 1, 0), (int4)( 0, 1, 0, 0), (int4)( 0, 1, 1, 0), // 0-3 + (int4)( 1, 0, 0, 0), (int4)( 1, 0, 1, 0), (int4)( 1, 1, 0, 0), (int4)( 1, 1, 1, 0), // 4-7 + (int4)(-1, 0, 0, 0), (int4)(-1, 0, 1, 0), (int4)(-1, 1, 0, 0), (int4)(-1, 1, 1, 0), // 8-11 + (int4)( 2, 0, 0, 0), (int4)( 2, 0, 1, 0), (int4)( 2, 1, 0, 0), (int4)( 2, 1, 1, 0), // 12-15 + (int4)( 0, -1, 0, 0), (int4)( 0, -1, 1, 0), (int4)( 1, -1, 0, 0), (int4)( 1, -1, 1, 0), // 16-19 + (int4)( 0, 2, 0, 0), (int4)( 0, 2, 1, 0), (int4)( 1, 2, 0, 0), (int4)( 1, 2, 1, 0), // 20-23 + (int4)( 0, 0, -1, 0), (int4)( 0, 1, -1, 0), (int4)( 1, 0, -1, 0), (int4)( 1, 1, -1, 0), // 24-27 + (int4)( 0, 0, 2, 0), (int4)( 0, 1, 2, 0), (int4)( 1, 0, 2, 0), (int4)( 1, 1, 2, 0), // 28-31 + }; + const int nVals = 32; + float vals[32]; +#endif + + for (int i = 0; i < nVals; i++) + { + int3 pt = iptVox + offsets[i].s012; + + // VoxelToVolumeUnitIdx() + int3 volumeUnitIdx = pt >> volumeUnitDegree; + + int3 vand = (volumeUnitIdx & 1); + int dictIdx = vand.s0 + vand.s1 * 2 + vand.s2 * 4; + + int it = iterMap[dictIdx]; + if (it < -1) + { + it = custom_find(volumeUnitIdx, hash_divisor, hashes, data); + iterMap[dictIdx] = it; + } + + struct TsdfVoxel tmp = atVolumeUnit(pt, volumeUnitIdx, it, volumeUnitDegree, volStrides, allVolumePtr, table_offset); + vals[i] = tsdfToFloat( tmp.tsdf ); + } + +#if !USE_INTERPOLATION_IN_GETNORMAL + float3 pv, nv; + + pv = (float3)(vals[0*2 ], vals[1*2 ], vals[2*2 ]); + nv = (float3)(vals[0*2+1], vals[1*2+1], vals[2*2+1]); + normal = pv - nv; +#else + + float cxv[8], cyv[8], czv[8]; + + // How these numbers were obtained: + // 1. Take the basic interpolation sequence: + // 000, 001, 010, 011, 100, 101, 110, 111 + // where each digit corresponds to shift by x, y, z axis respectively. + // 2. Add +1 for next or -1 for prev to each coordinate to corresponding axis + // 3. Search corresponding values in offsets + const int idxxn[8] = { 8, 9, 10, 11, 0, 1, 2, 3 }; + const int idxxp[8] = { 4, 5, 6, 7, 12, 13, 14, 15 }; + const int idxyn[8] = { 16, 17, 0, 1, 18, 19, 4, 5 }; + const int idxyp[8] = { 2, 3, 20, 21, 6, 7, 22, 23 }; + const int idxzn[8] = { 24, 0, 25, 2, 26, 4, 27, 6 }; + const int idxzp[8] = { 1, 28, 3, 29, 5, 30, 7, 31 }; + + float vcxp[8], vcxn[8]; + float vcyp[8], vcyn[8]; + float vczp[8], vczn[8]; + + for (int i = 0; i < 8; i++) + { + vcxp[i] = vals[idxxp[i]]; vcxn[i] = vals[idxxn[i]]; + vcyp[i] = vals[idxyp[i]]; vcyn[i] = vals[idxyn[i]]; + vczp[i] = vals[idxzp[i]]; vczn[i] = vals[idxzn[i]]; + } + + float8 cxp = vload8(0, vcxp), cxn = vload8(0, vcxn); + float8 cyp = vload8(0, vcyp), cyn = vload8(0, vcyn); + float8 czp = vload8(0, vczp), czn = vload8(0, vczn); + float8 cx = cxp - cxn; + float8 cy = cyp - cyn; + float8 cz = czp - czn; + + float3 tv = ptVox - fip; + normal.x = interpolate(tv, cx); + normal.y = interpolate(tv, cy); + normal.z = interpolate(tv, cz); +#endif + + float norm = sqrt(dot(normal, normal)); + return norm < 0.0001f ? nan((uint)0) : normal / norm; +} + +typedef float4 ptype; + +__kernel void raycast( + __global const int* hashes, + __global const int4* data, + __global char * pointsptr, + int points_step, int points_offset, + __global char * normalsptr, + int normals_step, int normals_offset, + const int2 frameSize, + __global const struct TsdfVoxel * allVolumePtr, + int table_step, int table_offset, + int table_rows, int table_cols, + float16 cam2volRotGPU, + float16 vol2camRotGPU, + float truncateThreshold, + const float2 fixy, const float2 cxy, + const float4 boxDown4, const float4 boxUp4, + const float tstep, + const float voxelSize, + const float voxelSizeInv, + float volumeUnitSize, + float truncDist, + int volumeUnitDegree, + int4 volStrides4 + ) +{ + const int hash_divisor = HASH_DIVISOR; + int x = get_global_id(0); + int y = get_global_id(1); + + if(x >= frameSize.x || y >= frameSize.y) + return; + + float3 point = nan((uint)0); + float3 normal = nan((uint)0); + + const float3 camRot0 = cam2volRotGPU.s012; + const float3 camRot1 = cam2volRotGPU.s456; + const float3 camRot2 = cam2volRotGPU.s89a; + const float3 camTrans = cam2volRotGPU.s37b; + + const float3 volRot0 = vol2camRotGPU.s012; + const float3 volRot1 = vol2camRotGPU.s456; + const float3 volRot2 = vol2camRotGPU.s89a; + const float3 volTrans = vol2camRotGPU.s37b; + + float3 planed = (float3)(((float2)(x, y) - cxy)*fixy, 1.f); + planed = (float3)(dot(planed, camRot0), + dot(planed, camRot1), + dot(planed, camRot2)); + + float3 orig = (float3) (camTrans.s0, camTrans.s1, camTrans.s2); + float3 dir = fast_normalize(planed); + float3 origScaled = orig * voxelSizeInv; + float3 dirScaled = dir * voxelSizeInv; + + float tmin = 0; + float tmax = truncateThreshold; + float tcurr = tmin; + float tprev = tcurr; + float prevTsdf = truncDist; + + int3 volStrides = volStrides4.xyz; + + while (tcurr < tmax) + { + float3 currRayPosVox = origScaled + tcurr * dirScaled; + + // VolumeToVolumeUnitIdx() + int3 currVoxel = convert_int3(floor(currRayPosVox)); + int3 currVolumeUnitIdx = currVoxel >> volumeUnitDegree; + + int row = custom_find(currVolumeUnitIdx, hash_divisor, hashes, data); + + float currTsdf = prevTsdf; + int currWeight = 0; + float stepSize = 0.5 * volumeUnitSize; + int3 volUnitLocalIdx; + + if (row >= 0) + { + volUnitLocalIdx = currVoxel - (currVolumeUnitIdx << volumeUnitDegree); + struct TsdfVoxel currVoxel = at(volUnitLocalIdx, row, volumeUnitDegree, volStrides, allVolumePtr, table_offset); + + currTsdf = tsdfToFloat(currVoxel.tsdf); + currWeight = currVoxel.weight; + stepSize = tstep; + } + + if (prevTsdf > 0.f && currTsdf <= 0.f && currWeight > 0) + { + float tInterp = (tcurr * prevTsdf - tprev * currTsdf) / (prevTsdf - currTsdf); + if ( !isnan(tInterp) && !isinf(tInterp) ) + { + float3 pvox = origScaled + tInterp * dirScaled; + float3 nv = getNormalVoxel( pvox, allVolumePtr, volumeUnitDegree, + hash_divisor, hashes, data, + volStrides, table_offset); + + if(!any(isnan(nv))) + { + //convert pv and nv to camera space + normal = (float3)(dot(nv, volRot0), + dot(nv, volRot1), + dot(nv, volRot2)); + // interpolation optimized a little + float3 pv = pvox * voxelSize; + point = (float3)(dot(pv, volRot0), + dot(pv, volRot1), + dot(pv, volRot2)) + volTrans; + } + } + break; + } + prevTsdf = currTsdf; + tprev = tcurr; + tcurr += stepSize; + } + + __global float* pts = (__global float*)(pointsptr + points_offset + y*points_step + x*sizeof(ptype)); + __global float* nrm = (__global float*)(normalsptr + normals_offset + y*normals_step + x*sizeof(ptype)); + vstore4((float4)(point, 0), 0, pts); + vstore4((float4)(normal, 0), 0, nrm); +} + + +__kernel void markActive ( + __global const int4* hashSetData, + + __global char* isActiveFlagsPtr, + int isActiveFlagsStep, int isActiveFlagsOffset, + int isActiveFlagsRows, int isActiveFlagsCols, + + __global char* lastVisibleIndicesPtr, + int lastVisibleIndicesStep, int lastVisibleIndicesOffset, + int lastVisibleIndicesRows, int lastVisibleIndicesCols, + + const float16 vol2cam, + const float2 fxy, + const float2 cxy, + const int2 frameSz, + const float volumeUnitSize, + const int lastVolIndex, + const float truncateThreshold, + const int frameId + ) +{ + const int hash_divisor = HASH_DIVISOR; + int row = get_global_id(0); + + if (row < lastVolIndex) + { + int3 idx = hashSetData[row].xyz; + + float3 volumeUnitPos = convert_float3(idx) * volumeUnitSize; + + float3 volUnitInCamSpace = (float3) (dot(volumeUnitPos, vol2cam.s012), + dot(volumeUnitPos, vol2cam.s456), + dot(volumeUnitPos, vol2cam.s89a)) + vol2cam.s37b; + + if (volUnitInCamSpace.z < 0 || volUnitInCamSpace.z > truncateThreshold) + { + *(isActiveFlagsPtr + isActiveFlagsOffset + row * isActiveFlagsStep) = 0; + return; + } + + float2 cameraPoint; + float invz = 1.f / volUnitInCamSpace.z; + cameraPoint = fxy * volUnitInCamSpace.xy * invz + cxy; + + if (all(cameraPoint >= 0) && all(cameraPoint < convert_float2(frameSz))) + { + *(__global int*)(lastVisibleIndicesPtr + lastVisibleIndicesOffset + row * lastVisibleIndicesStep) = frameId; + *(isActiveFlagsPtr + isActiveFlagsOffset + row * isActiveFlagsStep) = 1; + } + } +} diff --git a/modules/rgbd/src/opencl/tsdf.cl b/modules/rgbd/src/opencl/tsdf.cl index b9186826236..3a13d3b8336 100644 --- a/modules/rgbd/src/opencl/tsdf.cl +++ b/modules/rgbd/src/opencl/tsdf.cl @@ -26,20 +26,6 @@ static inline float tsdfToFloat(TsdfType num) return ( (float) num ) / (-128); } -__kernel void preCalculationPixNorm (__global float * pixNorms, - int pix_step, int pix_offset, - int pix_rows, int pix_cols, - const __global float * xx, - const __global float * yy, - int width, int height) -{ - int i = get_global_id(0); - int j = get_global_id(1); - int idx = i*width + j; - if(i < height && j < width && idx < pix_cols) - pixNorms[idx] = sqrt(xx[j] * xx[j] + yy[i] * yy[i] + 1.0f); -} - __kernel void integrate(__global const char * depthptr, int depth_step, int depth_offset, int depth_rows, int depth_cols, diff --git a/modules/rgbd/src/opencl/tsdf_functions.cl b/modules/rgbd/src/opencl/tsdf_functions.cl new file mode 100644 index 00000000000..40efeca57c8 --- /dev/null +++ b/modules/rgbd/src/opencl/tsdf_functions.cl @@ -0,0 +1,19 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +// This code is also subject to the license terms in the LICENSE_KinectFusion.md file found in this module's directory + +__kernel void preCalculationPixNorm (__global char * pixNormsPtr, + int pixNormsStep, int pixNormsOffset, + int pixNormsRows, int pixNormsCols, + const __global float * xx, + const __global float * yy) +{ + int i = get_global_id(0); + int j = get_global_id(1); + if (i < pixNormsRows && j < pixNormsCols) + { + *(__global float*)(pixNormsPtr + pixNormsOffset + i*pixNormsStep + j*sizeof(float)) = sqrt(xx[j] * xx[j] + yy[i] * yy[i] + 1.0f); + } +} \ No newline at end of file diff --git a/modules/rgbd/src/submap.hpp b/modules/rgbd/src/submap.hpp index 65a2bed6229..b40023b6a56 100644 --- a/modules/rgbd/src/submap.hpp +++ b/modules/rgbd/src/submap.hpp @@ -42,7 +42,7 @@ class Submap Submap(int _id, const VolumeParams& volumeParams, const cv::Affine3f& _pose = cv::Affine3f::Identity(), int _startFrameId = 0) - : id(_id), pose(_pose), cameraPose(Affine3f::Identity()), startFrameId(_startFrameId), volume(volumeParams) + : id(_id), pose(_pose), cameraPose(Affine3f::Identity()), startFrameId(_startFrameId), volume(makeHashTSDFVolume(volumeParams)) { std::cout << "Created volume\n"; } @@ -53,10 +53,10 @@ class Submap OutputArray points, OutputArray normals); virtual void updatePyrPointsNormals(const int pyramidLevels); - virtual int getTotalAllocatedBlocks() const { return int(volume.getTotalVolumeUnits()); }; + virtual int getTotalAllocatedBlocks() const { return int(volume->getTotalVolumeUnits()); }; virtual int getVisibleBlocks(int currFrameId) const { - return volume.getVisibleBlocks(currFrameId, FRAME_VISIBILITY_THRESHOLD); + return volume->getVisibleBlocks(currFrameId, FRAME_VISIBILITY_THRESHOLD); } float calcVisibilityRatio(int currFrameId) const @@ -91,7 +91,7 @@ class Submap //! TODO: Add support for GPU arrays (UMat) std::vector pyrPoints; std::vector pyrNormals; - HashTSDFVolumeCPU volume; + std::shared_ptr volume; }; template @@ -100,14 +100,14 @@ void Submap::integrate(InputArray _depth, float depthFactor, const cv:: const int currFrameId) { CV_Assert(currFrameId >= startFrameId); - volume.integrate(_depth, depthFactor, cameraPose.matrix, intrinsics, currFrameId); + volume->integrate(_depth, depthFactor, cameraPose.matrix, intrinsics, currFrameId); } template void Submap::raycast(const cv::Affine3f& _cameraPose, const cv::kinfu::Intr& intrinsics, cv::Size frameSize, OutputArray points, OutputArray normals) { - volume.raycast(_cameraPose.matrix, intrinsics, frameSize, points, normals); + volume->raycast(_cameraPose.matrix, intrinsics, frameSize, points, normals); } template diff --git a/modules/rgbd/src/tsdf.cpp b/modules/rgbd/src/tsdf.cpp index 412c2fe789e..0a0fcfacf05 100644 --- a/modules/rgbd/src/tsdf.cpp +++ b/modules/rgbd/src/tsdf.cpp @@ -102,10 +102,7 @@ TsdfVoxel TSDFVolumeCPU::at(const Vec3i& volumeIdx) const (volumeIdx[1] >= volResolution.y || volumeIdx[1] < 0) || (volumeIdx[2] >= volResolution.z || volumeIdx[2] < 0)) { - TsdfVoxel dummy; - dummy.tsdf = floatToTsdf(1.0f); - dummy.weight = 0; - return dummy; + return TsdfVoxel(floatToTsdf(1.f), 0); } const TsdfVoxel* volData = volume.ptr(); @@ -836,47 +833,6 @@ void TSDFVolumeGPU::reset() volume.setTo(Scalar(0, 0)); } -static void preCalculationPixNormGPU(int depth_rows, int depth_cols, Vec2f fxy, Vec2f cxy, UMat& pixNorm) -{ - Mat x(1, depth_cols, CV_32F); - Mat y(1, depth_rows, CV_32F); - pixNorm.create(1, depth_rows * depth_cols, CV_32F); - - for (int i = 0; i < depth_cols; i++) - x.at(0, i) = (i - cxy[0]) / fxy[0]; - for (int i = 0; i < depth_rows; i++) - y.at(0, i) = (i - cxy[1]) / fxy[1]; - - cv::String errorStr; - cv::String name = "preCalculationPixNorm"; - ocl::ProgramSource source = ocl::rgbd::tsdf_oclsrc; - cv::String options = "-cl-mad-enable"; - ocl::Kernel kk; - kk.create(name.c_str(), source, options, &errorStr); - - - if (kk.empty()) - throw std::runtime_error("Failed to create kernel: " + errorStr); - - AccessFlag af = ACCESS_READ; - UMat xx = x.getUMat(af); - UMat yy = y.getUMat(af); - - kk.args(ocl::KernelArg::ReadWrite(pixNorm), - ocl::KernelArg::PtrReadOnly(xx), - ocl::KernelArg::PtrReadOnly(yy), - depth_cols, depth_rows); - - size_t globalSize[2]; - globalSize[0] = depth_rows; - globalSize[1] = depth_cols; - - if (!kk.run(2, globalSize, NULL, true)) - throw std::runtime_error("Failed to run kernel"); - - return; -} - // use depth instead of distance (optimization) void TSDFVolumeGPU::integrate(InputArray _depth, float depthFactor, const Matx44f& cameraPose, const Intr& intrinsics, const int frameId) @@ -901,15 +857,13 @@ void TSDFVolumeGPU::integrate(InputArray _depth, float depthFactor, float dfac = 1.f/depthFactor; Vec4i volResGpu(volResolution.x, volResolution.y, volResolution.z); Vec2f fxy(intrinsics.fx, intrinsics.fy), cxy(intrinsics.cx, intrinsics.cy); - if (!(frameParams[0] == depth.rows && frameParams[1] == depth.cols && - frameParams[2] == intrinsics.fx && frameParams[3] == intrinsics.fy && - frameParams[4] == intrinsics.cx && frameParams[5] == intrinsics.cy)) + Vec6f newParams((float)depth.rows, (float)depth.cols, + intrinsics.fx, intrinsics.fy, + intrinsics.cx, intrinsics.cy); + if (!(frameParams == newParams)) { - frameParams[0] = (float)depth.rows; frameParams[1] = (float)depth.cols; - frameParams[2] = intrinsics.fx; frameParams[3] = intrinsics.fy; - frameParams[4] = intrinsics.cx; frameParams[5] = intrinsics.cy; - - preCalculationPixNormGPU(depth.rows, depth.cols, fxy, cxy, pixNorms); + frameParams = newParams; + pixNorms = preCalculationPixNormGPU(depth, intrinsics); } // TODO: optimization possible diff --git a/modules/rgbd/src/tsdf.hpp b/modules/rgbd/src/tsdf.hpp index 009c4a8466c..67361aa59d9 100644 --- a/modules/rgbd/src/tsdf.hpp +++ b/modules/rgbd/src/tsdf.hpp @@ -17,15 +17,15 @@ namespace cv { namespace kinfu { -// TODO: Optimization possible: -// * TsdfType can be FP16 -// * WeightType can be uint16 typedef int8_t TsdfType; typedef uchar WeightType; struct TsdfVoxel { + TsdfVoxel(TsdfType _tsdf, WeightType _weight) : + tsdf(_tsdf), weight(_weight) + { } TsdfType tsdf; WeightType weight; }; @@ -107,8 +107,7 @@ class TSDFVolumeGPU : public TSDFVolume UMat pixNorms; // See zFirstMemOrder arg of parent class constructor // for the array layout info - // Array elem is CV_32FC2, read as (float, int) - // TODO: optimization possible to (fp16, int16), see Voxel definition + // Array elem is CV_8UC2, read as (int8, uint8) UMat volume; }; #endif diff --git a/modules/rgbd/src/tsdf_functions.cpp b/modules/rgbd/src/tsdf_functions.cpp index ff40dce248d..b0e5276cba7 100644 --- a/modules/rgbd/src/tsdf_functions.cpp +++ b/modules/rgbd/src/tsdf_functions.cpp @@ -6,6 +6,7 @@ #include "precomp.hpp" #include "tsdf_functions.hpp" +#include "opencl_kernels_rgbd.hpp" namespace cv { @@ -35,6 +36,51 @@ cv::Mat preCalculationPixNorm(Depth depth, const Intr& intrinsics) return pixNorm; } +#ifdef HAVE_OPENCL +cv::UMat preCalculationPixNormGPU(const UMat& depth, const Intr& intrinsics) +{ + int depth_cols = depth.cols; + int depth_rows = depth.rows; + Point2f fl(intrinsics.fx, intrinsics.fy); + Point2f pp(intrinsics.cx, intrinsics.cy); + Mat x(1, depth_cols, CV_32FC1); + Mat y(1, depth_rows, CV_32FC1); + UMat pixNorm(depth_rows, depth_cols, CV_32F); + + for (int i = 0; i < depth_cols; i++) + x.at(i) = (i - pp.x) / fl.x; + for (int i = 0; i < depth_rows; i++) + y.at(i) = (i - pp.y) / fl.y; + + cv::String errorStr; + cv::String name = "preCalculationPixNorm"; + ocl::ProgramSource source = ocl::rgbd::tsdf_functions_oclsrc; + cv::String options = "-cl-mad-enable"; + ocl::Kernel kk; + kk.create(name.c_str(), source, options, &errorStr); + + if (kk.empty()) + throw std::runtime_error("Failed to create kernel: " + errorStr); + + AccessFlag af = ACCESS_READ; + UMat xx = x.getUMat(af); + UMat yy = y.getUMat(af); + + kk.args(ocl::KernelArg::WriteOnly(pixNorm), + ocl::KernelArg::PtrReadOnly(xx), + ocl::KernelArg::PtrReadOnly(yy)); + + size_t globalSize[2]; + globalSize[0] = depth_rows; + globalSize[1] = depth_cols; + + if (!kk.run(2, globalSize, NULL, true)) + throw std::runtime_error("Failed to run kernel"); + + return pixNorm; +} +#endif + const bool fixMissingData = false; depthType bilinearDepth(const Depth& m, cv::Point2f pt) { @@ -254,7 +300,7 @@ void integrateVolumeUnit( if (!(_u >= 0 && _u < depth.cols && _v >= 0 && _v < depth.rows)) continue; float pixNorm = pixNorms.at(_v, _u); - // float pixNorm = sqrt(v_reduce_sum(camPixVec*camPixVec)); + //float pixNorm = sqrt(v_reduce_sum(camPixVec*camPixVec)); // difference between distances of point and of surface to camera float sdf = pixNorm * (v * dfac - zCamSpace); // possible alternative is: diff --git a/modules/rgbd/src/tsdf_functions.hpp b/modules/rgbd/src/tsdf_functions.hpp index 6d86595118f..09efecf1252 100644 --- a/modules/rgbd/src/tsdf_functions.hpp +++ b/modules/rgbd/src/tsdf_functions.hpp @@ -35,6 +35,8 @@ inline float tsdfToFloat(TsdfType num) } cv::Mat preCalculationPixNorm(Depth depth, const Intr& intrinsics); +cv::UMat preCalculationPixNormGPU(const UMat& depth, const Intr& intrinsics); + depthType bilinearDepth(const Depth& m, cv::Point2f pt); void integrateVolumeUnit( @@ -43,6 +45,246 @@ void integrateVolumeUnit( InputArray _depth, float depthFactor, const cv::Matx44f& cameraPose, const cv::kinfu::Intr& intrinsics, InputArray _pixNorms, InputArray _volume); + +class CustomHashSet +{ +public: + static const int hashDivisor = 32768; + static const int startCapacity = 2048; + + std::vector hashes; + // 0-3 for key, 4th for internal use + // don't keep keep value + std::vector data; + int capacity; + int last; + + CustomHashSet() + { + hashes.resize(hashDivisor); + for (int i = 0; i < hashDivisor; i++) + hashes[i] = -1; + capacity = startCapacity; + + data.resize(capacity); + for (int i = 0; i < capacity; i++) + data[i] = { 0, 0, 0, -1 }; + + last = 0; + } + + ~CustomHashSet() { } + + inline size_t calc_hash(Vec3i x) const + { + uint32_t seed = 0; + constexpr uint32_t GOLDEN_RATIO = 0x9e3779b9; + for (int i = 0; i < 3; i++) + { + seed ^= x[i] + GOLDEN_RATIO + (seed << 6) + (seed >> 2); + } + return seed; + } + + // should work on existing elements too + // 0 - need resize + // 1 - idx is inserted + // 2 - idx already exists + int insert(Vec3i idx) + { + if (last < capacity) + { + int hash = int(calc_hash(idx) % hashDivisor); + int place = hashes[hash]; + if (place >= 0) + { + int oldPlace = place; + while (place >= 0) + { + if (data[place][0] == idx[0] && + data[place][1] == idx[1] && + data[place][2] == idx[2]) + return 2; + else + { + oldPlace = place; + place = data[place][3]; + //std::cout << "place=" << place << std::endl; + } + } + + // found, create here + data[oldPlace][3] = last; + } + else + { + // insert at last + hashes[hash] = last; + } + + data[last][0] = idx[0]; + data[last][1] = idx[1]; + data[last][2] = idx[2]; + data[last][3] = -1; + last++; + + return 1; + } + else + return 0; + } + + int find(Vec3i idx) const + { + int hash = int(calc_hash(idx) % hashDivisor); + int place = hashes[hash]; + // search a place + while (place >= 0) + { + if (data[place][0] == idx[0] && + data[place][1] == idx[1] && + data[place][2] == idx[2]) + break; + else + { + place = data[place][3]; + } + } + + return place; + } +}; + +// TODO: remove this structure as soon as HashTSDFGPU data is completely on GPU; +// until then CustomHashTable can be replaced by this one if needed + +const int NAN_ELEMENT = -2147483647; + +struct Volume_NODE +{ + Vec4i idx = Vec4i(NAN_ELEMENT); + int32_t row = -1; + int32_t nextVolumeRow = -1; + int32_t dummy = 0; + int32_t dummy2 = 0; +}; + +const int _hash_divisor = 32768; +const int _list_size = 4; + +class VolumesTable +{ +public: + const int hash_divisor = _hash_divisor; + const int list_size = _list_size; + const int32_t free_row = -1; + const int32_t free_isActive = 0; + + const cv::Vec4i nan4 = cv::Vec4i(NAN_ELEMENT); + + int bufferNums; + cv::Mat volumes; + + VolumesTable() : bufferNums(1) + { + this->volumes = cv::Mat(hash_divisor * list_size, 1, rawType()); + for (int i = 0; i < volumes.size().height; i++) + { + Volume_NODE* v = volumes.ptr(i); + v->idx = nan4; + v->row = -1; + v->nextVolumeRow = -1; + } + } + const VolumesTable& operator=(const VolumesTable& vt) + { + this->volumes = vt.volumes; + this->bufferNums = vt.bufferNums; + return *this; + } + ~VolumesTable() {}; + + bool insert(Vec3i idx, int row) + { + CV_Assert(row >= 0); + + int bufferNum = 0; + int hash = int(calc_hash(idx) % hash_divisor); + int start = getPos(idx, bufferNum); + int i = start; + + while (i >= 0) + { + Volume_NODE* v = volumes.ptr(i); + + if (v->idx[0] == NAN_ELEMENT) + { + Vec4i idx4(idx[0], idx[1], idx[2], 0); + + bool extend = false; + if (i != start && i % list_size == 0) + { + if (bufferNum >= bufferNums - 1) + { + extend = true; + volumes.resize(hash_divisor * bufferNums); + bufferNums++; + } + bufferNum++; + v->nextVolumeRow = (bufferNum * hash_divisor + hash) * list_size; + } + else + { + v->nextVolumeRow = i + 1; + } + + v->idx = idx4; + v->row = row; + + return extend; + } + + i = v->nextVolumeRow; + } + return false; + } + int findRow(Vec3i idx) const + { + int bufferNum = 0; + int i = getPos(idx, bufferNum); + + while (i >= 0) + { + const Volume_NODE* v = volumes.ptr(i); + + if (v->idx == Vec4i(idx[0], idx[1], idx[2], 0)) + return v->row; + else + i = v->nextVolumeRow; + } + + return -1; + } + + inline int getPos(Vec3i idx, int bufferNum) const + { + int hash = int(calc_hash(idx) % hash_divisor); + return (bufferNum * hash_divisor + hash) * list_size; + } + + inline size_t calc_hash(Vec3i x) const + { + uint32_t seed = 0; + constexpr uint32_t GOLDEN_RATIO = 0x9e3779b9; + for (int i = 0; i < 3; i++) + { + seed ^= x[i] + GOLDEN_RATIO + (seed << 6) + (seed >> 2); + } + return seed; + } +}; + + } // namespace kinfu } // namespace cv #endif diff --git a/modules/rgbd/src/volume.cpp b/modules/rgbd/src/volume.cpp index 8177213e8ab..88c46fe4b67 100644 --- a/modules/rgbd/src/volume.cpp +++ b/modules/rgbd/src/volume.cpp @@ -68,7 +68,7 @@ Ptr makeVolume(const VolumeParams& _volumeParams) if(_volumeParams.type == VolumeType::TSDF) return kinfu::makeTSDFVolume(_volumeParams); else if(_volumeParams.type == VolumeType::HASHTSDF) - return kinfu::makeHashTSDFVolume(_volumeParams); + return kinfu::makeHashTSDFVolume(_volumeParams); CV_Error(Error::StsBadArg, "Invalid VolumeType does not have parameters"); } @@ -79,13 +79,11 @@ Ptr makeVolume(VolumeType _volumeType, float _voxelSize, Matx44f _pose, Point3i _presolution = _resolution; if (_volumeType == VolumeType::TSDF) { - return makeTSDFVolume(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, - _presolution); + return makeTSDFVolume(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, _presolution); } else if (_volumeType == VolumeType::HASHTSDF) { - return makeHashTSDFVolume( - _voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, _truncateThreshold); + return makeHashTSDFVolume(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, _truncateThreshold); } CV_Error(Error::StsBadArg, "Invalid VolumeType does not have parameters"); } diff --git a/modules/rgbd/test/test_tsdf.cpp b/modules/rgbd/test/test_tsdf.cpp index 99da9579697..5a9b23afb95 100644 --- a/modules/rgbd/test/test_tsdf.cpp +++ b/modules/rgbd/test/test_tsdf.cpp @@ -358,7 +358,7 @@ void normal_test(bool isHashTSDF, bool isRaycast, bool isFetchPointsNormals, boo points = _points.getMat(af); renderPointsNormals(points, normals, image, _params->lightPose); imshow("render", image); - waitKey(20000); + waitKey(2000); } if (isRaycast) @@ -384,7 +384,7 @@ void normal_test(bool isHashTSDF, bool isRaycast, bool isFetchPointsNormals, boo points = _newPoints.getMat(af); renderPointsNormals(points, normals, image, _params->lightPose); imshow("render", image); - waitKey(20000); + waitKey(2000); } } @@ -448,7 +448,7 @@ void valid_points_test(bool isHashTSDF) imshow("depth", depth * (1.f / _params->depthFactor / 4.f)); renderPointsNormals(points, normals, image, _params->lightPose); imshow("render", image); - waitKey(20000); + waitKey(2000); } volume->raycast(poses[17].matrix, _params->intr, _params->frameSize, _newPoints, _newNormals); @@ -463,10 +463,13 @@ void valid_points_test(bool isHashTSDF) imshow("depth", depth * (1.f / _params->depthFactor / 4.f)); renderPointsNormals(points, normals, image, _params->lightPose); imshow("render", image); - waitKey(20000); + waitKey(2000); } - float percentValidity = float(profile) / float(anfas); + float percentValidity; + if (profile == 0) percentValidity = -0.5; + else if (anfas == 0) percentValidity = 0; + else percentValidity = float(profile) / float(anfas); ASSERT_LT(0.5 - percentValidity, 0.3); } @@ -510,4 +513,5 @@ TEST(HashTSDF, valid_points) valid_points_test(true); } + }} // namespace From 61e0a2de7317651a8e2747714f6227ee36ea5ad7 Mon Sep 17 00:00:00 2001 From: "amir.tulegenov" Date: Mon, 15 Feb 2021 15:19:23 +0600 Subject: [PATCH 49/70] [moved from opencv] fix getDefaultName() original commit: https://github.com/opencv/opencv/commit/cbb230fdfc0223a33cf2629963fe0253b91ca3eb --- modules/cudaoptflow/src/brox.cpp | 2 ++ modules/cudaoptflow/src/farneback.cpp | 2 ++ modules/cudaoptflow/src/pyrlk.cpp | 4 ++++ modules/cudaoptflow/src/tvl1flow.cpp | 3 +++ modules/optflow/src/tvl1flow.cpp | 2 ++ 5 files changed, 13 insertions(+) diff --git a/modules/cudaoptflow/src/brox.cpp b/modules/cudaoptflow/src/brox.cpp index 11c541906be..70c8a4e0bd3 100644 --- a/modules/cudaoptflow/src/brox.cpp +++ b/modules/cudaoptflow/src/brox.cpp @@ -64,6 +64,8 @@ namespace { { } + virtual String getDefaultName() const { return "DenseOpticalFlow.BroxOpticalFlow"; } + virtual void calc(InputArray I0, InputArray I1, InputOutputArray flow, Stream& stream); virtual double getFlowSmoothness() const { return alpha_; } diff --git a/modules/cudaoptflow/src/farneback.cpp b/modules/cudaoptflow/src/farneback.cpp index 69ea437ec41..7cc8373f72b 100644 --- a/modules/cudaoptflow/src/farneback.cpp +++ b/modules/cudaoptflow/src/farneback.cpp @@ -129,6 +129,8 @@ namespace virtual void calc(InputArray I0, InputArray I1, InputOutputArray flow, Stream& stream); + virtual String getDefaultName() const { return "DenseOpticalFlow.FarnebackOpticalFlow"; } + private: int numLevels_; double pyrScale_; diff --git a/modules/cudaoptflow/src/pyrlk.cpp b/modules/cudaoptflow/src/pyrlk.cpp index 10209779033..d7447ae71bc 100644 --- a/modules/cudaoptflow/src/pyrlk.cpp +++ b/modules/cudaoptflow/src/pyrlk.cpp @@ -347,6 +347,8 @@ namespace sparse(prevImg, nextImg, prevPts, nextPts, status, err, stream); } } + + virtual String getDefaultName() const { return "SparseOpticalFlow.SparsePyrLKOpticalFlow"; } }; class DensePyrLKOpticalFlowImpl : public DensePyrLKOpticalFlow, private PyrLKOpticalFlowBase @@ -388,6 +390,8 @@ namespace GpuMat flows[] = {u, v}; cuda::merge(flows, 2, _flow, stream); } + + virtual String getDefaultName() const { return "DenseOpticalFlow.DensePyrLKOpticalFlow"; } }; } diff --git a/modules/cudaoptflow/src/tvl1flow.cpp b/modules/cudaoptflow/src/tvl1flow.cpp index abc6c2e318f..5f28d4c6174 100644 --- a/modules/cudaoptflow/src/tvl1flow.cpp +++ b/modules/cudaoptflow/src/tvl1flow.cpp @@ -119,6 +119,9 @@ namespace virtual void calc(InputArray I0, InputArray I1, InputOutputArray flow, Stream& stream); + virtual String getDefaultName() const { return "DenseOpticalFlow.OpticalFlowDual_TVL1"; } + + private: double tau_; double lambda_; diff --git a/modules/optflow/src/tvl1flow.cpp b/modules/optflow/src/tvl1flow.cpp index 4131ac59a11..5af85d539fb 100644 --- a/modules/optflow/src/tvl1flow.cpp +++ b/modules/optflow/src/tvl1flow.cpp @@ -99,6 +99,8 @@ class OpticalFlowDual_TVL1 : public DualTVL1OpticalFlow } OpticalFlowDual_TVL1(); + virtual String getDefaultName() const CV_OVERRIDE { return "DenseOpticalFlow.DualTVL1OpticalFlow"; } + void calc(InputArray I0, InputArray I1, InputOutputArray flow) CV_OVERRIDE; void collectGarbage() CV_OVERRIDE; From fdf80d7fcbc02655b9d126c13de944fdfe782dcd Mon Sep 17 00:00:00 2001 From: Tomoaki Teshima Date: Mon, 15 Feb 2021 21:01:41 +0900 Subject: [PATCH 50/70] [moved from opencv] remove danger race condition original commit: https://github.com/opencv/opencv/commit/5d1540f4fc1ba1736cb248561ad1915abdef7246 --- modules/cudafilters/src/cuda/median_filter.cu | 89 ++++++++----------- 1 file changed, 38 insertions(+), 51 deletions(-) diff --git a/modules/cudafilters/src/cuda/median_filter.cu b/modules/cudafilters/src/cuda/median_filter.cu index f6be58d62cb..ed46eb4bf94 100644 --- a/modules/cudafilters/src/cuda/median_filter.cu +++ b/modules/cudafilters/src/cuda/median_filter.cu @@ -55,9 +55,6 @@ namespace cv { namespace cuda { namespace device { - // // namespace imgproc - // { - __device__ void histogramAddAndSub8(int* H, const int * hist_colAdd,const int * hist_colSub){ int tx = threadIdx.x; if (tx<8){ @@ -125,6 +122,25 @@ namespace cv { namespace cuda { namespace device luc[tx]=0; } +#define scanNeighbor(array, range, index, threadIndex) \ + { \ + int v = 0; \ + if (index <= threadIndex && threadIndex < range) \ + v = array[threadIndex] + array[threadIndex-index]; \ + __syncthreads(); \ + if (index <= threadIndex && threadIndex < range) \ + array[threadIndex] = v; \ + } +#define findMedian(array, range, threadIndex, result, count, position) \ + if (threadIndex < range) \ + { \ + if (array[threadIndex+1] > position && array[threadIndex] <= position) \ + { \ + *result = threadIndex+1; \ + *count = array[threadIndex]; \ + } \ + } + __device__ void histogramMedianPar8LookupOnly(int* H,int* Hscan, const int medPos,int* retval, int* countAtMed){ int tx=threadIdx.x; *retval=*countAtMed=0; @@ -132,28 +148,14 @@ namespace cv { namespace cuda { namespace device Hscan[tx]=H[tx]; } __syncthreads(); - if (1 <= tx && tx < 8 ) - Hscan[tx]+=Hscan[tx-1]; + scanNeighbor(Hscan, 8, 1, tx); __syncthreads(); - if (2 <= tx && tx < 8 ) - Hscan[tx]+=Hscan[tx-2]; + scanNeighbor(Hscan, 8, 2, tx); __syncthreads(); - if (4 <= tx && tx < 8 ) - Hscan[tx]+=Hscan[tx-4]; + scanNeighbor(Hscan, 8, 4, tx); __syncthreads(); - if(tx<7){ - if(Hscan[tx+1] > medPos && Hscan[tx] < medPos){ - *retval=tx+1; - *countAtMed=Hscan[tx]; - } - else if(Hscan[tx]==medPos){ - if(Hscan[tx+1]>medPos){ - *retval=tx+1; - *countAtMed=Hscan[tx]; - } - } - } + findMedian(Hscan, 7, tx, retval, countAtMed, medPos); } __device__ void histogramMedianPar32LookupOnly(int* H,int* Hscan, const int medPos,int* retval, int* countAtMed){ @@ -163,33 +165,18 @@ namespace cv { namespace cuda { namespace device Hscan[tx]=H[tx]; } __syncthreads(); - if ( 1 <= tx && tx < 32 ) - Hscan[tx]+=Hscan[tx-1]; + scanNeighbor(Hscan, 32, 1, tx); __syncthreads(); - if ( 2 <= tx && tx < 32 ) - Hscan[tx]+=Hscan[tx-2]; + scanNeighbor(Hscan, 32, 2, tx); __syncthreads(); - if ( 4 <= tx && tx < 32 ) - Hscan[tx]+=Hscan[tx-4]; + scanNeighbor(Hscan, 32, 4, tx); __syncthreads(); - if ( 8 <= tx && tx < 32 ) - Hscan[tx]+=Hscan[tx-8]; + scanNeighbor(Hscan, 32, 8, tx); __syncthreads(); - if ( 16 <= tx && tx < 32 ) - Hscan[tx]+=Hscan[tx-16]; + scanNeighbor(Hscan, 32, 16, tx); __syncthreads(); - if(tx<31){ - if(Hscan[tx+1] > medPos && Hscan[tx] < medPos){ - *retval=tx+1; - *countAtMed=Hscan[tx]; - } - else if(Hscan[tx]==medPos){ - if(Hscan[tx+1]>medPos){ - *retval=tx+1; - *countAtMed=Hscan[tx]; - } - } - } + + findMedian(Hscan, 31, tx, retval, countAtMed, medPos); } __global__ void cuMedianFilterMultiBlock(PtrStepSzb src, PtrStepSzb dest, PtrStepSzi histPar, PtrStepSzi coarseHistGrid,int r, int medPos_) @@ -288,7 +275,6 @@ namespace cv { namespace cuda { namespace device __syncthreads(); histogramMultipleAdd8(HCoarse,histCoarse, 2*r+1); -// __syncthreads(); int cols_m_1=cols-1; for(int j=r;j=0){ From 0165d48da48b169420cd4d70867b991bd48543e8 Mon Sep 17 00:00:00 2001 From: Dimitrios Psychogyios Date: Mon, 1 Mar 2021 11:52:49 +0000 Subject: [PATCH 51/70] Merge pull request #2869 from dimitrisPs:QDS_getDisparity Remove normalization in cv::stereo::QuasiDenseStereo getDisparity(). * Remove normalization in getDisparity. The function returns CV_32F disparity maps with nan values for unknown matches. * Switch to GTEST and replace license header --- .../opencv2/stereo/quasi_dense_stereo.hpp | 3 +- modules/stereo/samples/dense_disparity.cpp | 3 +- modules/stereo/samples/sample_quasi_dense.py | 2 +- modules/stereo/src/quasi_dense_stereo.cpp | 42 +---------- modules/stereo/test/test_block_matching.cpp | 2 +- modules/stereo/test/test_qds_matching.cpp | 69 +++++++++++++++++++ 6 files changed, 76 insertions(+), 45 deletions(-) create mode 100644 modules/stereo/test/test_qds_matching.cpp diff --git a/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp b/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp index 197d762b6c7..b2290e3768c 100644 --- a/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp +++ b/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp @@ -176,13 +176,12 @@ class CV_EXPORTS_W QuasiDenseStereo /** * @brief Compute and return the disparity map based on the correspondences found in the "process" method. - * @param[in] disparityLvls The level of detail in output disparity image. * @note Default level is 50 * @return cv::Mat containing a the disparity image in grayscale. * @sa computeDisparity * @sa quantizeDisparity */ - CV_WRAP virtual cv::Mat getDisparity(uint8_t disparityLvls=50) = 0; + CV_WRAP virtual cv::Mat getDisparity() = 0; CV_WRAP static cv::Ptr create(cv::Size monoImgSize, cv::String paramFilepath = cv::String()); diff --git a/modules/stereo/samples/dense_disparity.cpp b/modules/stereo/samples/dense_disparity.cpp index e7822366623..b72d567588a 100644 --- a/modules/stereo/samples/dense_disparity.cpp +++ b/modules/stereo/samples/dense_disparity.cpp @@ -31,9 +31,8 @@ int main() //! [disp] - uint8_t displvl = 80; cv::Mat disp; - disp = stereo->getDisparity(displvl); + disp = stereo->getDisparity(); cv::namedWindow("disparity map"); cv::imshow("disparity map", disp); //! [disp] diff --git a/modules/stereo/samples/sample_quasi_dense.py b/modules/stereo/samples/sample_quasi_dense.py index a6059e70179..bd11e1d2898 100644 --- a/modules/stereo/samples/sample_quasi_dense.py +++ b/modules/stereo/samples/sample_quasi_dense.py @@ -8,7 +8,7 @@ stereo = cv.stereo.QuasiDenseStereo_create(frame_size[::-1]) stereo.process(left_img, right_img) -disp = stereo.getDisparity(80) +disp = stereo.getDisparity() cv.imshow("disparity", disp) cv.waitKey() dense_matches = stereo.getDenseMatches() diff --git a/modules/stereo/src/quasi_dense_stereo.cpp b/modules/stereo/src/quasi_dense_stereo.cpp index a4d196ae11b..dfc48f52937 100644 --- a/modules/stereo/src/quasi_dense_stereo.cpp +++ b/modules/stereo/src/quasi_dense_stereo.cpp @@ -34,7 +34,6 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo ssum1 = cv::Mat_(integralSize); // the disparity image. disparity = cv::Mat_(monoImgSize); - disparityImg = cv::Mat_(monoImgSize); // texture images. textureDescLeft = cv::Mat_ (monoImgSize); textureDescRight = cv::Mat_ (monoImgSize); @@ -55,7 +54,6 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo ssum1.release(); // the disparity image. disparity.release(); - disparityImg.release(); // texture images. textureDescLeft.release(); textureDescRight.release(); @@ -248,7 +246,6 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo * @param[in] matchMap A matrix of points, the same size as the left channel. Each cell of this * matrix stores the location of the corresponding point in the right image. * @param[out] dispMat The disparity map. - * @sa quantizeDisparity * @sa getDisparity */ void computeDisparity(const cv::Mat_ &matchMap, @@ -262,7 +259,7 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo if (matchMap.at(tmpPoint) == NO_MATCH) { - dispMat.at(tmpPoint) = 200; + dispMat.at(tmpPoint) = NAN; continue; } //if a match is found, compute the difference in location of the match and current @@ -276,37 +273,6 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo } - /** - * @brief Disparity map normalization for display purposes. If needed specify the quantization - * level as input argument. - * @param[in] dispMat The disparity Map. - * @param[in] lvls The quantization level of the output disparity map. - * @return Disparity image. - * @note Stores the output in the disparityImage class variable. - * @sa computeDisparity - * @sa getDisparity - */ - cv::Mat quantiseDisparity(const cv::Mat_ &dispMat, const int lvls) - { - float tmpPixelVal ; - double min, max; -// minMaxLoc(disparity, &min, &max); - min = 0; - max = lvls; - for(int row=0; row(row, col); - tmpPixelVal = (float) (255. - 255.0*(tmpPixelVal-min)/(max-min)); - - disparityImg.at(row, col) = (uint8_t) tmpPixelVal; - } - } - return disparityImg; - } - - /** * @brief Compute the Zero-mean Normalized Cross-correlation. * @@ -635,10 +601,10 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo return refMap.at(y, x); } - cv::Mat getDisparity(uint8_t disparityLvls) override + cv::Mat getDisparity() override { computeDisparity(refMap, disparity); - return quantiseDisparity(disparity, disparityLvls); + return disparity; } // Variables used at sparse feature extraction. @@ -663,8 +629,6 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo cv::Mat_ ssum1; // Container to store the disparity un-normalized cv::Mat_ disparity; - // Container to store the disparity image. - cv::Mat_ disparityImg; // Containers to store textures descriptors. cv::Mat_ textureDescLeft; cv::Mat_ textureDescRight; diff --git a/modules/stereo/test/test_block_matching.cpp b/modules/stereo/test/test_block_matching.cpp index 159be4907cb..c678d42d724 100644 --- a/modules/stereo/test/test_block_matching.cpp +++ b/modules/stereo/test/test_block_matching.cpp @@ -166,7 +166,7 @@ void CV_SGBlockMatchingTest::run(int ) image2 = imread(ts->get_data_path() + "stereomatching/datasets/tsukuba/im6.png", IMREAD_GRAYSCALE); gt = imread(ts->get_data_path() + "stereomatching/datasets/tsukuba/disp2.png", IMREAD_GRAYSCALE); - + ts->printf(cvtest::TS::LOG,(ts->get_data_path() + "stereomatching/datasets/tsukuba/im2.png").c_str()); if(image1.empty() || image2.empty() || gt.empty()) { ts->printf(cvtest::TS::LOG, "Wrong input data \n"); diff --git a/modules/stereo/test/test_qds_matching.cpp b/modules/stereo/test/test_qds_matching.cpp new file mode 100644 index 00000000000..9b550a80da2 --- /dev/null +++ b/modules/stereo/test/test_qds_matching.cpp @@ -0,0 +1,69 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + + +#include "test_precomp.hpp" + +namespace opencv_test { namespace { + + +static float disparity_MAE(const Mat &reference, const Mat &estimation) +{ + int elems=0; + float error=0; + float ref_invalid=0; + for (int row=0; row< reference.rows; row++){ + for (int col=0; col(row, col); + float estimated_val = estimation.at(row, col); + if (ref_val == 0){ + ref_invalid++; + } + // filter out pixels with unknown reference value and pixels whose disparity did not get estimated. + if (estimated_val == 0 || ref_val == 0 || std::isnan(estimated_val)){ + continue; + } + else{ + error+=abs(ref_val - estimated_val); + elems+=1; + } + } + } + return error/elems; +} + + +// void CV_QdsMatchingTest::run(int) +TEST(qds_getDisparity, accuracy) +{ + //load data + Mat image1, image2, gt; + image1 = imread(cvtest::TS::ptr()->get_data_path() + "stereomatching/datasets/cones/im2.png", IMREAD_GRAYSCALE); + image2 = imread(cvtest::TS::ptr()->get_data_path() + "stereomatching/datasets/cones/im6.png", IMREAD_GRAYSCALE); + gt = imread(cvtest::TS::ptr()->get_data_path() + "stereomatching/datasets/cones/disp2.png", IMREAD_GRAYSCALE); + + // reference scale factor is based on this https://github.com/opencv/opencv_extra/blob/master/testdata/cv/stereomatching/datasets/datasets.xml + gt.convertTo(gt, CV_32F); + gt =gt/4; + + //test inputs + ASSERT_FALSE(image1.empty() || image2.empty() || gt.empty()) << "Issue with input data"; + + //configure disparity algorithm + cv::Size frameSize = image1.size(); + Ptr qds_matcher = stereo::QuasiDenseStereo::create(frameSize); + + + //compute disparity + qds_matcher->process(image1, image2); + Mat outDisp = qds_matcher->getDisparity(); + + // test input output size consistency + ASSERT_EQ(gt.size(), outDisp.size()) << "Mismatch input/output dimensions"; + ASSERT_LT(disparity_MAE(gt, outDisp),2) << "EPE should be 1.1053 for this sample/hyperparamters (Tested on version 4.5.1)"; +} + + + +}} // namespace \ No newline at end of file From 9240579e721fba50f67531bfd2eec966b2764f6d Mon Sep 17 00:00:00 2001 From: dzyGIT <124340156@qq.com> Date: Thu, 11 Mar 2021 20:07:07 +0800 Subject: [PATCH 52/70] Update omnidir.cpp Fixed R2 --- modules/ccalib/src/omnidir.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ccalib/src/omnidir.cpp b/modules/ccalib/src/omnidir.cpp index a67a6bd6c07..6c54a527c1e 100644 --- a/modules/ccalib/src/omnidir.cpp +++ b/modules/ccalib/src/omnidir.cpp @@ -2222,7 +2222,7 @@ void cv::omnidir::stereoRectify(InputArray R, InputArray T, OutputArray R1, Outp e1.copyTo(_R1.row(0)); e2.copyTo(_R1.row(1)); e3.copyTo(_R1.row(2)); - _R2 = R21 * _R1; + _R2 = _R1 * R21; } From 22cbfd671831cd6f1df3f0eff3641a3d8426a947 Mon Sep 17 00:00:00 2001 From: berak Date: Wed, 17 Mar 2021 09:08:40 +0100 Subject: [PATCH 53/70] ximgproc: add an assert in thinning() --- modules/ximgproc/src/thinning.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/ximgproc/src/thinning.cpp b/modules/ximgproc/src/thinning.cpp index c62c13e6d96..b28784d2894 100644 --- a/modules/ximgproc/src/thinning.cpp +++ b/modules/ximgproc/src/thinning.cpp @@ -69,6 +69,7 @@ static void thinningIteration(Mat img, int iter, int thinningType){ // Apply the thinning procedure to a given image void thinning(InputArray input, OutputArray output, int thinningType){ Mat processed = input.getMat().clone(); + CV_CheckTypeEQ(processed.type(), CV_8UC1, ""); // Enforce the range of the input image to be in between 0 - 255 processed /= 255; From bf067b4c4378a8c1bbfbd55a0e169152fc404edd Mon Sep 17 00:00:00 2001 From: Pavel Rojtberg Date: Fri, 19 Mar 2021 13:58:31 +0100 Subject: [PATCH 54/70] ovis: add software-renderer based unit testing --- modules/ovis/misc/python/test_ovis.py | 39 +++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 modules/ovis/misc/python/test_ovis.py diff --git a/modules/ovis/misc/python/test_ovis.py b/modules/ovis/misc/python/test_ovis.py new file mode 100644 index 00000000000..c5bf6547e6a --- /dev/null +++ b/modules/ovis/misc/python/test_ovis.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +import os +import numpy as np +import cv2 as cv + +from tests_common import NewOpenCVTests +#from unittest import TestCase as NewOpenCVTests + + +class ovis_contrib_test(NewOpenCVTests): + + def setUp(self): + super().setUp() + # use software rendering + os.environ["OPENCV_OVIS_RENDERSYSTEM"] = "Tiny Rendering Subsystem" + # in case something goes wrong + os.environ["OPENCV_OVIS_VERBOSE_LOG"] = "1" + + def test_multiWindow(self): + win0 = cv.ovis.createWindow("main", (1, 1)) + win1 = cv.ovis.createWindow("other", (1, 1)) + del win1 + win1 = cv.ovis.createWindow("other", (1, 1)) + del win1 + + def test_addResourceLocation(self): + win0 = cv.ovis.createWindow("main", (1, 1)) + with self.assertRaises(cv.error): + # must be called before the first createWindow + cv.ovis.addResourceLocation(".") + + def test_texStride(self): + win = cv.ovis.createWindow("main", (1, 1)) + data = np.zeros((200, 200), dtype=np.uint8) + cv.ovis.createPlaneMesh("plane", (1, 1), data[50:-50, 50:-50]) + + +if __name__ == '__main__': + NewOpenCVTests.bootstrap() From ba281ca1044793913b365adcd0cebd8b05a33b74 Mon Sep 17 00:00:00 2001 From: chester Date: Sun, 21 Mar 2021 16:45:11 +0800 Subject: [PATCH 55/70] fix: add a safe score threshold(1E-5) to wechat_qrcode detector --- modules/wechat_qrcode/src/detector/ssd_detector.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/wechat_qrcode/src/detector/ssd_detector.cpp b/modules/wechat_qrcode/src/detector/ssd_detector.cpp index e55e6ef1f29..dca1851ae0d 100644 --- a/modules/wechat_qrcode/src/detector/ssd_detector.cpp +++ b/modules/wechat_qrcode/src/detector/ssd_detector.cpp @@ -31,7 +31,8 @@ vector SSDDetector::forward(Mat img, const int target_width, const int targ const float* prob_score = prob.ptr(0, 0, row); // prob_score[0] is not used. // prob_score[1]==1 stands for qrcode - if (prob_score[1] == 1) { + if (prob_score[1] == 1 && prob_score[2] > 1E-5) { + // add a safe score threshold due to https://github.com/opencv/opencv_contrib/issues/2877 // prob_score[2] is the probability of the qrcode, which is not used. auto point = Mat(4, 2, CV_32FC1); float x0 = CLIP(prob_score[3] * img_w, 0.0f, img_w - 1.0f); From 068b6c700e8990d499715f63d685e712ad0dd8b2 Mon Sep 17 00:00:00 2001 From: Jan-Kristian Herring Date: Wed, 24 Mar 2021 15:23:30 +0200 Subject: [PATCH 56/70] Fix aruco _filterTooCloseCandidates _filterTooCloseCandidates was occasionally not choosing the biggest contour of a candidate group. Firstly, if the biggest contour of a candidate group happened to be the first candidate of that group, it would never be chosen due to the comparison loop skipping it. Secondly, contour sizes were being compared by their circumference (via counting the number of points in a contour) rather than area. In some cases it is possible for a smaller candidate to have more points than a bigger one. This was fixed by comparing the contour area as defined by the contour's corners. --- modules/aruco/src/aruco.cpp | 42 +++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/modules/aruco/src/aruco.cpp b/modules/aruco/src/aruco.cpp index 4d8f2b85f01..ca515bb3515 100644 --- a/modules/aruco/src/aruco.cpp +++ b/modules/aruco/src/aruco.cpp @@ -306,34 +306,36 @@ static void _filterTooCloseCandidates(const vector< vector< Point2f > > &candida vector< vector< Point > > smallerContours; // save possible candidates - for( unsigned int i = 0; i < groupedCandidates.size(); i++ ) { - int smallerIdx = groupedCandidates[i][0]; - int biggerIdx = -1; + for(unsigned int i = 0; i < groupedCandidates.size(); i++) { + unsigned int smallerIdx = groupedCandidates[i][0]; + unsigned int biggerIdx = smallerIdx; + double smallerArea = contourArea(candidatesIn[smallerIdx]); + double biggerArea = smallerArea; // evaluate group elements - for( unsigned int j = 1; j < groupedCandidates[i].size(); j++ ) { - size_t currPerim = contoursIn[ groupedCandidates[i][j] ].size(); + for(unsigned int j = 1; j < groupedCandidates[i].size(); j++) { + unsigned int currIdx = groupedCandidates[i][j]; + double currArea = contourArea(candidatesIn[currIdx]); // check if current contour is bigger - if ( biggerIdx < 0 ) - biggerIdx = groupedCandidates[i][j]; - else if(currPerim >= contoursIn[ biggerIdx ].size()) - biggerIdx = groupedCandidates[i][j]; + if(currArea >= biggerArea) { + biggerIdx = currIdx; + biggerArea = currArea; + } // check if current contour is smaller - if(currPerim < contoursIn[ smallerIdx ].size() && detectInvertedMarker) - smallerIdx = groupedCandidates[i][j]; + if(currArea < smallerArea && detectInvertedMarker) { + smallerIdx = currIdx; + smallerArea = currArea; + } } - // add contours und candidates - if(biggerIdx > -1){ - biggerCandidates.push_back(candidatesIn[biggerIdx]); - biggerContours.push_back(contoursIn[biggerIdx]); - - if( detectInvertedMarker ){ - smallerCandidates.push_back(alignContourOrder(candidatesIn[biggerIdx][0], candidatesIn[smallerIdx])); - smallerContours.push_back(contoursIn[smallerIdx]); - } + // add contours and candidates + biggerCandidates.push_back(candidatesIn[biggerIdx]); + biggerContours.push_back(contoursIn[biggerIdx]); + if(detectInvertedMarker) { + smallerCandidates.push_back(alignContourOrder(candidatesIn[biggerIdx][0], candidatesIn[smallerIdx])); + smallerContours.push_back(contoursIn[smallerIdx]); } } // to preserve the structure :: candidateSet< defaultCandidates, whiteCandidates > From 0042b40c2989b9594496c15a7cfee761e87d1a8e Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 31 Mar 2021 21:33:47 +0000 Subject: [PATCH 57/70] cmake: fix tracking detail headers --- .../tracking/include/opencv2/tracking/tracking_internals.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/tracking/include/opencv2/tracking/tracking_internals.hpp b/modules/tracking/include/opencv2/tracking/tracking_internals.hpp index 92ca54a992a..7d224928645 100644 --- a/modules/tracking/include/opencv2/tracking/tracking_internals.hpp +++ b/modules/tracking/include/opencv2/tracking/tracking_internals.hpp @@ -15,7 +15,7 @@ * */ -#include "opencv2/video/detail/tracking.private.hpp" +#include "opencv2/video/detail/tracking.detail.hpp" #include "feature.hpp" // CvHaarEvaluator #include "onlineBoosting.hpp" // StrongClassifierDirectSelection From 4b20db546bb5ad43933016ea847e25c8384ca740 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 1 Apr 2021 02:26:59 +0000 Subject: [PATCH 58/70] [moved from opencv] cuda: fix inplace condition in cv::cuda::flip original commit: https://github.com/opencv/opencv/commit/e4b0251e9d1885e2f6665643a8dbc1346ce34327 --- modules/cudaarithm/src/core.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/cudaarithm/src/core.cpp b/modules/cudaarithm/src/core.cpp index 368c2fcc419..223929881fd 100644 --- a/modules/cudaarithm/src/core.cpp +++ b/modules/cudaarithm/src/core.cpp @@ -163,7 +163,7 @@ void cv::cuda::flip(InputArray _src, OutputArray _dst, int flipCode, Stream& str _dst.create(src.size(), src.type()); GpuMat dst = getOutputMat(_dst, src.size(), src.type(), stream); - bool isInplace = (src.data == dst.data) || (src.refcount == dst.refcount); + bool isInplace = (src.data == dst.data); bool isSizeOdd = (src.cols & 1) == 1 || (src.rows & 1) == 1; if (isInplace && isSizeOdd) CV_Error(Error::BadROISize, "In-place version of flip only accepts even width/height"); From c287e526f11be833e9243a4ef8d6406a7728acdf Mon Sep 17 00:00:00 2001 From: DumDereDum <46279571+DumDereDum@users.noreply.github.com> Date: Thu, 1 Apr 2021 17:45:13 +0300 Subject: [PATCH 59/70] Merge pull request #2854 from DumDereDum:rgbd_test_refactoring RGBD tests refactoring * add GPU and CPU versions of tests * reduce time for demo show * add assert info and scene minor fix * add display to perf_test * replace extra code by function in perf_test * add display func for tests * add settings class for test * remove extra code * replace scene in perf test * main dug fixed * fix the same bug * fix * docs fix * minor fix * namespace fix * tsdf cpu getnormal fix * add folder ocl with simple tests * build error fix --- modules/rgbd/perf/perf_tsdf.cpp | 182 +++++++++-- modules/rgbd/src/opencl/tsdf.cl | 4 +- modules/rgbd/src/tsdf.cpp | 16 +- modules/rgbd/test/ocl/test_tsdf.cpp | 470 ++++++++++++++++++++++++++++ modules/rgbd/test/test_tsdf.cpp | 317 ++++++++++--------- 5 files changed, 795 insertions(+), 194 deletions(-) create mode 100644 modules/rgbd/test/ocl/test_tsdf.cpp diff --git a/modules/rgbd/perf/perf_tsdf.cpp b/modules/rgbd/perf/perf_tsdf.cpp index ebca91587d6..149093665ce 100644 --- a/modules/rgbd/perf/perf_tsdf.cpp +++ b/modules/rgbd/perf/perf_tsdf.cpp @@ -32,12 +32,13 @@ template struct RenderInvoker : ParallelLoopBody { RenderInvoker(Mat_& _frame, Affine3f _pose, - Reprojector _reproj, - float _depthFactor) : ParallelLoopBody(), + Reprojector _reproj, float _depthFactor, bool _onlySemisphere) + : ParallelLoopBody(), frame(_frame), pose(_pose), reproj(_reproj), - depthFactor(_depthFactor) + depthFactor(_depthFactor), + onlySemisphere(_onlySemisphere) { } virtual void operator ()(const cv::Range& r) const @@ -64,7 +65,7 @@ struct RenderInvoker : ParallelLoopBody for (int step = 0; step < maxSteps && t < maxDepth; step++) { Point3f p = orig + dir * t; - float d = Scene::map(p); + float d = Scene::map(p, onlySemisphere); if (d < 0.000001f) { float depth = std::sqrt(t * t * xyt); @@ -83,12 +84,13 @@ struct RenderInvoker : ParallelLoopBody Affine3f pose; Reprojector reproj; float depthFactor; + bool onlySemisphere; }; struct Scene { virtual ~Scene() {} - static Ptr create(Size sz, Matx33f _intr, float _depthFactor); + static Ptr create(Size sz, Matx33f _intr, float _depthFactor, bool onlySemisphere); virtual Mat depth(Affine3f pose) = 0; virtual std::vector getPoses() = 0; }; @@ -96,40 +98,36 @@ struct Scene struct SemisphereScene : Scene { const int framesPerCycle = 72; - const float nCycles = 1.0f; - const Affine3f startPose = Affine3f(Vec3f(0.f, 0.f, 0.f), Vec3f(1.5f, 0.3f, -1.5f)); + const float nCycles = 0.25f; + const Affine3f startPose = Affine3f(Vec3f(0.f, 0.f, 0.f), Vec3f(1.5f, 0.3f, -2.1f)); Size frameSize; Matx33f intr; float depthFactor; + bool onlySemisphere; - SemisphereScene(Size sz, Matx33f _intr, float _depthFactor) : - frameSize(sz), intr(_intr), depthFactor(_depthFactor) + SemisphereScene(Size sz, Matx33f _intr, float _depthFactor, bool _onlySemisphere) : + frameSize(sz), intr(_intr), depthFactor(_depthFactor), onlySemisphere(_onlySemisphere) { } - static float map(Point3f p) + static float map(Point3f p, bool onlySemisphere) { float plane = p.y + 0.5f; - - Point3f boxPose = p - Point3f(-0.0f, 0.3f, 0.5f); - float boxSize = 0.5f; - float roundness = 0.08f; - Point3f boxTmp; - boxTmp.x = max(abs(boxPose.x) - boxSize, 0.0f); - boxTmp.y = max(abs(boxPose.y) - boxSize, 0.0f); - boxTmp.z = max(abs(boxPose.z) - boxSize, 0.0f); - float roundBox = (float)cv::norm(boxTmp) - roundness; - - Point3f spherePose = p - Point3f(-0.0f, 0.3f, 0.0f); + Point3f spherePose = p - Point3f(-0.0f, 0.3f, 1.1f); float sphereRadius = 0.5f; float sphere = (float)cv::norm(spherePose) - sphereRadius; - float sphereMinusBox = max(sphere, -roundBox); + float sphereMinusBox = sphere; float subSphereRadius = 0.05f; Point3f subSpherePose = p - Point3f(0.3f, -0.1f, -0.3f); float subSphere = (float)cv::norm(subSpherePose) - subSphereRadius; - float res = min({ sphereMinusBox, subSphere, plane }); + float res; + if (!onlySemisphere) + res = min({ sphereMinusBox, subSphere, plane }); + else + res = sphereMinusBox; + return res; } @@ -139,7 +137,7 @@ struct SemisphereScene : Scene Reprojector reproj(intr); Range range(0, frame.rows); - parallel_for_(range, RenderInvoker(frame, pose, reproj, depthFactor)); + parallel_for_(range, RenderInvoker(frame, pose, reproj, depthFactor, onlySemisphere)); return std::move(frame); } @@ -152,7 +150,7 @@ struct SemisphereScene : Scene float angle = (float)(CV_2PI * i / framesPerCycle); Affine3f pose; pose = pose.rotate(startPose.rotation()); - pose = pose.rotate(Vec3f(0.f, -1.f, 0.f) * angle); + pose = pose.rotate(Vec3f(0.f, -0.5f, 0.f) * angle); pose = pose.translate(Vec3f(startPose.translation()[0] * sin(angle), startPose.translation()[1], startPose.translation()[2] * cos(angle))); @@ -164,11 +162,117 @@ struct SemisphereScene : Scene }; -Ptr Scene::create(Size sz, Matx33f _intr, float _depthFactor) +Ptr Scene::create(Size sz, Matx33f _intr, float _depthFactor, bool _onlySemisphere) +{ + return makePtr(sz, _intr, _depthFactor, _onlySemisphere); +} + +// this is a temporary solution +// ---------------------------- + +typedef cv::Vec4f ptype; +typedef cv::Mat_< ptype > Points; +typedef Points Normals; +typedef Size2i Size; + +template +inline float specPow(float x) +{ + if (p % 2 == 0) + { + float v = specPow

(x); + return v * v; + } + else + { + float v = specPow<(p - 1) / 2>(x); + return v * v * x; + } +} + +template<> +inline float specPow<0>(float /*x*/) { - return makePtr(sz, _intr, _depthFactor); + return 1.f; } +template<> +inline float specPow<1>(float x) +{ + return x; +} + +inline cv::Vec3f fromPtype(const ptype& x) +{ + return cv::Vec3f(x[0], x[1], x[2]); +} + +inline Point3f normalize(const Vec3f& v) +{ + double nv = sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]); + return v * (nv ? 1. / nv : 0.); +} + +void renderPointsNormals(InputArray _points, InputArray _normals, OutputArray image, Affine3f lightPose) +{ + Size sz = _points.size(); + image.create(sz, CV_8UC4); + + Points points = _points.getMat(); + Normals normals = _normals.getMat(); + + Mat_ img = image.getMat(); + + Range range(0, sz.height); + const int nstripes = -1; + parallel_for_(range, [&](const Range&) + { + for (int y = range.start; y < range.end; y++) + { + Vec4b* imgRow = img[y]; + const ptype* ptsRow = points[y]; + const ptype* nrmRow = normals[y]; + + for (int x = 0; x < sz.width; x++) + { + Point3f p = fromPtype(ptsRow[x]); + Point3f n = fromPtype(nrmRow[x]); + + Vec4b color; + + if (cvIsNaN(p.x) || cvIsNaN(p.y) || cvIsNaN(p.z) ) + { + color = Vec4b(0, 32, 0, 0); + } + else + { + const float Ka = 0.3f; //ambient coeff + const float Kd = 0.5f; //diffuse coeff + const float Ks = 0.2f; //specular coeff + const int sp = 20; //specular power + + const float Ax = 1.f; //ambient color, can be RGB + const float Dx = 1.f; //diffuse color, can be RGB + const float Sx = 1.f; //specular color, can be RGB + const float Lx = 1.f; //light color + + Point3f l = normalize(lightPose.translation() - Vec3f(p)); + Point3f v = normalize(-Vec3f(p)); + Point3f r = normalize(Vec3f(2.f * n * n.dot(l) - l)); + + uchar ix = (uchar)((Ax * Ka * Dx + Lx * Kd * Dx * max(0.f, n.dot(l)) + + Lx * Ks * Sx * specPow(max(0.f, r.dot(v)))) * 255.f); + color = Vec4b(ix, ix, ix, 0); + } + + imgRow[x] = color; + } + } + }, nstripes); +} + +// ---------------------------- + class Settings { public: @@ -188,11 +292,27 @@ class Settings _params->raycast_step_factor, _params->tsdf_trunc_dist, _params->tsdf_max_weight, _params->truncateThreshold, _params->volumeDims); - scene = Scene::create(_params->frameSize, _params->intr, _params->depthFactor); + scene = Scene::create(_params->frameSize, _params->intr, _params->depthFactor, true); poses = scene->getPoses(); } }; +void displayImage(Mat depth, UMat _points, UMat _normals, float depthFactor, Vec3f lightPose) +{ + Mat points, normals, image; + AccessFlag af = ACCESS_READ; + normals = _normals.getMat(af); + points = _points.getMat(af); + patchNaNs(points); + + imshow("depth", depth * (1.f / depthFactor / 4.f)); + renderPointsNormals(points, normals, image, lightPose); + imshow("render", image); + waitKey(2000); +} + +static const bool display = false; + PERF_TEST(Perf_TSDF, integrate) { Settings settings(false); @@ -220,6 +340,9 @@ PERF_TEST(Perf_TSDF, raycast) startTimer(); settings.volume->raycast(pose, settings._params->intr, settings._params->frameSize, _points, _normals); stopTimer(); + + if (display) + displayImage(depth, _points, _normals, settings._params->depthFactor, settings._params->lightPose); } SANITY_CHECK_NOTHING(); } @@ -252,6 +375,9 @@ PERF_TEST(Perf_HashTSDF, raycast) startTimer(); settings.volume->raycast(pose, settings._params->intr, settings._params->frameSize, _points, _normals); stopTimer(); + + if (display) + displayImage(depth, _points, _normals, settings._params->depthFactor, settings._params->lightPose); } SANITY_CHECK_NOTHING(); } diff --git a/modules/rgbd/src/opencl/tsdf.cl b/modules/rgbd/src/opencl/tsdf.cl index 3a13d3b8336..4f9841b291d 100644 --- a/modules/rgbd/src/opencl/tsdf.cl +++ b/modules/rgbd/src/opencl/tsdf.cl @@ -227,8 +227,8 @@ inline float3 getNormalVoxel(float3 p, __global const struct TsdfVoxel* volumePt float vaz[8]; for(int i = 0; i < 8; i++) - vaz[i] = tsdfToFloat(volumePtr[nco[i] + dim].tsdf - - volumePtr[nco[i] - dim].tsdf); + vaz[i] = tsdfToFloat(volumePtr[nco[i] + dim].tsdf) - + tsdfToFloat(volumePtr[nco[i] - dim].tsdf); float8 vz = vload8(0, vaz); diff --git a/modules/rgbd/src/tsdf.cpp b/modules/rgbd/src/tsdf.cpp index 0a0fcfacf05..991d525b9d6 100644 --- a/modules/rgbd/src/tsdf.cpp +++ b/modules/rgbd/src/tsdf.cpp @@ -259,13 +259,13 @@ inline v_float32x4 TSDFVolumeCPU::getNormalVoxel(const v_float32x4& p) const const int dim = volDims[c]; float& nv = an[c]; - TsdfType vx[8]; + float vx[8]; for(int i = 0; i < 8; i++) - vx[i] = volData[neighbourCoords[i] + coordBase + 1*dim].tsdf - - volData[neighbourCoords[i] + coordBase - 1*dim].tsdf; + vx[i] = tsdfToFloat(volData[neighbourCoords[i] + coordBase + 1*dim].tsdf) - + tsdfToFloat(volData[neighbourCoords[i] + coordBase - 1*dim].tsdf); - v_float32x4 v0246 = tsdfToFloat_INTR(v_int32x4(vx[0], vx[2], vx[4], vx[6])); - v_float32x4 v1357 = tsdfToFloat_INTR(v_int32x4(vx[1], vx[3], vx[5], vx[7])); + v_float32x4 v0246 (vx[0], vx[2], vx[4], vx[6]); + v_float32x4 v1357 (vx[1], vx[3], vx[5], vx[7]); v_float32x4 vxx = v0246 + v_setall_f32(tz)*(v1357 - v0246); v_float32x4 v00_10 = vxx; @@ -312,9 +312,9 @@ inline Point3f TSDFVolumeCPU::getNormalVoxel(const Point3f& p) const float& nv = an[c]; float vx[8]; - for(int i = 0; i < 8; i++) - vx[i] = tsdfToFloat(volData[neighbourCoords[i] + coordBase + 1*dim].tsdf - - volData[neighbourCoords[i] + coordBase - 1*dim].tsdf); + for (int i = 0; i < 8; i++) + vx[i] = tsdfToFloat(volData[neighbourCoords[i] + coordBase + 1 * dim].tsdf) - + tsdfToFloat(volData[neighbourCoords[i] + coordBase - 1 * dim].tsdf); float v00 = vx[0] + tz*(vx[1] - vx[0]); float v01 = vx[2] + tz*(vx[3] - vx[2]); diff --git a/modules/rgbd/test/ocl/test_tsdf.cpp b/modules/rgbd/test/ocl/test_tsdf.cpp new file mode 100644 index 00000000000..1c55e1f4001 --- /dev/null +++ b/modules/rgbd/test/ocl/test_tsdf.cpp @@ -0,0 +1,470 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#include "../test_precomp.hpp" +#include "opencv2/ts/ocl_test.hpp" + +#ifdef HAVE_OPENCL + +namespace opencv_test { +namespace { + +using namespace cv; + +/** Reprojects screen point to camera space given z coord. */ +struct Reprojector +{ + Reprojector() {} + inline Reprojector(Matx33f intr) + { + fxinv = 1.f / intr(0, 0), fyinv = 1.f / intr(1, 1); + cx = intr(0, 2), cy = intr(1, 2); + } + template + inline cv::Point3_ operator()(cv::Point3_ p) const + { + T x = p.z * (p.x - cx) * fxinv; + T y = p.z * (p.y - cy) * fyinv; + return cv::Point3_(x, y, p.z); + } + + float fxinv, fyinv, cx, cy; +}; + +template +struct RenderInvoker : ParallelLoopBody +{ + RenderInvoker(Mat_& _frame, Affine3f _pose, + Reprojector _reproj, float _depthFactor, bool _onlySemisphere) + : ParallelLoopBody(), + frame(_frame), + pose(_pose), + reproj(_reproj), + depthFactor(_depthFactor), + onlySemisphere(_onlySemisphere) + { } + + virtual void operator ()(const cv::Range& r) const + { + for (int y = r.start; y < r.end; y++) + { + float* frameRow = frame[y]; + for (int x = 0; x < frame.cols; x++) + { + float pix = 0; + + Point3f orig = pose.translation(); + // direction through pixel + Point3f screenVec = reproj(Point3f((float)x, (float)y, 1.f)); + float xyt = 1.f / (screenVec.x * screenVec.x + + screenVec.y * screenVec.y + 1.f); + Point3f dir = normalize(Vec3f(pose.rotation() * screenVec)); + // screen space axis + dir.y = -dir.y; + + const float maxDepth = 20.f; + const float maxSteps = 256; + float t = 0.f; + for (int step = 0; step < maxSteps && t < maxDepth; step++) + { + Point3f p = orig + dir * t; + float d = Scene::map(p, onlySemisphere); + if (d < 0.000001f) + { + float depth = std::sqrt(t * t * xyt); + pix = depth * depthFactor; + break; + } + t += d; + } + + frameRow[x] = pix; + } + } + } + + Mat_& frame; + Affine3f pose; + Reprojector reproj; + float depthFactor; + bool onlySemisphere; +}; + +struct Scene +{ + virtual ~Scene() {} + static Ptr create(Size sz, Matx33f _intr, float _depthFactor, bool onlySemisphere); + virtual Mat depth(Affine3f pose) = 0; + virtual std::vector getPoses() = 0; +}; + +struct SemisphereScene : Scene +{ + const int framesPerCycle = 72; + const float nCycles = 0.25f; + const Affine3f startPose = Affine3f(Vec3f(0.f, 0.f, 0.f), Vec3f(1.5f, 0.3f, -2.1f)); + + Size frameSize; + Matx33f intr; + float depthFactor; + bool onlySemisphere; + + SemisphereScene(Size sz, Matx33f _intr, float _depthFactor, bool _onlySemisphere) : + frameSize(sz), intr(_intr), depthFactor(_depthFactor), onlySemisphere(_onlySemisphere) + { } + + static float map(Point3f p, bool onlySemisphere) + { + float plane = p.y + 0.5f; + Point3f spherePose = p - Point3f(-0.0f, 0.3f, 1.1f); + float sphereRadius = 0.5f; + float sphere = (float)cv::norm(spherePose) - sphereRadius; + float sphereMinusBox = sphere; + + float subSphereRadius = 0.05f; + Point3f subSpherePose = p - Point3f(0.3f, -0.1f, -0.3f); + float subSphere = (float)cv::norm(subSpherePose) - subSphereRadius; + + float res; + if (!onlySemisphere) + res = min({ sphereMinusBox, subSphere, plane }); + else + res = sphereMinusBox; + + return res; + } + + Mat depth(Affine3f pose) override + { + Mat_ frame(frameSize); + Reprojector reproj(intr); + + Range range(0, frame.rows); + parallel_for_(range, RenderInvoker(frame, pose, reproj, depthFactor, onlySemisphere)); + + return std::move(frame); + } + + std::vector getPoses() override + { + std::vector poses; + for (int i = 0; i < framesPerCycle * nCycles; i++) + { + float angle = (float)(CV_2PI * i / framesPerCycle); + Affine3f pose; + pose = pose.rotate(startPose.rotation()); + pose = pose.rotate(Vec3f(0.f, -0.5f, 0.f) * angle); + pose = pose.translate(Vec3f(startPose.translation()[0] * sin(angle), + startPose.translation()[1], + startPose.translation()[2] * cos(angle))); + poses.push_back(pose); + } + + return poses; + } + +}; + +Ptr Scene::create(Size sz, Matx33f _intr, float _depthFactor, bool _onlySemisphere) +{ + return makePtr(sz, _intr, _depthFactor, _onlySemisphere); +} + +// this is a temporary solution +// ---------------------------- + +typedef cv::Vec4f ptype; +typedef cv::Mat_< ptype > Points; +typedef Points Normals; +typedef Size2i Size; + +template +inline float specPow(float x) +{ + if (p % 2 == 0) + { + float v = specPow

(x); + return v * v; + } + else + { + float v = specPow<(p - 1) / 2>(x); + return v * v * x; + } +} + +template<> +inline float specPow<0>(float /*x*/) +{ + return 1.f; +} + +template<> +inline float specPow<1>(float x) +{ + return x; +} + +inline cv::Vec3f fromPtype(const ptype& x) +{ + return cv::Vec3f(x[0], x[1], x[2]); +} + +inline Point3f normalize(const Vec3f& v) +{ + double nv = sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]); + return v * (nv ? 1. / nv : 0.); +} + +void renderPointsNormals(InputArray _points, InputArray _normals, OutputArray image, Affine3f lightPose) +{ + Size sz = _points.size(); + image.create(sz, CV_8UC4); + + Points points = _points.getMat(); + Normals normals = _normals.getMat(); + + Mat_ img = image.getMat(); + + Range range(0, sz.height); + const int nstripes = -1; + parallel_for_(range, [&](const Range&) + { + for (int y = range.start; y < range.end; y++) + { + Vec4b* imgRow = img[y]; + const ptype* ptsRow = points[y]; + const ptype* nrmRow = normals[y]; + + for (int x = 0; x < sz.width; x++) + { + Point3f p = fromPtype(ptsRow[x]); + Point3f n = fromPtype(nrmRow[x]); + + Vec4b color; + + if (cvIsNaN(p.x) || cvIsNaN(p.y) || cvIsNaN(p.z)) + { + color = Vec4b(0, 32, 0, 0); + } + else + { + const float Ka = 0.3f; //ambient coeff + const float Kd = 0.5f; //diffuse coeff + const float Ks = 0.2f; //specular coeff + const int sp = 20; //specular power + + const float Ax = 1.f; //ambient color, can be RGB + const float Dx = 1.f; //diffuse color, can be RGB + const float Sx = 1.f; //specular color, can be RGB + const float Lx = 1.f; //light color + + Point3f l = normalize(lightPose.translation() - Vec3f(p)); + Point3f v = normalize(-Vec3f(p)); + Point3f r = normalize(Vec3f(2.f * n * n.dot(l) - l)); + + uchar ix = (uchar)((Ax * Ka * Dx + Lx * Kd * Dx * max(0.f, n.dot(l)) + + Lx * Ks * Sx * specPow(max(0.f, r.dot(v)))) * 255.f); + color = Vec4b(ix, ix, ix, 0); + } + + imgRow[x] = color; + } + } + }, nstripes); +} +// ---------------------------- + +static const bool display = false; +static const bool parallelCheck = false; + +class Settings +{ +public: + Ptr params; + Ptr volume; + Ptr scene; + std::vector poses; + + Settings(bool useHashTSDF, bool onlySemisphere) + { + if (useHashTSDF) + params = kinfu::Params::hashTSDFParams(true); + else + params = kinfu::Params::coarseParams(); + + volume = kinfu::makeVolume(params->volumeType, params->voxelSize, params->volumePose.matrix, + params->raycast_step_factor, params->tsdf_trunc_dist, params->tsdf_max_weight, + params->truncateThreshold, params->volumeDims); + + scene = Scene::create(params->frameSize, params->intr, params->depthFactor, onlySemisphere); + poses = scene->getPoses(); + } +}; + +void displayImage(Mat depth, Mat points, Mat normals, float depthFactor, Vec3f lightPose) +{ + Mat image; + patchNaNs(points); + imshow("depth", depth * (1.f / depthFactor / 4.f)); + renderPointsNormals(points, normals, image, lightPose); + imshow("render", image); + waitKey(2000); +} + +void normalsCheck(Mat normals) +{ + Vec4f vector; + for (auto pvector = normals.begin(); pvector < normals.end(); pvector++) + { + vector = *pvector; + if (!cvIsNaN(vector[0])) + { + float length = vector[0] * vector[0] + + vector[1] * vector[1] + + vector[2] * vector[2]; + ASSERT_LT(abs(1 - length), 0.0001f) << "There is normal with length != 1"; + } + } +} + +int counterOfValid(Mat points) +{ + Vec4f* v; + int i, j; + int count = 0; + for (i = 0; i < points.rows; ++i) + { + v = (points.ptr(i)); + for (j = 0; j < points.cols; ++j) + { + if ((v[j])[0] != 0 || + (v[j])[1] != 0 || + (v[j])[2] != 0) + { + count++; + } + } + } + return count; +} + +void normal_test(bool isHashTSDF, bool isRaycast, bool isFetchPointsNormals, bool isFetchNormals) +{ + auto normalCheck = [](Vec4f& vector, const int*) + { + if (!cvIsNaN(vector[0])) + { + float length = vector[0] * vector[0] + + vector[1] * vector[1] + + vector[2] * vector[2]; + ASSERT_LT(abs(1 - length), 0.0001f) << "There is normal with length != 1"; + } + }; + + Settings settings(isHashTSDF, false); + + Mat depth = settings.scene->depth(settings.poses[0]); + UMat _points, _normals, _tmpnormals; + UMat _newPoints, _newNormals; + Mat points, normals; + AccessFlag af = ACCESS_READ; + + settings.volume->integrate(depth, settings.params->depthFactor, settings.poses[0].matrix, settings.params->intr); + + if (isRaycast) + { + settings.volume->raycast(settings.poses[0].matrix, settings.params->intr, settings.params->frameSize, _points, _normals); + } + if (isFetchPointsNormals) + { + settings.volume->fetchPointsNormals(_points, _normals); + } + if (isFetchNormals) + { + settings.volume->fetchPointsNormals(_points, _tmpnormals); + settings.volume->fetchNormals(_points, _normals); + } + + normals = _normals.getMat(af); + points = _points.getMat(af); + + if (parallelCheck) + normals.forEach(normalCheck); + else + normalsCheck(normals); + + if (isRaycast && display) + displayImage(depth, points, normals, settings.params->depthFactor, settings.params->lightPose); + + if (isRaycast) + { + settings.volume->raycast(settings.poses[17].matrix, settings.params->intr, settings.params->frameSize, _newPoints, _newNormals); + normals = _newNormals.getMat(af); + points = _newPoints.getMat(af); + normalsCheck(normals); + + if (parallelCheck) + normals.forEach(normalCheck); + else + normalsCheck(normals); + + if (display) + displayImage(depth, points, normals, settings.params->depthFactor, settings.params->lightPose); + } + + points.release(); normals.release(); +} + +void valid_points_test(bool isHashTSDF) +{ + Settings settings(isHashTSDF, true); + + Mat depth = settings.scene->depth(settings.poses[0]); + UMat _points, _normals, _newPoints, _newNormals; + AccessFlag af = ACCESS_READ; + Mat points, normals; + int anfas, profile; + + settings.volume->integrate(depth, settings.params->depthFactor, settings.poses[0].matrix, settings.params->intr); + settings.volume->raycast(settings.poses[0].matrix, settings.params->intr, settings.params->frameSize, _points, _normals); + normals = _normals.getMat(af); + points = _points.getMat(af); + patchNaNs(points); + anfas = counterOfValid(points); + + if (display) + displayImage(depth, points, normals, settings.params->depthFactor, settings.params->lightPose); + + settings.volume->raycast(settings.poses[17].matrix, settings.params->intr, settings.params->frameSize, _newPoints, _newNormals); + normals = _newNormals.getMat(af); + points = _newPoints.getMat(af); + patchNaNs(points); + profile = counterOfValid(points); + + if (display) + displayImage(depth, points, normals, settings.params->depthFactor, settings.params->lightPose); + + // TODO: why profile == 2*anfas ? + float percentValidity = float(anfas) / float(profile); + + ASSERT_NE(profile, 0) << "There is no points in profile"; + ASSERT_NE(anfas, 0) << "There is no points in anfas"; + ASSERT_LT(abs(0.5 - percentValidity), 0.3) << "percentValidity out of [0.3; 0.7] (percentValidity=" << percentValidity << ")"; +} + +TEST(TSDF_GPU, raycast_normals) { normal_test(false, true, false, false); } +TEST(TSDF_GPU, fetch_points_normals) { normal_test(false, false, true, false); } +TEST(TSDF_GPU, fetch_normals) { normal_test(false, false, false, true); } +TEST(TSDF_GPU, valid_points) { valid_points_test(false); } + +TEST(HashTSDF_GPU, raycast_normals) { normal_test(true, true, false, false); } +TEST(HashTSDF_GPU, fetch_points_normals) { normal_test(true, false, true, false); } +TEST(HashTSDF_GPU, fetch_normals) { normal_test(true, false, false, true); } +TEST(HashTSDF_GPU, valid_points) { valid_points_test(true); } + +} +} // namespace + +#endif diff --git a/modules/rgbd/test/test_tsdf.cpp b/modules/rgbd/test/test_tsdf.cpp index 5a9b23afb95..31a137854c3 100644 --- a/modules/rgbd/test/test_tsdf.cpp +++ b/modules/rgbd/test/test_tsdf.cpp @@ -4,7 +4,8 @@ #include "test_precomp.hpp" -namespace opencv_test { namespace { +namespace opencv_test { +namespace { using namespace cv; @@ -32,12 +33,13 @@ template struct RenderInvoker : ParallelLoopBody { RenderInvoker(Mat_& _frame, Affine3f _pose, - Reprojector _reproj, - float _depthFactor) : ParallelLoopBody(), + Reprojector _reproj, float _depthFactor, bool _onlySemisphere) + : ParallelLoopBody(), frame(_frame), pose(_pose), reproj(_reproj), - depthFactor(_depthFactor) + depthFactor(_depthFactor), + onlySemisphere(_onlySemisphere) { } virtual void operator ()(const cv::Range& r) const @@ -64,7 +66,7 @@ struct RenderInvoker : ParallelLoopBody for (int step = 0; step < maxSteps && t < maxDepth; step++) { Point3f p = orig + dir * t; - float d = Scene::map(p); + float d = Scene::map(p, onlySemisphere); if (d < 0.000001f) { float depth = std::sqrt(t * t * xyt); @@ -83,12 +85,13 @@ struct RenderInvoker : ParallelLoopBody Affine3f pose; Reprojector reproj; float depthFactor; + bool onlySemisphere; }; struct Scene { virtual ~Scene() {} - static Ptr create(Size sz, Matx33f _intr, float _depthFactor); + static Ptr create(Size sz, Matx33f _intr, float _depthFactor, bool onlySemisphere); virtual Mat depth(Affine3f pose) = 0; virtual std::vector getPoses() = 0; }; @@ -97,39 +100,35 @@ struct SemisphereScene : Scene { const int framesPerCycle = 72; const float nCycles = 0.25f; - const Affine3f startPose = Affine3f(Vec3f(0.f, 0.f, 0.f), Vec3f(1.5f, 0.3f, -2.3f)); + const Affine3f startPose = Affine3f(Vec3f(0.f, 0.f, 0.f), Vec3f(1.5f, 0.3f, -2.1f)); Size frameSize; Matx33f intr; float depthFactor; + bool onlySemisphere; - SemisphereScene(Size sz, Matx33f _intr, float _depthFactor) : - frameSize(sz), intr(_intr), depthFactor(_depthFactor) + SemisphereScene(Size sz, Matx33f _intr, float _depthFactor, bool _onlySemisphere) : + frameSize(sz), intr(_intr), depthFactor(_depthFactor), onlySemisphere(_onlySemisphere) { } - static float map(Point3f p) + static float map(Point3f p, bool onlySemisphere) { float plane = p.y + 0.5f; - - Point3f boxPose = p - Point3f(-0.0f, 0.3f, 0.5f); - float boxSize = 0.5f; - float roundness = 0.08f; - Point3f boxTmp; - boxTmp.x = max(abs(boxPose.x) - boxSize, 0.0f); - boxTmp.y = max(abs(boxPose.y) - boxSize, 0.0f); - boxTmp.z = max(abs(boxPose.z) - boxSize, 0.0f); - float roundBox = (float)cv::norm(boxTmp) - roundness; - - Point3f spherePose = p - Point3f(-0.0f, 0.3f, 0.0f); + Point3f spherePose = p - Point3f(-0.0f, 0.3f, 1.1f); float sphereRadius = 0.5f; float sphere = (float)cv::norm(spherePose) - sphereRadius; - float sphereMinusBox = max(sphere, -roundBox); + float sphereMinusBox = sphere; float subSphereRadius = 0.05f; Point3f subSpherePose = p - Point3f(0.3f, -0.1f, -0.3f); float subSphere = (float)cv::norm(subSpherePose) - subSphereRadius; - float res = min({sphereMinusBox, subSphere, plane}); + float res; + if (!onlySemisphere) + res = min({ sphereMinusBox, subSphere, plane }); + else + res = sphereMinusBox; + return res; } @@ -139,7 +138,7 @@ struct SemisphereScene : Scene Reprojector reproj(intr); Range range(0, frame.rows); - parallel_for_(range, RenderInvoker(frame, pose, reproj, depthFactor)); + parallel_for_(range, RenderInvoker(frame, pose, reproj, depthFactor, onlySemisphere)); return std::move(frame); } @@ -152,7 +151,7 @@ struct SemisphereScene : Scene float angle = (float)(CV_2PI * i / framesPerCycle); Affine3f pose; pose = pose.rotate(startPose.rotation()); - pose = pose.rotate(Vec3f(0.f, -1.f, 0.f) * angle); + pose = pose.rotate(Vec3f(0.f, -0.5f, 0.f) * angle); pose = pose.translate(Vec3f(startPose.translation()[0] * sin(angle), startPose.translation()[1], startPose.translation()[2] * cos(angle))); @@ -164,9 +163,9 @@ struct SemisphereScene : Scene }; -Ptr Scene::create(Size sz, Matx33f _intr, float _depthFactor) +Ptr Scene::create(Size sz, Matx33f _intr, float _depthFactor, bool _onlySemisphere) { - return makePtr(sz, _intr, _depthFactor); + return makePtr(sz, _intr, _depthFactor, _onlySemisphere); } // this is a temporary solution @@ -242,7 +241,7 @@ void renderPointsNormals(InputArray _points, InputArray _normals, OutputArray im Vec4b color; - if (cvIsNaN(p.x) || cvIsNaN(p.y) || cvIsNaN(p.z) ) + if (cvIsNaN(p.x) || cvIsNaN(p.y) || cvIsNaN(p.z)) { color = Vec4b(0, 32, 0, 0); } @@ -277,6 +276,40 @@ void renderPointsNormals(InputArray _points, InputArray _normals, OutputArray im static const bool display = false; static const bool parallelCheck = false; +class Settings +{ +public: + Ptr params; + Ptr volume; + Ptr scene; + std::vector poses; + + Settings(bool useHashTSDF, bool onlySemisphere) + { + if (useHashTSDF) + params = kinfu::Params::hashTSDFParams(true); + else + params = kinfu::Params::coarseParams(); + + volume = kinfu::makeVolume(params->volumeType, params->voxelSize, params->volumePose.matrix, + params->raycast_step_factor, params->tsdf_trunc_dist, params->tsdf_max_weight, + params->truncateThreshold, params->volumeDims); + + scene = Scene::create(params->frameSize, params->intr, params->depthFactor, onlySemisphere); + poses = scene->getPoses(); + } +}; + +void displayImage(Mat depth, Mat points, Mat normals, float depthFactor, Vec3f lightPose) +{ + Mat image; + patchNaNs(points); + imshow("depth", depth * (1.f / depthFactor / 4.f)); + renderPointsNormals(points, normals, image, lightPose); + imshow("render", image); + waitKey(2000); +} + void normalsCheck(Mat normals) { Vec4f vector; @@ -288,29 +321,34 @@ void normalsCheck(Mat normals) float length = vector[0] * vector[0] + vector[1] * vector[1] + vector[2] * vector[2]; - ASSERT_LT(abs(1 - length), 0.0001f); + ASSERT_LT(abs(1 - length), 0.0001f) << "There is normal with length != 1"; } } } -void normal_test(bool isHashTSDF, bool isRaycast, bool isFetchPointsNormals, bool isFetchNormals) +int counterOfValid(Mat points) { - Ptr _params; - if (isHashTSDF) - _params = kinfu::Params::hashTSDFParams(true); - else - _params = kinfu::Params::coarseParams(); - - Ptr scene = Scene::create(_params->frameSize, _params->intr, _params->depthFactor); - std::vector poses = scene->getPoses(); - - Mat depth = scene->depth(poses[0]); - UMat _points, _normals, _tmpnormals; - UMat _newPoints, _newNormals; - Mat points, normals; - Mat image; - AccessFlag af = ACCESS_READ; + Vec4f* v; + int i, j; + int count = 0; + for (i = 0; i < points.rows; ++i) + { + v = (points.ptr(i)); + for (j = 0; j < points.cols; ++j) + { + if ((v[j])[0] != 0 || + (v[j])[1] != 0 || + (v[j])[2] != 0) + { + count++; + } + } + } + return count; +} +void normal_test(bool isHashTSDF, bool isRaycast, bool isFetchPointsNormals, bool isFetchNormals) +{ auto normalCheck = [](Vec4f& vector, const int*) { if (!cvIsNaN(vector[0])) @@ -318,200 +356,167 @@ void normal_test(bool isHashTSDF, bool isRaycast, bool isFetchPointsNormals, boo float length = vector[0] * vector[0] + vector[1] * vector[1] + vector[2] * vector[2]; - ASSERT_LT(abs(1 - length), 0.0001f); + ASSERT_LT(abs(1 - length), 0.0001f) << "There is normal with length != 1"; } }; - Ptr volume = kinfu::makeVolume(_params->volumeType, _params->voxelSize, _params->volumePose.matrix, - _params->raycast_step_factor, _params->tsdf_trunc_dist, _params->tsdf_max_weight, - _params->truncateThreshold, _params->volumeDims); - volume->integrate(depth, _params->depthFactor, poses[0].matrix, _params->intr); + Settings settings(isHashTSDF, false); + + Mat depth = settings.scene->depth(settings.poses[0]); + UMat _points, _normals, _tmpnormals; + UMat _newPoints, _newNormals; + Mat points, normals; + AccessFlag af = ACCESS_READ; + + settings.volume->integrate(depth, settings.params->depthFactor, settings.poses[0].matrix, settings.params->intr); if (isRaycast) { - volume->raycast(poses[0].matrix, _params->intr, _params->frameSize, _points, _normals); + settings.volume->raycast(settings.poses[0].matrix, settings.params->intr, settings.params->frameSize, _points, _normals); } if (isFetchPointsNormals) { - volume->fetchPointsNormals(_points, _normals); + settings.volume->fetchPointsNormals(_points, _normals); } if (isFetchNormals) { - volume->fetchPointsNormals(_points, _tmpnormals); - volume->fetchNormals(_points, _normals); + settings.volume->fetchPointsNormals(_points, _tmpnormals); + settings.volume->fetchNormals(_points, _normals); } normals = _normals.getMat(af); + points = _points.getMat(af); if (parallelCheck) - { normals.forEach(normalCheck); - } else - { normalsCheck(normals); - } if (isRaycast && display) - { - imshow("depth", depth * (1.f / _params->depthFactor / 4.f)); - points = _points.getMat(af); - renderPointsNormals(points, normals, image, _params->lightPose); - imshow("render", image); - waitKey(2000); - } + displayImage(depth, points, normals, settings.params->depthFactor, settings.params->lightPose); if (isRaycast) { - volume->raycast(poses[17].matrix, _params->intr, _params->frameSize, _newPoints, _newNormals); - + settings.volume->raycast(settings.poses[17].matrix, settings.params->intr, settings.params->frameSize, _newPoints, _newNormals); normals = _newNormals.getMat(af); + points = _newPoints.getMat(af); normalsCheck(normals); if (parallelCheck) - { normals.forEach(normalCheck); - } else - { normalsCheck(normals); - } - if (display) - { - imshow("depth", depth * (1.f / _params->depthFactor / 4.f)); - points = _newPoints.getMat(af); - renderPointsNormals(points, normals, image, _params->lightPose); - imshow("render", image); - waitKey(2000); - } - + displayImage(depth, points, normals, settings.params->depthFactor, settings.params->lightPose); } points.release(); normals.release(); } -int counterOfValid(Mat points) -{ - Vec4f* v; - int i, j; - int count = 0; - for (i = 0; i < points.rows; ++i) - { - v = (points.ptr(i)); - for (j = 0; j < points.cols; ++j) - { - if ((v[j])[0] != 0 || - (v[j])[1] != 0 || - (v[j])[2] != 0) - { - count++; - } - } - } - return count; -} - void valid_points_test(bool isHashTSDF) { - Ptr _params; - if (isHashTSDF) - _params = kinfu::Params::hashTSDFParams(true); - else - _params = kinfu::Params::coarseParams(); + Settings settings(isHashTSDF, true); - Ptr scene = Scene::create(_params->frameSize, _params->intr, _params->depthFactor); - std::vector poses = scene->getPoses(); - - Mat depth = scene->depth(poses[0]); - UMat _points, _normals; - UMat _newPoints, _newNormals; + Mat depth = settings.scene->depth(settings.poses[0]); + UMat _points, _normals, _newPoints, _newNormals; + AccessFlag af = ACCESS_READ; Mat points, normals; - Mat image; int anfas, profile; - AccessFlag af = ACCESS_READ; - - Ptr volume = kinfu::makeVolume(_params->volumeType, _params->voxelSize, _params->volumePose.matrix, - _params->raycast_step_factor, _params->tsdf_trunc_dist, _params->tsdf_max_weight, - _params->truncateThreshold, _params->volumeDims); - volume->integrate(depth, _params->depthFactor, poses[0].matrix, _params->intr); - volume->raycast(poses[0].matrix, _params->intr, _params->frameSize, _points, _normals); + settings.volume->integrate(depth, settings.params->depthFactor, settings.poses[0].matrix, settings.params->intr); + settings.volume->raycast(settings.poses[0].matrix, settings.params->intr, settings.params->frameSize, _points, _normals); normals = _normals.getMat(af); points = _points.getMat(af); patchNaNs(points); anfas = counterOfValid(points); if (display) - { - imshow("depth", depth * (1.f / _params->depthFactor / 4.f)); - renderPointsNormals(points, normals, image, _params->lightPose); - imshow("render", image); - waitKey(2000); - } - - volume->raycast(poses[17].matrix, _params->intr, _params->frameSize, _newPoints, _newNormals); + displayImage(depth, points, normals, settings.params->depthFactor, settings.params->lightPose); + settings.volume->raycast(settings.poses[17].matrix, settings.params->intr, settings.params->frameSize, _newPoints, _newNormals); normals = _newNormals.getMat(af); points = _newPoints.getMat(af); patchNaNs(points); profile = counterOfValid(points); if (display) - { - imshow("depth", depth * (1.f / _params->depthFactor / 4.f)); - renderPointsNormals(points, normals, image, _params->lightPose); - imshow("render", image); - waitKey(2000); - } + displayImage(depth, points, normals, settings.params->depthFactor, settings.params->lightPose); + + // TODO: why profile == 2*anfas ? + float percentValidity = float(anfas) / float(profile); - float percentValidity; - if (profile == 0) percentValidity = -0.5; - else if (anfas == 0) percentValidity = 0; - else percentValidity = float(profile) / float(anfas); - ASSERT_LT(0.5 - percentValidity, 0.3); + ASSERT_NE(profile, 0) << "There is no points in profile"; + ASSERT_NE(anfas, 0) << "There is no points in anfas"; + ASSERT_LT(abs(0.5 - percentValidity), 0.3) << "percentValidity out of [0.3; 0.7] (percentValidity=" << percentValidity << ")"; } -TEST(TSDF, raycast_normals) +#ifndef HAVE_OPENCL +TEST(TSDF, raycast_normals) { normal_test(false, true, false, false); } +TEST(TSDF, fetch_points_normals) { normal_test(false, false, true, false); } +TEST(TSDF, fetch_normals) { normal_test(false, false, false, true); } +TEST(TSDF, valid_points) { valid_points_test(false); } + +TEST(HashTSDF, raycast_normals) { normal_test(true, true, false, false); } +TEST(HashTSDF, fetch_points_normals) { normal_test(true, false, true, false); } +TEST(HashTSDF, fetch_normals) { normal_test(true, false, false, true); } +TEST(HashTSDF, valid_points) { valid_points_test(true); } +#else +TEST(TSDF_CPU, raycast_normals) { + cv::ocl::setUseOpenCL(false); normal_test(false, true, false, false); + cv::ocl::setUseOpenCL(true); } -TEST(HashTSDF, raycast_normals) +TEST(TSDF_CPU, fetch_points_normals) { - normal_test(true, true, false, false); + cv::ocl::setUseOpenCL(false); + normal_test(false, false, true, false); + cv::ocl::setUseOpenCL(true); } -TEST(TSDF, fetch_points_normals) +TEST(TSDF_CPU, fetch_normals) { - normal_test(false, false, true, false); + cv::ocl::setUseOpenCL(false); + normal_test(false, false, false, true); + cv::ocl::setUseOpenCL(true); } -TEST(HashTSDF, fetch_points_normals) +TEST(TSDF_CPU, valid_points) { - normal_test(true, false, true, false); + cv::ocl::setUseOpenCL(false); + valid_points_test(false); + cv::ocl::setUseOpenCL(true); } -TEST(TSDF, fetch_normals) +TEST(HashTSDF_CPU, raycast_normals) { - normal_test(false, false, false, true); + cv::ocl::setUseOpenCL(false); + normal_test(true, true, false, false); + cv::ocl::setUseOpenCL(true); } -TEST(HashTSDF, fetch_normals) +TEST(HashTSDF_CPU, fetch_points_normals) { - normal_test(true, false, false, true); + cv::ocl::setUseOpenCL(false); + normal_test(true, false, true, false); + cv::ocl::setUseOpenCL(true); } -TEST(TSDF, valid_points) +TEST(HashTSDF_CPU, fetch_normals) { - valid_points_test(false); + cv::ocl::setUseOpenCL(false); + normal_test(true, false, false, true); + cv::ocl::setUseOpenCL(true); } -TEST(HashTSDF, valid_points) +TEST(HashTSDF_CPU, valid_points) { + cv::ocl::setUseOpenCL(false); valid_points_test(true); + cv::ocl::setUseOpenCL(true); } - - -}} // namespace +#endif +} +} // namespace From 1341c05766fab3920eeebf0234f006fc04086152 Mon Sep 17 00:00:00 2001 From: Rostislav Vasilikhin Date: Thu, 1 Apr 2021 23:59:16 +0300 Subject: [PATCH 60/70] Merge pull request #2892 from savuor:posegraph_no_ceres Pose Graph rewritten without Ceres * Works well on g2o dataset, cumulative: 1. Pose3d done w/o Eigen types; 2. PoseGraph nodes vector -> map 3. Eigen is not used in cost function or parametrization 4. Cost function debugged & fixed (original one was wrong), rewritten from Automatic to Analytic * g2o dataset reading added to PoseGraph * sparse solver fixes from DynaFu draft * Eigen cost function and parametrization removed + g2o reading fixed * refactored: pose error, pose graph edge, pose graph node * sparse solve: templated * MyOptimize(): 1st version * several fixes and TODOs for future * sparse block matrix: val functions, template type * works at Ceres quality (cleanup needed) * MyOptimize() is set to default optimizer * Ceres thrown away, PoseGraph class and header/source code reorganized * pose, node, edge -> nested for PoseGraph * warnings fixed * jacobiScaling disabled for better performance + minors * trailing whitespace fixed * more warnings fixed * message added: Eigen is required for build + minors * trying to fix warning * try to fix "unreachable code" warning * trying to fix unreachable code, pt.3 * trying to fix unreachable code, pt. 5 * trying to fix unreachable code, pt. the worst + minors * try to fix unreachable code, pt. the ugliest * trying to fix unreachable code, pt. the grumpiest * cout -> CV_LOG_INFO * quat matrix functions moved outside cv and kinfu namespaces * unused function fix * pose graph made public (but in detail namespace) + test for pose graph * minor: prints * Pose Graph interface settled * Pose graph interface and its use updated * cos -> std::cos * cout -> CV_LOG_INFO * pose graph interface updated: implementation * Pose Graph Node and Edge: extra fields dropped * more minor refactor-like fixes * return and finish condition fixed * more updates to test * test disabled for Debug builds because 400 sec is too much * whitespace * Disable pose graph test if there's no Eigen * more unused vars * fixing unused function warning * less includes * "verbose" removed * write obj to file only when debug level is raised * License + include guard * skip test using tags and SkipTestException * suppress "unused function" warning * minor --- modules/rgbd/CMakeLists.txt | 12 +- modules/rgbd/include/opencv2/rgbd.hpp | 1 + .../opencv2/rgbd/detail/pose_graph.hpp | 62 ++ modules/rgbd/src/fast_icp.cpp | 4 +- modules/rgbd/src/large_kinfu.cpp | 25 +- modules/rgbd/src/nonrigid_icp.cpp | 2 +- modules/rgbd/src/pose_graph.cpp | 914 ++++++++++++++++-- modules/rgbd/src/pose_graph.hpp | 321 ------ modules/rgbd/src/precomp.hpp | 2 + modules/rgbd/src/sparse_block_matrix.hpp | 159 +-- modules/rgbd/src/submap.hpp | 33 +- modules/rgbd/test/test_pose_graph.cpp | 151 +++ 12 files changed, 1177 insertions(+), 509 deletions(-) create mode 100644 modules/rgbd/include/opencv2/rgbd/detail/pose_graph.hpp delete mode 100644 modules/rgbd/src/pose_graph.hpp create mode 100644 modules/rgbd/test/test_pose_graph.cpp diff --git a/modules/rgbd/CMakeLists.txt b/modules/rgbd/CMakeLists.txt index 143cdf913af..b0a7c73294e 100644 --- a/modules/rgbd/CMakeLists.txt +++ b/modules/rgbd/CMakeLists.txt @@ -1,15 +1,7 @@ set(the_description "RGBD algorithms") -find_package(Ceres QUIET) ocv_define_module(rgbd opencv_core opencv_calib3d opencv_imgproc OPTIONAL opencv_viz WRAP python) -if(Ceres_FOUND) - ocv_target_compile_definitions(${the_module} PUBLIC CERES_FOUND) - ocv_target_link_libraries(${the_module} ${CERES_LIBRARIES}) - if(Ceres_VERSION VERSION_LESS 2.0.0) - ocv_include_directories("${CERES_INCLUDE_DIRS}") - endif() - add_definitions(/DGLOG_NO_ABBREVIATED_SEVERITIES) # avoid ERROR macro conflict in glog (ceres dependency) -else() - message(STATUS "rgbd: CERES support is disabled. Ceres Solver is Required for Posegraph optimization") +if(NOT HAVE_EIGEN) + message(STATUS "rgbd: Eigen support is disabled. Eigen is Required for Posegraph optimization") endif() diff --git a/modules/rgbd/include/opencv2/rgbd.hpp b/modules/rgbd/include/opencv2/rgbd.hpp index d4ac749c2a5..550d0cc8398 100755 --- a/modules/rgbd/include/opencv2/rgbd.hpp +++ b/modules/rgbd/include/opencv2/rgbd.hpp @@ -14,6 +14,7 @@ #include "opencv2/rgbd/kinfu.hpp" #include "opencv2/rgbd/dynafu.hpp" #include "opencv2/rgbd/large_kinfu.hpp" +#include "opencv2/rgbd/detail/pose_graph.hpp" /** @defgroup rgbd RGB-Depth Processing diff --git a/modules/rgbd/include/opencv2/rgbd/detail/pose_graph.hpp b/modules/rgbd/include/opencv2/rgbd/detail/pose_graph.hpp new file mode 100644 index 00000000000..a611ffa9b81 --- /dev/null +++ b/modules/rgbd/include/opencv2/rgbd/detail/pose_graph.hpp @@ -0,0 +1,62 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#ifndef OPENCV_RGBD_POSE_GRAPH_HPP +#define OPENCV_RGBD_POSE_GRAPH_HPP + +#include "opencv2/core/affine.hpp" +#include "opencv2/core/quaternion.hpp" + +namespace cv +{ +namespace kinfu +{ +namespace detail +{ + +// ATTENTION! This class is used internally in Large KinFu. +// It has been pushed to publicly available headers for tests only. +// Source compatibility of this API is not guaranteed in the future. + +// This class provides tools to solve so-called pose graph problem often arisen in SLAM problems +// The pose graph format, cost function and optimization techniques +// repeat the ones used in Ceres 3D Pose Graph Optimization: +// http://ceres-solver.org/nnls_tutorial.html#other-examples, pose_graph_3d.cc bullet +class CV_EXPORTS_W PoseGraph +{ +public: + static Ptr create(); + virtual ~PoseGraph(); + + // Node may have any id >= 0 + virtual void addNode(size_t _nodeId, const Affine3d& _pose, bool fixed) = 0; + virtual bool isNodeExist(size_t nodeId) const = 0; + virtual bool setNodeFixed(size_t nodeId, bool fixed) = 0; + virtual bool isNodeFixed(size_t nodeId) const = 0; + virtual Affine3d getNodePose(size_t nodeId) const = 0; + virtual std::vector getNodesIds() const = 0; + virtual size_t getNumNodes() const = 0; + + // Edges have consequent indices starting from 0 + virtual void addEdge(size_t _sourceNodeId, size_t _targetNodeId, const Affine3f& _transformation, + const Matx66f& _information = Matx66f::eye()) = 0; + virtual size_t getEdgeStart(size_t i) const = 0; + virtual size_t getEdgeEnd(size_t i) const = 0; + virtual size_t getNumEdges() const = 0; + + // checks if graph is connected and each edge connects exactly 2 nodes + virtual bool isValid() const = 0; + + // Termination criteria are max number of iterations and min relative energy change to current energy + // Returns number of iterations elapsed or -1 if max number of iterations was reached or failed to optimize + virtual int optimize(const cv::TermCriteria& tc = cv::TermCriteria(TermCriteria::COUNT+TermCriteria::EPS, 100, 1e-6)) = 0; + + // calculate cost function based on current nodes parameters + virtual double calcEnergy() const = 0; +}; + +} // namespace detail +} // namespace kinfu +} // namespace cv +#endif /* ifndef OPENCV_RGBD_POSE_GRAPH_HPP */ diff --git a/modules/rgbd/src/fast_icp.cpp b/modules/rgbd/src/fast_icp.cpp index ab25eb94f0d..07eb84d19fd 100644 --- a/modules/rgbd/src/fast_icp.cpp +++ b/modules/rgbd/src/fast_icp.cpp @@ -504,7 +504,7 @@ void ICPImpl::getAb(const Mat& oldPts, const Mat& oldNrm, const Mat& newPts const Normals np(newPts), nn(newNrm); GetAbInvoker invoker(sumAB, mutex, op, on, np, nn, pose, intrinsics.scale(level).makeProjector(), - distanceThreshold*distanceThreshold, cos(angleThreshold)); + distanceThreshold*distanceThreshold, std::cos(angleThreshold)); Range range(0, newPts.rows); const int nstripes = -1; parallel_for_(range, invoker, nstripes); @@ -590,7 +590,7 @@ void ICPImpl::getAb(const UMat& oldPts, const UMat& oldNrm, const UMat& ne sizeof(pose.matrix.val)), fxy.val, cxy.val, distanceThreshold*distanceThreshold, - cos(angleThreshold), + std::cos(angleThreshold), ocl::KernelArg::Local(lsz), ocl::KernelArg::WriteOnlyNoSize(groupedSumGpu) ); diff --git a/modules/rgbd/src/large_kinfu.cpp b/modules/rgbd/src/large_kinfu.cpp index 16006713c53..8babd45fd60 100644 --- a/modules/rgbd/src/large_kinfu.cpp +++ b/modules/rgbd/src/large_kinfu.cpp @@ -208,6 +208,7 @@ bool LargeKinfuImpl::update(InputArray _depth) } } + template bool LargeKinfuImpl::updateT(const MatType& _depth) { @@ -224,14 +225,14 @@ bool LargeKinfuImpl::updateT(const MatType& _depth) params.bilateral_sigma_depth, params.bilateral_sigma_spatial, params.bilateral_kernel_size, params.truncateThreshold); - std::cout << "Current frameID: " << frameCounter << "\n"; + CV_LOG_INFO(NULL, "Current frameID: " << frameCounter); for (const auto& it : submapMgr->activeSubmaps) { int currTrackingId = it.first; auto submapData = it.second; Ptr> currTrackingSubmap = submapMgr->getSubmap(currTrackingId); Affine3f affine; - std::cout << "Current tracking ID: " << currTrackingId << std::endl; + CV_LOG_INFO(NULL, "Current tracking ID: " << currTrackingId); if(frameCounter == 0) //! Only one current tracking map { @@ -248,7 +249,7 @@ bool LargeKinfuImpl::updateT(const MatType& _depth) currTrackingSubmap->composeCameraPose(affine); else { - std::cout << "Tracking failed" << std::endl; + CV_LOG_INFO(NULL, "Tracking failed"); continue; } @@ -267,8 +268,8 @@ bool LargeKinfuImpl::updateT(const MatType& _depth) currTrackingSubmap->updatePyrPointsNormals(params.pyramidLevels); - std::cout << "Submap: " << currTrackingId << " Total allocated blocks: " << currTrackingSubmap->getTotalAllocatedBlocks() << "\n"; - std::cout << "Submap: " << currTrackingId << " Visible blocks: " << currTrackingSubmap->getVisibleBlocks(frameCounter) << "\n"; + CV_LOG_INFO(NULL, "Submap: " << currTrackingId << " Total allocated blocks: " << currTrackingSubmap->getTotalAllocatedBlocks()); + CV_LOG_INFO(NULL, "Submap: " << currTrackingId << " Visible blocks: " << currTrackingSubmap->getVisibleBlocks(frameCounter)); } //4. Update map @@ -277,13 +278,19 @@ bool LargeKinfuImpl::updateT(const MatType& _depth) if(isMapUpdated) { // TODO: Convert constraints to posegraph - PoseGraph poseGraph = submapMgr->MapToPoseGraph(); - std::cout << "Created posegraph\n"; - Optimizer::optimize(poseGraph); + Ptr poseGraph = submapMgr->MapToPoseGraph(); + CV_LOG_INFO(NULL, "Created posegraph"); + int iters = poseGraph->optimize(); + if (iters < 0) + { + CV_LOG_INFO(NULL, "Failed to perform pose graph optimization"); + return false; + } + submapMgr->PoseGraphToMap(poseGraph); } - std::cout << "Number of submaps: " << submapMgr->submapList.size() << "\n"; + CV_LOG_INFO(NULL, "Number of submaps: " << submapMgr->submapList.size()); frameCounter++; return true; diff --git a/modules/rgbd/src/nonrigid_icp.cpp b/modules/rgbd/src/nonrigid_icp.cpp index f2bbc5ffb2a..9bac595a4c2 100644 --- a/modules/rgbd/src/nonrigid_icp.cpp +++ b/modules/rgbd/src/nonrigid_icp.cpp @@ -350,7 +350,7 @@ bool ICPImpl::estimateWarpNodes(WarpField& currentWarp, const Affine3f &pose, Vec3f diff = oldPoints.at(y, x) - Vec3f(newP); if(diff.dot(diff) > 0.0004f) continue; - if(abs(newN.dot(oldNormals.at(y, x))) < cos((float)CV_PI / 2)) continue; + if(abs(newN.dot(oldNormals.at(y, x))) < std::cos((float)CV_PI / 2)) continue; float rd = newN.dot(diff); diff --git a/modules/rgbd/src/pose_graph.cpp b/modules/rgbd/src/pose_graph.cpp index b525e455129..78a331a2d94 100644 --- a/modules/rgbd/src/pose_graph.cpp +++ b/modules/rgbd/src/pose_graph.cpp @@ -2,59 +2,390 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html -#include "pose_graph.hpp" +#include "precomp.hpp" -#include -#include -#include -#include +#include "sparse_block_matrix.hpp" -#if defined(CERES_FOUND) -#include +// matrix form of conjugation +static const cv::Matx44d M_Conj{ 1, 0, 0, 0, + 0, -1, 0, 0, + 0, 0, -1, 0, + 0, 0, 0, -1 }; + +// matrix form of quaternion multiplication from left side +static inline cv::Matx44d m_left(cv::Quatd q) +{ + // M_left(a)* V(b) = + // = (I_4 * a0 + [ 0 | -av [ 0 | 0_1x3 + // av | 0_3] + 0_3x1 | skew(av)]) * V(b) + + double w = q.w, x = q.x, y = q.y, z = q.z; + return { w, -x, -y, -z, + x, w, -z, y, + y, z, w, -x, + z, -y, x, w }; +} + +// matrix form of quaternion multiplication from right side +static inline cv::Matx44d m_right(cv::Quatd q) +{ + // M_right(b)* V(a) = + // = (I_4 * b0 + [ 0 | -bv [ 0 | 0_1x3 + // bv | 0_3] + 0_3x1 | skew(-bv)]) * V(a) + + double w = q.w, x = q.x, y = q.y, z = q.z; + return { w, -x, -y, -z, + x, w, z, -y, + y, -z, w, x, + z, y, -x, w }; +} + +// precaution against "unused function" warning when there's no Eigen +#if defined(HAVE_EIGEN) +// jacobian of quaternionic (exp(x)*q) : R_3 -> H near x == 0 +static inline cv::Matx43d expQuatJacobian(cv::Quatd q) +{ + double w = q.w, x = q.x, y = q.y, z = q.z; + return cv::Matx43d(-x, -y, -z, + w, z, -y, + -z, w, x, + y, -x, w); +} #endif +// concatenate matrices vertically +template static inline +cv::Matx<_Tp, m + k, n> concatVert(const cv::Matx<_Tp, m, n>& a, const cv::Matx<_Tp, k, n>& b) +{ + cv::Matx<_Tp, m + k, n> res; + for (int i = 0; i < m; i++) + { + for (int j = 0; j < n; j++) + { + res(i, j) = a(i, j); + } + } + for (int i = 0; i < k; i++) + { + for (int j = 0; j < n; j++) + { + res(m + i, j) = b(i, j); + } + } + return res; +} + +// concatenate matrices horizontally +template static inline +cv::Matx<_Tp, m, n + k> concatHor(const cv::Matx<_Tp, m, n>& a, const cv::Matx<_Tp, m, k>& b) +{ + cv::Matx<_Tp, m, n + k> res; + + for (int i = 0; i < m; i++) + { + for (int j = 0; j < n; j++) + { + res(i, j) = a(i, j); + } + } + for (int i = 0; i < m; i++) + { + for (int j = 0; j < k; j++) + { + res(i, n + j) = b(i, j); + } + } + return res; +} + namespace cv { namespace kinfu { -bool PoseGraph::isValid() const +namespace detail +{ + +class PoseGraphImpl : public detail::PoseGraph { - int numNodes = getNumNodes(); - int numEdges = getNumEdges(); + struct Pose3d + { + Vec3d t; + Quatd q; + + Pose3d() : t(), q(1, 0, 0, 0) { } + + Pose3d(const Matx33d& rotation, const Vec3d& translation) + : t(translation), q(Quatd::createFromRotMat(rotation).normalize()) + { } + + explicit Pose3d(const Matx44d& pose) : + Pose3d(pose.get_minor<3, 3>(0, 0), Vec3d(pose(0, 3), pose(1, 3), pose(2, 3))) + { } + + inline Pose3d operator*(const Pose3d& otherPose) const + { + Pose3d out(*this); + out.t += q.toRotMat3x3(QUAT_ASSUME_UNIT) * otherPose.t; + out.q = out.q * otherPose.q; + return out; + } + + Affine3d getAffine() const + { + return Affine3d(q.toRotMat3x3(QUAT_ASSUME_UNIT), t); + } + + inline Pose3d inverse() const + { + Pose3d out; + out.q = q.conjugate(); + out.t = -(out.q.toRotMat3x3(QUAT_ASSUME_UNIT) * t); + return out; + } - if (numNodes <= 0 || numEdges <= 0) + inline void normalizeRotation() + { + q = q.normalize(); + } + }; + + /*! \class GraphNode + * \brief Defines a node/variable that is optimizable in a posegraph + * + * Detailed description + */ + struct Node + { + public: + explicit Node(size_t _nodeId, const Affine3d& _pose) + : id(_nodeId), isFixed(false), pose(_pose.rotation(), _pose.translation()) + { } + + Affine3d getPose() const + { + return pose.getAffine(); + } + void setPose(const Affine3d& _pose) + { + pose = Pose3d(_pose.rotation(), _pose.translation()); + } + + public: + size_t id; + bool isFixed; + Pose3d pose; + }; + + /*! \class PoseGraphEdge + * \brief Defines the constraints between two PoseGraphNodes + * + * Detailed description + */ + struct Edge + { + public: + explicit Edge(size_t _sourceNodeId, size_t _targetNodeId, const Affine3f& _transformation, + const Matx66f& _information = Matx66f::eye()); + + bool operator==(const Edge& edge) + { + if ((edge.sourceNodeId == sourceNodeId && edge.targetNodeId == targetNodeId) || + (edge.sourceNodeId == targetNodeId && edge.targetNodeId == sourceNodeId)) + return true; + return false; + } + + public: + size_t sourceNodeId; + size_t targetNodeId; + Pose3d pose; + Matx66f sqrtInfo; + }; + + +public: + PoseGraphImpl() : nodes(), edges() + { } + virtual ~PoseGraphImpl() CV_OVERRIDE + { } + + // Node may have any id >= 0 + virtual void addNode(size_t _nodeId, const Affine3d& _pose, bool fixed) CV_OVERRIDE; + virtual bool isNodeExist(size_t nodeId) const CV_OVERRIDE + { + return (nodes.find(nodeId) != nodes.end()); + } + + virtual bool setNodeFixed(size_t nodeId, bool fixed) CV_OVERRIDE + { + auto it = nodes.find(nodeId); + if (it != nodes.end()) + { + it->second.isFixed = fixed; + return true; + } + else + return false; + } + + virtual bool isNodeFixed(size_t nodeId) const CV_OVERRIDE + { + auto it = nodes.find(nodeId); + if (it != nodes.end()) + return it->second.isFixed; + else + return false; + } + + virtual Affine3d getNodePose(size_t nodeId) const CV_OVERRIDE + { + auto it = nodes.find(nodeId); + if (it != nodes.end()) + return it->second.getPose(); + else + return Affine3d(); + } + + virtual std::vector getNodesIds() const CV_OVERRIDE + { + std::vector ids; + for (const auto& it : nodes) + { + ids.push_back(it.first); + } + return ids; + } + + virtual size_t getNumNodes() const CV_OVERRIDE + { + return nodes.size(); + } + + // Edges have consequent indices starting from 0 + virtual void addEdge(size_t _sourceNodeId, size_t _targetNodeId, const Affine3f& _transformation, + const Matx66f& _information = Matx66f::eye()) CV_OVERRIDE + { + Edge e(_sourceNodeId, _targetNodeId, _transformation, _information); + edges.push_back(e); + } + + virtual size_t getEdgeStart(size_t i) const CV_OVERRIDE + { + return edges[i].sourceNodeId; + } + + virtual size_t getEdgeEnd(size_t i) const CV_OVERRIDE + { + return edges[i].targetNodeId; + } + + virtual size_t getNumEdges() const CV_OVERRIDE + { + return edges.size(); + } + + // checks if graph is connected and each edge connects exactly 2 nodes + virtual bool isValid() const CV_OVERRIDE; + + // calculate cost function based on current nodes parameters + virtual double calcEnergy() const CV_OVERRIDE; + + // calculate cost function based on provided nodes parameters + double calcEnergyNodes(const std::map& newNodes) const; + + // Termination criteria are max number of iterations and min relative energy change to current energy + // Returns number of iterations elapsed or -1 if max number of iterations was reached or failed to optimize + virtual int optimize(const cv::TermCriteria& tc = cv::TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 100, 1e-6)) CV_OVERRIDE; + + std::map nodes; + std::vector edges; +}; + + +void PoseGraphImpl::addNode(size_t _nodeId, const Affine3d& _pose, bool fixed) +{ + Node node(_nodeId, _pose); + node.isFixed = fixed; + + size_t id = node.id; + const auto& it = nodes.find(id); + if (it != nodes.end()) + { + std::cout << "duplicated node, id=" << id << std::endl; + nodes.insert(it, { id, node }); + } + else + { + nodes.insert({ id, node }); + } +} + + +// Cholesky decomposition of symmetrical 6x6 matrix +static inline cv::Matx66d llt6(Matx66d m) +{ + Matx66d L; + for (int i = 0; i < 6; i++) + { + for (int j = 0; j < (i + 1); j++) + { + double sum = 0; + for (int k = 0; k < j; k++) + sum += L(i, k) * L(j, k); + + if (i == j) + L(i, i) = sqrt(m(i, i) - sum); + else + L(i, j) = (1.0 / L(j, j) * (m(i, j) - sum)); + } + } + return L; +} + +PoseGraphImpl::Edge::Edge(size_t _sourceNodeId, size_t _targetNodeId, const Affine3f& _transformation, + const Matx66f& _information) : + sourceNodeId(_sourceNodeId), + targetNodeId(_targetNodeId), + pose(_transformation.rotation(), _transformation.translation()), + sqrtInfo(llt6(_information)) +{ } + + +bool PoseGraphImpl::isValid() const +{ + size_t numNodes = getNumNodes(); + size_t numEdges = getNumEdges(); + + if (!numNodes || !numEdges) return false; - std::unordered_set nodesVisited; - std::vector nodesToVisit; + std::unordered_set nodesVisited; + std::vector nodesToVisit; - nodesToVisit.push_back(nodes.at(0).getId()); + nodesToVisit.push_back(nodes.begin()->first); bool isGraphConnected = false; while (!nodesToVisit.empty()) { - int currNodeId = nodesToVisit.back(); + size_t currNodeId = nodesToVisit.back(); nodesToVisit.pop_back(); - std::cout << "Visiting node: " << currNodeId << "\n"; nodesVisited.insert(currNodeId); // Since each node does not maintain its neighbor list - for (int i = 0; i < numEdges; i++) + for (size_t i = 0; i < numEdges; i++) { - const PoseGraphEdge& potentialEdge = edges.at(i); - int nextNodeId = -1; + const Edge& potentialEdge = edges.at(i); + size_t nextNodeId = (size_t)(-1); - if (potentialEdge.getSourceNodeId() == currNodeId) + if (potentialEdge.sourceNodeId == currNodeId) { - nextNodeId = potentialEdge.getTargetNodeId(); + nextNodeId = potentialEdge.targetNodeId; } - else if (potentialEdge.getTargetNodeId() == currNodeId) + else if (potentialEdge.targetNodeId == currNodeId) { - nextNodeId = potentialEdge.getSourceNodeId(); + nextNodeId = potentialEdge.sourceNodeId; } - if (nextNodeId != -1) + if (nextNodeId != (size_t)(-1)) { - std::cout << "Next node: " << nextNodeId << " " << nodesVisited.count(nextNodeId) - << std::endl; if (nodesVisited.count(nextNodeId) == 0) { nodesToVisit.push_back(nextNodeId); @@ -63,16 +394,17 @@ bool PoseGraph::isValid() const } } - isGraphConnected = (int(nodesVisited.size()) == numNodes); - std::cout << "nodesVisited: " << nodesVisited.size() - << " IsGraphConnected: " << isGraphConnected << std::endl; + isGraphConnected = (nodesVisited.size() == numNodes); + + CV_LOG_INFO(NULL, "nodesVisited: " << nodesVisited.size() << " IsGraphConnected: " << isGraphConnected); + bool invalidEdgeNode = false; - for (int i = 0; i < numEdges; i++) + for (size_t i = 0; i < numEdges; i++) { - const PoseGraphEdge& edge = edges.at(i); + const Edge& edge = edges.at(i); // edges have spurious source/target nodes - if ((nodesVisited.count(edge.getSourceNodeId()) != 1) || - (nodesVisited.count(edge.getTargetNodeId()) != 1)) + if ((nodesVisited.count(edge.sourceNodeId) != 1) || + (nodesVisited.count(edge.targetNodeId) != 1)) { invalidEdgeNode = true; break; @@ -81,89 +413,499 @@ bool PoseGraph::isValid() const return isGraphConnected && !invalidEdgeNode; } -#if defined(CERES_FOUND) && defined(HAVE_EIGEN) -void Optimizer::createOptimizationProblem(PoseGraph& poseGraph, ceres::Problem& problem) + +////////////////////////// +// Optimization itself // +//////////////////////// + +static inline double poseError(Quatd sourceQuat, Vec3d sourceTrans, Quatd targetQuat, Vec3d targetTrans, + Quatd rotMeasured, Vec3d transMeasured, Matx66d sqrtInfoMatrix, bool needJacobians, + Matx& sqj, Matx& stj, + Matx& tqj, Matx& ttj, + Vec6d& res) { - int numEdges = poseGraph.getNumEdges(); - int numNodes = poseGraph.getNumNodes(); - if (numEdges == 0) + // err_r = 2*Im(conj(rel_r) * measure_r) = 2*Im(conj(target_r) * source_r * measure_r) + // err_t = conj(source_r) * (target_t - source_t) * source_r - measure_t + + Quatd sourceQuatInv = sourceQuat.conjugate(); + Vec3d deltaTrans = targetTrans - sourceTrans; + + Quatd relativeQuat = sourceQuatInv * targetQuat; + Vec3d relativeTrans = sourceQuatInv.toRotMat3x3(cv::QUAT_ASSUME_UNIT) * deltaTrans; + + //! Definition should actually be relativeQuat * rotMeasured.conjugate() + Quatd deltaRot = relativeQuat.conjugate() * rotMeasured; + + Vec3d terr = relativeTrans - transMeasured; + Vec3d rerr = 2.0 * Vec3d(deltaRot.x, deltaRot.y, deltaRot.z); + Vec6d rterr(terr[0], terr[1], terr[2], rerr[0], rerr[1], rerr[2]); + + res = sqrtInfoMatrix * rterr; + + if (needJacobians) { - CV_Error(Error::StsBadArg, "PoseGraph has no edges, no optimization to be done"); - return; + // d(err_r) = 2*Im(d(conj(target_r) * source_r * measure_r)) = < measure_r is constant > = + // 2*Im((conj(d(target_r)) * source_r + conj(target_r) * d(source_r)) * measure_r) + // d(target_r) == 0: + // # d(err_r) = 2*Im(conj(target_r) * d(source_r) * measure_r) + // # V(d(err_r)) = 2 * M_Im * M_right(measure_r) * M_left(conj(target_r)) * V(d(source_r)) + // # d(err_r) / d(source_r) = 2 * M_Im * M_right(measure_r) * M_left(conj(target_r)) + Matx34d drdsq = 2.0 * (m_right(rotMeasured) * m_left(targetQuat.conjugate())).get_minor<3, 4>(1, 0); + + // d(source_r) == 0: + // # d(err_r) = 2*Im(conj(d(target_r)) * source_r * measure_r) + // # V(d(err_r)) = 2 * M_Im * M_right(source_r * measure_r) * M_Conj * V(d(target_r)) + // # d(err_r) / d(target_r) = 2 * M_Im * M_right(source_r * measure_r) * M_Conj + Matx34d drdtq = 2.0 * (m_right(sourceQuat * rotMeasured) * M_Conj).get_minor<3, 4>(1, 0); + + // d(err_t) = d(conj(source_r) * (target_t - source_t) * source_r) = + // conj(source_r) * (d(target_t) - d(source_t)) * source_r + + // conj(d(source_r)) * (target_t - source_t) * source_r + + // conj(source_r) * (target_t - source_t) * d(source_r) = + // + // conj(source_r) * (d(target_t) - d(source_t)) * source_r + + // 2 * Im(conj(source_r) * (target_t - source_t) * d(source_r)) + // d(*_t) == 0: + // # d(err_t) = 2 * Im(conj(source_r) * (target_t - source_t) * d(source_r)) + // # V(d(err_t)) = 2 * M_Im * M_left(conj(source_r) * (target_t - source_t)) * V(d(source_r)) + // # d(err_t) / d(source_r) = 2 * M_Im * M_left(conj(source_r) * (target_t - source_t)) + Matx34d dtdsq = 2 * m_left(sourceQuatInv * Quatd(0, deltaTrans[0], deltaTrans[1], deltaTrans[2])).get_minor<3, 4>(1, 0); + // deltaTrans is rotated by sourceQuatInv, so the jacobian is rot matrix of sourceQuatInv by +1 or -1 + Matx33d dtdtt = sourceQuatInv.toRotMat3x3(QUAT_ASSUME_UNIT); + Matx33d dtdst = -dtdtt; + + Matx33d z; + sqj = concatVert(dtdsq, drdsq); + tqj = concatVert(Matx34d(), drdtq); + stj = concatVert(dtdst, z); + ttj = concatVert(dtdtt, z); + + stj = sqrtInfoMatrix * stj; + ttj = sqrtInfoMatrix * ttj; + sqj = sqrtInfoMatrix * sqj; + tqj = sqrtInfoMatrix * tqj; } - ceres::LossFunction* lossFunction = nullptr; - // TODO: Experiment with SE3 parameterization - ceres::LocalParameterization* quatLocalParameterization = - new ceres::EigenQuaternionParameterization; + return res.ddot(res); +} + + +double PoseGraphImpl::calcEnergy() const +{ + return calcEnergyNodes(nodes); +} + - for (int currEdgeNum = 0; currEdgeNum < numEdges; ++currEdgeNum) +// estimate current energy +double PoseGraphImpl::calcEnergyNodes(const std::map& newNodes) const +{ + double totalErr = 0; + for (const auto& e : edges) { - const PoseGraphEdge& currEdge = poseGraph.edges.at(currEdgeNum); - int sourceNodeId = currEdge.getSourceNodeId(); - int targetNodeId = currEdge.getTargetNodeId(); - Pose3d& sourcePose = poseGraph.nodes.at(sourceNodeId).se3Pose; - Pose3d& targetPose = poseGraph.nodes.at(targetNodeId).se3Pose; + Pose3d srcP = newNodes.at(e.sourceNodeId).pose; + Pose3d tgtP = newNodes.at(e.targetNodeId).pose; - const Matx66f& informationMatrix = currEdge.information; + Vec6d res; + Matx stj, ttj; + Matx sqj, tqj; + double err = poseError(srcP.q, srcP.t, tgtP.q, tgtP.t, e.pose.q, e.pose.t, e.sqrtInfo, + /* needJacobians = */ false, sqj, stj, tqj, ttj, res); + + totalErr += err; + } + return totalErr * 0.5; +}; - ceres::CostFunction* costFunction = Pose3dErrorFunctor::create( - Pose3d(currEdge.transformation.rotation(), currEdge.transformation.translation()), - informationMatrix); - problem.AddResidualBlock(costFunction, lossFunction, sourcePose.t.data(), - sourcePose.r.coeffs().data(), targetPose.t.data(), - targetPose.r.coeffs().data()); - problem.SetParameterization(sourcePose.r.coeffs().data(), quatLocalParameterization); - problem.SetParameterization(targetPose.r.coeffs().data(), quatLocalParameterization); +#if defined(HAVE_EIGEN) + +// from Ceres, equation energy change: +// eq. energy = 1/2 * (residuals + J * step)^2 = +// 1/2 * ( residuals^2 + 2 * residuals^T * J * step + (J*step)^T * J * step) +// eq. energy change = 1/2 * residuals^2 - eq. energy = +// residuals^T * J * step + 1/2 * (J*step)^T * J * step = +// (residuals^T * J + 1/2 * step^T * J^T * J) * step = +// step^T * ((residuals^T * J)^T + 1/2 * (step^T * J^T * J)^T) = +// 1/2 * step^T * (2 * J^T * residuals + J^T * J * step) = +// 1/2 * step^T * (2 * J^T * residuals + (J^T * J + LMDiag - LMDiag) * step) = +// 1/2 * step^T * (2 * J^T * residuals + (J^T * J + LMDiag) * step - LMDiag * step) = +// 1/2 * step^T * (J^T * residuals - LMDiag * step) = +// 1/2 * x^T * (jtb - lmDiag^T * x) +static inline double calcJacCostChange(const std::vector& jtb, + const std::vector& x, + const std::vector& lmDiag) +{ + double jdiag = 0.0; + for (size_t i = 0; i < x.size(); i++) + { + jdiag += x[i] * (jtb[i] - lmDiag[i] * x[i]); } + double costChange = jdiag * 0.5; + return costChange; +}; + - for (int currNodeId = 0; currNodeId < numNodes; ++currNodeId) +// J := J * d_inv, d_inv = make_diag(di) +// J^T*J := (J * d_inv)^T * J * d_inv = diag(di)* (J^T * J)* diag(di) = eltwise_mul(J^T*J, di*di^T) +// J^T*b := (J * d_inv)^T * b = d_inv^T * J^T*b = eltwise_mul(J^T*b, di) +static inline void doJacobiScaling(BlockSparseMat& jtj, std::vector& jtb, const std::vector& di) +{ + // scaling J^T*J + for (auto& ijv : jtj.ijValue) { - PoseGraphNode& currNode = poseGraph.nodes.at(currNodeId); - if (currNode.isPoseFixed()) + Point2i bpt = ijv.first; + Matx66d& m = ijv.second; + for (int i = 0; i < 6; i++) { - problem.SetParameterBlockConstant(currNode.se3Pose.t.data()); - problem.SetParameterBlockConstant(currNode.se3Pose.r.coeffs().data()); + for (int j = 0; j < 6; j++) + { + Point2i pt(bpt.x * 6 + i, bpt.y * 6 + j); + m(i, j) *= di[pt.x] * di[pt.y]; + } } } + + // scaling J^T*b + for (size_t i = 0; i < di.size(); i++) + { + jtb[i] *= di[i]; + } } -#endif -void Optimizer::optimize(PoseGraph& poseGraph) + +int PoseGraphImpl::optimize(const cv::TermCriteria& tc) { - PoseGraph poseGraphOriginal = poseGraph; + if (!isValid()) + { + CV_LOG_INFO(NULL, "Invalid PoseGraph that is either not connected or has invalid nodes"); + return -1; + } + + size_t numNodes = getNumNodes(); + size_t numEdges = getNumEdges(); + + // Allocate indices for nodes + std::vector placesIds; + std::map idToPlace; + for (const auto& ni : nodes) + { + if (!ni.second.isFixed) + { + idToPlace[ni.first] = placesIds.size(); + placesIds.push_back(ni.first); + } + } + + size_t nVarNodes = placesIds.size(); + if (!nVarNodes) + { + CV_LOG_INFO(NULL, "PoseGraph contains no non-constant nodes, skipping optimization"); + return -1; + } - if (!poseGraphOriginal.isValid()) + if (numEdges == 0) { - CV_Error(Error::StsBadArg, - "Invalid PoseGraph that is either not connected or has invalid nodes"); - return; + CV_LOG_INFO(NULL, "PoseGraph has no edges, no optimization to be done"); + return -1; } - int numNodes = poseGraph.getNumNodes(); - int numEdges = poseGraph.getNumEdges(); - std::cout << "Optimizing PoseGraph with " << numNodes << " nodes and " << numEdges << " edges" - << std::endl; + CV_LOG_INFO(NULL, "Optimizing PoseGraph with " << numNodes << " nodes and " << numEdges << " edges"); + + size_t nVars = nVarNodes * 6; + BlockSparseMat jtj(nVarNodes); + std::vector jtb(nVars); + + double energy = calcEnergyNodes(nodes); + double oldEnergy = energy; + + CV_LOG_INFO(NULL, "#s" << " energy: " << energy); + + // options + // stop conditions + bool checkIterations = (tc.type & TermCriteria::COUNT); + bool checkEps = (tc.type & TermCriteria::EPS); + const unsigned int maxIterations = tc.maxCount; + const double minGradientTolerance = 1e-6; + const double stepNorm2Tolerance = 1e-6; + const double relEnergyDeltaTolerance = tc.epsilon; + // normalize jacobian columns for better conditioning + // slows down sparse solver, but maybe this'd be useful for some other solver + const bool jacobiScaling = false; + const double minDiag = 1e-6; + const double maxDiag = 1e32; + + const double initialLambdaLevMarq = 0.0001; + const double initialLmUpFactor = 2.0; + const double initialLmDownFactor = 3.0; + + // finish reasons + bool tooLong = false; // => not found + bool smallGradient = false; // => found + bool smallStep = false; // => found + bool smallEnergyDelta = false; // => found + + // column scale inverted, for jacobian scaling + std::vector di(nVars); + + double lmUpFactor = initialLmUpFactor; + double lambdaLevMarq = initialLambdaLevMarq; + + unsigned int iter = 0; + bool done = false; + while (!done) + { + jtj.clear(); + std::fill(jtb.begin(), jtb.end(), 0.0); + + // caching nodes jacobians + std::vector> cachedJac; + for (auto id : placesIds) + { + Pose3d p = nodes.at(id).pose; + Matx43d qj = expQuatJacobian(p.q); + // x node layout is (rot_x, rot_y, rot_z, trans_x, trans_y, trans_z) + // pose layout is (q_w, q_x, q_y, q_z, trans_x, trans_y, trans_z) + Matx j = concatVert(concatHor(qj, Matx43d()), + concatHor(Matx33d(), Matx33d::eye())); + cachedJac.push_back(j); + } + + // fill jtj and jtb + for (const auto& e : edges) + { + size_t srcId = e.sourceNodeId, dstId = e.targetNodeId; + const Node& srcNode = nodes.at(srcId); + const Node& dstNode = nodes.at(dstId); -#if defined(CERES_FOUND) && defined(HAVE_EIGEN) - ceres::Problem problem; - createOptimizationProblem(poseGraph, problem); + Pose3d srcP = srcNode.pose; + Pose3d tgtP = dstNode.pose; + bool srcFixed = srcNode.isFixed; + bool dstFixed = dstNode.isFixed; - ceres::Solver::Options options; - options.max_num_iterations = 100; - options.linear_solver_type = ceres::SPARSE_NORMAL_CHOLESKY; + Vec6d res; + Matx stj, ttj; + Matx sqj, tqj; + poseError(srcP.q, srcP.t, tgtP.q, tgtP.t, e.pose.q, e.pose.t, e.sqrtInfo, + /* needJacobians = */ true, sqj, stj, tqj, ttj, res); - ceres::Solver::Summary summary; - ceres::Solve(options, &problem, &summary); + size_t srcPlace = (size_t)(-1), dstPlace = (size_t)(-1); + Matx66d sj, tj; + if (!srcFixed) + { + srcPlace = idToPlace.at(srcId); + sj = concatHor(sqj, stj) * cachedJac[srcPlace]; + + jtj.refBlock(srcPlace, srcPlace) += sj.t() * sj; + + Vec6f jtbSrc = sj.t() * res; + for (int i = 0; i < 6; i++) + { + jtb[6 * srcPlace + i] += -jtbSrc[i]; + } + } + + if (!dstFixed) + { + dstPlace = idToPlace.at(dstId); + tj = concatHor(tqj, ttj) * cachedJac[dstPlace]; + + jtj.refBlock(dstPlace, dstPlace) += tj.t() * tj; + + Vec6f jtbDst = tj.t() * res; + for (int i = 0; i < 6; i++) + { + jtb[6 * dstPlace + i] += -jtbDst[i]; + } + } + + if (!(srcFixed || dstFixed)) + { + Matx66d sjttj = sj.t() * tj; + jtj.refBlock(srcPlace, dstPlace) += sjttj; + jtj.refBlock(dstPlace, srcPlace) += sjttj.t(); + } + } - std::cout << summary.FullReport() << '\n'; + CV_LOG_INFO(NULL, "#LM#s" << " energy: " << energy); + + // do the jacobian conditioning improvement used in Ceres + if (jacobiScaling) + { + // L2-normalize each jacobian column + // vec d = {d_j = sum(J_ij^2) for each column j of J} = get_diag{ J^T * J } + // di = { 1/(1+sqrt(d_j)) }, extra +1 to avoid div by zero + if (iter == 0) + { + for (size_t i = 0; i < nVars; i++) + { + double ds = sqrt(jtj.valElem(i, i)) + 1.0; + di[i] = 1.0 / ds; + } + } + + doJacobiScaling(jtj, jtb, di); + } + + double gradientMax = 0.0; + // gradient max + for (size_t i = 0; i < nVars; i++) + { + gradientMax = std::max(gradientMax, abs(jtb[i])); + } + + // Save original diagonal of jtj matrix for LevMarq + std::vector diag(nVars); + for (size_t i = 0; i < nVars; i++) + { + diag[i] = jtj.valElem(i, i); + } + + // Solve using LevMarq and get delta transform + bool enoughLm = false; + + decltype(nodes) tempNodes = nodes; + + while (!enoughLm && !done) + { + // form LevMarq matrix + std::vector lmDiag(nVars); + for (size_t i = 0; i < nVars; i++) + { + double v = diag[i]; + double ld = std::min(max(v * lambdaLevMarq, minDiag), maxDiag); + lmDiag[i] = ld; + jtj.refElem(i, i) = v + ld; + } + + CV_LOG_INFO(NULL, "sparse solve..."); + + // use double or convert everything to float + std::vector x; + bool solved = jtj.sparseSolve(jtb, x, false); + + CV_LOG_INFO(NULL, (solved ? "OK" : "FAIL")); + + double costChange = 0.0; + double jacCostChange = 0.0; + double stepQuality = 0.0; + double xNorm2 = 0.0; + if (solved) + { + jacCostChange = calcJacCostChange(jtb, x, lmDiag); + + // x squared norm + for (size_t i = 0; i < nVars; i++) + { + xNorm2 += x[i] * x[i]; + } + + // undo jacobi scaling + if (jacobiScaling) + { + for (size_t i = 0; i < nVars; i++) + { + x[i] *= di[i]; + } + } + + tempNodes = nodes; + + // Update temp nodes using x + for (size_t i = 0; i < nVarNodes; i++) + { + Vec6d dx(&x[i * 6]); + Vec3d deltaRot(dx[0], dx[1], dx[2]), deltaTrans(dx[3], dx[4], dx[5]); + Pose3d& p = tempNodes.at(placesIds[i]).pose; + + p.q = Quatd(0, deltaRot[0], deltaRot[1], deltaRot[2]).exp() * p.q; + p.t += deltaTrans; + } + + // calc energy with temp nodes + energy = calcEnergyNodes(tempNodes); + + costChange = oldEnergy - energy; + + stepQuality = costChange / jacCostChange; + + CV_LOG_INFO(NULL, "#LM#" << iter + << " energy: " << energy + << " deltaEnergy: " << costChange + << " deltaEqEnergy: " << jacCostChange + << " max(J^T*b): " << gradientMax + << " norm2(x): " << xNorm2 + << " deltaEnergy/energy: " << costChange / energy); + } + + if (!solved || costChange < 0) + { + // failed to optimize, increase lambda and repeat + + lambdaLevMarq *= lmUpFactor; + lmUpFactor *= 2.0; + + CV_LOG_INFO(NULL, "LM goes up, lambda: " << lambdaLevMarq << ", old energy: " << oldEnergy); + } + else + { + // optimized successfully, decrease lambda and set variables for next iteration + enoughLm = true; + + lambdaLevMarq *= std::max(1.0 / initialLmDownFactor, 1.0 - pow(2.0 * stepQuality - 1.0, 3)); + lmUpFactor = initialLmUpFactor; + + smallGradient = (gradientMax < minGradientTolerance); + smallStep = (xNorm2 < stepNorm2Tolerance); + smallEnergyDelta = (costChange / energy < relEnergyDeltaTolerance); + + nodes = tempNodes; + + CV_LOG_INFO(NULL, "#" << iter << " energy: " << energy); + + oldEnergy = energy; + + CV_LOG_INFO(NULL, "LM goes down, lambda: " << lambdaLevMarq << " step quality: " << stepQuality); + } + + iter++; + + tooLong = (iter >= maxIterations); + + done = ( (checkIterations && tooLong) || smallGradient || smallStep || (checkEps && smallEnergyDelta) ); + } + } + + // if maxIterations is given as a stop criteria, let's consider tooLong as a successful finish + bool found = (smallGradient || smallStep || smallEnergyDelta || (checkIterations && tooLong)); + + CV_LOG_INFO(NULL, "Finished: " << (found ? "" : "not") << "found"); + if (smallGradient) + CV_LOG_INFO(NULL, "Finish reason: gradient max val dropped below threshold"); + if (smallStep) + CV_LOG_INFO(NULL, "Finish reason: step size dropped below threshold"); + if (smallEnergyDelta) + CV_LOG_INFO(NULL, "Finish reason: relative energy change between iterations dropped below threshold"); + if (tooLong) + CV_LOG_INFO(NULL, "Finish reason: max number of iterations reached"); + + return (found ? iter : -1); +} - std::cout << "Is solution usable: " << summary.IsSolutionUsable() << std::endl; #else - CV_Error(Error::StsNotImplemented, "Ceres and Eigen required for Pose Graph optimization"); +int PoseGraphImpl::optimize(const cv::TermCriteria& /*tc*/) +{ + CV_Error(Error::StsNotImplemented, "Eigen library required for sparse matrix solve during pose graph optimization, dense solver is not implemented"); +} #endif + + +Ptr detail::PoseGraph::create() +{ + return makePtr(); } +PoseGraph::~PoseGraph() { } + +} // namespace detail } // namespace kinfu } // namespace cv diff --git a/modules/rgbd/src/pose_graph.hpp b/modules/rgbd/src/pose_graph.hpp deleted file mode 100644 index e8b7c34c53b..00000000000 --- a/modules/rgbd/src/pose_graph.hpp +++ /dev/null @@ -1,321 +0,0 @@ -#ifndef OPENCV_RGBD_GRAPH_NODE_H -#define OPENCV_RGBD_GRAPH_NODE_H - -#include -#include - -#include "opencv2/core/affine.hpp" -#if defined(HAVE_EIGEN) -#include -#include -#include "opencv2/core/eigen.hpp" -#endif - -#if defined(CERES_FOUND) -#include -#endif - -namespace cv -{ -namespace kinfu -{ -/*! \class GraphNode - * \brief Defines a node/variable that is optimizable in a posegraph - * - * Detailed description - */ -#if defined(HAVE_EIGEN) -struct Pose3d -{ - EIGEN_MAKE_ALIGNED_OPERATOR_NEW - - Eigen::Vector3d t; - Eigen::Quaterniond r; - - Pose3d() - { - t.setZero(); - r.setIdentity(); - }; - Pose3d(const Eigen::Matrix3d& rotation, const Eigen::Vector3d& translation) - : t(translation), r(Eigen::Quaterniond(rotation)) - { - normalizeRotation(); - } - - Pose3d(const Matx33d& rotation, const Vec3d& translation) - { - Eigen::Matrix3d R; - cv2eigen(rotation, R); - cv2eigen(translation, t); - r = Eigen::Quaterniond(R); - normalizeRotation(); - } - - explicit Pose3d(const Matx44f& pose) - { - Matx33d rotation(pose.val[0], pose.val[1], pose.val[2], pose.val[4], pose.val[5], - pose.val[6], pose.val[8], pose.val[9], pose.val[10]); - Vec3d translation(pose.val[3], pose.val[7], pose.val[11]); - Pose3d(rotation, translation); - } - - // NOTE: Eigen overloads quaternion multiplication appropriately - inline Pose3d operator*(const Pose3d& otherPose) const - { - Pose3d out(*this); - out.t += r * otherPose.t; - out.r *= otherPose.r; - out.normalizeRotation(); - return out; - } - - inline Pose3d& operator*=(const Pose3d& otherPose) - { - t += otherPose.t; - r *= otherPose.r; - normalizeRotation(); - return *this; - } - - inline Pose3d inverse() const - { - Pose3d out; - out.r = r.conjugate(); - out.t = out.r * (t * -1.0); - return out; - } - - inline void normalizeRotation() - { - if (r.w() < 0) - r.coeffs() *= -1.0; - r.normalize(); - } -}; -#endif - -struct PoseGraphNode -{ - public: - explicit PoseGraphNode(int _nodeId, const Affine3f& _pose) - : nodeId(_nodeId), isFixed(false), pose(_pose) - { -#if defined(HAVE_EIGEN) - se3Pose = Pose3d(_pose.rotation(), _pose.translation()); -#endif - } - virtual ~PoseGraphNode() = default; - - int getId() const { return nodeId; } - inline Affine3f getPose() const - { - return pose; - } - void setPose(const Affine3f& _pose) - { - pose = _pose; -#if defined(HAVE_EIGEN) - se3Pose = Pose3d(pose.rotation(), pose.translation()); -#endif - } -#if defined(HAVE_EIGEN) - void setPose(const Pose3d& _pose) - { - se3Pose = _pose; - const Eigen::Matrix3d& rotation = se3Pose.r.toRotationMatrix(); - const Eigen::Vector3d& translation = se3Pose.t; - Matx33d rot; - Vec3d trans; - eigen2cv(rotation, rot); - eigen2cv(translation, trans); - Affine3d poseMatrix(rot, trans); - pose = poseMatrix; - } -#endif - void setFixed(bool val = true) { isFixed = val; } - bool isPoseFixed() const { return isFixed; } - - public: - int nodeId; - bool isFixed; - Affine3f pose; -#if defined(HAVE_EIGEN) - Pose3d se3Pose; -#endif -}; - -/*! \class PoseGraphEdge - * \brief Defines the constraints between two PoseGraphNodes - * - * Detailed description - */ -struct PoseGraphEdge -{ - public: - PoseGraphEdge(int _sourceNodeId, int _targetNodeId, const Affine3f& _transformation, - const Matx66f& _information = Matx66f::eye()) - : sourceNodeId(_sourceNodeId), - targetNodeId(_targetNodeId), - transformation(_transformation), - information(_information) - { - } - virtual ~PoseGraphEdge() = default; - - int getSourceNodeId() const { return sourceNodeId; } - int getTargetNodeId() const { return targetNodeId; } - - bool operator==(const PoseGraphEdge& edge) - { - if ((edge.getSourceNodeId() == sourceNodeId && edge.getTargetNodeId() == targetNodeId) || - (edge.getSourceNodeId() == targetNodeId && edge.getTargetNodeId() == sourceNodeId)) - return true; - return false; - } - - public: - int sourceNodeId; - int targetNodeId; - Affine3f transformation; - Matx66f information; -}; - -//! @brief Reference: A tutorial on SE(3) transformation parameterizations and on-manifold -//! optimization Jose Luis Blanco Compactly represents the jacobian of the SE3 generator -// clang-format off -/* static const std::array generatorJacobian = { */ -/* // alpha */ -/* Matx44f(0, 0, 0, 0, */ -/* 0, 0, -1, 0, */ -/* 0, 1, 0, 0, */ -/* 0, 0, 0, 0), */ -/* // beta */ -/* Matx44f( 0, 0, 1, 0, */ -/* 0, 0, 0, 0, */ -/* -1, 0, 0, 0, */ -/* 0, 0, 0, 0), */ -/* // gamma */ -/* Matx44f(0, -1, 0, 0, */ -/* 1, 0, 0, 0, */ -/* 0, 0, 0, 0, */ -/* 0, 0, 0, 0), */ -/* // x */ -/* Matx44f(0, 0, 0, 1, */ -/* 0, 0, 0, 0, */ -/* 0, 0, 0, 0, */ -/* 0, 0, 0, 0), */ -/* // y */ -/* Matx44f(0, 0, 0, 0, */ -/* 0, 0, 0, 1, */ -/* 0, 0, 0, 0, */ -/* 0, 0, 0, 0), */ -/* // z */ -/* Matx44f(0, 0, 0, 0, */ -/* 0, 0, 0, 0, */ -/* 0, 0, 0, 1, */ -/* 0, 0, 0, 0) */ -/* }; */ -// clang-format on - -class PoseGraph -{ - public: - typedef std::vector NodeVector; - typedef std::vector EdgeVector; - - explicit PoseGraph(){}; - virtual ~PoseGraph() = default; - - //! PoseGraph can be copied/cloned - PoseGraph(const PoseGraph&) = default; - PoseGraph& operator=(const PoseGraph&) = default; - - void addNode(const PoseGraphNode& node) { nodes.push_back(node); } - void addEdge(const PoseGraphEdge& edge) { edges.push_back(edge); } - - bool nodeExists(int nodeId) const - { - return std::find_if(nodes.begin(), nodes.end(), [nodeId](const PoseGraphNode& currNode) { - return currNode.getId() == nodeId; - }) != nodes.end(); - } - - bool isValid() const; - - int getNumNodes() const { return int(nodes.size()); } - int getNumEdges() const { return int(edges.size()); } - - public: - NodeVector nodes; - EdgeVector edges; -}; - -namespace Optimizer -{ -void optimize(PoseGraph& poseGraph); - -#if defined(CERES_FOUND) -void createOptimizationProblem(PoseGraph& poseGraph, ceres::Problem& problem); - -//! Error Functor required for Ceres to obtain an auto differentiable cost function -class Pose3dErrorFunctor -{ - public: - EIGEN_MAKE_ALIGNED_OPERATOR_NEW - Pose3dErrorFunctor(const Pose3d& _poseMeasurement, const Matx66d& _sqrtInformation) - : poseMeasurement(_poseMeasurement) - { - cv2eigen(_sqrtInformation, sqrtInfo); - } - Pose3dErrorFunctor(const Pose3d& _poseMeasurement, - const Eigen::Matrix& _sqrtInformation) - : poseMeasurement(_poseMeasurement), sqrtInfo(_sqrtInformation) - { - } - - template - bool operator()(const T* const _pSourceTrans, const T* const _pSourceQuat, - const T* const _pTargetTrans, const T* const _pTargetQuat, T* _pResidual) const - { - Eigen::Map> sourceTrans(_pSourceTrans); - Eigen::Map> targetTrans(_pTargetTrans); - Eigen::Map> sourceQuat(_pSourceQuat); - Eigen::Map> targetQuat(_pTargetQuat); - Eigen::Map> residual(_pResidual); - - Eigen::Quaternion targetQuatInv = targetQuat.conjugate(); - - Eigen::Quaternion relativeQuat = targetQuatInv * sourceQuat; - Eigen::Matrix relativeTrans = targetQuatInv * (targetTrans - sourceTrans); - - //! Definition should actually be relativeQuat * poseMeasurement.r.conjugate() - Eigen::Quaternion deltaRot = - poseMeasurement.r.template cast() * relativeQuat.conjugate(); - - residual.template block<3, 1>(0, 0) = relativeTrans - poseMeasurement.t.template cast(); - residual.template block<3, 1>(3, 0) = T(2.0) * deltaRot.vec(); - - residual.applyOnTheLeft(sqrtInfo.template cast()); - - return true; - } - - static ceres::CostFunction* create(const Pose3d& _poseMeasurement, - const Matx66f& _sqrtInformation) - { - return new ceres::AutoDiffCostFunction( - new Pose3dErrorFunctor(_poseMeasurement, _sqrtInformation)); - } - - private: - const Pose3d poseMeasurement; - Eigen::Matrix sqrtInfo; -}; -#endif - -} // namespace Optimizer - -} // namespace kinfu -} // namespace cv -#endif /* ifndef OPENCV_RGBD_GRAPH_NODE_H */ diff --git a/modules/rgbd/src/precomp.hpp b/modules/rgbd/src/precomp.hpp index bad630705d7..4852274f3b0 100644 --- a/modules/rgbd/src/precomp.hpp +++ b/modules/rgbd/src/precomp.hpp @@ -11,8 +11,10 @@ #define __OPENCV_PRECOMP_H__ #include +#include #include #include +#include #include #include "opencv2/core/utility.hpp" diff --git a/modules/rgbd/src/sparse_block_matrix.hpp b/modules/rgbd/src/sparse_block_matrix.hpp index 0e607af639d..2826f948664 100644 --- a/modules/rgbd/src/sparse_block_matrix.hpp +++ b/modules/rgbd/src/sparse_block_matrix.hpp @@ -7,6 +7,7 @@ #include "opencv2/core/base.hpp" #include "opencv2/core/types.hpp" +#include "opencv2/core/utils/logger.hpp" #if defined(HAVE_EIGEN) #include @@ -24,7 +25,7 @@ namespace kinfu * \class BlockSparseMat * Naive implementation of Sparse Block Matrix */ -template +template struct BlockSparseMat { struct Point2iHash @@ -41,55 +42,78 @@ struct BlockSparseMat typedef Matx<_Tp, blockM, blockN> MatType; typedef std::unordered_map IDtoBlockValueMap; - BlockSparseMat(int _nBlocks) : nBlocks(_nBlocks), ijValue() {} + BlockSparseMat(size_t _nBlocks) : nBlocks(_nBlocks), ijValue() {} - MatType& refBlock(int i, int j) + void clear() { - Point2i p(i, j); + ijValue.clear(); + } + + inline MatType& refBlock(size_t i, size_t j) + { + Point2i p((int)i, (int)j); auto it = ijValue.find(p); if (it == ijValue.end()) { - it = ijValue.insert({ p, Matx<_Tp, blockM, blockN>::zeros() }).first; + it = ijValue.insert({ p, MatType::zeros() }).first; } return it->second; } - Mat diagonal() + inline _Tp& refElem(size_t i, size_t j) + { + Point2i ib((int)(i / blockM), (int)(j / blockN)); + Point2i iv((int)(i % blockM), (int)(j % blockN)); + return refBlock(ib.x, ib.y)(iv.x, iv.y); + } + + inline MatType valBlock(size_t i, size_t j) const + { + Point2i p((int)i, (int)j); + auto it = ijValue.find(p); + if (it == ijValue.end()) + return MatType::zeros(); + else + return it->second; + } + + inline _Tp valElem(size_t i, size_t j) const + { + Point2i ib((int)(i / blockM), (int)(j / blockN)); + Point2i iv((int)(i % blockM), (int)(j % blockN)); + return valBlock(ib.x, ib.y)(iv.x, iv.y); + } + + Mat diagonal() const { // Diagonal max length is the number of columns in the sparse matrix int diagLength = blockN * nBlocks; - cv::Mat diag = cv::Mat::zeros(diagLength, 1, CV_32F); + cv::Mat diag = cv::Mat::zeros(diagLength, 1, cv::DataType<_Tp>::type); for (int i = 0; i < diagLength; i++) { - diag.at(i, 0) = refElem(i, i); + diag.at<_Tp>(i, 0) = valElem(i, i); } return diag; } - float& refElem(int i, int j) - { - Point2i ib(i / blockM, j / blockN), iv(i % blockM, j % blockN); - return refBlock(ib.x, ib.y)(iv.x, iv.y); - } - #if defined(HAVE_EIGEN) Eigen::SparseMatrix<_Tp> toEigen() const { - std::vector> tripletList; + std::vector> tripletList; tripletList.reserve(ijValue.size() * blockM * blockN); - for (auto ijv : ijValue) + for (const auto& ijv : ijValue) { int xb = ijv.first.x, yb = ijv.first.y; MatType vblock = ijv.second; - for (int i = 0; i < blockM; i++) + for (size_t i = 0; i < blockM; i++) { - for (int j = 0; j < blockN; j++) + for (size_t j = 0; j < blockN; j++) { - float val = vblock(i, j); + _Tp val = vblock((int)i, (int)j); if (abs(val) >= NON_ZERO_VAL_THRESHOLD) { - tripletList.push_back(Eigen::Triplet(blockM * xb + i, blockN * yb + j, val)); + tripletList.push_back(Eigen::Triplet<_Tp>((int)(blockM * xb + i), (int)(blockN * yb + j), val)); } } } @@ -101,59 +125,76 @@ struct BlockSparseMat return EigenMat; } #endif - size_t nonZeroBlocks() const { return ijValue.size(); } + inline size_t nonZeroBlocks() const { return ijValue.size(); } - static constexpr float NON_ZERO_VAL_THRESHOLD = 0.0001f; - int nBlocks; - IDtoBlockValueMap ijValue; -}; + BlockSparseMat<_Tp, blockM, blockN>& operator+=(const BlockSparseMat<_Tp, blockM, blockN>& other) + { + for (const auto& oijv : other.ijValue) + { + Point2i p = oijv.first; + MatType vblock = oijv.second; + this->refBlock(p.x, p.y) += vblock; + } -//! Function to solve a sparse linear system of equations HX = B -//! Requires Eigen -static bool sparseSolve(const BlockSparseMat& H, const Mat& B, Mat& X, Mat& predB) -{ - bool result = false; -#if defined(HAVE_EIGEN) - Eigen::SparseMatrix bigA = H.toEigen(); - Eigen::VectorXf bigB; - cv2eigen(B, bigB); + return *this; + } - Eigen::SparseMatrix bigAtranspose = bigA.transpose(); - if(!bigA.isApprox(bigAtranspose)) +#if defined(HAVE_EIGEN) + //! Function to solve a sparse linear system of equations HX = B + //! Requires Eigen + bool sparseSolve(InputArray B, OutputArray X, bool checkSymmetry = true, OutputArray predB = cv::noArray()) const { - CV_Error(Error::StsBadArg, "H matrix is not symmetrical"); - return result; - } + Eigen::SparseMatrix<_Tp> bigA = toEigen(); + Mat mb = B.getMat().t(); + Eigen::Matrix<_Tp, -1, 1> bigB; + cv2eigen(mb, bigB); - Eigen::SimplicialLDLT> solver; + Eigen::SparseMatrix<_Tp> bigAtranspose = bigA.transpose(); + if(checkSymmetry && !bigA.isApprox(bigAtranspose)) + { + CV_Error(Error::StsBadArg, "H matrix is not symmetrical"); + return false; + } - solver.compute(bigA); - if (solver.info() != Eigen::Success) - { - std::cout << "failed to eigen-decompose" << std::endl; - result = false; - } - else - { - Eigen::VectorXf solutionX = solver.solve(bigB); - Eigen::VectorXf predBEigen = bigA * solutionX; + Eigen::SimplicialLDLT> solver; + + solver.compute(bigA); if (solver.info() != Eigen::Success) { - std::cout << "failed to eigen-solve" << std::endl; - result = false; + CV_LOG_INFO(NULL, "Failed to eigen-decompose"); + return false; } else { - eigen2cv(solutionX, X); - eigen2cv(predBEigen, predB); - result = true; + Eigen::Matrix<_Tp, -1, 1> solutionX = solver.solve(bigB); + if (solver.info() != Eigen::Success) + { + CV_LOG_INFO(NULL, "Failed to eigen-solve"); + return false; + } + else + { + eigen2cv(solutionX, X); + if (predB.needed()) + { + Eigen::Matrix<_Tp, -1, 1> predBEigen = bigA * solutionX; + eigen2cv(predBEigen, predB); + } + return true; + } } } #else - std::cout << "no eigen library" << std::endl; - CV_Error(Error::StsNotImplemented, "Eigen library required for matrix solve, dense solver is not implemented"); + bool sparseSolve(InputArray /*B*/, OutputArray /*X*/, bool /*checkSymmetry*/ = true, OutputArray /*predB*/ = cv::noArray()) const + { + CV_Error(Error::StsNotImplemented, "Eigen library required for matrix solve, dense solver is not implemented"); + } #endif - return result; -} + + static constexpr _Tp NON_ZERO_VAL_THRESHOLD = _Tp(0.0001); + size_t nBlocks; + IDtoBlockValueMap ijValue; +}; + } // namespace kinfu } // namespace cv diff --git a/modules/rgbd/src/submap.hpp b/modules/rgbd/src/submap.hpp index b40023b6a56..ad41c47cb34 100644 --- a/modules/rgbd/src/submap.hpp +++ b/modules/rgbd/src/submap.hpp @@ -13,7 +13,7 @@ #include "hash_tsdf.hpp" #include "opencv2/core/mat.inl.hpp" -#include "pose_graph.hpp" +#include "opencv2/rgbd/detail/pose_graph.hpp" namespace cv { @@ -166,15 +166,15 @@ class SubmapManager int estimateConstraint(int fromSubmapId, int toSubmapId, int& inliers, Affine3f& inlierPose); bool updateMap(int _frameId, std::vector _framePoints, std::vector _frameNormals); - PoseGraph MapToPoseGraph(); - void PoseGraphToMap(const PoseGraph& updatedPoseGraph); + Ptr MapToPoseGraph(); + void PoseGraphToMap(const Ptr& updatedPoseGraph); VolumeParams volumeParams; std::vector> submapList; IdToActiveSubmaps activeSubmaps; - PoseGraph poseGraph; + Ptr poseGraph; }; template @@ -494,10 +494,9 @@ bool SubmapManager::updateMap(int _frameId, std::vector _frame } template -PoseGraph SubmapManager::MapToPoseGraph() +Ptr SubmapManager::MapToPoseGraph() { - PoseGraph localPoseGraph; - + Ptr localPoseGraph = detail::PoseGraph::create(); for(const auto& currSubmap : submapList) { @@ -507,34 +506,26 @@ PoseGraph SubmapManager::MapToPoseGraph() // TODO: Handle case with duplicate constraints A -> B and B -> A /* Matx66f informationMatrix = Matx66f::eye() * (currConstraintPair.second.weight/10); */ Matx66f informationMatrix = Matx66f::eye(); - PoseGraphEdge currEdge(currSubmap->id, currConstraintPair.first, currConstraintPair.second.estimatedPose, informationMatrix); - localPoseGraph.addEdge(currEdge); + localPoseGraph->addEdge(currSubmap->id, currConstraintPair.first, currConstraintPair.second.estimatedPose, informationMatrix); } } for(const auto& currSubmap : submapList) { - PoseGraphNode currNode(currSubmap->id, currSubmap->pose); - if(currSubmap->id == 0) - { - currNode.setFixed(); - } - localPoseGraph.addNode(currNode); + localPoseGraph->addNode(currSubmap->id, currSubmap->pose, (currSubmap->id == 0)); } - - return localPoseGraph; } template -void SubmapManager::PoseGraphToMap(const PoseGraph &updatedPoseGraph) +void SubmapManager::PoseGraphToMap(const Ptr& updatedPoseGraph) { for(const auto& currSubmap : submapList) { - const PoseGraphNode& currNode = updatedPoseGraph.nodes.at(currSubmap->id); - if(!currNode.isPoseFixed()) - currSubmap->pose = currNode.getPose(); + Affine3d pose = updatedPoseGraph->getNodePose(currSubmap->id); + if(!updatedPoseGraph->isNodeFixed(currSubmap->id)) + currSubmap->pose = pose; std::cout << "Current node: " << currSubmap->id << " Updated Pose: \n" << currSubmap->pose.matrix << std::endl; } } diff --git a/modules/rgbd/test/test_pose_graph.cpp b/modules/rgbd/test/test_pose_graph.cpp new file mode 100644 index 00000000000..a675e343c27 --- /dev/null +++ b/modules/rgbd/test/test_pose_graph.cpp @@ -0,0 +1,151 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#include "test_precomp.hpp" + +namespace opencv_test { namespace { + +using namespace cv; + +static Affine3d readAffine(std::istream& input) +{ + Vec3d p; + Vec4d q; + input >> p[0] >> p[1] >> p[2]; + input >> q[1] >> q[2] >> q[3] >> q[0]; + // Normalize the quaternion to account for precision loss due to + // serialization. + return Affine3d(Quatd(q).toRotMat3x3(), p); +}; + +// Rewritten from Ceres pose graph demo: https://ceres-solver.org/ +static Ptr readG2OFile(const std::string& g2oFileName) +{ + Ptr pg = kinfu::detail::PoseGraph::create(); + + // for debugging purposes + size_t minId = 0, maxId = 1 << 30; + + std::ifstream infile(g2oFileName.c_str()); + if (!infile) + { + CV_Error(cv::Error::StsError, "failed to open file"); + } + + while (infile.good()) + { + std::string data_type; + // Read whether the type is a node or a constraint + infile >> data_type; + if (data_type == "VERTEX_SE3:QUAT") + { + size_t id; + infile >> id; + Affine3d pose = readAffine(infile); + + if (id < minId || id >= maxId) + continue; + + bool fixed = (id == minId); + + // Ensure we don't have duplicate poses + if (pg->isNodeExist(id)) + { + CV_LOG_INFO(NULL, "duplicated node, id=" << id); + } + pg->addNode(id, pose, fixed); + } + else if (data_type == "EDGE_SE3:QUAT") + { + size_t startId, endId; + infile >> startId >> endId; + Affine3d pose = readAffine(infile); + + Matx66d info; + for (int i = 0; i < 6 && infile.good(); ++i) + { + for (int j = i; j < 6 && infile.good(); ++j) + { + infile >> info(i, j); + if (i != j) + { + info(j, i) = info(i, j); + } + } + } + + if ((startId >= minId && startId < maxId) && (endId >= minId && endId < maxId)) + { + pg->addEdge(startId, endId, pose, info); + } + } + else + { + CV_Error(cv::Error::StsError, "unknown tag"); + } + + // Clear any trailing whitespace from the line + infile >> std::ws; + } + + return pg; +} + + +TEST( PoseGraph, sphereG2O ) +{ + // Test takes 15+ sec in Release mode and 400+ sec in Debug mode + applyTestTag(CV_TEST_TAG_LONG, CV_TEST_TAG_DEBUG_VERYLONG); + + // The dataset was taken from here: https://lucacarlone.mit.edu/datasets/ + // Connected paper: + // L.Carlone, R.Tron, K.Daniilidis, and F.Dellaert. + // Initialization Techniques for 3D SLAM : a Survey on Rotation Estimation and its Use in Pose Graph Optimization. + // In IEEE Intl.Conf.on Robotics and Automation(ICRA), pages 4597 - 4604, 2015. + + std::string filename = cvtest::TS::ptr()->get_data_path() + "rgbd/sphere_bignoise_vertex3.g2o"; + Ptr pg = readG2OFile(filename); + +#ifdef HAVE_EIGEN + // You may change logging level to view detailed optimization report + // For example, set env. variable like this: OPENCV_LOG_LEVEL=INFO + + int iters = pg->optimize(); + + ASSERT_GE(iters, 0); + ASSERT_LE(iters, 36); // should converge in 36 iterations + + double energy = pg->calcEnergy(); + + ASSERT_LE(energy, 1.47723e+06); // should converge to 1.47722e+06 or less + + // Add the "--test_debug" to arguments to see resulting pose graph nodes positions + if (cvtest::debugLevel > 0) + { + // Write edge-only model of how nodes are located in space + std::string fname = "pgout.obj"; + std::fstream of(fname, std::fstream::out); + std::vector ids = pg->getNodesIds(); + for (const size_t& id : ids) + { + Point3d d = pg->getNodePose(id).translation(); + of << "v " << d.x << " " << d.y << " " << d.z << std::endl; + } + + size_t esz = pg->getNumEdges(); + for (size_t i = 0; i < esz; i++) + { + size_t sid = pg->getEdgeStart(i), tid = pg->getEdgeEnd(i); + of << "l " << sid + 1 << " " << tid + 1 << std::endl; + } + + of.close(); + } +#else + throw SkipTestException("Build with Eigen required for pose graph optimization"); +#endif +} + + +}} // namespace \ No newline at end of file From f7efcbab611b140a4a6a58e3f55cf5f1abca9d4a Mon Sep 17 00:00:00 2001 From: Suleyman TURKMEN Date: Fri, 2 Apr 2021 04:38:00 +0300 Subject: [PATCH 61/70] Merge pull request #2313 from sturkmen72:test_Edge_Drawing_Algos * EdgeDrawing algorithms (ED EDPF EDLines EDCircles) * eliminate build warning --- modules/ximgproc/doc/ximgproc.bib | 44 + modules/ximgproc/include/opencv2/ximgproc.hpp | 3 + .../include/opencv2/ximgproc/edge_drawing.hpp | 115 + .../opencv2/ximgproc/fast_line_detector.hpp | 2 +- .../misc/python/pyopencv_ximgproc.hpp | 3 + modules/ximgproc/samples/edge_drawing.py | 85 + modules/ximgproc/samples/fld_lines.cpp | 77 +- modules/ximgproc/src/edge_drawing.cpp | 5830 +++++++++++++++++ modules/ximgproc/src/edge_drawing_common.hpp | 569 ++ modules/ximgproc/test/test_fld.cpp | 82 +- 10 files changed, 6797 insertions(+), 13 deletions(-) create mode 100644 modules/ximgproc/include/opencv2/ximgproc/edge_drawing.hpp create mode 100644 modules/ximgproc/misc/python/pyopencv_ximgproc.hpp create mode 100644 modules/ximgproc/samples/edge_drawing.py create mode 100644 modules/ximgproc/src/edge_drawing.cpp create mode 100644 modules/ximgproc/src/edge_drawing_common.hpp diff --git a/modules/ximgproc/doc/ximgproc.bib b/modules/ximgproc/doc/ximgproc.bib index b1c7ae0c44a..c7e6714630d 100644 --- a/modules/ximgproc/doc/ximgproc.bib +++ b/modules/ximgproc/doc/ximgproc.bib @@ -310,3 +310,47 @@ @inproceedings{BarronPoole2016 publisher={Springer International Publishing}, pages={617--632}, } + +@article{topal2012edge, + title={Edge drawing: a combined real-time edge and segment detector}, + author={Topal, Cihan and Akinlar, Cuneyt}, + journal={Journal of Visual Communication and Image Representation}, + volume={23}, + number={6}, + pages={862--872}, + year={2012}, + publisher={Elsevier} +} + +@article{akinlar2011edlines, + title={EDLines: A real-time line segment detector with a false detection control}, + author={Akinlar, Cuneyt and Topal, Cihan}, + journal={Pattern Recognition Letters}, + volume={32}, + number={13}, + pages={1633--1642}, + year={2011}, + publisher={Elsevier} +} + +@article{akinlar2012edpf, + title={EDPF: a real-time parameter-free edge segment detector with a false detection control}, + author={Akinlar, Cuneyt and Topal, Cihan}, + journal={International Journal of Pattern Recognition and Artificial Intelligence}, + volume={26}, + number={01}, + pages={1255002}, + year={2012}, + publisher={World Scientific} +} + +@article{akinlar2013edcircles, + title={EDCircles: A real-time circle detector with a false detection control}, + author={Akinlar, Cuneyt and Topal, Cihan}, + journal={Pattern Recognition}, + volume={46}, + number={3}, + pages={725--740}, + year={2013}, + publisher={Elsevier} +} diff --git a/modules/ximgproc/include/opencv2/ximgproc.hpp b/modules/ximgproc/include/opencv2/ximgproc.hpp index 58446f19c47..a1d1becf12d 100644 --- a/modules/ximgproc/include/opencv2/ximgproc.hpp +++ b/modules/ximgproc/include/opencv2/ximgproc.hpp @@ -42,6 +42,7 @@ #include "ximgproc/sparse_match_interpolator.hpp" #include "ximgproc/structured_edge_detection.hpp" #include "ximgproc/edgeboxes.hpp" +#include "ximgproc/edge_drawing.hpp" #include "ximgproc/seeds.hpp" #include "ximgproc/segmentation.hpp" #include "ximgproc/fast_hough_transform.hpp" @@ -75,6 +76,8 @@ i.e. algorithms which somehow takes into account pixel affinities in natural ima @defgroup ximgproc_fast_line_detector Fast line detector + @defgroup ximgproc_edge_drawing EdgeDrawing + @defgroup ximgproc_fourier Fourier descriptors @} */ diff --git a/modules/ximgproc/include/opencv2/ximgproc/edge_drawing.hpp b/modules/ximgproc/include/opencv2/ximgproc/edge_drawing.hpp new file mode 100644 index 00000000000..cb0bc735aad --- /dev/null +++ b/modules/ximgproc/include/opencv2/ximgproc/edge_drawing.hpp @@ -0,0 +1,115 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef __OPENCV_EDGE_DRAWING_HPP__ +#define __OPENCV_EDGE_DRAWING_HPP__ + +#include + +namespace cv +{ +namespace ximgproc +{ + +//! @addtogroup ximgproc_edge_drawing +//! @{ + +/** @brief Class implementing the ED (EdgeDrawing) @cite topal2012edge, EDLines @cite akinlar2011edlines, EDPF @cite akinlar2012edpf and EDCircles @cite akinlar2013edcircles algorithms + +EDGE DRAWING LIBRARY FOR GEOMETRIC FEATURE EXTRACTION AND VALIDATION + +Edge Drawing (ED) algorithm is an proactive approach on edge detection problem. In contrast to many other existing edge detection algorithms which follow a subtractive +approach (i.e. after applying gradient filters onto an image eliminating pixels w.r.t. several rules, e.g. non-maximal suppression and hysteresis in Canny), ED algorithm +works via an additive strategy, i.e. it picks edge pixels one by one, hence the name Edge Drawing. Then we process those random shaped edge segments to extract higher level +edge features, i.e. lines, circles, ellipses, etc. The popular method of extraction edge pixels from the thresholded gradient magnitudes is non-maximal supressiun that tests +every pixel whether it has the maximum gradient response along its gradient direction and eliminates if it does not. However, this method does not check status of the +neighboring pixels, and therefore might result low quality (in terms of edge continuity, smoothness, thinness, localization) edge segments. Instead of non-maximal supression, +ED points a set of edge pixels and join them by maximizing the total gradient response of edge segments. Therefore it can extract high quality edge segments without need for +an additional hysteresis step. +*/ + +class CV_EXPORTS_W EdgeDrawing : public Algorithm +{ +public: + + enum GradientOperator + { + PREWITT = 0, + SOBEL = 1, + SCHARR = 2, + LSD = 3 + }; + + struct CV_EXPORTS_W_SIMPLE Params + { + CV_WRAP Params(); + //! Parameter Free mode will be activated when this value is true. + CV_PROP_RW bool PFmode; + //! indicates the operator used for gradient calculation.The following operation flags are available(cv::ximgproc::EdgeDrawing::GradientOperator) + CV_PROP_RW int EdgeDetectionOperator; + //! threshold value used to create gradient image. + CV_PROP_RW int GradientThresholdValue; + //! threshold value used to create gradient image. + CV_PROP_RW int AnchorThresholdValue; + CV_PROP_RW int ScanInterval; + //! minimun connected pixels length processed to create an edge segment. + CV_PROP_RW int MinPathLength; + //! sigma value for internal GaussianBlur() function. + CV_PROP_RW float Sigma; + CV_PROP_RW bool SumFlag; + //! when this value is true NFA (Number of False Alarms) algorithm will be used for line and ellipse validation. + CV_PROP_RW bool NFAValidation; + //! minimun line length to detect. + CV_PROP_RW int MinLineLength; + CV_PROP_RW double MaxDistanceBetweenTwoLines; + CV_PROP_RW double LineFitErrorThreshold; + CV_PROP_RW double MaxErrorThreshold; + + void read(const FileNode& fn); + void write(FileStorage& fs) const; + }; + + /** @brief Detects edges and prepares them to detect lines and ellipses. + + @param src input image + */ + CV_WRAP virtual void detectEdges(InputArray src) = 0; + CV_WRAP virtual void getEdgeImage(OutputArray dst) = 0; + CV_WRAP virtual void getGradientImage(OutputArray dst) = 0; + + CV_WRAP virtual std::vector > getSegments() = 0; + + /** @brief Detects lines. + + @param lines output Vec<4f> contains start point and end point of detected lines. + @note you should call detectEdges() method before call this. + */ + CV_WRAP virtual void detectLines(OutputArray lines) = 0; + + /** @brief Detects circles and ellipses. + + @param ellipses output Vec<6d> contains center point and perimeter for circles. + @note you should call detectEdges() method before call this. + */ + CV_WRAP virtual void detectEllipses(OutputArray ellipses) = 0; + + CV_WRAP Params params; + + /** @brief sets parameters. + + this function is meant to be used for parameter setting in other languages than c++. + */ + CV_WRAP void setParams(const EdgeDrawing::Params& parameters); + virtual ~EdgeDrawing() { } +}; + +/** @brief Creates a smart pointer to a EdgeDrawing object and initializes it +*/ +CV_EXPORTS_W Ptr createEdgeDrawing(); +//! @} + +} +} + +#endif /* __OPENCV_EDGE_DRAWING_HPP__ */ diff --git a/modules/ximgproc/include/opencv2/ximgproc/fast_line_detector.hpp b/modules/ximgproc/include/opencv2/ximgproc/fast_line_detector.hpp index 1df555865eb..78ec43a61b3 100644 --- a/modules/ximgproc/include/opencv2/ximgproc/fast_line_detector.hpp +++ b/modules/ximgproc/include/opencv2/ximgproc/fast_line_detector.hpp @@ -68,7 +68,7 @@ class CV_EXPORTS_W FastLineDetector : public Algorithm @param _canny_aperture_size 3 - Aperturesize for the sobel operator in Canny() @param _do_merge false - If true, incremental merging of segments - will be perfomred + will be performed */ CV_EXPORTS_W Ptr createFastLineDetector( int _length_threshold = 10, float _distance_threshold = 1.414213562f, diff --git a/modules/ximgproc/misc/python/pyopencv_ximgproc.hpp b/modules/ximgproc/misc/python/pyopencv_ximgproc.hpp new file mode 100644 index 00000000000..c7d0a195ea1 --- /dev/null +++ b/modules/ximgproc/misc/python/pyopencv_ximgproc.hpp @@ -0,0 +1,3 @@ +#ifdef HAVE_OPENCV_XIMGPROC +typedef cv::ximgproc::EdgeDrawing::Params EdgeDrawing_Params; +#endif diff --git a/modules/ximgproc/samples/edge_drawing.py b/modules/ximgproc/samples/edge_drawing.py new file mode 100644 index 00000000000..43b33344713 --- /dev/null +++ b/modules/ximgproc/samples/edge_drawing.py @@ -0,0 +1,85 @@ +#!/usr/bin/python + +''' +This example illustrates how to use cv.ximgproc.EdgeDrawing class. + +Usage: + ed.py [] + image argument defaults to board.jpg +''' + +# Python 2/3 compatibility +from __future__ import print_function + +import numpy as np +import cv2 as cv +import random as rng +import sys + +rng.seed(12345) + +def main(): + try: + fn = sys.argv[1] + except IndexError: + fn = 'board.jpg' + + src = cv.imread(cv.samples.findFile(fn)) + gray = cv.cvtColor(src, cv.COLOR_BGR2GRAY) + cv.imshow("source", src) + + ssrc = src.copy()*0 + lsrc = src.copy() + esrc = src.copy() + + ed = cv.ximgproc.createEdgeDrawing() + + # you can change parameters (refer the documentation to see all parameters) + EDParams = cv.ximgproc_EdgeDrawing_Params() + EDParams.MinPathLength = 50 # try changing this value between 5 to 1000 + EDParams.PFmode = False # defaut value try to swich it to True + EDParams.MinLineLength = 10 # try changing this value between 5 to 100 + EDParams.NFAValidation = True # defaut value try to swich it to False + + ed.setParams(EDParams) + + # Detect edges + # you should call this before detectLines() and detectEllipses() + ed.detectEdges(gray) + segments = ed.getSegments() + lines = ed.detectLines() + ellipses = ed.detectEllipses() + + #Draw detected edge segments + for i in range(len(segments)): + color = (rng.randint(0,256), rng.randint(0,256), rng.randint(0,256)) + cv.polylines(ssrc, [segments[i]], False, color, 1, cv.LINE_8) + + cv.imshow("detected edge segments", ssrc) + + #Draw detected lines + if lines is not None: # Check if the lines have been found and only then iterate over these and add them to the image + lines = np.uint16(np.around(lines)) + for i in range(len(lines)): + cv.line(lsrc, (lines[i][0][0], lines[i][0][1]), (lines[i][0][2], lines[i][0][3]), (0, 0, 255), 1, cv.LINE_AA) + + cv.imshow("detected lines", lsrc) + + #Draw detected circles and ellipses + if ellipses is not None: # Check if circles and ellipses have been found and only then iterate over these and add them to the image + ellipses = np.uint16(np.around(ellipses)) + for i in range(len(ellipses)): + color = (0, 0, 255) + if ellipses[i][0][2] == 0: + color = (0, 255, 0) + cv.ellipse(esrc, (ellipses[i][0][0], ellipses[i][0][1]), (ellipses[i][0][2]+ellipses[i][0][3],ellipses[i][0][2]+ellipses[i][0][4]),ellipses[i][0][5],0, 360, color, 2, cv.LINE_AA) + + cv.imshow("detected ellipses", esrc) + cv.waitKey(0) + print('Done') + + +if __name__ == '__main__': + print(__doc__) + main() + cv.destroyAllWindows() diff --git a/modules/ximgproc/samples/fld_lines.cpp b/modules/ximgproc/samples/fld_lines.cpp index 62a2d48cb39..acb4e32ac52 100644 --- a/modules/ximgproc/samples/fld_lines.cpp +++ b/modules/ximgproc/samples/fld_lines.cpp @@ -11,14 +11,14 @@ using namespace cv::ximgproc; int main(int argc, char** argv) { - std::string in; - cv::CommandLineParser parser(argc, argv, "{@input|../samples/data/corridor.jpg|input image}{help h||show help message}"); + string in; + CommandLineParser parser(argc, argv, "{@input|corridor.jpg|input image}{help h||show help message}"); if (parser.has("help")) { parser.printMessage(); return 0; } - in = parser.get("@input"); + in = samples::findFile(parser.get("@input")); Mat image = imread(in, IMREAD_GRAYSCALE); @@ -40,7 +40,7 @@ int main(int argc, char** argv) // canny_aperture_size 3 - Aperturesize for the sobel // operator in Canny() // do_merge false - If true, incremental merging of segments - // will be perfomred + // will be performed int length_threshold = 10; float distance_threshold = 1.41421356f; double canny_th1 = 50.0; @@ -50,27 +50,84 @@ int main(int argc, char** argv) Ptr fld = createFastLineDetector(length_threshold, distance_threshold, canny_th1, canny_th2, canny_aperture_size, do_merge); - vector lines_fld; + vector lines; // Because of some CPU's power strategy, it seems that the first running of // an algorithm takes much longer. So here we run the algorithm 10 times // to see the algorithm's processing time with sufficiently warmed-up // CPU performance. - for(int run_count = 0; run_count < 10; run_count++) { + for (int run_count = 0; run_count < 5; run_count++) { double freq = getTickFrequency(); - lines_fld.clear(); + lines.clear(); int64 start = getTickCount(); // Detect the lines with FLD - fld->detect(image, lines_fld); + fld->detect(image, lines); double duration_ms = double(getTickCount() - start) * 1000 / freq; - std::cout << "Elapsed time for FLD " << duration_ms << " ms." << std::endl; + cout << "Elapsed time for FLD " << duration_ms << " ms." << endl; } // Show found lines with FLD Mat line_image_fld(image); - fld->drawSegments(line_image_fld, lines_fld); + fld->drawSegments(line_image_fld, lines); imshow("FLD result", line_image_fld); + waitKey(1); + + Ptr ed = createEdgeDrawing(); + ed->params.EdgeDetectionOperator = EdgeDrawing::SOBEL; + ed->params.GradientThresholdValue = 38; + ed->params.AnchorThresholdValue = 8; + + vector ellipses; + + for (int run_count = 0; run_count < 5; run_count++) { + double freq = getTickFrequency(); + lines.clear(); + int64 start = getTickCount(); + + // Detect edges + //you should call this before detectLines() and detectEllipses() + ed->detectEdges(image); + + // Detect lines + ed->detectLines(lines); + double duration_ms = double(getTickCount() - start) * 1000 / freq; + cout << "Elapsed time for EdgeDrawing detectLines " << duration_ms << " ms." << endl; + + start = getTickCount(); + // Detect circles and ellipses + ed->detectEllipses(ellipses); + duration_ms = double(getTickCount() - start) * 1000 / freq; + cout << "Elapsed time for EdgeDrawing detectEllipses " << duration_ms << " ms." << endl; + } + + Mat edge_image_ed = Mat::zeros(image.size(), CV_8UC3); + vector > segments = ed->getSegments(); + + for (size_t i = 0; i < segments.size(); i++) + { + const Point* pts = &segments[i][0]; + int n = (int)segments[i].size(); + polylines(edge_image_ed, &pts, &n, 1, false, Scalar((rand() & 255), (rand() & 255), (rand() & 255)), 1); + } + + imshow("EdgeDrawing detected edges", edge_image_ed); + + Mat line_image_ed(image); + fld->drawSegments(line_image_ed, lines); + + // Draw circles and ellipses + for (size_t i = 0; i < ellipses.size(); i++) + { + Point center((int)ellipses[i][0], (int)ellipses[i][1]); + Size axes((int)ellipses[i][2] + (int)ellipses[i][3], (int)ellipses[i][2] + (int)ellipses[i][4]); + double angle(ellipses[i][5]); + Scalar color = ellipses[i][2] == 0 ? Scalar(255, 255, 0) : Scalar(0, 255, 0); + + ellipse(line_image_ed, center, axes, angle, 0, 360, color, 2, LINE_AA); + } + + imshow("EdgeDrawing result", line_image_ed); waitKey(); return 0; } diff --git a/modules/ximgproc/src/edge_drawing.cpp b/modules/ximgproc/src/edge_drawing.cpp new file mode 100644 index 00000000000..628893be513 --- /dev/null +++ b/modules/ximgproc/src/edge_drawing.cpp @@ -0,0 +1,5830 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "precomp.hpp" +#include "edge_drawing_common.hpp" + +using namespace std; + +namespace cv +{ +namespace ximgproc +{ + +class EdgeDrawingImpl : public EdgeDrawing +{ +public: + + enum EllipseFittingMethods + { + BOOKSTEIN = 0, + FPF = 1 + }; + + EdgeDrawingImpl(); + void detectEdges(InputArray src); + void getEdgeImage(OutputArray dst); + void getGradientImage(OutputArray dst); + + vector > getSegments(); + void detectLines(OutputArray lines); + void detectEllipses(OutputArray ellipses); + + virtual void read(const FileNode& fn) CV_OVERRIDE; + virtual void write(FileStorage& fs) const CV_OVERRIDE; + +protected: + int width; // width of source image + int height; // height of source image + uchar *srcImg; + vector > segmentPoints; + Mat smoothImage; + uchar *edgeImg; // pointer to edge image data + uchar *smoothImg; // pointer to smoothed image data + int segmentNos; + Mat srcImage; + + double divForTestSegment; + double* dH; + int np; + +private: + void ComputeGradient(); + void ComputeAnchorPoints(); + void JoinAnchorPointsUsingSortedAnchors(); + int* sortAnchorsByGradValue1(); + + static int LongestChain(Chain *chains, int root); + static int RetrieveChainNos(Chain *chains, int root, int chainNos[]); + + int anchorNos; + vector anchorPoints; + vector edgePoints; + + Mat edgeImage; + Mat gradImage; + + uchar *dirImg; // pointer to direction image data + short *gradImg; // pointer to gradient image data + + int op; // edge detection operator + int gradThresh; // gradient threshold + int anchorThresh; // anchor point threshold + + + static void SplitSegment2Lines(double* x, double* y, int noPixels, int segmentNo, std::vector& lines, int min_line_len = 6, double line_error = 1.0); + std::vector lines; + std::vector invalidLines; + int linesNo; + int min_line_len; + double line_error; + double max_distance_between_two_lines; + double max_error; + double precision; + NFALUT* nfa; + + int ComputeMinLineLength(); + void SplitSegment2Lines(double* x, double* y, int noPixels, int segmentNo); + void JoinCollinearLines(); + + void ValidateLineSegments(); + bool ValidateLineSegmentRect(int* x, int* y, EDLineSegment* ls); + bool TryToJoinTwoLineSegments(EDLineSegment* ls1, EDLineSegment* ls2, int changeIndex); + + static double ComputeMinDistance(double x1, double y1, double a, double b, int invert); + static void ComputeClosestPoint(double x1, double y1, double a, double b, int invert, double& xOut, double& yOut); + static void LineFit(double* x, double* y, int count, double& a, double& b, int invert); + static void LineFit(double* x, double* y, int count, double& a, double& b, double& e, int& invert); + static double ComputeMinDistanceBetweenTwoLines(EDLineSegment* ls1, EDLineSegment* ls2, int* pwhich); + static void UpdateLineParameters(EDLineSegment* ls); + static void EnumerateRectPoints(double sx, double sy, double ex, double ey, int ptsx[], int ptsy[], int* pNoPoints); + + void TestSegment(int i, int index1, int index2); + void ExtractNewSegments(); + double NFA(double prob, int len); + + int noEllipses; + int noCircles; + std::vector Circles; + std::vector Ellipses; + + Circle* circles1; + Circle* circles2; + Circle* circles3; + int noCircles1; + int noCircles2; + int noCircles3; + + EDArcs* edarcs1; + EDArcs* edarcs2; + EDArcs* edarcs3; + EDArcs* edarcs4; + + int* segmentStartLines; + BufferManager* bm; + Info* info; + + void GenerateCandidateCircles(); + void DetectArcs(); + void ValidateCircles(bool validate); + void JoinCircles(); + void JoinArcs1(); + void JoinArcs2(); + void JoinArcs3(); + + // circle utility functions + static Circle* addCircle(Circle* circles, int& noCircles, double xc, double yc, double r, double circleFitError, double* x, double* y, int noPixels); + static Circle* addCircle(Circle* circles, int& noCircles, double xc, double yc, double r, double circleFitError, EllipseEquation* pEq, double ellipseFitError, double* x, double* y, int noPixels); + static void sortCircles(Circle* circles, int noCircles); + static bool CircleFit(double* x, double* y, int N, double* pxc, double* pyc, double* pr, double* pe); + static void ComputeCirclePoints(double xc, double yc, double r, double* px, double* py, int* noPoints); + static void sortCircle(Circle* circles, int noCircles); + + // ellipse utility functions + static bool EllipseFit(double* x, double* y, int noPoints, EllipseEquation* pResult, int mode = FPF); + static double** AllocateMatrix(int noRows, int noColumns); + static void A_TperB(double** A_, double** B_, double** _res, int _righA, int _colA, int _righB, int _colB); + static void choldc(double** a, int n, double** l); + static int inverse(double** TB, double** InvB, int N); + static void DeallocateMatrix(double** m, int noRows); + static void AperB_T(double** A_, double** B_, double** _res, int _righA, int _colA, int _righB, int _colB); + static void AperB(double** A_, double** B_, double** _res, int _righA, int _colA, int _righB, int _colB); + static void jacobi(double** a, int n, double d[], double** v, int nrot); + static void ROTATE(double** a, int i, int j, int k, int l, double tau, double s); + static double computeEllipsePerimeter(EllipseEquation* eq); + static double ComputeEllipseError(EllipseEquation* eq, double* px, double* py, int noPoints); + static double ComputeEllipseCenterAndAxisLengths(EllipseEquation* eq, double* pxc, double* pyc, double* pmajorAxisLength, double* pminorAxisLength); + static void ComputeEllipsePoints(double* pvec, double* px, double* py, int noPoints); + + // arc utility functions + static void joinLastTwoArcs(MyArc* arcs, int& noArcs); + static void addArc(MyArc* arcs, int& noArchs, double xc, double yc, double r, double circleFitError, // Circular arc + double sTheta, double eTheta, int turn, int segmentNo, + int sx, int sy, int ex, int ey, + double* x, double* y, int noPixels, double overlapRatio = 0.0); + static void addArc(MyArc* arcs, int& noArchs, double xc, double yc, double r, double circleFitError, // Elliptic arc + double sTheta, double eTheta, int turn, int segmentNo, + EllipseEquation* pEq, double ellipseFitError, + int sx, int sy, int ex, int ey, + double* x, double* y, int noPixels, double overlapRatio = 0.0); + + static void ComputeStartAndEndAngles(double xc, double yc, double r, + double* x, double* y, int len, + double* psTheta, double* peTheta); + + static void sortArc(MyArc* arcs, int noArcs); +}; + +Ptr createEdgeDrawing() +{ + return makePtr(); +} + +EdgeDrawing::Params::Params() +{ + PFmode = false; + EdgeDetectionOperator = PREWITT; + GradientThresholdValue = 20; + AnchorThresholdValue = 0; + ScanInterval = 1; + MinPathLength = 10; + Sigma = 1.0; + SumFlag = true; + NFAValidation = true; + MinLineLength = -1; + MaxDistanceBetweenTwoLines = 6.0; + LineFitErrorThreshold = 1.0; + MaxErrorThreshold = 1.3; +} + +void EdgeDrawing::setParams(const EdgeDrawing::Params& parameters) +{ + params = parameters; +} + +void EdgeDrawing::Params::read(const cv::FileNode& fn) +{ + PFmode = (int)fn["PFmode"] != 0 ? true : false; + EdgeDetectionOperator = fn["EdgeDetectionOperator"]; + GradientThresholdValue = fn["GradientThresholdValue"]; + AnchorThresholdValue = fn["AnchorThresholdValue"]; + ScanInterval = fn["ScanInterval"]; + MinPathLength = fn["MinPathLength"]; + Sigma = fn["Sigma"]; + SumFlag = (int)fn["SumFlag"] != 0 ? true : false; + NFAValidation = (int)fn["NFAValidation"] != 0 ? true : false; + MinLineLength = fn["MinLineLength"]; + MaxDistanceBetweenTwoLines = fn["MaxDistanceBetweenTwoLines"]; + LineFitErrorThreshold = fn["LineFitErrorThreshold"]; + MaxErrorThreshold = fn["MaxErrorThreshold"]; +} + +void EdgeDrawing::Params::write(cv::FileStorage& fs) const +{ + fs << "PFmode" << PFmode; + fs << "EdgeDetectionOperator" << EdgeDetectionOperator; + fs << "GradientThresholdValue" << GradientThresholdValue; + fs << "AnchorThresholdValue" << AnchorThresholdValue; + fs << "ScanInterval" << ScanInterval; + fs << "MinPathLength" << MinPathLength; + fs << "Sigma" << Sigma; + fs << "SumFlag" << SumFlag; + fs << "NFAValidation" << NFAValidation; + fs << "MinLineLength" << MinLineLength; + fs << "MaxDistanceBetweenTwoLines" << MaxDistanceBetweenTwoLines; + fs << "LineFitErrorThreshold" << LineFitErrorThreshold; + fs << "MaxErrorThreshold" << MaxErrorThreshold; +} + +void EdgeDrawingImpl::read(const cv::FileNode& fn) +{ + params.read(fn); +} + +void EdgeDrawingImpl::write(cv::FileStorage& fs) const +{ + writeFormat(fs); + params.write(fs); +} + +EdgeDrawingImpl::EdgeDrawingImpl() +{ + params = EdgeDrawing::Params(); +} + +void EdgeDrawingImpl::detectEdges(InputArray src) +{ + gradThresh = params.GradientThresholdValue; + anchorThresh = params.AnchorThresholdValue; + op = params.EdgeDetectionOperator; + + if (params.PFmode) + { + op = PREWITT; + gradThresh = 11; + anchorThresh = 3; + } + + // Check parameters for sanity + if (op < 0 || op > 3) + op = 0; + + if (gradThresh < 1) + gradThresh = 1; + + if (anchorThresh < 0) + anchorThresh = 0; + + segmentNos = 0; + anchorNos = 0; + anchorPoints.clear(); + lines.clear(); + invalidLines.clear(); + segmentPoints.clear(); + segmentPoints.push_back(vector()); // create empty vector of points for segments + srcImage = src.getMat(); + srcImg = srcImage.data; + height = srcImage.rows; + width = srcImage.cols; + + edgeImage = Mat(height, width, CV_8UC1, Scalar(0)); // initialize edge Image + gradImage = Mat(height, width, CV_16SC1); // gradImage contains short values + + if (params.Sigma < 1.0) + smoothImage = srcImage; + else if (params.Sigma == 1.0) + GaussianBlur(srcImage, smoothImage, Size(5, 5), params.Sigma); + else + GaussianBlur(srcImage, smoothImage, Size(), params.Sigma); // calculate kernel from sigma + + // Assign Pointers from Mat's data + smoothImg = smoothImage.data; + gradImg = (short*)gradImage.data; + edgeImg = edgeImage.data; + + Mat _dirImg = Mat(height, width, CV_8UC1); + dirImg = _dirImg.data; + + ComputeGradient(); // COMPUTE GRADIENT & EDGE DIRECTION MAPS + ComputeAnchorPoints(); // COMPUTE ANCHORS + JoinAnchorPointsUsingSortedAnchors(); // JOIN ANCHORS + + if (params.PFmode) + { + divForTestSegment = 2.25; // Some magic number :-) + memset(edgeImg, 0, width * height); // clear edge image + np = 0; + for (int i = 0; i < segmentNos; i++) + { + int len = (int)segmentPoints[i].size(); + np += (len * (len - 1)) / 2; + } + + // Validate segments + for (int i = 0; i < segmentNos; i++) + TestSegment(i, 0, (int)segmentPoints[i].size() - 1); + + ExtractNewSegments(); + } +} + +void EdgeDrawingImpl::getEdgeImage(OutputArray _dst) +{ + if (!edgeImage.empty()) + edgeImage.copyTo(_dst); +} + + +void EdgeDrawingImpl::getGradientImage(OutputArray _dst) +{ + if (!gradImage.empty()) + convertScaleAbs(gradImage, _dst); +} + + +std::vector > EdgeDrawingImpl::getSegments() +{ + return segmentPoints; +} + +void EdgeDrawingImpl::ComputeGradient() +{ +#define MAX_GRAD_VALUE 128*256 + dH = new double[MAX_GRAD_VALUE]; + memset(dH, 0, sizeof(double) * MAX_GRAD_VALUE); + + int* grads = new int[MAX_GRAD_VALUE]; + memset(grads, 0, sizeof(int) * MAX_GRAD_VALUE); + + // Initialize gradient image for row = 0, row = height-1, column=0, column=width-1 + for (int j = 0; j < width; j++) + { + gradImg[j] = gradImg[(height - 1) * width + j] = (short)gradThresh - 1; + } + + for (int i = 1; i < height - 1; i++) + { + gradImg[i * width] = gradImg[(i + 1) * width - 1] = (short)gradThresh - 1; + } + + for (int i = 1; i < height - 1; i++) + { + for (int j = 1; j < width - 1; j++) + { + int com1 = smoothImg[(i + 1) * width + j + 1] - smoothImg[(i - 1) * width + j - 1]; + int com2 = smoothImg[(i - 1) * width + j + 1] - smoothImg[(i + 1) * width + j - 1]; + + int gx(0); + int gy(0); + + switch (op) + { + case PREWITT: + gx = abs(com1 + com2 + (smoothImg[i * width + j + 1] - smoothImg[i * width + j - 1])); + gy = abs(com1 - com2 + (smoothImg[(i + 1) * width + j] - smoothImg[(i - 1) * width + j])); + break; + case SOBEL: + gx = abs(com1 + com2 + 2 * (smoothImg[i * width + j + 1] - smoothImg[i * width + j - 1])); + gy = abs(com1 - com2 + 2 * (smoothImg[(i + 1) * width + j] - smoothImg[(i - 1) * width + j])); + break; + case SCHARR: + gx = abs(3 * (com1 + com2) + 10 * (smoothImg[i * width + j + 1] - smoothImg[i * width + j - 1])); + gy = abs(3 * (com1 - com2) + 10 * (smoothImg[(i + 1) * width + j] - smoothImg[(i - 1) * width + j])); + case LSD: + // com1 and com2 differs from previous operators, because LSD has 2x2 kernel + com1 = smoothImg[(i + 1) * width + j + 1] - smoothImg[i * width + j]; + com2 = smoothImg[i * width + j + 1] - smoothImg[(i + 1) * width + j]; + + gx = abs(com1 + com2); + gy = abs(com1 - com2); + } + + int sum; + + if (params.SumFlag) + sum = gx + gy; + else + sum = (int)sqrt((double)gx * gx + gy * gy); + + int index = i * width + j; + gradImg[index] = (short)sum; + + grads[(int)sum]++; + + if (sum >= gradThresh) + { + if (gx >= gy) + dirImg[index] = EDGE_VERTICAL; + else + dirImg[index] = EDGE_HORIZONTAL; + } + } + } + + // Compute probability function H + int size = (width - 2) * (height - 2); + + for (int i = MAX_GRAD_VALUE - 1; i > 0; i--) + grads[i - 1] += grads[i]; + + for (int i = 0; i < MAX_GRAD_VALUE; i++) + dH[i] = (double)grads[i] / ((double)size); + +#undef MAX_GRAD_VALUE +} + +void EdgeDrawingImpl::ComputeAnchorPoints() +{ + for (int i = 2; i < height - 2; i++) + { + int start = 2; + int inc = 1; + if (i % params.ScanInterval != 0) + { + start = params.ScanInterval; + inc = params.ScanInterval; + } + + for (int j = start; j < width - 2; j += inc) + { + if (gradImg[i * width + j] < gradThresh) + continue; + + if (dirImg[i * width + j] == EDGE_VERTICAL) + { + // vertical edge + int diff1 = gradImg[i * width + j] - gradImg[i * width + j - 1]; + int diff2 = gradImg[i * width + j] - gradImg[i * width + j + 1]; + if (diff1 >= anchorThresh && diff2 >= anchorThresh) + { + edgeImg[i * width + j] = ANCHOR_PIXEL; + anchorPoints.push_back(Point(j, i)); + } + } + else + { + // horizontal edge + int diff1 = gradImg[i * width + j] - gradImg[(i - 1) * width + j]; + int diff2 = gradImg[i * width + j] - gradImg[(i + 1) * width + j]; + if (diff1 >= anchorThresh && diff2 >= anchorThresh) + { + edgeImg[i * width + j] = ANCHOR_PIXEL; + anchorPoints.push_back(Point(j, i)); + } + } + } + } + + anchorNos = (int)anchorPoints.size(); // get the total number of anchor points +} + +void EdgeDrawingImpl::JoinAnchorPointsUsingSortedAnchors() +{ + int* chainNos = new int[(width + height) * 8]; + + Point* pixels = new Point[width * height]; + StackNode* stack = new StackNode[width * height]; + Chain* chains = new Chain[width * height]; + + // sort the anchor points by their gradient value in decreasing order + int* pAnchors = sortAnchorsByGradValue1(); + + // Now join the anchors starting with the anchor having the greatest gradient value + + for (int k0 = anchorNos - 1; k0 >= 0; k0--) + { + int pixelOffset = pAnchors[k0]; + + int i = pixelOffset / width; + int j = pixelOffset % width; + + if (edgeImg[i * width + j] != ANCHOR_PIXEL) + continue; + + chains[0].len = 0; + chains[0].parent = -1; + chains[0].dir = 0; + chains[0].children[0] = chains[0].children[1] = -1; + chains[0].pixels = NULL; + + int noChains = 1; + int len = 0; + int duplicatePixelCount = 0; + int top = -1; // top of the stack + + if (dirImg[i * width + j] == EDGE_VERTICAL) + { + stack[++top].r = i; + stack[top].c = j; + stack[top].dir = DOWN; + stack[top].parent = 0; + + stack[++top].r = i; + stack[top].c = j; + stack[top].dir = UP; + stack[top].parent = 0; + + } + else + { + stack[++top].r = i; + stack[top].c = j; + stack[top].dir = RIGHT; + stack[top].parent = 0; + + stack[++top].r = i; + stack[top].c = j; + stack[top].dir = LEFT; + stack[top].parent = 0; + } + + // While the stack is not empty +StartOfWhile: + while (top >= 0) + { + int r = stack[top].r; + int c = stack[top].c; + int dir = stack[top].dir; + int parent = stack[top].parent; + top--; + + if (edgeImg[r * width + c] != EDGE_PIXEL) + duplicatePixelCount++; + + chains[noChains].dir = dir; // traversal direction + chains[noChains].parent = parent; + chains[noChains].children[0] = chains[noChains].children[1] = -1; + + int chainLen = 0; + + chains[noChains].pixels = &pixels[len]; + + pixels[len].y = r; + pixels[len].x = c; + len++; + chainLen++; + + if (dir == LEFT) + { + while (dirImg[r * width + c] == EDGE_HORIZONTAL) + { + edgeImg[r * width + c] = EDGE_PIXEL; + + // The edge is horizontal. Look LEFT + // + // A + // B x + // C + // + // cleanup up & down pixels + if (edgeImg[(r - 1) * width + c] == ANCHOR_PIXEL) + edgeImg[(r - 1) * width + c] = 0; + if (edgeImg[(r + 1) * width + c] == ANCHOR_PIXEL) + edgeImg[(r + 1) * width + c] = 0; + + // Look if there is an edge pixel in the neighbors + if (edgeImg[r * width + c - 1] >= ANCHOR_PIXEL) + { + c--; + } + else if (edgeImg[(r - 1) * width + c - 1] >= ANCHOR_PIXEL) + { + r--; + c--; + } + else if (edgeImg[(r + 1) * width + c - 1] >= ANCHOR_PIXEL) + { + r++; + c--; + } + else + { + // else -- follow max. pixel to the LEFT + int A = gradImg[(r - 1) * width + c - 1]; + int B = gradImg[r * width + c - 1]; + int C = gradImg[(r + 1) * width + c - 1]; + + if (A > B) + { + if (A > C) + r--; + else + r++; + } + else if (C > B) + r++; + c--; + } + + if (edgeImg[r * width + c] == EDGE_PIXEL || gradImg[r * width + c] < gradThresh) + { + if (chainLen > 0) + { + chains[noChains].len = chainLen; + chains[parent].children[0] = noChains; + noChains++; + } + goto StartOfWhile; + } + + pixels[len].y = r; + pixels[len].x = c; + len++; + chainLen++; + } + + stack[++top].r = r; + stack[top].c = c; + stack[top].dir = DOWN; + stack[top].parent = noChains; + + stack[++top].r = r; + stack[top].c = c; + stack[top].dir = UP; + stack[top].parent = noChains; + + len--; + chainLen--; + + chains[noChains].len = chainLen; + chains[parent].children[0] = noChains; + noChains++; + } + else if (dir == RIGHT) + { + while (dirImg[r * width + c] == EDGE_HORIZONTAL) + { + edgeImg[r * width + c] = EDGE_PIXEL; + + // The edge is horizontal. Look RIGHT + // + // A + // x B + // C + // + // cleanup up&down pixels + if (edgeImg[(r + 1) * width + c] == ANCHOR_PIXEL) + edgeImg[(r + 1) * width + c] = 0; + if (edgeImg[(r - 1) * width + c] == ANCHOR_PIXEL) + edgeImg[(r - 1) * width + c] = 0; + + // Look if there is an edge pixel in the neighbors + if (edgeImg[r * width + c + 1] >= ANCHOR_PIXEL) + { + c++; + } + else if (edgeImg[(r + 1) * width + c + 1] >= ANCHOR_PIXEL) + { + r++; + c++; + } + else if (edgeImg[(r - 1) * width + c + 1] >= ANCHOR_PIXEL) + { + r--; + c++; + } + else + { + // else -- follow max. pixel to the RIGHT + int A = gradImg[(r - 1) * width + c + 1]; + int B = gradImg[r * width + c + 1]; + int C = gradImg[(r + 1) * width + c + 1]; + + if (A > B) + { + if (A > C) + r--; // A + else + r++; // C + } + else if (C > B) + r++; // C + c++; + } + + if (edgeImg[r * width + c] == EDGE_PIXEL || gradImg[r * width + c] < gradThresh) + { + if (chainLen > 0) + { + chains[noChains].len = chainLen; + chains[parent].children[1] = noChains; + noChains++; + } + goto StartOfWhile; + } + + pixels[len].y = r; + pixels[len].x = c; + len++; + chainLen++; + } + + stack[++top].r = r; + stack[top].c = c; + stack[top].dir = DOWN; // Go down + stack[top].parent = noChains; + + stack[++top].r = r; + stack[top].c = c; + stack[top].dir = UP; // Go up + stack[top].parent = noChains; + + len--; + chainLen--; + + chains[noChains].len = chainLen; + chains[parent].children[1] = noChains; + noChains++; + + } + else if (dir == UP) + { + while (dirImg[r * width + c] == EDGE_VERTICAL) + { + edgeImg[r * width + c] = EDGE_PIXEL; + + // The edge is vertical. Look UP + // + // A B C + // x + // + // Cleanup left & right pixels + if (edgeImg[r * width + c - 1] == ANCHOR_PIXEL) + edgeImg[r * width + c - 1] = 0; + if (edgeImg[r * width + c + 1] == ANCHOR_PIXEL) + edgeImg[r * width + c + 1] = 0; + + // Look if there is an edge pixel in the neighbors + if (edgeImg[(r - 1) * width + c] >= ANCHOR_PIXEL) + { + r--; + } + else if (edgeImg[(r - 1) * width + c - 1] >= ANCHOR_PIXEL) + { + r--; + c--; + } + else if (edgeImg[(r - 1) * width + c + 1] >= ANCHOR_PIXEL) + { + r--; + c++; + } + else + { + // else -- follow the max. pixel UP + int A = gradImg[(r - 1) * width + c - 1]; + int B = gradImg[(r - 1) * width + c]; + int C = gradImg[(r - 1) * width + c + 1]; + + if (A > B) + { + if (A > C) + c--; + else + c++; + } + else if (C > B) + c++; + r--; + } + + if (edgeImg[r * width + c] == EDGE_PIXEL || gradImg[r * width + c] < gradThresh) + { + if (chainLen > 0) + { + chains[noChains].len = chainLen; + chains[parent].children[0] = noChains; + noChains++; + } + goto StartOfWhile; + } + + pixels[len].y = r; + pixels[len].x = c; + + len++; + chainLen++; + } + + stack[++top].r = r; + stack[top].c = c; + stack[top].dir = RIGHT; + stack[top].parent = noChains; + + stack[++top].r = r; + stack[top].c = c; + stack[top].dir = LEFT; + stack[top].parent = noChains; + + len--; + chainLen--; + + chains[noChains].len = chainLen; + chains[parent].children[0] = noChains; + noChains++; + } + else // dir == DOWN + { + while (dirImg[r * width + c] == EDGE_VERTICAL) + { + edgeImg[r * width + c] = EDGE_PIXEL; + + // The edge is vertical + // + // x + // A B C + // + // cleanup side pixels + if (edgeImg[r * width + c + 1] == ANCHOR_PIXEL) + edgeImg[r * width + c + 1] = 0; + if (edgeImg[r * width + c - 1] == ANCHOR_PIXEL) + edgeImg[r * width + c - 1] = 0; + + // Look if there is an edge pixel in the neighbors + if (edgeImg[(r + 1) * width + c] >= ANCHOR_PIXEL) + { + r++; + } + else if (edgeImg[(r + 1) * width + c + 1] >= ANCHOR_PIXEL) + { + r++; + c++; + } + else if (edgeImg[(r + 1) * width + c - 1] >= ANCHOR_PIXEL) + { + r++; + c--; + } + else + { + // else -- follow the max. pixel DOWN + int A = gradImg[(r + 1) * width + c - 1]; + int B = gradImg[(r + 1) * width + c]; + int C = gradImg[(r + 1) * width + c + 1]; + + if (A > B) + { + if (A > C) + c--; // A + else + c++; // C + } + else if (C > B) + c++; // C + r++; + } + + if (edgeImg[r * width + c] == EDGE_PIXEL || gradImg[r * width + c] < gradThresh) + { + if (chainLen > 0) + { + chains[noChains].len = chainLen; + chains[parent].children[1] = noChains; + noChains++; + } + goto StartOfWhile; + } + + pixels[len].y = r; + pixels[len].x = c; + + len++; + chainLen++; + } + + stack[++top].r = r; + stack[top].c = c; + stack[top].dir = RIGHT; + stack[top].parent = noChains; + + stack[++top].r = r; + stack[top].c = c; + stack[top].dir = LEFT; + stack[top].parent = noChains; + + len--; + chainLen--; + + chains[noChains].len = chainLen; + chains[parent].children[1] = noChains; + noChains++; + } + } + + if (len - duplicatePixelCount < params.MinPathLength) + { + for (int k1 = 0; k1 < len; k1++) + { + edgeImg[pixels[k1].y * width + pixels[k1].x] = 0; + edgeImg[pixels[k1].y * width + pixels[k1].x] = 0; + } + } + else + { + int noSegmentPixels = 0; + int totalLen = LongestChain(chains, chains[0].children[1]); + + if (totalLen > 0) + { + // Retrieve the chainNos + int count = RetrieveChainNos(chains, chains[0].children[1], chainNos); + + // Copy these pixels in the reverse order + for (int k2 = count - 1; k2 >= 0; k2--) + { + int chainNo = chainNos[k2]; + + /* See if we can erase some pixels from the last chain. This is for cleanup */ + + int fr = chains[chainNo].pixels[chains[chainNo].len - 1].y; + int fc = chains[chainNo].pixels[chains[chainNo].len - 1].x; + + int index = noSegmentPixels - 2; + while (index >= 0) + { + int dr = abs(fr - segmentPoints[segmentNos][index].y); + int dc = abs(fc - segmentPoints[segmentNos][index].x); + + if (dr <= 1 && dc <= 1) + { + // neighbors. Erase last pixel + segmentPoints[segmentNos].pop_back(); + noSegmentPixels--; + index--; + } + else + break; + } + + if (chains[chainNo].len > 1 && noSegmentPixels > 0) + { + fr = chains[chainNo].pixels[chains[chainNo].len - 2].y; + fc = chains[chainNo].pixels[chains[chainNo].len - 2].x; + + int dr = abs(fr - segmentPoints[segmentNos][noSegmentPixels - 1].y); + int dc = abs(fc - segmentPoints[segmentNos][noSegmentPixels - 1].x); + + if (dr <= 1 && dc <= 1) + chains[chainNo].len--; + } + + for (int l = chains[chainNo].len - 1; l >= 0; l--) + { + segmentPoints[segmentNos].push_back(chains[chainNo].pixels[l]); + noSegmentPixels++; + } + + chains[chainNo].len = 0; // Mark as copied + } + } + + totalLen = LongestChain(chains, chains[0].children[0]); + if (totalLen > 1) + { + // Retrieve the chainNos + int count = RetrieveChainNos(chains, chains[0].children[0], chainNos); + + // Copy these chains in the forward direction. Skip the first pixel of the first chain + // due to repetition with the last pixel of the previous chain + int lastChainNo = chainNos[0]; + chains[lastChainNo].pixels++; + chains[lastChainNo].len--; + + for (int k3 = 0; k3 < count; k3++) + { + int chainNo = chainNos[k3]; + + /* See if we can erase some pixels from the last chain. This is for cleanup */ + int fr = chains[chainNo].pixels[0].y; + int fc = chains[chainNo].pixels[0].x; + + int index = noSegmentPixels - 2; + while (index >= 0) + { + int dr = abs(fr - segmentPoints[segmentNos][index].y); + int dc = abs(fc - segmentPoints[segmentNos][index].x); + + if (dr <= 1 && dc <= 1) + { + // neighbors. Erase last pixel + segmentPoints[segmentNos].pop_back(); + noSegmentPixels--; + index--; + } + else + break; + } + + int startIndex = 0; + int chainLen = chains[chainNo].len; + if (chainLen > 1 && noSegmentPixels > 0) + { + fr = chains[chainNo].pixels[1].y; + fc = chains[chainNo].pixels[1].x; + + int dr = abs(fr - segmentPoints[segmentNos][noSegmentPixels - 1].y); + int dc = abs(fc - segmentPoints[segmentNos][noSegmentPixels - 1].x); + + if (dr <= 1 && dc <= 1) + { + startIndex = 1; + } + } + + /* Start a new chain & copy pixels from the new chain */ + for (int l = startIndex; l < chains[chainNo].len; l++) + { + segmentPoints[segmentNos].push_back(chains[chainNo].pixels[l]); + noSegmentPixels++; + } + + chains[chainNo].len = 0; // Mark as copied + } + } + + // See if the first pixel can be cleaned up + int fr = segmentPoints[segmentNos][1].y; + int fc = segmentPoints[segmentNos][1].x; + + int dr = abs(fr - segmentPoints[segmentNos][noSegmentPixels - 1].y); + int dc = abs(fc - segmentPoints[segmentNos][noSegmentPixels - 1].x); + + if (dr <= 1 && dc <= 1) + { + segmentPoints[segmentNos].erase(segmentPoints[segmentNos].begin()); + noSegmentPixels--; + } + + segmentNos++; + segmentPoints.push_back(vector()); // create empty vector of points for segments + + // Copy the rest of the long chains here + for (int k4 = 2; k4 < noChains; k4++) + { + if (chains[k4].len < 2) + continue; + + totalLen = LongestChain(chains, k4); + + if (totalLen >= 10) + { + // Retrieve the chainNos + int count = RetrieveChainNos(chains, k4, chainNos); + + // Copy the pixels + noSegmentPixels = 0; + for (int k5 = 0; k5 < count; k5++) + { + int chainNo = chainNos[k5]; + + /* See if we can erase some pixels from the last chain. This is for cleanup */ + fr = chains[chainNo].pixels[0].y; + fc = chains[chainNo].pixels[0].x; + + int index = noSegmentPixels - 2; + while (index >= 0) + { + dr = abs(fr - segmentPoints[segmentNos][index].y); + dc = abs(fc - segmentPoints[segmentNos][index].x); + + if (dr <= 1 && dc <= 1) + { + // neighbors. Erase last pixel + segmentPoints[segmentNos].pop_back(); + noSegmentPixels--; + index--; + } + else + break; + } + + int startIndex = 0; + int chainLen = chains[chainNo].len; + if (chainLen > 1 && noSegmentPixels > 0) + { + fr = chains[chainNo].pixels[1].y; + fc = chains[chainNo].pixels[1].x; + + dr = abs(fr - segmentPoints[segmentNos][noSegmentPixels - 1].y); + dc = abs(fc - segmentPoints[segmentNos][noSegmentPixels - 1].x); + + if (dr <= 1 && dc <= 1) + { + startIndex = 1; + } + } + + /* Start a new chain & copy pixels from the new chain */ + for (int l = startIndex; l < chains[chainNo].len; l++) + { + segmentPoints[segmentNos].push_back(chains[chainNo].pixels[l]); + noSegmentPixels++; + } + + chains[chainNo].len = 0; // Mark as copied + } + segmentPoints.push_back(vector()); // create empty vector of points for segments + segmentNos++; + } + } + } + } + + // pop back last segment from vector + // because of one preallocation in the beginning, it will always empty + segmentPoints.pop_back(); + + // Clean up + delete[] pAnchors; + delete[] chains; + delete[] stack; + delete[] chainNos; + delete[] pixels; +} + +int* EdgeDrawingImpl::sortAnchorsByGradValue1() +{ + int SIZE = 128 * 256; + int* C = new int[SIZE]; + memset(C, 0, sizeof(int) * SIZE); + + // Count the number of grad values + for (int i = 1; i < height - 1; i++) + { + for (int j = 1; j < width - 1; j++) + { + if (edgeImg[i * width + j] != ANCHOR_PIXEL) + continue; + + int grad = gradImg[i * width + j]; + C[grad]++; + } + } + + // Compute indices + for (int i = 1; i < SIZE; i++) + C[i] += C[i - 1]; + + int noAnchors = C[SIZE - 1]; + int* A = new int[noAnchors]; + + for (int i = 1; i < height - 1; i++) + { + for (int j = 1; j < width - 1; j++) + { + if (edgeImg[i * width + j] != ANCHOR_PIXEL) + continue; + + int grad = gradImg[i * width + j]; + int index = --C[grad]; + A[index] = i * width + j; // anchor's offset + } + } + + delete[] C; + return A; +} + +int EdgeDrawingImpl::LongestChain(Chain* chains, int root) +{ + if (root == -1 || chains[root].len == 0) + return 0; + + int len0 = 0; + if (chains[root].children[0] != -1) + len0 = LongestChain(chains, chains[root].children[0]); + + int len1 = 0; + if (chains[root].children[1] != -1) + len1 = LongestChain(chains, chains[root].children[1]); + + int max = 0; + + if (len0 >= len1) + { + max = len0; + chains[root].children[1] = -1; + } + else + { + max = len1; + chains[root].children[0] = -1; + } + + return chains[root].len + max; +} + +int EdgeDrawingImpl::RetrieveChainNos(Chain* chains, int root, int chainNos[]) +{ + int count = 0; + + while (root != -1) + { + chainNos[count] = root; + count++; + + if (chains[root].children[0] != -1) + root = chains[root].children[0]; + else + root = chains[root].children[1]; + } + + return count; +} + +void EdgeDrawingImpl::detectLines(OutputArray _lines) +{ + if (segmentPoints.size() < 1) + return; + + min_line_len = params.MinLineLength; + line_error = params.LineFitErrorThreshold; + max_distance_between_two_lines = params.MaxDistanceBetweenTwoLines; + max_error = params.MaxErrorThreshold; + + if (min_line_len == -1) // If no initial value given, compute it + min_line_len = ComputeMinLineLength(); + + if (min_line_len < 9) // avoids small line segments in the result. Might be deleted! + min_line_len = 9; + + // Temporary buffers used during line fitting + double* x = new double[(width + height) * 8]; + double* y = new double[(width + height) * 8]; + + linesNo = 0; + + // Use the whole segment + for (size_t segmentNumber = 0; segmentNumber < segmentPoints.size(); segmentNumber++) + { + std::vector segment = segmentPoints[segmentNumber]; + for (int k = 0; k < (int)segment.size(); k++) + { + x[k] = segment[k].x; + y[k] = segment[k].y; + } + SplitSegment2Lines(x, y, (int)segment.size(), (int)segmentNumber); + } + + JoinCollinearLines(); + +#define PRECISON_ANGLE 22.5 + precision = (PRECISON_ANGLE / 180) * CV_PI; + double prob = 0.125; +#undef PRECISON_ANGLE + + if (params.NFAValidation) + { + int lutSize = (width + height) / 8; + nfa = new NFALUT(lutSize, prob, width, height); + ValidateLineSegments(); + } + + // Delete redundant space from lines + // Pop them back + int size = (int)lines.size(); + for (int i = 1; i <= size - linesNo; i++) + lines.pop_back(); + + std::vector linePoints; + for (int i = 0; i < linesNo; i++) + { + Vec4f line((float)lines[i].sx, (float)lines[i].sy, (float)lines[i].ex, (float)lines[i].ey); + linePoints.push_back(line); + } + + delete[] x; + delete[] y; + Mat(linePoints).copyTo(_lines); +} + +// Computes the minimum line length using the NFA formula given width & height values +int EdgeDrawingImpl::ComputeMinLineLength() +{ + // The reason we are dividing the theoretical minimum line length by 2 is because + // we now test short line segments by a line support region rectangle having width=2. + // This means that within a line support region rectangle for a line segment of length "l" + // there are "2*l" many pixels. Thus, a line segment of length "l" has a chance of getting + // validated by NFA. + + double logNT = 2.0 * (log10((double)width) + log10((double)height)); + return (int)round((-logNT / log10(0.125)) * 0.5); +} //end-ComputeMinLineLength + +//----------------------------------------------------------------- +// Given a full segment of pixels, splits the chain to lines +// This code is used when we use the whole segment of pixels +// +void EdgeDrawingImpl::SplitSegment2Lines(double* x, double* y, int noPixels, int segmentNo) +{ + // First pixel of the line segment within the segment of points + int firstPixelIndex = 0; + + while (noPixels >= min_line_len) + { + // Start by fitting a line to MIN_LINE_LEN pixels + bool valid = false; + double lastA(0), lastB(0), error; + int lastInvert(0); + + while (noPixels >= min_line_len) + { + LineFit(x, y, min_line_len, lastA, lastB, error, lastInvert); + if (error <= 0.5) + { + valid = true; + break; + } + + noPixels -= 1; // Go slowly + x += 1; + y += 1; + firstPixelIndex += 1; + } + + if (valid == false) + return; + + // Now try to extend this line + int index = min_line_len; + int len = min_line_len; + + while (index < noPixels) + { + int startIndex = index; + int lastGoodIndex = index - 1; + int goodPixelCount = 0; + int badPixelCount = 0; + + while (index < noPixels) + { + double d = ComputeMinDistance(x[index], y[index], lastA, lastB, lastInvert); + + if (d <= line_error) + { + lastGoodIndex = index; + goodPixelCount++; + badPixelCount = 0; + } + else + { + badPixelCount++; + if (badPixelCount >= 5) + break; + } + index++; + } + + if (goodPixelCount >= 2) + { + len += lastGoodIndex - startIndex + 1; + LineFit(x, y, len, lastA, lastB, lastInvert); // faster LineFit + index = lastGoodIndex + 1; + } + + if (goodPixelCount < 2 || index >= noPixels) + { + // End of a line segment. Compute the end points + double sx, sy, ex, ey; + + index = 0; + while (ComputeMinDistance(x[index], y[index], lastA, lastB, lastInvert) > line_error) + index++; + ComputeClosestPoint(x[index], y[index], lastA, lastB, lastInvert, sx, sy); + int noSkippedPixels = index; + + index = lastGoodIndex; + while (ComputeMinDistance(x[index], y[index], lastA, lastB, lastInvert) > line_error) + index--; + ComputeClosestPoint(x[index], y[index], lastA, lastB, lastInvert, ex, ey); + + // Add the line segment to lines + lines.push_back(EDLineSegment(lastA, lastB, lastInvert, sx, sy, ex, ey, segmentNo, firstPixelIndex + noSkippedPixels, index - noSkippedPixels + 1)); + linesNo++; + len = index + 1; + break; + } + } + + noPixels -= len; + x += len; + y += len; + firstPixelIndex += len; + } +} + +//------------------------------------------------------------------ +// Goes over the original line segments and combines collinear lines that belong to the same segment +// +void EdgeDrawingImpl::JoinCollinearLines() +{ + int lastLineIndex = -1; //Index of the last line in the joined lines + int i = 0; + while (i < linesNo) + { + int segmentNo = lines[i].segmentNo; + + lastLineIndex++; + if (lastLineIndex != i) + lines[lastLineIndex] = lines[i]; + + int firstLineIndex = lastLineIndex; // Index of the first line in this segment + + int count = 1; + for (int j = i + 1; j < linesNo; j++) + { + if (lines[j].segmentNo != segmentNo) + break; + + // Try to combine this line with the previous line in this segment + if (TryToJoinTwoLineSegments(&lines[lastLineIndex], &lines[j], + lastLineIndex) == false) + { + lastLineIndex++; + if (lastLineIndex != j) + lines[lastLineIndex] = lines[j]; + } + count++; + } + + // Try to join the first & last line of this segment + if (firstLineIndex != lastLineIndex) + { + if (TryToJoinTwoLineSegments(&lines[firstLineIndex], &lines[lastLineIndex], + firstLineIndex)) + { + lastLineIndex--; + } + } + i += count; + } + linesNo = lastLineIndex + 1; +} + +void EdgeDrawingImpl::ValidateLineSegments() +{ + int* x = new int[(width + height) * 4]; + int* y = new int[(width + height) * 4]; + + int noValidLines = 0; + + for (int i = 0; i < linesNo; i++) + { + EDLineSegment* ls = &lines[i]; + + // Compute Line's angle + double lineAngle; + + if (ls->invert == 0) + { + // y = a + bx + lineAngle = atan(ls->b); + } + else + { + // x = a + by + lineAngle = atan(1.0 / ls->b); + } + + if (lineAngle < 0) + lineAngle += CV_PI; + + Point* pixels = &(segmentPoints[ls->segmentNo][0]); + int noPixels = ls->len; + + bool valid = false; + + // Accept very long lines without testing. They are almost never invalidated. + if (ls->len >= 80) + { + valid = true; + // Validate short line segments by a line support region rectangle having width=2 + } + else if (ls->len <= 25) + { + valid = ValidateLineSegmentRect(x, y, ls); + } + else + { + // Longer line segments are first validated by a line support region rectangle having width=1 (for speed) + // If the line segment is still invalid, then a line support region rectangle having width=2 is tried + // If the line segment fails both tests, it is discarded + int aligned = 0; + int count = 0; + for (int j = 0; j < noPixels; j++) + { + int r = pixels[j].x; + int c = pixels[j].y; + + if (r <= 0 || r >= height - 1 || c <= 0 || c >= width - 1) + continue; + + count++; + + // compute gx & gy using the simple [-1 -1 -1] + // [ 1 1 1] filter in both directions + // Faster method below + // A B C + // D x E + // F G H + // gx = (C-A) + (E-D) + (H-F) + // gy = (F-A) + (G-B) + (H-C) + // + // To make this faster: + // com1 = (H-A) + // com2 = (C-F) + // Then: gx = com1 + com2 + (E-D) = (H-A) + (C-F) + (E-D) = (C-A) + (E-D) + (H-F) + // gy = com2 - com1 + (G-B) = (H-A) - (C-F) + (G-B) = (F-A) + (G-B) + (H-C) + // + int com1 = srcImg[(r + 1) * width + c + 1] - srcImg[(r - 1) * width + c - 1]; + int com2 = srcImg[(r - 1) * width + c + 1] - srcImg[(r + 1) * width + c - 1]; + + int gx = com1 + com2 + srcImg[r * width + c + 1] - srcImg[r * width + c - 1]; + int gy = com1 - com2 + srcImg[(r + 1) * width + c] - srcImg[(r - 1) * width + c]; + + double pixelAngle = nfa->myAtan2((double)gx, (double)-gy); + double diff = fabs(lineAngle - pixelAngle); + + if (diff <= precision || diff >= CV_PI - precision) + aligned++; + } + + // Check validation by NFA computation (fast due to LUT) + valid = nfa->checkValidationByNFA(count, aligned); + if (valid == false) + valid = ValidateLineSegmentRect(x, y, ls); + } + + if (valid) + { + if (i != noValidLines) + lines[noValidLines] = lines[i]; + noValidLines++; + } + else + { + invalidLines.push_back(lines[i]); + } + } + + linesNo = noValidLines; + + delete[] x; + delete[] y; +} + +bool EdgeDrawingImpl::ValidateLineSegmentRect(int* x, int* y, EDLineSegment* ls) +{ + // Compute Line's angle + double lineAngle; + + if (ls->invert == 0) + { + // y = a + bx + lineAngle = atan(ls->b); + } + else + { + // x = a + by + lineAngle = atan(1.0 / ls->b); + } + + if (lineAngle < 0) + lineAngle += CV_PI; + + int noPoints = 0; + + // Enumerate all pixels that fall within the bounding rectangle + EnumerateRectPoints(ls->sx, ls->sy, ls->ex, ls->ey, x, y, &noPoints); + + int count = 0; + int aligned = 0; + + for (int i = 0; i < noPoints; i++) + { + int r = y[i]; + int c = x[i]; + + if (r <= 0 || r >= height - 1 || c <= 0 || c >= width - 1) + continue; + + count++; + + // compute gx & gy using the simple [-1 -1 -1] + // [ 1 1 1] filter in both directions + // Faster method below + // A B C + // D x E + // F G H + // gx = (C-A) + (E-D) + (H-F) + // gy = (F-A) + (G-B) + (H-C) + // + // To make this faster: + // com1 = (H-A) + // com2 = (C-F) + // Then: gx = com1 + com2 + (E-D) = (H-A) + (C-F) + (E-D) = (C-A) + (E-D) + (H-F) + // gy = com2 - com1 + (G-B) = (H-A) - (C-F) + (G-B) = (F-A) + (G-B) + (H-C) + // + int com1 = srcImg[(r + 1) * width + c + 1] - srcImg[(r - 1) * width + c - 1]; + int com2 = srcImg[(r - 1) * width + c + 1] - srcImg[(r + 1) * width + c - 1]; + + int gx = com1 + com2 + srcImg[r * width + c + 1] - srcImg[r * width + c - 1]; + int gy = com1 - com2 + srcImg[(r + 1) * width + c] - srcImg[(r - 1) * width + c]; + double pixelAngle = nfa->myAtan2((double)gx, (double)-gy); + + double diff = fabs(lineAngle - pixelAngle); + + if (diff <= precision || diff >= CV_PI - precision) + aligned++; + } + return nfa->checkValidationByNFA(count, aligned); +} + +double EdgeDrawingImpl::ComputeMinDistance(double x1, double y1, double a, double b, int invert) +{ + double x2, y2; + + if (invert == 0) + { + if (b == 0) + { + x2 = x1; + y2 = a; + } + else + { + // Let the line passing through (x1, y1) that is perpendicular to a+bx be c+dx + double d = -1.0 / (b); + double c = y1 - d * x1; + + x2 = (a - c) / (d - b); + y2 = a + b * x2; + } + } + else + { + /// invert = 1 + if (b == 0) + { + x2 = a; + y2 = y1; + } + else + { + // Let the line passing through (x1, y1) that is perpendicular to a+by be c+dy + double d = -1.0 / (b); + double c = x1 - d * y1; + + y2 = (a - c) / (d - b); + x2 = a + b * y2; + } + } + + return sqrt((x1 - x2) * (x1 - x2) + (y1 - y2) * (y1 - y2)); +} + +//--------------------------------------------------------------------------------- +// Given a point (x1, y1) and a line equation y=a+bx (invert=0) OR x=a+by (invert=1) +// Computes the (x2, y2) on the line that is closest to (x1, y1) +// +void EdgeDrawingImpl::ComputeClosestPoint(double x1, double y1, double a, double b, int invert, double& xOut, double& yOut) +{ + double x2, y2; + + if (invert == 0) + { + if (b == 0) + { + x2 = x1; + y2 = a; + } + else + { + // Let the line passing through (x1, y1) that is perpendicular to a+bx be c+dx + double d = -1.0 / (b); + double c = y1 - d * x1; + + x2 = (a - c) / (d - b); + y2 = a + b * x2; + } + } + else + { + /// invert = 1 + if (b == 0) + { + x2 = a; + y2 = y1; + } + else + { + // Let the line passing through (x1, y1) that is perpendicular to a+by be c+dy + double d = -1.0 / (b); + double c = x1 - d * y1; + + y2 = (a - c) / (d - b); + x2 = a + b * y2; + } + } + + xOut = x2; + yOut = y2; +} + +//----------------------------------------------------------------------------------- +// Fits a line of the form y=a+bx (invert == 0) OR x=a+by (invert == 1) +// Assumes that the direction of the line is known by a previous computation +// +void EdgeDrawingImpl::LineFit(double* x, double* y, int count, double& a, double& b, int invert) +{ + if (count < 2) + return; + + double S = count, Sx = 0.0, Sy = 0.0, Sxx = 0.0, Sxy = 0.0; + for (int i = 0; i < count; i++) + { + Sx += x[i]; + Sy += y[i]; + } + + if (invert) + { + // Vertical line. Swap x & y, Sx & Sy + double* t = x; + x = y; + y = t; + + double d = Sx; + Sx = Sy; + Sy = d; + } + + // Now compute Sxx & Sxy + for (int i = 0; i < count; i++) + { + Sxx += x[i] * x[i]; + Sxy += x[i] * y[i]; + } + + double D = S * Sxx - Sx * Sx; + a = (Sxx * Sy - Sx * Sxy) / D; + b = (S * Sxy - Sx * Sy) / D; +} + +//----------------------------------------------------------------------------------- +// Fits a line of the form y=a+bx (invert == 0) OR x=a+by (invert == 1) +// +void EdgeDrawingImpl::LineFit(double* x, double* y, int count, double& a, double& b, double& e, int& invert) +{ + if (count < 2) + return; + + double S = count, Sx = 0.0, Sy = 0.0, Sxx = 0.0, Sxy = 0.0; + for (int i = 0; i < count; i++) + { + Sx += x[i]; + Sy += y[i]; + } + + double mx = Sx / count; + double my = Sy / count; + + double dx = 0.0; + double dy = 0.0; + + for (int i = 0; i < count; i++) + { + dx += (x[i] - mx) * (x[i] - mx); + dy += (y[i] - my) * (y[i] - my); + } + + if (dx < dy) + { + // Vertical line. Swap x & y, Sx & Sy + invert = 1; + double* t = x; + x = y; + y = t; + + double d = Sx; + Sx = Sy; + Sy = d; + } + else + { + invert = 0; + } + + // Now compute Sxx & Sxy + for (int i = 0; i < count; i++) + { + Sxx += x[i] * x[i]; + Sxy += x[i] * y[i]; + } + + double D = S * Sxx - Sx * Sx; + a = (Sxx * Sy - Sx * Sxy) / D; + b = (S * Sxy - Sx * Sy) / D; + + if (b == 0.0) + { + // Vertical or horizontal line + double error = 0.0; + for (int i = 0; i < count; i++) + { + error += fabs((a)-y[i]); + } + e = error / count; + } + else + { + double error = 0.0; + for (int i = 0; i < count; i++) + { + // Let the line passing through (x[i], y[i]) that is perpendicular to a+bx be c+dx + double d = -1.0 / (b); + double c = y[i] - d * x[i]; + double x2 = ((a)-c) / (d - (b)); + double y2 = (a)+(b)*x2; + + double dist = (x[i] - x2) * (x[i] - x2) + (y[i] - y2) * (y[i] - y2); + error += dist; + } + e = sqrt(error / count); + } +} + +//----------------------------------------------------------------- +// Checks if the given line segments are collinear & joins them if they are +// In case of a join, ls1 is updated. ls2 is NOT changed +// Returns true if join is successful, false otherwise +// +bool EdgeDrawingImpl::TryToJoinTwoLineSegments(EDLineSegment* ls1, EDLineSegment* ls2, int changeIndex) +{ + int which; + double dist = ComputeMinDistanceBetweenTwoLines(ls1, ls2, &which); + if (dist > max_distance_between_two_lines) + return false; + + // Compute line lengths. Use the longer one as the ground truth + double dx = ls1->sx - ls1->ex; + double dy = ls1->sy - ls1->ey; + double prevLen = sqrt(dx * dx + dy * dy); + + dx = ls2->sx - ls2->ex; + dy = ls2->sy - ls2->ey; + double nextLen = sqrt(dx * dx + dy * dy); + + // Use the longer line as the ground truth + EDLineSegment* shorter = ls1; + EDLineSegment* longer = ls2; + + if (prevLen > nextLen) + { + shorter = ls2; + longer = ls1; + } + + // Just use 3 points to check for collinearity + dist = ComputeMinDistance(shorter->sx, shorter->sy, longer->a, longer->b, longer->invert); + dist += ComputeMinDistance((shorter->sx + shorter->ex) / 2.0, (shorter->sy + shorter->ey) / 2.0, longer->a, longer->b, longer->invert); + dist += ComputeMinDistance(shorter->ex, shorter->ey, longer->a, longer->b, longer->invert); + + dist /= 3.0; + + if (dist > max_error) + return false; + + /// 4 cases: 1:(s1, s2), 2:(s1, e2), 3:(e1, s2), 4:(e1, e2) + + /// case 1: (s1, s2) + dx = fabs(ls1->sx - ls2->sx); + dy = fabs(ls1->sy - ls2->sy); + double d = dx + dy; + double max = d; + which = 1; + + /// case 2: (s1, e2) + dx = fabs(ls1->sx - ls2->ex); + dy = fabs(ls1->sy - ls2->ey); + d = dx + dy; + if (d > max) + { + max = d; + which = 2; + } + + /// case 3: (e1, s2) + dx = fabs(ls1->ex - ls2->sx); + dy = fabs(ls1->ey - ls2->sy); + d = dx + dy; + if (d > max) + { + max = d; + which = 3; + } + + /// case 4: (e1, e2) + dx = fabs(ls1->ex - ls2->ex); + dy = fabs(ls1->ey - ls2->ey); + d = dx + dy; + if (d > max) + { + max = d; + which = 4; + } + + if (which == 1) + { + // (s1, s2) + ls1->ex = ls2->sx; + ls1->ey = ls2->sy; + } + else if (which == 2) + { + // (s1, e2) + ls1->ex = ls2->ex; + ls1->ey = ls2->ey; + } + else if (which == 3) + { + // (e1, s2) + ls1->sx = ls2->sx; + ls1->sy = ls2->sy; + } + else + { + // (e1, e2) + ls1->sx = ls1->ex; + ls1->sy = ls1->ey; + + ls1->ex = ls2->ex; + ls1->ey = ls2->ey; + } + + // Update the first line's parameters + if (ls1->firstPixelIndex + ls1->len + 5 >= ls2->firstPixelIndex) + ls1->len += ls2->len; + else if (ls2->len > ls1->len) + { + ls1->firstPixelIndex = ls2->firstPixelIndex; + ls1->len = ls2->len; + } + + UpdateLineParameters(ls1); + lines[changeIndex] = *ls1; + + return true; +} + +//------------------------------------------------------------------------------- +// Computes the minimum distance between the end points of two lines +// +double EdgeDrawingImpl::ComputeMinDistanceBetweenTwoLines(EDLineSegment* ls1, EDLineSegment* ls2, int* pwhich) +{ + double dx = ls1->sx - ls2->sx; + double dy = ls1->sy - ls2->sy; + double d = sqrt(dx * dx + dy * dy); + double min = d; + int which = SOUTH_SOUTH; + + dx = ls1->sx - ls2->ex; + dy = ls1->sy - ls2->ey; + d = sqrt(dx * dx + dy * dy); + if (d < min) + { + min = d; + which = SOUTH_EAST; + } + + dx = ls1->ex - ls2->sx; + dy = ls1->ey - ls2->sy; + d = sqrt(dx * dx + dy * dy); + if (d < min) + { + min = d; + which = EAST_SOUTH; + } + + dx = ls1->ex - ls2->ex; + dy = ls1->ey - ls2->ey; + d = sqrt(dx * dx + dy * dy); + if (d < min) + { + min = d; + which = EAST_EAST; + } + + if (pwhich) + *pwhich = which; + return min; +} + +//----------------------------------------------------------------------------------- +// Uses the two end points (sx, sy)----(ex, ey) of the line segment +// and computes the line that passes through these points (a, b, invert) +// +void EdgeDrawingImpl::UpdateLineParameters(EDLineSegment* ls) +{ + double dx = ls->ex - ls->sx; + double dy = ls->ey - ls->sy; + + if (fabs(dx) >= fabs(dy)) + { + /// Line will be of the form y = a + bx + ls->invert = 0; + if (fabs(dy) < 1e-3) + { + ls->b = 0; + ls->a = (ls->sy + ls->ey) / 2; + } + else + { + ls->b = dy / dx; + ls->a = ls->sy - (ls->b) * ls->sx; + } + } + else + { + /// Line will be of the form x = a + by + ls->invert = 1; + if (fabs(dx) < 1e-3) + { + ls->b = 0; + ls->a = (ls->sx + ls->ex) / 2; + } + else + { + ls->b = dx / dy; + ls->a = ls->sx - (ls->b) * ls->sy; + } + } +} + +void EdgeDrawingImpl::EnumerateRectPoints(double sx, double sy, double ex, double ey, int ptsx[], int ptsy[], int* pNoPoints) +{ + double vxTmp[4], vyTmp[4]; + double vx[4], vy[4]; + int n, offset; + + double x1 = sx; + double y1 = sy; + double x2 = ex; + double y2 = ey; + double width = 2; + + double dx = x2 - x1; + double dy = y2 - y1; + double vLen = sqrt(dx * dx + dy * dy); + + // make unit vector + dx = dx / vLen; + dy = dy / vLen; + + /* build list of rectangle corners ordered + in a circular way around the rectangle */ + vxTmp[0] = x1 - dy * width / 2.0; + vyTmp[0] = y1 + dx * width / 2.0; + vxTmp[1] = x2 - dy * width / 2.0; + vyTmp[1] = y2 + dx * width / 2.0; + vxTmp[2] = x2 + dy * width / 2.0; + vyTmp[2] = y2 - dx * width / 2.0; + vxTmp[3] = x1 + dy * width / 2.0; + vyTmp[3] = y1 - dx * width / 2.0; + + /* compute rotation of index of corners needed so that the first + point has the smaller x. + + if one side is vertical, thus two corners have the same smaller x + value, the one with the largest y value is selected as the first. + */ + if (x1 < x2 && y1 <= y2) + offset = 0; + else if (x1 >= x2 && y1 < y2) + offset = 1; + else if (x1 > x2&& y1 >= y2) + offset = 2; + else + offset = 3; + + /* apply rotation of index. */ + for (n = 0; n < 4; n++) + { + vx[n] = vxTmp[(offset + n) % 4]; + vy[n] = vyTmp[(offset + n) % 4]; + } + + /* Set a initial condition. + + The values are set to values that will cause 'ri_inc' (that will + be called immediately) to initialize correctly the first 'column' + and compute the limits 'ys' and 'ye'. + + 'y' is set to the integer value of vy[0], the starting corner. + + 'ys' and 'ye' are set to very small values, so 'ri_inc' will + notice that it needs to start a new 'column'. + + The smaller integer coordinate inside of the rectangle is + 'ceil(vx[0])'. The current 'x' value is set to that value minus + one, so 'ri_inc' (that will increase x by one) will advance to + the first 'column'. + */ + int x = (int)ceil(vx[0]) - 1; + int y = (int)ceil(vy[0]); + double ys = -DBL_MAX, ye = -DBL_MAX; + + int noPoints = 0; + while (1) + { + /* if not at end of exploration, + increase y value for next pixel in the 'column' */ + y++; + + /* if the end of the current 'column' is reached, + and it is not the end of exploration, + advance to the next 'column' */ + while (y > ye&& x <= vx[2]) + { + /* increase x, next 'column' */ + x++; + + /* if end of exploration, return */ + if (x > vx[2]) + break; + + /* update lower y limit (start) for the new 'column'. + + We need to interpolate the y value that corresponds to the + lower side of the rectangle. The first thing is to decide if + the corresponding side is + + vx[0],vy[0] to vx[3],vy[3] or + vx[3],vy[3] to vx[2],vy[2] + + Then, the side is interpolated for the x value of the + 'column'. But, if the side is vertical (as it could happen if + the rectangle is vertical and we are dealing with the first + or last 'columns') then we pick the lower value of the side + by using 'inter_low'. + */ + if ((double)x < vx[3]) + { + /* interpolation */ + if (fabs(vx[0] - vx[3]) <= 0.01) + { + if (vy[0] < vy[3]) + ys = vy[0]; + else if (vy[0] > vy[3]) + ys = vy[3]; + else + ys = vy[0] + (x - vx[0]) * (vy[3] - vy[0]) / (vx[3] - vx[0]); + } + else + ys = vy[0] + (x - vx[0]) * (vy[3] - vy[0]) / (vx[3] - vx[0]); + } + else + { + /* interpolation */ + if (fabs(vx[3] - vx[2]) <= 0.01) + { + if (vy[3] < vy[2]) + ys = vy[3]; + else if (vy[3] > vy[2]) + ys = vy[2]; + else + ys = vy[3] + (x - vx[3]) * (y2 - vy[3]) / (vx[2] - vx[3]); + } + else + ys = vy[3] + (x - vx[3]) * (vy[2] - vy[3]) / (vx[2] - vx[3]); + } + + /* update upper y limit (end) for the new 'column'. + + We need to interpolate the y value that corresponds to the + upper side of the rectangle. The first thing is to decide if + the corresponding side is + + vx[0],vy[0] to vx[1],vy[1] or + vx[1],vy[1] to vx[2],vy[2] + + Then, the side is interpolated for the x value of the + 'column'. But, if the side is vertical (as it could happen if + the rectangle is vertical and we are dealing with the first + or last 'columns') then we pick the lower value of the side + by using 'inter_low'. + */ + if ((double)x < vx[1]) + { + /* interpolation */ + if (fabs(vx[0] - vx[1]) <= 0.01) + { + if (vy[0] < vy[1]) + ye = vy[1]; + else if (vy[0] > vy[1]) + ye = vy[0]; + else + ye = vy[0] + (x - vx[0]) * (vy[1] - vy[0]) / (vx[1] - vx[0]); + } + else + ye = vy[0] + (x - vx[0]) * (vy[1] - vy[0]) / (vx[1] - vx[0]); + } + else + { + /* interpolation */ + if (fabs(vx[1] - vx[2]) <= 0.01) + { + if (vy[1] < vy[2]) + ye = vy[2]; + else if (vy[1] > vy[2]) + ye = vy[1]; + else + ye = vy[1] + (x - vx[1]) * (vy[2] - vy[1]) / (vx[2] - vx[1]); + } + else + ye = vy[1] + (x - vx[1]) * (vy[2] - vy[1]) / (vx[2] - vx[1]); + } + /* new y */ + y = (int)ceil(ys); + } + + // Are we done? + if (x > vx[2]) + break; + + ptsx[noPoints] = x; + ptsy[noPoints] = y; + noPoints++; + } + *pNoPoints = noPoints; +} + +void EdgeDrawingImpl::SplitSegment2Lines(double* x, double* y, int noPixels, int segmentNo, vector& lines, int min_line_len, double line_error) +{ + // First pixel of the line segment within the segment of points + int firstPixelIndex = 0; + + while (noPixels >= min_line_len) + { + // Start by fitting a line to MIN_LINE_LEN pixels + bool valid = false; + double lastA(0), lastB(0), error; + int lastInvert(0); + + while (noPixels >= min_line_len) + { + LineFit(x, y, min_line_len, lastA, lastB, error, lastInvert); + if (error <= 0.5) + { + valid = true; + break; + } + + noPixels -= 1; // Go slowly + x += 1; + y += 1; + firstPixelIndex += 1; + } + + if (valid == false) + return; + + // Now try to extend this line + int index = min_line_len; + int len = min_line_len; + + while (index < noPixels) + { + int startIndex = index; + int lastGoodIndex = index - 1; + int goodPixelCount = 0; + int badPixelCount = 0; + while (index < noPixels) + { + double d = ComputeMinDistance(x[index], y[index], lastA, lastB, lastInvert); + + if (d <= line_error) + { + lastGoodIndex = index; + goodPixelCount++; + badPixelCount = 0; + } + else + { + badPixelCount++; + if (badPixelCount >= 5) + break; + } + index++; + } + + if (goodPixelCount >= 2) + { + len += lastGoodIndex - startIndex + 1; + LineFit(x, y, len, lastA, lastB, lastInvert); + index = lastGoodIndex + 1; + } + + if (goodPixelCount < 2 || index >= noPixels) + { + // End of a line segment. Compute the end points + double sx, sy, ex, ey; + + index = 0; + while (ComputeMinDistance(x[index], y[index], lastA, lastB, lastInvert) > line_error) + index++; + ComputeClosestPoint(x[index], y[index], lastA, lastB, lastInvert, sx, sy); + int noSkippedPixels = index; + + index = lastGoodIndex; + while (ComputeMinDistance(x[index], y[index], lastA, lastB, lastInvert) > line_error) + index--; + ComputeClosestPoint(x[index], y[index], lastA, lastB, lastInvert, ex, ey); + + // Add the line segment to lines + lines.push_back(EDLineSegment(lastA, lastB, lastInvert, sx, sy, ex, ey, segmentNo, firstPixelIndex + noSkippedPixels, index - noSkippedPixels + 1)); + len = index + 1; + break; + } + } + + noPixels -= len; + x += len; + y += len; + firstPixelIndex += len; + } +} + +/*--------------------------------------EDPF----------------------------------------*/ + +//---------------------------------------------------------------------------------- +// Resursive validation using half of the pixels as suggested by DMM algorithm +// We take pixels at Nyquist distance, i.e., 2 (as suggested by DMM) +// +void EdgeDrawingImpl::TestSegment(int i, int index1, int index2) +{ + int chainLen = index2 - index1 + 1; + if (chainLen < params.MinPathLength) + return; + + // Test from index1 to index2. If OK, then we are done. Otherwise, split into two and + // recursively test the left & right halves + + // First find the min. gradient along the segment + int minGrad = 1 << 30; + int minGradIndex = 0; + + for (int k = index1; k <= index2; k++) { + int r = segmentPoints[i][k].y; + int c = segmentPoints[i][k].x; + if (gradImg[r * width + c] < minGrad) { minGrad = gradImg[r * width + c]; minGradIndex = k; } + } + + double nfa0 = NFA(dH[minGrad], (int)(chainLen / divForTestSegment)); + + if (nfa0 <= 1.0) { + for (int k = index1; k <= index2; k++) { + int r = segmentPoints[i][k].y; + int c = segmentPoints[i][k].x; + + edgeImg[r * width + c] = 255; + } + return; + } + + // Split into two halves. We divide at the point where the gradient is the minimum + int end = minGradIndex - 1; + while (end > index1) { + int r = segmentPoints[i][end].y; + int c = segmentPoints[i][end].x; + + if (gradImg[r * width + c] <= minGrad) end--; + else break; + } + + int start = minGradIndex + 1; + while (start < index2) { + int r = segmentPoints[i][start].y; + int c = segmentPoints[i][start].x; + + if (gradImg[r * width + c] <= minGrad) start++; + else break; + } + + TestSegment(i, index1, end); + TestSegment(i, start, index2); +} + +//---------------------------------------------------------------------------------------------- +// After the validation of the edge segments, extracts the valid ones +// In other words, updates the valid segments' pixel arrays and their lengths +void EdgeDrawingImpl::ExtractNewSegments() +{ + vector< vector > validSegments; + int noSegments = 0; + + for (int i = 0; i < segmentNos; i++) { + int start = 0; + while (start < (int)segmentPoints[i].size()) { + + while (start < (int)segmentPoints[i].size()) { + int r = segmentPoints[i][start].y; + int c = segmentPoints[i][start].x; + + if (edgeImg[r * width + c]) break; + start++; + } + + int end = start + 1; + while (end < (int)segmentPoints[i].size()) { + int r = segmentPoints[i][end].y; + int c = segmentPoints[i][end].x; + + if (edgeImg[r * width + c] == 0) break; + end++; + } + + int len = end - start; + if (len >= 10) { + // A new segment. Accepted only only long enough (whatever that means) + //segments[noSegments].pixels = &map->segments[i].pixels[start]; + //segments[noSegments].noPixels = len; + validSegments.push_back(vector()); + vector subVec(&segmentPoints[i][start], &segmentPoints[i][end - 1]); + validSegments[noSegments] = subVec; + noSegments++; + } + start = end + 1; + } + } + + segmentPoints = validSegments; + segmentNos = noSegments; +} + +double EdgeDrawingImpl::NFA(double prob, int len) +{ + double nfa0 = np; + for (int i = 0; i 1.0; i++) + nfa0 *= prob; + + return nfa0; +} + +/*---------------------------------EDCircle--------------------------------------------*/ + +void EdgeDrawingImpl::detectEllipses(OutputArray ellipses) +{ + if (segmentPoints.size() < 1) + return; + + vector _ellipses; + Circles.clear(); + Ellipses.clear(); + // Arcs & circles to be detected + // If the end-points of the segment is very close to each other, + // then directly fit a circle/ellipse instread of line fitting + noCircles1 = 0; + circles1 = new Circle[(width + height) * 8]; + + // ----------------------------------- DETECT LINES --------------------------------- + int bufferSize = 0; + for (int i = 0; i < (int)segmentPoints.size(); i++) + bufferSize += (int)segmentPoints[i].size(); + + // Compute the starting line number for each segment + segmentStartLines = new int[segmentNos + 1]; + + bm = new BufferManager(bufferSize * 8); + +#define CIRCLE_MIN_LINE_LEN 6 + + for (int i = 0; i < segmentNos; i++) + { + // Make note of the starting line number for this segment + segmentStartLines[i] = (int)lines.size(); + + int noPixels = (int)segmentPoints[i].size(); + + if (noPixels < 2 * CIRCLE_MIN_LINE_LEN) + continue; + + double* x = bm->getX(); + double* y = bm->getY(); + + for (int j = 0; j < noPixels; j++) + { + x[j] = segmentPoints[i][j].x; + y[j] = segmentPoints[i][j].y; + } + + // If the segment is reasonably long, then see if the segment traverses the boundary of a closed shape + if (noPixels >= 4 * CIRCLE_MIN_LINE_LEN) + { + // If the end-points of the segment is close to each other, then assume a circular/elliptic structure + double dx = x[0] - x[noPixels - 1]; + double dy = y[0] - y[noPixels - 1]; + double d = sqrt(dx * dx + dy * dy); + double r = noPixels / CV_2PI; // Assume a complete circle + + double maxDistanceBetweenEndPoints = std::max(3.0, r / 4.0); + + // If almost closed loop, then try to fit a circle/ellipse + if (d <= maxDistanceBetweenEndPoints) + { + double xc, yc, circleFitError = 1e10; + + CircleFit(x, y, noPixels, &xc, &yc, &r, &circleFitError); + + EllipseEquation eq; + double ellipseFitError = 1e10; + + if (circleFitError > LONG_ARC_ERROR) + { + // Try fitting an ellipse + if (EllipseFit(x, y, noPixels, &eq)) + ellipseFitError = ComputeEllipseError(&eq, x, y, noPixels); + } + + if (circleFitError <= LONG_ARC_ERROR) + { + addCircle(circles1, noCircles1, xc, yc, r, circleFitError, x, y, noPixels); + bm->move(noPixels); + continue; + } + else if (ellipseFitError <= ELLIPSE_ERROR) + { + double major, minor; + ComputeEllipseCenterAndAxisLengths(&eq, &xc, &yc, &major, &minor); + + // Assume major is longer. Otherwise, swap + if (minor > major) + { + double tmp = major; + major = minor; + minor = tmp; + } + + if (major < 8 * minor) + { + addCircle(circles1, noCircles1, xc, yc, r, circleFitError, &eq, ellipseFitError, x, y, noPixels); + bm->move(noPixels); + } + continue; + } + } + } + // Otherwise, split to lines + SplitSegment2Lines(x, y, noPixels, i, lines); + } + + segmentStartLines[segmentNos] = (int)lines.size(); + + // ------------------------------- DETECT ARCS --------------------------------- + + info = new Info[lines.size()]; + + // Compute the angle information for each line segment + for (int i = 0; i < segmentNos; i++) + { + for (int j = segmentStartLines[i]; j < segmentStartLines[i + 1]; j++) + { + EDLineSegment* l1 = &lines[j]; + EDLineSegment* l2; + + if (j == segmentStartLines[i + 1] - 1) + l2 = &lines[segmentStartLines[i]]; + else + l2 = &lines[j + 1]; + + // If the end points of the lines are far from each other, then stop at this line + double dx = l1->ex - l2->sx; + double dy = l1->ey - l2->sy; + double d = sqrt(dx * dx + dy * dy); + if (d >= 15) + { + info[j].angle = 10; + info[j].sign = 2; + info[j].taken = false; + continue; + } + + // Compute the angle between the lines & their turn direction + double v1x = l1->ex - l1->sx; + double v1y = l1->ey - l1->sy; + double v1Len = sqrt(v1x * v1x + v1y * v1y); + + double v2x = l2->ex - l2->sx; + double v2y = l2->ey - l2->sy; + double v2Len = sqrt(v2x * v2x + v2y * v2y); + + double dotProduct = (v1x * v2x + v1y * v2y) / (v1Len * v2Len); + if (dotProduct > 1.0) + dotProduct = 1.0; + else if (dotProduct < -1.0) + dotProduct = -1.0; + + info[j].angle = acos(dotProduct); + info[j].sign = (v1x * v2y - v2x * v1y) >= 0 ? 1 : -1; // compute cross product + info[j].taken = false; + } + } + + // This is how much space we will allocate for circles buffers + int maxNoOfCircles = (int)lines.size() / 3 + noCircles1 * 2; + + edarcs1 = new EDArcs(maxNoOfCircles); + DetectArcs(); // Detect all arcs + + // Try to join arcs that are almost perfectly circular. + // Use the distance between the arc end-points as a metric in choosing in choosing arcs to join + edarcs2 = new EDArcs(maxNoOfCircles); + JoinArcs1(); + + // Try to join arcs that belong to the same segment + edarcs3 = new EDArcs(maxNoOfCircles); + JoinArcs2(); + + // Try to combine arcs that belong to different segments + edarcs4 = new EDArcs(maxNoOfCircles); // The remaining arcs + JoinArcs3(); + + // Finally, go over the arcs & circles, and generate candidate circles + GenerateCandidateCircles(); + + //----------------------------- VALIDATE CIRCLES -------------------------- + noCircles2 = 0; + circles2 = new Circle[maxNoOfCircles]; + GaussianBlur(srcImage, smoothImage, Size(), 0.50); // calculate kernel from sigma; + + ValidateCircles(params.NFAValidation); + + //----------------------------- JOIN CIRCLES -------------------------- + noCircles3 = 0; + circles3 = new Circle[maxNoOfCircles]; + JoinCircles(); + + noCircles = 0; + noEllipses = 0; + for (int i = 0; i < noCircles3; i++) + { + if (circles3[i].isEllipse) + noEllipses++; + else + noCircles++; + } + + for (int i = 0; i < noCircles3; i++) + { + if (circles3[i].isEllipse) + { + EllipseEquation eq = circles3[i].eq; + double xc; + double yc; + double a; + double b; + double theta = ComputeEllipseCenterAndAxisLengths(&eq, &xc, &yc, &a, &b); + Ellipses.push_back(mEllipse(Point2d(xc, yc), Size2d(a, b), theta)); + _ellipses.push_back(Vec6d(xc, yc, 0, a, b, theta * 180 / CV_PI)); + } + else + { + double r = circles3[i].r; + double xc = circles3[i].xc; + double yc = circles3[i].yc; + + Circles.push_back(mCircle(Point2d(xc, yc), r)); + _ellipses.push_back(Vec6d(xc, yc, r, 0, 0, 0)); + } + } + + Mat(_ellipses).copyTo(ellipses); + + delete edarcs1; + delete edarcs2; + delete edarcs3; + delete edarcs4; + + delete[] circles1; + delete[] circles2; + delete[] circles3; + + delete bm; + delete[] segmentStartLines; + delete[] info; +} + +void EdgeDrawingImpl::GenerateCandidateCircles() +{ + // Now, go over the circular arcs & add them to circles1 + MyArc* arcs = edarcs4->arcs; + for (int i = 0; i < edarcs4->noArcs; i++) + { + if (arcs[i].isEllipse) + { + // Ellipse + if (arcs[i].coverRatio >= CANDIDATE_ELLIPSE_RATIO && arcs[i].ellipseFitError <= ELLIPSE_ERROR) + { + addCircle(circles1, noCircles1, arcs[i].xc, arcs[i].yc, arcs[i].r, arcs[i].circleFitError, &arcs[i].eq, arcs[i].ellipseFitError, + arcs[i].x, arcs[i].y, arcs[i].noPixels); + } + else + { + double coverRatio = MAX(ArcLength(arcs[i].sTheta, arcs[i].eTheta) / CV_2PI, arcs[i].coverRatio); + if ((coverRatio >= FULL_CIRCLE_RATIO && arcs[i].circleFitError <= LONG_ARC_ERROR) || + (coverRatio >= HALF_CIRCLE_RATIO && arcs[i].circleFitError <= HALF_ARC_ERROR) || + (coverRatio >= CANDIDATE_CIRCLE_RATIO2 && arcs[i].circleFitError <= SHORT_ARC_ERROR)) + { + addCircle(circles1, noCircles1, arcs[i].xc, arcs[i].yc, arcs[i].r, arcs[i].circleFitError, arcs[i].x, arcs[i].y, arcs[i].noPixels); + } + } + } + else + { + // If a very short arc, ignore + if (arcs[i].coverRatio < CANDIDATE_CIRCLE_RATIO1) + continue; + + // If the arc is long enough and the circleFitError is small enough, assume a circle + if ((arcs[i].coverRatio >= FULL_CIRCLE_RATIO && arcs[i].circleFitError <= LONG_ARC_ERROR) || + (arcs[i].coverRatio >= HALF_CIRCLE_RATIO && arcs[i].circleFitError <= HALF_ARC_ERROR) || + (arcs[i].coverRatio >= CANDIDATE_CIRCLE_RATIO2 && arcs[i].circleFitError <= SHORT_ARC_ERROR)) + { + + addCircle(circles1, noCircles1, arcs[i].xc, arcs[i].yc, arcs[i].r, arcs[i].circleFitError, arcs[i].x, arcs[i].y, arcs[i].noPixels); + + continue; + } + + if (arcs[i].coverRatio < CANDIDATE_CIRCLE_RATIO2) + continue; + + // Circle is not possible. Try an ellipse + EllipseEquation eq; + double ellipseFitError = 1e10; + double coverRatio(0); + + int noPixels = arcs[i].noPixels; + if (EllipseFit(arcs[i].x, arcs[i].y, noPixels, &eq)) + { + ellipseFitError = ComputeEllipseError(&eq, arcs[i].x, arcs[i].y, noPixels); + coverRatio = noPixels / computeEllipsePerimeter(&eq); + } + + if (arcs[i].coverRatio > coverRatio) + coverRatio = arcs[i].coverRatio; + + if (coverRatio >= CANDIDATE_ELLIPSE_RATIO && ellipseFitError <= ELLIPSE_ERROR) + { + addCircle(circles1, noCircles1, arcs[i].xc, arcs[i].yc, arcs[i].r, arcs[i].circleFitError, &eq, ellipseFitError, arcs[i].x, arcs[i].y, arcs[i].noPixels); + } + } + } +} + +void EdgeDrawingImpl::DetectArcs() +{ + + double maxLineLengthThreshold = MAX(width, height) / 5; + + double MIN_ANGLE = CV_PI / 30; // 6 degrees + double MAX_ANGLE = CV_PI / 3; // 60 degrees + + for (int iter = 1; iter <= 2; iter++) + { + if (iter == 2) + MAX_ANGLE = CV_PI / 1.9; // 95 degrees + + for (int curSegmentNo = 0; curSegmentNo < segmentNos; curSegmentNo++) + { + int firstLine = segmentStartLines[curSegmentNo]; + int stopLine = segmentStartLines[curSegmentNo + 1]; + + // We need at least 2 line segments + if (stopLine - firstLine <= 1) + continue; + + // Process the info for the lines of this segment + while (firstLine < stopLine - 1) + { + // If the line is already taken during the previous step, continue + if (info[firstLine].taken) + { + firstLine++; + continue; + } + + // very long lines cannot be part of an arc + if (lines[firstLine].len >= maxLineLengthThreshold) + { + firstLine++; + continue; + } + + // Skip lines that cannot be part of an arc + if (info[firstLine].angle < MIN_ANGLE || info[firstLine].angle > MAX_ANGLE) + { + firstLine++; + continue; + } + + // Find a group of lines (at least 3) with the same sign & angle < MAX_ANGLE degrees + int lastLine = firstLine + 1; + while (lastLine < stopLine - 1) + { + if (info[lastLine].taken) + break; + if (info[lastLine].sign != info[firstLine].sign) + break; + + if (lines[lastLine].len >= maxLineLengthThreshold) + break; // very long lines cannot be part of an arc + if (info[lastLine].angle < MIN_ANGLE) + break; + if (info[lastLine].angle > MAX_ANGLE) + break; + + lastLine++; + } + + bool specialCase = false; + int wrapCase = -1; // 1: wrap the first two lines with the last line, 2: wrap the last two lines with the first line + + if (lastLine - firstLine == 1) + { + // Just 2 lines. If long enough, then try to combine. Angle between 15 & 45 degrees. Min. length = 40 + int totalLineLength = lines[firstLine].len + lines[firstLine + 1].len; + int shorterLen = lines[firstLine].len; + int longerLen = lines[firstLine + 1].len; + + if (lines[firstLine + 1].len < shorterLen) + { + shorterLen = lines[firstLine + 1].len; + longerLen = lines[firstLine].len; + } + + if (info[firstLine].angle >= CV_PI / 12 && info[firstLine].angle <= CV_PI / 4 && totalLineLength >= 40 && shorterLen * 2 >= longerLen) + { + specialCase = true; + } + + // If the two lines do not make up for arc generation, then try to wrap the lines to the first OR last line. + // There are two wrapper cases: + if (specialCase == false) + { + // Case 1: Combine the first two lines with the last line of the segment + if (firstLine == segmentStartLines[curSegmentNo] && info[stopLine - 1].angle >= MIN_ANGLE && info[stopLine - 1].angle <= MAX_ANGLE) + { + wrapCase = 1; + specialCase = true; + } + + // Case 2: Combine the last two lines with the first line of the segment + else if (lastLine == stopLine - 1 && info[lastLine].angle >= MIN_ANGLE && info[lastLine].angle <= MAX_ANGLE) + { + wrapCase = 2; + specialCase = true; + } + } + + // If still not enough for arc generation, then skip + if (specialCase == false) + { + firstLine = lastLine; + continue; + } + } + + // Copy the pixels of this segment to an array + int noPixels = 0; + double* x = bm->getX(); + double* y = bm->getY(); + + // wrapCase 1: Combine the first two lines with the last line of the segment + if (wrapCase == 1) + { + int index = lines[stopLine - 1].firstPixelIndex; + + for (int n = 0; n < lines[stopLine - 1].len; n++) + { + x[noPixels] = segmentPoints[curSegmentNo][index + n].x; + y[noPixels] = segmentPoints[curSegmentNo][index + n].y; + noPixels++; + } + } + + for (int m = firstLine; m <= lastLine; m++) + { + int index = lines[m].firstPixelIndex; + + for (int n = 0; n < lines[m].len; n++) + { + x[noPixels] = segmentPoints[curSegmentNo][index + n].x; + y[noPixels] = segmentPoints[curSegmentNo][index + n].y; + noPixels++; + } + } + + // wrapCase 2: Combine the last two lines with the first line of the segment + if (wrapCase == 2) + { + int index = lines[segmentStartLines[curSegmentNo]].firstPixelIndex; + + for (int n = 0; n < lines[segmentStartLines[curSegmentNo]].len; n++) + { + x[noPixels] = segmentPoints[curSegmentNo][index + n].x; + y[noPixels] = segmentPoints[curSegmentNo][index + n].y; + noPixels++; + } + } + + // Move buffer pointers + bm->move(noPixels); + + // Try to fit a circle to the entire arc of lines + double xc, yc, radius, circleFitError; + CircleFit(x, y, noPixels, &xc, &yc, &radius, &circleFitError); + + double coverage = noPixels / (CV_2PI * radius); + + // In the case of the special case, the arc must cover at least 22.5 degrees + if (specialCase && coverage < 1.0 / 16) + { + info[firstLine].taken = true; + firstLine = lastLine; + continue; + } + + // If only 3 lines, use the SHORT_ARC_ERROR + double MYERROR = SHORT_ARC_ERROR; + if (lastLine - firstLine >= 3) + MYERROR = LONG_ARC_ERROR; + if (circleFitError <= MYERROR) + { + // Add this to the list of arcs + if (wrapCase == 1) + { + x += lines[stopLine - 1].len; + y += lines[stopLine - 1].len; + noPixels -= lines[stopLine - 1].len; + } + else if (wrapCase == 2) + { + noPixels -= lines[segmentStartLines[curSegmentNo]].len; + } + + if ((coverage >= FULL_CIRCLE_RATIO && circleFitError <= LONG_ARC_ERROR)) + { + addCircle(circles1, noCircles1, xc, yc, radius, circleFitError, x, y, noPixels); + } + else + { + double sTheta, eTheta; + ComputeStartAndEndAngles(xc, yc, radius, x, y, noPixels, &sTheta, &eTheta); + + addArc(edarcs1->arcs, edarcs1->noArcs, xc, yc, radius, circleFitError, sTheta, eTheta, info[firstLine].sign, curSegmentNo, + (int)x[0], (int)y[0], (int)x[noPixels - 1], (int)y[noPixels - 1], x, y, noPixels); + } + + for (int m = firstLine; m < lastLine; m++) + info[m].taken = true; + firstLine = lastLine; + continue; + } + + // Check if this is an almost closed loop (i.e, if 60% of the circle is present). If so, try to fit an ellipse to the entire arc of lines + double dx = x[0] - x[noPixels - 1]; + double dy = y[0] - y[noPixels - 1]; + double distanceBetweenEndPoints = sqrt(dx * dx + dy * dy); + + bool isAlmostClosedLoop = (distanceBetweenEndPoints <= 1.72 * radius && coverage >= FULL_CIRCLE_RATIO); + if (isAlmostClosedLoop || (iter == 1 && coverage >= 0.25)) // an arc covering at least 90 degrees + { + EllipseEquation eq; + double ellipseFitError = 1e10; + + bool valid = EllipseFit(x, y, noPixels, &eq); + if (valid) + ellipseFitError = ComputeEllipseError(&eq, x, y, noPixels); + + MYERROR = ELLIPSE_ERROR; + if (isAlmostClosedLoop == false) + MYERROR = 0.75; + + if (ellipseFitError <= MYERROR) + { + // Add this to the list of arcs + if (wrapCase == 1) + { + x += lines[stopLine - 1].len; + y += lines[stopLine - 1].len; + noPixels -= lines[stopLine - 1].len; + } + else if (wrapCase == 2) + { + noPixels -= lines[segmentStartLines[curSegmentNo]].len; + } + + if (isAlmostClosedLoop) + { + addCircle(circles1, noCircles1, xc, yc, radius, circleFitError, &eq, ellipseFitError, x, y, noPixels); // Add an ellipse for validation + } + else + { + double sTheta, eTheta; + ComputeStartAndEndAngles(xc, yc, radius, x, y, noPixels, &sTheta, &eTheta); + + addArc(edarcs1->arcs, edarcs1->noArcs, xc, yc, radius, circleFitError, sTheta, eTheta, info[firstLine].sign, curSegmentNo, &eq, ellipseFitError, + (int)x[0], (int)y[0], (int)x[noPixels - 1], (int)y[noPixels - 1], x, y, noPixels); + } + + for (int m = firstLine; m < lastLine; m++) + info[m].taken = true; + firstLine = lastLine; + continue; + } + } + + if (specialCase) + { + info[firstLine].taken = true; + firstLine = lastLine; + continue; + } + + // Continue until we finish all lines that belong to arc of lines + while (firstLine <= lastLine - 2) + { + // Fit an initial arc and extend it + int curLine = firstLine + 2; + + // Fit a circle to the pixels of these lines and see if the error is less than a threshold + double XC(0), YC(0), R(0), Error = 1e10; + bool found = false; + + noPixels = 0; + while (curLine <= lastLine) + { + noPixels = 0; + for (int m = firstLine; m <= curLine; m++) + noPixels += lines[m].len; + + // Fit circle + CircleFit(x, y, noPixels, &XC, &YC, &R, &Error); + if (Error <= SHORT_ARC_ERROR) + { + found = true; // found if the error is smaller than the threshold + break; + } + + // Not found. Move to the next set of lines + x += lines[firstLine].len; + y += lines[firstLine].len; + + firstLine++; + curLine++; + } + + // If no initial arc found, then we are done with this arc of lines + if (!found) + break; + + // If we found an initial arc, then extend it + for (int m = curLine - 2; m <= curLine; m++) + info[m].taken = true; + curLine++; + + while (curLine <= lastLine) + { + int noPixelsSave = noPixels; + + noPixels += lines[curLine].len; + + double r, error; + CircleFit(x, y, noPixels, &xc, &yc, &r, &error); + if (error > LONG_ARC_ERROR) + { + noPixels = noPixelsSave; // Adding this line made the error big. So, we do not use this line + break; + } + + // OK. Longer arc + XC = xc; + YC = yc; + R = r; + Error = error; + + info[curLine].taken = true; + curLine++; + } + + coverage = noPixels / (CV_2PI * radius); + if ((coverage >= FULL_CIRCLE_RATIO && circleFitError <= LONG_ARC_ERROR)) + { + addCircle(circles1, noCircles1, XC, YC, R, Error, x, y, noPixels); + + } + else + { + // Add this to the list of arcs + double sTheta, eTheta; + ComputeStartAndEndAngles(XC, YC, R, x, y, noPixels, &sTheta, &eTheta); + + addArc(edarcs1->arcs, edarcs1->noArcs, XC, YC, R, Error, sTheta, eTheta, info[firstLine].sign, curSegmentNo, + (int)x[0], (int)y[0], (int)x[noPixels - 1], (int)y[noPixels - 1], x, y, noPixels); + } + + x += noPixels; + y += noPixels; + + firstLine = curLine; + info[curLine].taken = false; // may reuse the last line? + } + firstLine = lastLine; + } + } + } +} + +// Go over all circles & ellipses and validate them +// The idea here is to look at all pixels of a circle/ellipse +// rather than only the pixels of the lines making up the circle/ellipse +void EdgeDrawingImpl::ValidateCircles(bool validate) +{ + precision = CV_PI / 16; // Alignment precision + double prob = 1.0 / 8; // probability of alignment + + double max = width; + if (height > max) + max = height; + double min = width; + if (height < min) + min = height; + + double* px = new double[8 * (width + height)]; + double* py = new double[8 * (width + height)]; + + int lutSize = (width + height) / 8; + nfa = new NFALUT(lutSize, prob, width, height); // create look up table + + // Validate circles & ellipses + bool validateAgain; + int count = 0; + for (int i = 0; i < noCircles1; ) + { + Circle* circle = &circles1[i]; + double xc = circle->xc; + double yc = circle->yc; + double radius = circle->r; + + // Skip potential invalid circles (sometimes these kinds of candidates get generated!) + if (radius > MAX(width, height)) + { + i++; + continue; + } + + validateAgain = false; + + int noPoints = 0; + + if (circle->isEllipse) + { + noPoints = std::min(static_cast(computeEllipsePerimeter(&circle->eq)), 8 * (width + height)); + + if (noPoints % 2) + noPoints--; + ComputeEllipsePoints(circle->eq.coeff, px, py, noPoints); + } + else + { + ComputeCirclePoints(xc, yc, radius, px, py, &noPoints); + } + + int pr = -1; // previous row + int pc = -1; // previous column + + int tr = -100; + int tc = -100; + int tcount = 0; + + int noPeripheryPixels = 0; + int noEdgePixels = 0; + int aligned = 0; + for (int j = 0; j < noPoints; j++) + { + int r = (int)(py[j] + 0.5); + int c = (int)(px[j] + 0.5); + + if (r == pr && c == pc) + continue; + noPeripheryPixels++; + + if (r <= 0 || r >= height - 1) + continue; + if (c <= 0 || c >= width - 1) + continue; + + pr = r; + pc = c; + + int dr = abs(r - tr); + int dc = abs(c - tc); + if (dr + dc >= 2) + { + tr = r; + tc = c; + tcount++; + } + + // + // See if there is an edge pixel within 1 pixel vicinity + // + if (edgeImg[r * width + c] != 255) + { + // y-cy=-x-cx y-cy=x-cx + // \ / + // \ IV. / + // \ / + // \ / + // III. + I. quadrant + // / \ + // / \ + // / II. \ + // / \ + // + // (x, y)-->(x-cx, y-cy) + // + + int x = c; + int y = r; + + int diff1 = (int)(y - yc - x + xc); + int diff2 = (int)(y - yc + x - xc); + + if (diff1 < 0) + { + if (diff2 > 0) + { + // I. quadrant + c = x - 1; + if (c >= 1 && edgeImg[r * width + c] == 255) + goto out; + c = x + 1; + if (c < width - 1 && edgeImg[r * width + c] == 255) + goto out; + + c = x - 2; + if (c >= 2 && edgeImg[r * width + c] == 255) + goto out; + c = x + 2; + if (c < width - 2 && edgeImg[r * width + c] == 255) + goto out; + } + else + { + // IV. quadrant + r = y - 1; + if (r >= 1 && edgeImg[r * width + c] == 255) + goto out; + r = y + 1; + if (r < height - 1 && edgeImg[r * width + c] == 255) + goto out; + + r = y - 2; + if (r >= 2 && edgeImg[r * width + c] == 255) + goto out; + r = y + 2; + if (r < height - 2 && edgeImg[r * width + c] == 255) + goto out; + } + } + else + { + if (diff2 > 0) + { + // II. quadrant + r = y - 1; + if (r >= 1 && edgeImg[r * width + c] == 255) + goto out; + r = y + 1; + if (r < height - 1 && edgeImg[r * width + c] == 255) + goto out; + + r = y - 2; + if (r >= 2 && edgeImg[r * width + c] == 255) + goto out; + r = y + 2; + if (r < height - 2 && edgeImg[r * width + c] == 255) + goto out; + } + else + { + // III. quadrant + c = x - 1; + if (c >= 1 && edgeImg[r * width + c] == 255) + goto out; + c = x + 1; + if (c < width - 1 && edgeImg[r * width + c] == 255) + goto out; + + c = x - 2; + if (c >= 2 && edgeImg[r * width + c] == 255) + goto out; + c = x + 2; + if (c < width - 2 && edgeImg[r * width + c] == 255) + goto out; + } + } + + r = pr; + c = pc; + continue; // Ignore non-edge pixels. + // This produces less false positives, but occationally misses on some valid circles + } + out: + if (edgeImg[r * width + c] == 255) + noEdgePixels++; + + // compute gx & gy + int com1 = smoothImg[(r + 1) * width + c + 1] - smoothImg[(r - 1) * width + c - 1]; + int com2 = smoothImg[(r - 1) * width + c + 1] - smoothImg[(r + 1) * width + c - 1]; + + int gx = com1 + com2 + smoothImg[r * width + c + 1] - smoothImg[r * width + c - 1]; + int gy = com1 - com2 + smoothImg[(r + 1) * width + c] - smoothImg[(r - 1) * width + c]; + double pixelAngle = nfa->myAtan2((double)gx, (double)-gy); + + double derivX, derivY; + if (circle->isEllipse) + { + // Ellipse + derivX = 2 * circle->eq.A() * c + circle->eq.B() * r + circle->eq.D(); + derivY = circle->eq.B() * c + 2 * circle->eq.C() * r + circle->eq.E(); + } + else + { + // circle + derivX = c - xc; + derivY = r - yc; + } + + double idealPixelAngle = nfa->myAtan2(derivX, -derivY); + double diff = fabs(pixelAngle - idealPixelAngle); + if (diff <= precision || diff >= CV_PI - precision) + aligned++; + } + + bool isValid = !validate || nfa->checkValidationByNFA(noPeripheryPixels, aligned); + + if (isValid) + { + circles2[count++] = circles1[i]; + } + else if (circle->isEllipse == false && circle->coverRatio >= CANDIDATE_ELLIPSE_RATIO) + { + // Fit an ellipse to this circle, and try to revalidate + double ellipseFitError = 1e10; + EllipseEquation eq; + + if (EllipseFit(circle->x, circle->y, circle->noPixels, &eq)) + { + ellipseFitError = ComputeEllipseError(&eq, circle->x, circle->y, circle->noPixels); + } + + if (ellipseFitError <= ELLIPSE_ERROR) + { + circle->isEllipse = true; + circle->ellipseFitError = ellipseFitError; + circle->eq = eq; + + validateAgain = true; + } + } + + if (validateAgain == false) + i++; + } + + noCircles2 = count; + + delete[] px; + delete[] py; + delete nfa; +} + +void EdgeDrawingImpl::JoinCircles() +{ + // Sort the circles wrt their radius + sortCircle(circles2, noCircles2); + + noCircles = noCircles2; + Circle* circles = circles2; + + vector taken; + vector candidateCircles; + int noCandidateCircles; + + for (int i = 0; i < noCircles; i++) + { + taken.push_back(false); + candidateCircles.push_back(0); + + if (circles[i].isEllipse) + { + ComputeEllipseCenterAndAxisLengths(&circles[i].eq, &circles[i].xc, &circles[i].yc, &circles[i].majorAxisLength, &circles[i].minorAxisLength); + } + } + + for (int i = 0; i < noCircles; i++) + { + if (taken[i]) + continue; + + // Current arc + double majorAxisLength, minorAxisLength; + + if (circles[i].isEllipse) + { + majorAxisLength = circles[i].majorAxisLength; + minorAxisLength = circles[i].minorAxisLength; + } + else + { + majorAxisLength = circles[i].r; + minorAxisLength = circles[i].r; + } + + // Find other circles to join with + noCandidateCircles = 0; + + for (int j = i + 1; j < noCircles; j++) + { + if (taken[j]) + continue; + +#define JOINED_SHORT_ARC_ERROR_THRESHOLD 2 +#define AXIS_LENGTH_DIFF_THRESHOLD 6 //(JOINED_SHORT_ARC_ERROR_THRESHOLD*2+1) +#define CENTER_DISTANCE_THRESHOLD 12 //(AXIS_LENGTH_DIFF_THRESHOLD*2) + + double dx = circles[i].xc - circles[j].xc; + double dy = circles[i].yc - circles[j].yc; + double centerDistance = sqrt(dx * dx + dy * dy); + if (centerDistance > CENTER_DISTANCE_THRESHOLD) + continue; + + double diff1, diff2; + if (circles[j].isEllipse) + { + diff1 = fabs(majorAxisLength - circles[j].majorAxisLength); + diff2 = fabs(minorAxisLength - circles[j].minorAxisLength); + } + else + { + diff1 = fabs(majorAxisLength - circles[j].r); + diff2 = fabs(minorAxisLength - circles[j].r); + } + + if (diff1 > AXIS_LENGTH_DIFF_THRESHOLD) + continue; + if (diff2 > AXIS_LENGTH_DIFF_THRESHOLD) + continue; + + // Add to candidates + candidateCircles[noCandidateCircles] = j; + noCandidateCircles++; + } + + // Try to join the current arc with the candidate arc (if there is one) + double XC = circles[i].xc; + double YC = circles[i].yc; + double R = circles[i].r; + + double CircleFitError = circles[i].circleFitError; + bool CircleFitValid = false; + + EllipseEquation Eq; + double EllipseFitError(0); + bool EllipseFitValid = false; + + if (noCandidateCircles > 0) + { + int noPixels = circles[i].noPixels; + double* x = bm->getX(); + double* y = bm->getY(); + memcpy(x, circles[i].x, noPixels * sizeof(double)); + memcpy(y, circles[i].y, noPixels * sizeof(double)); + + for (int j = 0; j < noCandidateCircles; j++) + { + int CandidateArcNo = candidateCircles[j]; + + int noPixelsSave = noPixels; + memcpy(x + noPixels, circles[CandidateArcNo].x, circles[CandidateArcNo].noPixels * sizeof(double)); + memcpy(y + noPixels, circles[CandidateArcNo].y, circles[CandidateArcNo].noPixels * sizeof(double)); + noPixels += circles[CandidateArcNo].noPixels; + + bool circleFitOK = false; + if (EllipseFitValid == false && circles[i].isEllipse == false && circles[CandidateArcNo].isEllipse == false) + { + double xc, yc, r, error = 1e10; + CircleFit(x, y, noPixels, &xc, &yc, &r, &error); + + if (error <= JOINED_SHORT_ARC_ERROR_THRESHOLD) + { + taken[CandidateArcNo] = true; + + XC = xc; + YC = yc; + R = r; + CircleFitError = error; + + circleFitOK = true; + CircleFitValid = true; + } + } + + bool ellipseFitOK = false; + if (circleFitOK == false) + { + // Try to fit an ellipse + double error = 1e10; + EllipseEquation eq; + if (EllipseFit(x, y, noPixels, &eq)) + { + error = ComputeEllipseError(&eq, x, y, noPixels); + } + + if (error <= JOINED_SHORT_ARC_ERROR_THRESHOLD) + { + taken[CandidateArcNo] = true; + + Eq = eq; + EllipseFitError = error; + + ellipseFitOK = true; + EllipseFitValid = true; + CircleFitValid = false; + } + } + + if (circleFitOK == false && ellipseFitOK == false) + { + noPixels = noPixelsSave; + } + } + } + + // Add the new circle/ellipse to circles2 + if (CircleFitValid) + { + addCircle(circles3, noCircles3, XC, YC, R, CircleFitError, NULL, NULL, 0); + } + else if (EllipseFitValid) + { + addCircle(circles3, noCircles3, XC, YC, R, CircleFitError, &Eq, EllipseFitError, NULL, NULL, 0); + } + else + { + circles3[noCircles3] = circles[i]; + noCircles3++; + } + } +} + +void EdgeDrawingImpl::JoinArcs1() +{ + AngleSet angles; + + // Sort the arcs with respect to their length so that longer arcs are at the beginning + sortArc(edarcs1->arcs, edarcs1->noArcs); + + int noArcs = edarcs1->noArcs; + MyArc* arcs = edarcs1->arcs; + + bool* taken = new bool[noArcs]; + for (int i = 0; i < noArcs; i++) + taken[i] = false; + + struct CandidateArc + { + int arcNo; + int which; // 1: (SX, SY)-(sx, sy), 2: (SX, SY)-(ex, ey), 3: (EX, EY)-(sx, sy), 4: (EX, EY)-(ex, ey) + double dist; // min distance between the end points + }; + + CandidateArc* candidateArcs = new CandidateArc[noArcs]; + int noCandidateArcs; + + for (int i = 0; i < noArcs; i++) + { + if (taken[i]) + continue; + if (arcs[i].isEllipse) + { + edarcs2->arcs[edarcs2->noArcs++] = arcs[i]; + continue; + } + + // Current arc + bool CircleEqValid = false; + double XC = arcs[i].xc; + double YC = arcs[i].yc; + double R = arcs[i].r; + double CircleFitError = arcs[i].circleFitError; + int Turn = arcs[i].turn; + int NoPixels = arcs[i].noPixels; + + int SX = arcs[i].sx; + int SY = arcs[i].sy; + int EX = arcs[i].ex; + int EY = arcs[i].ey; + + // Take the pixels making up this arc + int noPixels = arcs[i].noPixels; + + double* x = bm->getX(); + double* y = bm->getY(); + memcpy(x, arcs[i].x, noPixels * sizeof(double)); + memcpy(y, arcs[i].y, noPixels * sizeof(double)); + + angles.clear(); + angles.set(arcs[i].sTheta, arcs[i].eTheta); + + while (1) + { + bool extendedArc = false; + + // Find other arcs to join with + noCandidateArcs = 0; + + for (int j = i + 1; j < noArcs; j++) + { + if (taken[j]) + continue; + if (arcs[j].isEllipse) + continue; + + double minR = MIN(R, arcs[j].r); + double radiusDiffThreshold = minR * 0.25; + + double diff = fabs(R - arcs[j].r); + if (diff > radiusDiffThreshold) + continue; + + // If 50% of the current arc overlaps with the existing arc, then ignore this arc + if (angles.overlap(arcs[j].sTheta, arcs[j].eTheta) >= 0.50) + continue; + + // Compute the distances + // 1: (SX, SY)-(sx, sy) + double dx = SX - arcs[j].sx; + double dy = SY - arcs[j].sy; + double d = sqrt(dx * dx + dy * dy); + int which = 1; + + // 2: (SX, SY)-(ex, ey) + dx = SX - arcs[j].ex; + dy = SY - arcs[j].ey; + double d2 = sqrt(dx * dx + dy * dy); + + if (d2 < d) + { + d = d2; + which = 2; + } + + // 3: (EX, EY)-(sx, sy) + dx = EX - arcs[j].sx; + dy = EY - arcs[j].sy; + d2 = sqrt(dx * dx + dy * dy); + + if (d2 < d) + { + d = d2; + which = 3; + } + + // 4: (EX, EY)-(ex, ey) + dx = EX - arcs[j].ex; + dy = EY - arcs[j].ey; + d2 = sqrt(dx * dx + dy * dy); + + if (d2 < d) + { + d = d2; + which = 4; + } + + // Endpoints must be very close + double maxDistanceBetweenEndpoints = minR * 1.75; //1.5; + if (d > maxDistanceBetweenEndpoints) + continue; + + // This is to give precedence to better matching arc + d += diff; + + // They have to turn in the same direction + if (which == 2 || which == 3) + { + if (Turn != arcs[j].turn) + continue; + } + else + { + if (Turn == arcs[j].turn) + continue; + } + + // Add to candidate arcs in sorted order. User insertion sort + int index = noCandidateArcs - 1; + + while (index >= 0) + { + if (candidateArcs[index].dist < d) + break; + + candidateArcs[index + 1] = candidateArcs[index]; + index--; + } + + // Add the new candidate arc to the candidate list + index++; + candidateArcs[index].arcNo = j; + candidateArcs[index].which = which; + candidateArcs[index].dist = d; + noCandidateArcs++; + } + + // Try to join the current arc with the candidate arc (if there is one) + if (noCandidateArcs > 0) + { + for (int j = 0; j < noCandidateArcs; j++) + { + int CandidateArcNo = candidateArcs[j].arcNo; + int Which = candidateArcs[j].which; + + int noPixelsSave = noPixels; + memcpy(x + noPixels, arcs[CandidateArcNo].x, arcs[CandidateArcNo].noPixels * sizeof(double)); + memcpy(y + noPixels, arcs[CandidateArcNo].y, arcs[CandidateArcNo].noPixels * sizeof(double)); + noPixels += arcs[CandidateArcNo].noPixels; + + double xc, yc, r, circleFitError; + CircleFit(x, y, noPixels, &xc, &yc, &r, &circleFitError); + + if (circleFitError > LONG_ARC_ERROR) + { + // No match. Continue with the next candidate + noPixels = noPixelsSave; + } + else + { + // Match. Take it + extendedArc = true; + CircleEqValid = true; + XC = xc; + YC = yc; + R = r; + CircleFitError = circleFitError; + NoPixels = noPixels; + + taken[CandidateArcNo] = true; + taken[i] = true; + + angles.set(arcs[CandidateArcNo].sTheta, arcs[CandidateArcNo].eTheta); + + // Update the end points of the new arc + switch (Which) + { + // (SX, SY)-(sy, sy) + case 1: + SX = EX, SY = EY; + EX = arcs[CandidateArcNo].ex; + EY = arcs[CandidateArcNo].ey; + if (Turn == 1) + Turn = -1; + else + Turn = 1; // reverse the turn direction + break; + + // (SX, SY)-(ex, ey) + case 2: + SX = EX, SY = EY; + EX = arcs[CandidateArcNo].sx; + EY = arcs[CandidateArcNo].sy; + if (Turn == 1) + Turn = -1; + else + Turn = 1; // reverse the turn direction + break; + + // (EX, EY)-(sx, sy) + case 3: + EX = arcs[CandidateArcNo].ex; + EY = arcs[CandidateArcNo].ey; + break; + + // (EX, EY)-(ex, ey) + case 4: + EX = arcs[CandidateArcNo].sx; + EY = arcs[CandidateArcNo].sy; + break; + } //end-switch + + break; // Do not look at the other candidates + } + } + } + + if (extendedArc == false) + break; + } + + if (CircleEqValid == false) + { + // Add to arcs + edarcs2->arcs[edarcs2->noArcs++] = arcs[i]; + } + else + { + // Add the current OR the extended arc to the new arcs + double sTheta, eTheta; + angles.computeStartEndTheta(sTheta, eTheta); + + double coverage = ArcLength(sTheta, eTheta) / CV_2PI; + if ((coverage >= FULL_CIRCLE_RATIO && CircleFitError <= LONG_ARC_ERROR)) + addCircle(circles1, noCircles1, XC, YC, R, CircleFitError, x, y, NoPixels); + else + addArc(edarcs2->arcs, edarcs2->noArcs, XC, YC, R, CircleFitError, sTheta, eTheta, Turn, arcs[i].segmentNo, SX, SY, EX, EY, x, y, NoPixels, angles.overlapRatio()); + + bm->move(NoPixels); + } + } + + delete[] taken; + delete[] candidateArcs; +} + +void EdgeDrawingImpl::JoinArcs2() +{ + AngleSet angles; + + // Sort the arcs with respect to their length so that longer arcs are at the beginning + sortArc(edarcs2->arcs, edarcs2->noArcs); + + int noArcs = edarcs2->noArcs; + MyArc* arcs = edarcs2->arcs; + + bool* taken = new bool[noArcs]; + for (int i = 0; i < noArcs; i++) + taken[i] = false; + + struct CandidateArc + { + int arcNo; + int which; // 1: (SX, SY)-(sx, sy), 2: (SX, SY)-(ex, ey), 3: (EX, EY)-(sx, sy), 4: (EX, EY)-(ex, ey) + double dist; // min distance between the end points + }; + + CandidateArc* candidateArcs = new CandidateArc[noArcs]; + int noCandidateArcs; + + for (int i = 0; i < noArcs; i++) + { + if (taken[i]) + continue; + + // Current arc + bool EllipseEqValid = false; + EllipseEquation Eq; + double EllipseFitError(0); + + double R = arcs[i].r; + int Turn = arcs[i].turn; + int NoPixels = arcs[i].noPixels; + + int SX = arcs[i].sx; + int SY = arcs[i].sy; + int EX = arcs[i].ex; + int EY = arcs[i].ey; + + // Take the pixels making up this arc + int noPixels = arcs[i].noPixels; + + double* x = bm->getX(); + double* y = bm->getY(); + memcpy(x, arcs[i].x, noPixels * sizeof(double)); + memcpy(y, arcs[i].y, noPixels * sizeof(double)); + + angles.clear(); + angles.set(arcs[i].sTheta, arcs[i].eTheta); + + while (1) + { + bool extendedArc = false; + + // Find other arcs to join with + noCandidateArcs = 0; + + for (int j = i + 1; j < noArcs; j++) + { + if (taken[j]) + continue; + if (arcs[j].segmentNo != arcs[i].segmentNo) + continue; + if (arcs[j].turn != Turn) + continue; + + double minR = MIN(R, arcs[j].r); + double radiusDiffThreshold = minR * 2.5; + + double diff = fabs(R - arcs[j].r); + if (diff > radiusDiffThreshold) + continue; + + // If 75% of the current arc overlaps with the existing arc, then ignore this arc + if (angles.overlap(arcs[j].sTheta, arcs[j].eTheta) >= 0.75) + continue; + + // Compute the distances + // 1: (SX, SY)-(sx, sy) + double dx = SX - arcs[j].sx; + double dy = SY - arcs[j].sy; + double d = sqrt(dx * dx + dy * dy); + int which = 1; + + // 2: (SX, SY)-(ex, ey) + dx = SX - arcs[j].ex; + dy = SY - arcs[j].ey; + double d2 = sqrt(dx * dx + dy * dy); + + if (d2 < d) + { + d = d2; + which = 2; + } + + // 3: (EX, EY)-(sx, sy) + dx = EX - arcs[j].sx; + dy = EY - arcs[j].sy; + d2 = sqrt(dx * dx + dy * dy); + + if (d2 < d) + { + d = d2; + which = 3; + } + + // 4: (EX, EY)-(ex, ey) + dx = EX - arcs[j].ex; + dy = EY - arcs[j].ey; + d2 = sqrt(dx * dx + dy * dy); + + if (d2 < d) + { + d = d2; + which = 4; + } + + // Endpoints must be very close + double maxDistanceBetweenEndpoints = 5; + if (d > maxDistanceBetweenEndpoints) + continue; + + // Add to candidate arcs in sorted order. User insertion sort + int index = noCandidateArcs - 1; + while (index >= 0) + { + if (candidateArcs[index].dist < d) + break; + + candidateArcs[index + 1] = candidateArcs[index]; + index--; + } + + // Add the new candidate arc to the candidate list + index++; + candidateArcs[index].arcNo = j; + candidateArcs[index].which = which; + candidateArcs[index].dist = d; + noCandidateArcs++; + } + + // Try to join the current arc with the candidate arc (if there is one) + if (noCandidateArcs > 0) + { + for (int j = 0; j < noCandidateArcs; j++) + { + int CandidateArcNo = candidateArcs[j].arcNo; + int Which = candidateArcs[j].which; + + int noPixelsSave = noPixels; + memcpy(x + noPixels, arcs[CandidateArcNo].x, arcs[CandidateArcNo].noPixels * sizeof(double)); + memcpy(y + noPixels, arcs[CandidateArcNo].y, arcs[CandidateArcNo].noPixels * sizeof(double)); + noPixels += arcs[CandidateArcNo].noPixels; + + // Directly fit an ellipse + EllipseEquation eq; + double ellipseFitError = 1e10; + if (EllipseFit(x, y, noPixels, &eq)) + ellipseFitError = ComputeEllipseError(&eq, x, y, noPixels); + + if (ellipseFitError > ELLIPSE_ERROR) + { + // No match. Continue with the next candidate + noPixels = noPixelsSave; + } + else + { + // Match. Take it + extendedArc = true; + EllipseEqValid = true; + Eq = eq; + EllipseFitError = ellipseFitError; + NoPixels = noPixels; + + taken[CandidateArcNo] = true; + taken[i] = true; + + R = (R + arcs[CandidateArcNo].r) / 2.0; + + angles.set(arcs[CandidateArcNo].sTheta, arcs[CandidateArcNo].eTheta); + + // Update the end points of the new arc + switch (Which) + { + // (SX, SY)-(sy, sy) + case 1: + SX = EX, SY = EY; + EX = arcs[CandidateArcNo].ex; + EY = arcs[CandidateArcNo].ey; + if (Turn == 1) + Turn = -1; + else + Turn = 1; // reverse the turn direction + break; + + // (SX, SY)-(ex, ey) + case 2: + SX = EX, SY = EY; + EX = arcs[CandidateArcNo].sx; + EY = arcs[CandidateArcNo].sy; + if (Turn == 1) + Turn = -1; + else + Turn = 1; // reverse the turn direction + break; + + // (EX, EY)-(sx, sy) + case 3: + EX = arcs[CandidateArcNo].ex; + EY = arcs[CandidateArcNo].ey; + break; + + // (EX, EY)-(ex, ey) + case 4: + EX = arcs[CandidateArcNo].sx; + EY = arcs[CandidateArcNo].sy; + break; + } + + break; // Do not look at the other candidates + } + } + } + + if (extendedArc == false) + break; + } + + if (EllipseEqValid == false) + { + // Add to arcs + edarcs3->arcs[edarcs3->noArcs++] = arcs[i]; + } + else + { + // Add the current OR the extended arc to the new arcs + double sTheta, eTheta; + angles.computeStartEndTheta(sTheta, eTheta); + + double XC, YC, CircleFitError; + CircleFit(x, y, NoPixels, &XC, &YC, &R, &CircleFitError); + + double coverage = ArcLength(sTheta, eTheta) / CV_2PI; + if ((coverage >= FULL_CIRCLE_RATIO && CircleFitError <= LONG_ARC_ERROR)) + addCircle(circles1, noCircles1, XC, YC, R, CircleFitError, x, y, NoPixels); + else + addArc(edarcs3->arcs, edarcs3->noArcs, XC, YC, R, CircleFitError, sTheta, eTheta, Turn, arcs[i].segmentNo, &Eq, EllipseFitError, SX, SY, EX, EY, x, y, NoPixels, angles.overlapRatio()); + + // Move buffer pointers + bm->move(NoPixels); + } + } + + delete[] taken; + delete[] candidateArcs; +} + +void EdgeDrawingImpl::JoinArcs3() +{ + AngleSet angles; + + // Sort the arcs with respect to their length so that longer arcs are at the beginning + sortArc(edarcs3->arcs, edarcs3->noArcs); + + int noArcs = edarcs3->noArcs; + MyArc* arcs = edarcs3->arcs; + + bool* taken = new bool[noArcs]; + for (int i = 0; i < noArcs; i++) + taken[i] = false; + + struct CandidateArc + { + int arcNo; + int which; // 1: (SX, SY)-(sx, sy), 2: (SX, SY)-(ex, ey), 3: (EX, EY)-(sx, sy), 4: (EX, EY)-(ex, ey) + double dist; // min distance between the end points + }; + + CandidateArc* candidateArcs = new CandidateArc[noArcs]; + int noCandidateArcs; + + for (int i = 0; i < noArcs; i++) + { + if (taken[i]) + continue; + + // Current arc + bool EllipseEqValid = false; + EllipseEquation Eq; + double EllipseFitError(0); + + double R = arcs[i].r; + int Turn = arcs[i].turn; + int NoPixels = arcs[i].noPixels; + + int SX = arcs[i].sx; + int SY = arcs[i].sy; + int EX = arcs[i].ex; + int EY = arcs[i].ey; + + // Take the pixels making up this arc + int noPixels = arcs[i].noPixels; + + double* x = bm->getX(); + double* y = bm->getY(); + memcpy(x, arcs[i].x, noPixels * sizeof(double)); + memcpy(y, arcs[i].y, noPixels * sizeof(double)); + + angles.clear(); + angles.set(arcs[i].sTheta, arcs[i].eTheta); + + while (1) + { + bool extendedArc = false; + + // Find other arcs to join with + noCandidateArcs = 0; + + for (int j = i + 1; j < noArcs; j++) + { + if (taken[j]) + continue; + + /****************************************************************** + * It seems that for minimum false detections, + * radiusDiffThreshold = minR*0.5 & maxDistanceBetweenEndpoints = minR*0.75. + * But these parameters results in many valid misses too! + ******************************************************************/ + + double minR = MIN(R, arcs[j].r); + double diff = fabs(R - arcs[j].r); + if (diff > minR) + continue; + + // If 50% of the current arc overlaps with the existing arc, then ignore this arc + if (angles.overlap(arcs[j].sTheta, arcs[j].eTheta) >= 0.50) + continue; + + // Compute the distances + // 1: (SX, SY)-(sx, sy) + double dx = SX - arcs[j].sx; + double dy = SY - arcs[j].sy; + double d = sqrt(dx * dx + dy * dy); + int which = 1; + + // 2: (SX, SY)-(ex, ey) + dx = SX - arcs[j].ex; + dy = SY - arcs[j].ey; + double d2 = sqrt(dx * dx + dy * dy); + + if (d2 < d) + { + d = d2; + which = 2; + } + + // 3: (EX, EY)-(sx, sy) + dx = EX - arcs[j].sx; + dy = EY - arcs[j].sy; + d2 = sqrt(dx * dx + dy * dy); + + if (d2 < d) + { + d = d2; + which = 3; + } + + // 4: (EX, EY)-(ex, ey) + dx = EX - arcs[j].ex; + dy = EY - arcs[j].ey; + d2 = sqrt(dx * dx + dy * dy); + + if (d2 < d) + { + d = d2; + which = 4; + } + + // Endpoints must be very close + if (diff <= 0.50 * minR) + { + if (d > minR * 0.75) + continue; + } + else if (diff <= 0.75 * minR) + { + if (d > minR * 0.50) + continue; + } + else if (diff <= 1.00 * minR) + { + if (d > minR * 0.25) + continue; + } + else + continue; + + // This is to allow more circular arcs a precedence + d += diff; + + // They have to turn in the same direction + if (which == 2 || which == 3) + { + if (Turn != arcs[j].turn) + continue; + } + else + { + if (Turn == arcs[j].turn) + continue; + } + + // Add to candidate arcs in sorted order. User insertion sort + int index = noCandidateArcs - 1; + while (index >= 0) + { + if (candidateArcs[index].dist < d) + break; + + candidateArcs[index + 1] = candidateArcs[index]; + index--; + } + + // Add the new candidate arc to the candidate list + index++; + candidateArcs[index].arcNo = j; + candidateArcs[index].which = which; + candidateArcs[index].dist = d; + noCandidateArcs++; + } + + // Try to join the current arc with the candidate arc (if there is one) + if (noCandidateArcs > 0) + { + for (int j = 0; j < noCandidateArcs; j++) + { + int CandidateArcNo = candidateArcs[j].arcNo; + int Which = candidateArcs[j].which; + + int noPixelsSave = noPixels; + memcpy(x + noPixels, arcs[CandidateArcNo].x, arcs[CandidateArcNo].noPixels * sizeof(double)); + memcpy(y + noPixels, arcs[CandidateArcNo].y, arcs[CandidateArcNo].noPixels * sizeof(double)); + noPixels += arcs[CandidateArcNo].noPixels; + + // Directly fit an ellipse + EllipseEquation eq; + double ellipseFitError = 1e10; + if (EllipseFit(x, y, noPixels, &eq)) + ellipseFitError = ComputeEllipseError(&eq, x, y, noPixels); + + if (ellipseFitError > ELLIPSE_ERROR) + { + // No match. Continue with the next candidate + noPixels = noPixelsSave; + } + else + { + // Match. Take it + extendedArc = true; + EllipseEqValid = true; + Eq = eq; + EllipseFitError = ellipseFitError; + NoPixels = noPixels; + + taken[CandidateArcNo] = true; + taken[i] = true; + + R = (R + arcs[CandidateArcNo].r) / 2.0; + + angles.set(arcs[CandidateArcNo].sTheta, arcs[CandidateArcNo].eTheta); + + // Update the end points of the new arc + switch (Which) + { + // (SX, SY)-(sy, sy) + case 1: + SX = EX, SY = EY; + EX = arcs[CandidateArcNo].ex; + EY = arcs[CandidateArcNo].ey; + if (Turn == 1) + Turn = -1; + else + Turn = 1; // reverse the turn direction + break; + + // (SX, SY)-(ex, ey) + case 2: + SX = EX, SY = EY; + EX = arcs[CandidateArcNo].sx; + EY = arcs[CandidateArcNo].sy; + if (Turn == 1) + Turn = -1; + else + Turn = 1; // reverse the turn direction + break; + + // (EX, EY)-(sx, sy) + case 3: + EX = arcs[CandidateArcNo].ex; + EY = arcs[CandidateArcNo].ey; + break; + + // (EX, EY)-(ex, ey) + case 4: + EX = arcs[CandidateArcNo].sx; + EY = arcs[CandidateArcNo].sy; + break; + } + break; // Do not look at the other candidates + } + } + } + + if (extendedArc == false) + break; + } + + if (EllipseEqValid == false) + { + // Add to arcs + edarcs4->arcs[edarcs4->noArcs++] = arcs[i]; + } + else + { + // Add the current OR the extended arc to the new arcs + double sTheta, eTheta; + angles.computeStartEndTheta(sTheta, eTheta); + + double XC, YC, CircleFitError; + CircleFit(x, y, NoPixels, &XC, &YC, &R, &CircleFitError); + + double coverage = ArcLength(sTheta, eTheta) / CV_2PI; + if ((coverage >= FULL_CIRCLE_RATIO && CircleFitError <= LONG_ARC_ERROR)) + addCircle(circles1, noCircles1, XC, YC, R, CircleFitError, x, y, NoPixels); + else + addArc(edarcs4->arcs, edarcs4->noArcs, XC, YC, R, CircleFitError, sTheta, eTheta, Turn, arcs[i].segmentNo, &Eq, EllipseFitError, SX, SY, EX, EY, x, y, NoPixels, angles.overlapRatio()); + + bm->move(NoPixels); + } + } + + delete[] taken; + delete[] candidateArcs; +} + +Circle* EdgeDrawingImpl::addCircle(Circle* circles, int& noCircles, double xc, double yc, double r, double circleFitError, double* x, double* y, int noPixels) +{ + circles[noCircles].xc = xc; + circles[noCircles].yc = yc; + circles[noCircles].r = r; + circles[noCircles].circleFitError = circleFitError; + circles[noCircles].coverRatio = noPixels / CV_2PI * r; + + circles[noCircles].x = x; + circles[noCircles].y = y; + circles[noCircles].noPixels = noPixels; + + circles[noCircles].isEllipse = false; + + noCircles++; + + return &circles[noCircles - 1]; +} + +Circle* EdgeDrawingImpl::addCircle(Circle* circles, int& noCircles, double xc, double yc, double r, double circleFitError, EllipseEquation* pEq, double ellipseFitError, double* x, double* y, int noPixels) +{ + circles[noCircles].xc = xc; + circles[noCircles].yc = yc; + circles[noCircles].r = r; + circles[noCircles].circleFitError = circleFitError; + circles[noCircles].coverRatio = noPixels / computeEllipsePerimeter(pEq); + + circles[noCircles].x = x; + circles[noCircles].y = y; + circles[noCircles].noPixels = noPixels; + + circles[noCircles].eq = *pEq; + circles[noCircles].ellipseFitError = ellipseFitError; + circles[noCircles].isEllipse = true; + + noCircles++; + + return &circles[noCircles - 1]; +} + +void EdgeDrawingImpl::sortCircles(Circle* circles, int noCircles) +{ + for (int i = 0; i < noCircles - 1; i++) + { + int max = i; + for (int j = i + 1; j < noCircles; j++) + { + if (circles[j].r > circles[max].r) + max = j; + } + + if (max != i) + { + Circle t = circles[i]; + circles[i] = circles[max]; + circles[max] = t; + } + } +} + + +// --------------------------------------------------------------------------- +// Given an ellipse equation, computes the length of the perimeter of the ellipse +// Calculates the ellipse perimeter wrt the Ramajunan II formula +// +double EdgeDrawingImpl::computeEllipsePerimeter(EllipseEquation* eq) +{ + double mult = 1; + + double A = eq->A() * mult; + double B = eq->B() * mult; + double C = eq->C() * mult; + double D = eq->D() * mult; + double E = eq->E() * mult; + double F = eq->F() * mult; + + double A2(0), C2(0), D2(0), E2(0), F2(0), theta(0); //rotated coefficients + double D3, E3, F3; //ellipse form coefficients + double cX, cY, a, b; //(cX,cY) center, a & b: semimajor & semiminor axes + double h; //h = (a-b)^2 / (a+b)^2 + bool rotation = false; + + //Normalize coefficients + B /= A; + C /= A; + D /= A; + E /= A; + F /= A; + A /= A; + + if (B == 0) //Then not need to rotate the axes + { + A2 = A; + C2 = C; + D2 = D; + E2 = E; + F2 = F; + } + + else if (B != 0) //Rotate the axes + { + rotation = true; + + //Determine the rotation angle (in radians) + theta = atan(B / (A - C)) / 2; + + //Compute the coefficients wrt the new coordinate system + A2 = 0.5 * (A * (1 + cos(2 * theta) + B * sin(2 * theta) + C * (1 - cos(2 * theta)))); + + C2 = 0.5 * (A * (1 - cos(2 * theta) - B * sin(2 * theta) + C * (1 + cos(2 * theta)))); + + D2 = D * cos(theta) + E * sin(theta); + + E2 = -D * sin(theta) + E * cos(theta); + + F2 = F; + } + + //Transform the conic equation into the ellipse form + D3 = D2 / A2; //normalize x term's coef + + E3 = E2 / C2; //normalize y term's coef + + cX = -(D3 / 2); //center X + cY = -(E3 / 2); //center Y + + F3 = A2 * pow(cX, 2.0) + C2 * pow(cY, 2.0) - F2; + + //semimajor axis + a = sqrt(F3 / A2); + //semiminor axis + b = sqrt(F3 / C2); + + //Center coordinates have to be re-transformed if rotation is applied! + if (rotation) + { + double tmpX = cX, tmpY = cY; + cX = tmpX * cos(theta) - tmpY * sin(theta); + cY = tmpX * sin(theta) + tmpY * cos(theta); + } + + //Perimeter Computation(s) + h = pow((a - b), 2.0) / pow((a + b), 2.0); + + //Ramajunan II + double P2 = CV_PI * (a + b) * (1 + 3 * h / (10 + sqrt(4 - 3 * h))); + + return P2; +} + +double EdgeDrawingImpl::ComputeEllipseError(EllipseEquation* eq, double* px, double* py, int noPoints) +{ + double error = 0; + + double A = eq->A(); + double B = eq->B(); + double C = eq->C(); + double D = eq->D(); + double E = eq->E(); + double F = eq->F(); + + double xc, yc, major, minor; + ComputeEllipseCenterAndAxisLengths(eq, &xc, &yc, &major, &minor); + + for (int i = 0; i < noPoints; i++) + { + double dx = px[i] - xc; + double dy = py[i] - yc; + + double min; + double xs; + + if (fabs(dx) > fabs(dy)) + { + // The line equation is of the form: y = mx+n + double m = dy / dx; + double n = yc - m * xc; + + // a*x^2 + b*x + c + double a = A + B * m + C * m * m; + double b = B * n + 2 * C * m * n + D + E * m; + double c = C * n * n + E * n + F; + double det = b * b - 4 * a * c; + if (det < 0) + det = 0; + double x1 = -(b + sqrt(det)) / (2 * a); + double x2 = -(b - sqrt(det)) / (2 * a); + + double y1 = m * x1 + n; + double y2 = m * x2 + n; + + dx = px[i] - x1; + dy = py[i] - y1; + double d1 = dx * dx + dy * dy; + + dx = px[i] - x2; + dy = py[i] - y2; + double d2 = dx * dx + dy * dy; + + if (d1 < d2) + { + min = d1; + xs = x1; + } + else + { + min = d2; + xs = x2; + } + } + else + { + // The line equation is of the form: x = my+n + double m = dx / dy; + double n = xc - m * yc; + + // a*y^2 + b*y + c + double a = A * m * m + B * m + C; + double b = 2 * A * m * n + B * n + D * m + E; + double c = A * n * n + D * n + F; + double det = b * b - 4 * a * c; + if (det < 0) + det = 0; + double y1 = -(b + sqrt(det)) / (2 * a); + double y2 = -(b - sqrt(det)) / (2 * a); + + double x1 = m * y1 + n; + double x2 = m * y2 + n; + + dx = px[i] - x1; + dy = py[i] - y1; + double d1 = dx * dx + dy * dy; + + dx = px[i] - x2; + dy = py[i] - y2; + double d2 = dx * dx + dy * dy; + + if (d1 < d2) + { + min = d1; + xs = x1; + } + else + { + min = d2; + xs = x2; + } + } + + // Refine the search in the vicinity of (xs, ys) + double delta = 0.5; + double x = xs; + while (1) + { + x += delta; + + double a = C; + double b = B * x + E; + double c = A * x * x + D * x + F; + double det = b * b - 4 * a * c; + if (det < 0) + det = 0; + + double y1 = -(b + sqrt(det)) / (2 * a); + double y2 = -(b - sqrt(det)) / (2 * a); + + dx = px[i] - x; + dy = py[i] - y1; + double d1 = dx * dx + dy * dy; + + dy = py[i] - y2; + double d2 = dx * dx + dy * dy; + + if (d1 <= min) + { + min = d1; + } + else if (d2 <= min) + { + min = d2; + } + else + break; + } + + x = xs; + while (1) + { + x -= delta; + + double a = C; + double b = B * x + E; + double c = A * x * x + D * x + F; + double det = b * b - 4 * a * c; + if (det < 0) + det = 0; + + double y1 = -(b + sqrt(det)) / (2 * a); + double y2 = -(b - sqrt(det)) / (2 * a); + + dx = px[i] - x; + dy = py[i] - y1; + double d1 = dx * dx + dy * dy; + + dy = py[i] - y2; + double d2 = dx * dx + dy * dy; + + if (d1 <= min) + { + min = d1; + } + else if (d2 <= min) + { + min = d2; + } + else + break; + } + error += min; + } + + error = sqrt(error / noPoints); + + return error; +} + +// also returns rotate angle theta +double EdgeDrawingImpl::ComputeEllipseCenterAndAxisLengths(EllipseEquation* eq, double* pxc, double* pyc, double* pmajorAxisLength, double* pminorAxisLength) +{ + double mult = 1; + + double A = eq->A() * mult; + double B = eq->B() * mult; + double C = eq->C() * mult; + double D = eq->D() * mult; + double E = eq->E() * mult; + double F = eq->F() * mult; + + double A2(0), C2(0), D2(0), E2(0), F2(0), theta(0); //rotated coefficients + double D3, E3, F3; //ellipse form coefficients + double cX, cY, a, b; //(cX,cY) center, a & b: semimajor & semiminor axes + bool rotation = false; + + //Normalize coefficients + B /= A; + C /= A; + D /= A; + E /= A; + F /= A; + A /= A; + + if (B == 0) //Then not need to rotate the axes + { + A2 = A; + C2 = C; + D2 = D; + E2 = E; + F2 = F; + } + else if (B != 0) //Rotate the axes + { + rotation = true; + + //Determine the rotation angle (in radians) + theta = atan(B / (A - C)) / 2; + + //Compute the coefficients wrt the new coordinate system + A2 = 0.5 * (A * (1 + cos(2 * theta) + B * sin(2 * theta) + C * (1 - cos(2 * theta)))); + + C2 = 0.5 * (A * (1 - cos(2 * theta) - B * sin(2 * theta) + C * (1 + cos(2 * theta)))); + + D2 = D * cos(theta) + E * sin(theta); + + E2 = -D * sin(theta) + E * cos(theta); + + F2 = F; + } + + //Transform the conic equation into the ellipse form + D3 = D2 / A2; //normalize x term's coef + + E3 = E2 / C2; //normalize y term's coef + + cX = -(D3 / 2); //center X + cY = -(E3 / 2); //center Y + + F3 = A2 * pow(cX, 2.0) + C2 * pow(cY, 2.0) - F2; + + //semimajor axis + a = sqrt(F3 / A2); + //semiminor axis + b = sqrt(F3 / C2); + + //Center coordinates have to be re-transformed if rotation is applied! + if (rotation) + { + double tmpX = cX, tmpY = cY; + cX = tmpX * cos(theta) - tmpY * sin(theta); + cY = tmpX * sin(theta) + tmpY * cos(theta); + } + + *pxc = cX; + *pyc = cY; + + *pmajorAxisLength = a; + *pminorAxisLength = b; + + return theta; +} + +// --------------------------------------------------------------------------- +// Given an ellipse equation, computes "noPoints" many consecutive points +// on the ellipse periferi. These points can be used to draw the ellipse +// noPoints must be an even number. +// +void EdgeDrawingImpl::ComputeEllipsePoints(double* pvec, double* px, double* py, int noPoints) +{ + if (noPoints % 2) + noPoints--; + int npts = noPoints / 2; + + double** u = AllocateMatrix(3, npts + 1); + double** Aiu = AllocateMatrix(3, npts + 1); + double** L = AllocateMatrix(3, npts + 1); + double** B = AllocateMatrix(3, npts + 1); + double** Xpos = AllocateMatrix(3, npts + 1); + double** Xneg = AllocateMatrix(3, npts + 1); + double** ss1 = AllocateMatrix(3, npts + 1); + double** ss2 = AllocateMatrix(3, npts + 1); + double* lambda = new double[npts + 1]; + double** uAiu = AllocateMatrix(3, npts + 1); + double** A = AllocateMatrix(3, 3); + double** Ai = AllocateMatrix(3, 3); + double** Aib = AllocateMatrix(3, 2); + double** b = AllocateMatrix(3, 2); + double** r1 = AllocateMatrix(2, 2); + double Ao, Ax, Ay, Axx, Ayy, Axy; + double theta; + int i; + int j; + double kk; + + memset(lambda, 0, sizeof(double) * (npts + 1)); + + Ao = pvec[6]; + Ax = pvec[4]; + Ay = pvec[5]; + Axx = pvec[1]; + Ayy = pvec[3]; + Axy = pvec[2]; + + A[1][1] = Axx; + A[1][2] = Axy / 2; + A[2][1] = Axy / 2; + A[2][2] = Ayy; + b[1][1] = Ax; + b[2][1] = Ay; + + // Generate normals linspace + for (i = 1, theta = 0.0; i <= npts; i++, theta += (CV_PI / npts)) + { + u[1][i] = cos(theta); + u[2][i] = sin(theta); + } + + inverse(A, Ai, 2); + + AperB(Ai, b, Aib, 2, 2, 2, 1); + A_TperB(b, Aib, r1, 2, 1, 2, 1); + r1[1][1] = r1[1][1] - 4 * Ao; + + AperB(Ai, u, Aiu, 2, 2, 2, npts); + for (i = 1; i <= 2; i++) + for (j = 1; j <= npts; j++) + uAiu[i][j] = u[i][j] * Aiu[i][j]; + + for (j = 1; j <= npts; j++) + { + if ((kk = (r1[1][1] / (uAiu[1][j] + uAiu[2][j]))) >= 0.0) + lambda[j] = sqrt(kk); + else + lambda[j] = -1.0; + } + + // Builds up B and L + for (j = 1; j <= npts; j++) + L[1][j] = L[2][j] = lambda[j]; + for (j = 1; j <= npts; j++) + { + B[1][j] = b[1][1]; + B[2][j] = b[2][1]; + } + + for (j = 1; j <= npts; j++) + { + ss1[1][j] = 0.5 * (L[1][j] * u[1][j] - B[1][j]); + ss1[2][j] = 0.5 * (L[2][j] * u[2][j] - B[2][j]); + ss2[1][j] = 0.5 * (-L[1][j] * u[1][j] - B[1][j]); + ss2[2][j] = 0.5 * (-L[2][j] * u[2][j] - B[2][j]); + } + + AperB(Ai, ss1, Xpos, 2, 2, 2, npts); + AperB(Ai, ss2, Xneg, 2, 2, 2, npts); + + for (j = 1; j <= npts; j++) + { + if (lambda[j] == -1.0) + { + px[j - 1] = -1; + py[j - 1] = -1; + px[j - 1 + npts] = -1; + py[j - 1 + npts] = -1; + } + else + { + px[j - 1] = Xpos[1][j]; + py[j - 1] = Xpos[2][j]; + px[j - 1 + npts] = Xneg[1][j]; + py[j - 1 + npts] = Xneg[2][j]; + } + } + + DeallocateMatrix(u, 3); + DeallocateMatrix(Aiu, 3); + DeallocateMatrix(L, 3); + DeallocateMatrix(B, 3); + DeallocateMatrix(Xpos, 3); + DeallocateMatrix(Xneg, 3); + DeallocateMatrix(ss1, 3); + DeallocateMatrix(ss2, 3); + delete[] lambda; + DeallocateMatrix(uAiu, 3); + DeallocateMatrix(A, 3); + DeallocateMatrix(Ai, 3); + DeallocateMatrix(Aib, 3); + DeallocateMatrix(b, 3); + DeallocateMatrix(r1, 2); +} + + +// Tries to join the last two arcs if their end-points are very close to each other +// and if they are part of the same segment. This is useful in cases where an arc on a segment +// is broken due to a noisy patch along the arc, and the long arc is broken into two or more arcs. +// This function will join such broken arcs +// +void EdgeDrawingImpl::joinLastTwoArcs(MyArc* arcs, int& noArcs) +{ + if (noArcs < 2) + return; + + int prev = noArcs - 2; + int last = noArcs - 1; + + if (arcs[prev].segmentNo != arcs[last].segmentNo) + return; + if (arcs[prev].turn != arcs[last].turn) + return; + if (arcs[prev].isEllipse || arcs[last].isEllipse) + return; + + // The radius difference between the arcs must be very small + double minR = MIN(arcs[prev].r, arcs[last].r); + double radiusDiffThreshold = minR * 0.25; + + double diff = fabs(arcs[prev].r - arcs[last].r); + if (diff > radiusDiffThreshold) + return; + + // End-point distance + double dx = arcs[prev].ex - arcs[last].sx; + double dy = arcs[prev].ey - arcs[last].sy; + double d = sqrt(dx * dx + dy * dy); + + double endPointDiffThreshold = 10; + if (d > endPointDiffThreshold) + return; + + // Try join + int noPixels = arcs[prev].noPixels + arcs[last].noPixels; + + double xc, yc, r, circleFitError; + CircleFit(arcs[prev].x, arcs[prev].y, noPixels, &xc, &yc, &r, &circleFitError); + + if (circleFitError <= LONG_ARC_ERROR) + { + arcs[prev].noPixels = noPixels; + arcs[prev].circleFitError = circleFitError; + + arcs[prev].xc = xc; + arcs[prev].yc = yc; + arcs[prev].r = r; + arcs[prev].ex = arcs[last].ex; + arcs[prev].ey = arcs[last].ey; + + AngleSet angles; + angles.set(arcs[prev].sTheta, arcs[prev].eTheta); + angles.set(arcs[last].sTheta, arcs[last].eTheta); + angles.computeStartEndTheta(arcs[prev].sTheta, arcs[prev].eTheta); + + arcs[prev].coverRatio = ArcLength(arcs[prev].sTheta, arcs[prev].eTheta) / (CV_2PI); + + noArcs--; + } +} + + +//----------------------------------------------------------------------- +// Add a new arc to arcs +// +void EdgeDrawingImpl::addArc(MyArc* arcs, int& noArcs, double xc, double yc, double r, double circleFitError, double sTheta, double eTheta, int turn, int segmentNo, int sx, int sy, int ex, int ey, double* x, double* y, int noPixels, double overlapRatio) +{ + CV_UNUSED(overlapRatio); + arcs[noArcs].xc = xc; + arcs[noArcs].yc = yc; + arcs[noArcs].r = r; + arcs[noArcs].circleFitError = circleFitError; + + arcs[noArcs].sTheta = sTheta; + arcs[noArcs].eTheta = eTheta; + arcs[noArcs].coverRatio = ArcLength(sTheta, eTheta) / CV_2PI; + + arcs[noArcs].turn = turn; + + arcs[noArcs].segmentNo = segmentNo; + + arcs[noArcs].isEllipse = false; + + arcs[noArcs].sx = sx; + arcs[noArcs].sy = sy; + arcs[noArcs].ex = ex; + arcs[noArcs].ey = ey; + + arcs[noArcs].x = x; + arcs[noArcs].y = y; + arcs[noArcs].noPixels = noPixels; + + noArcs++; + + // See if you can join the last two arcs + joinLastTwoArcs(arcs, noArcs); +} + +//------------------------------------------------------------------------- +// Add an elliptic arc to the list of arcs +// +void EdgeDrawingImpl::addArc(MyArc* arcs, int& noArcs, double xc, double yc, double r, double circleFitError, double sTheta, double eTheta, int turn, int segmentNo, EllipseEquation* pEq, double ellipseFitError, int sx, int sy, int ex, int ey, double* x, double* y, int noPixels, double overlapRatio) +{ + arcs[noArcs].xc = xc; + arcs[noArcs].yc = yc; + arcs[noArcs].r = r; + arcs[noArcs].circleFitError = circleFitError; + + arcs[noArcs].sTheta = sTheta; + arcs[noArcs].eTheta = eTheta; + arcs[noArcs].coverRatio = (double)((1.0 - overlapRatio) * noPixels) / computeEllipsePerimeter(pEq); + arcs[noArcs].turn = turn; + + arcs[noArcs].segmentNo = segmentNo; + + arcs[noArcs].isEllipse = true; + arcs[noArcs].eq = *pEq; + arcs[noArcs].ellipseFitError = ellipseFitError; + + arcs[noArcs].sx = sx; + arcs[noArcs].sy = sy; + arcs[noArcs].ex = ex; + arcs[noArcs].ey = ey; + + arcs[noArcs].x = x; + arcs[noArcs].y = y; + arcs[noArcs].noPixels = noPixels; + + noArcs++; +} + +//-------------------------------------------------------------- +// Given a circular arc, computes the start & end angles of the arc in radians +// +void EdgeDrawingImpl::ComputeStartAndEndAngles(double xc, double yc, double r, double* x, double* y, int len, double* psTheta, double* peTheta) +{ + double sx = x[0]; + double sy = y[0]; + double ex = x[len - 1]; + double ey = y[len - 1]; + double mx = x[len / 2]; + double my = y[len / 2]; + + double d = (sx - xc) / r; + if (d > 1.0) + d = 1.0; + else if (d < -1.0) + d = -1.0; + double theta1 = acos(d); + + double sTheta; + if (sx >= xc) + { + if (sy >= yc) + { + // I. quadrant + sTheta = theta1; + } + else + { + // IV. quadrant + sTheta = CV_2PI - theta1; + } + } + else + { + if (sy >= yc) + { + // II. quadrant + sTheta = theta1; + } + else + { + // III. quadrant + sTheta = CV_2PI - theta1; + } + } + + d = (ex - xc) / r; + if (d > 1.0) + d = 1.0; + else if (d < -1.0) + d = -1.0; + theta1 = acos(d); + + double eTheta; + if (ex >= xc) + { + if (ey >= yc) + { + // I. quadrant + eTheta = theta1; + } + else + { + // IV. quadrant + eTheta = CV_2PI - theta1; + } + } + else + { + if (ey >= yc) + { + // II. quadrant + eTheta = theta1; + } + else + { + // III. quadrant + eTheta = CV_2PI - theta1; + } + } + + // Determine whether the arc is clockwise (CW) or counter-clockwise (CCW) + double circumference = CV_2PI * r; + double ratio = len / circumference; + + if (ratio <= 0.25 || ratio >= 0.75) + { + double angle1, angle2; + + if (eTheta > sTheta) + { + angle1 = eTheta - sTheta; + angle2 = CV_2PI - eTheta + sTheta; + } + else + { + angle1 = sTheta - eTheta; + angle2 = CV_2PI - sTheta + eTheta; + } + + angle1 = angle1 / CV_2PI; + angle2 = angle2 / CV_2PI; + + double diff1 = fabs(ratio - angle1); + double diff2 = fabs(ratio - angle2); + + if (diff1 < diff2) + { + // angle1 is correct + if (eTheta > sTheta) + { + ; + } + else + { + double tmp = sTheta; + sTheta = eTheta; + eTheta = tmp; + } + } + else + { + // angle2 is correct + if (eTheta > sTheta) + { + double tmp = sTheta; + sTheta = eTheta; + eTheta = tmp; + } + } + } + else + { + double v1x = mx - sx; + double v1y = my - sy; + double v2x = ex - mx; + double v2y = ey - my; + + // cross product + double cross = v1x * v2y - v1y * v2x; + if (cross < 0) + { + // swap sTheta & eTheta + double tmp = sTheta; + sTheta = eTheta; + eTheta = tmp; + } + } + + double diff = fabs(sTheta - eTheta); + if (diff < (CV_2PI / 120)) + { + sTheta = 0; + eTheta = 6.26; // 359 degrees + } + + // Round the start & etheta to 0 if very close to 6.28 or 0 + if (sTheta >= 6.26) + sTheta = 0; + if (eTheta < 1.0 / CV_2PI) + eTheta = 6.28; // if less than 1 degrees, then round to 6.28 + + *psTheta = sTheta; + *peTheta = eTheta; +} + +void EdgeDrawingImpl::sortArc(MyArc* arcs, int noArcs) +{ + for (int i = 0; i < noArcs - 1; i++) + { + int max = i; + for (int j = i + 1; j < noArcs; j++) + { + if (arcs[j].coverRatio > arcs[max].coverRatio) + max = j; + } + + if (max != i) + { + MyArc t = arcs[i]; + arcs[i] = arcs[max]; + arcs[max] = t; + } + } +} + + +//--------------------------------------------------------------------- +// Fits a circle to a given set of points. There must be at least 2 points +// The circle equation is of the form: (x-xc)^2 + (y-yc)^2 = r^2 +// Returns true if there is a fit, false in case no circles can be fit +// +bool EdgeDrawingImpl::CircleFit(double* x, double* y, int N, double* pxc, double* pyc, double* pr, double* pe) +{ + *pe = 1e20; + if (N < 3) + return false; + + double xAvg = 0; + double yAvg = 0; + + for (int i = 0; i < N; i++) + { + xAvg += x[i]; + yAvg += y[i]; + } + + xAvg /= N; + yAvg /= N; + + double Suu = 0; + double Suv = 0; + double Svv = 0; + double Suuu = 0; + double Suvv = 0; + double Svvv = 0; + double Svuu = 0; + + for (int i = 0; i < N; i++) + { + double u = x[i] - xAvg; + double v = y[i] - yAvg; + + Suu += u * u; + Suv += u * v; + Svv += v * v; + Suuu += u * u * u; + Suvv += u * v * v; + Svvv += v * v * v; + Svuu += v * u * u; + } + + // Now, we solve for the following linear system of equations + // Av = b, where v = (uc, vc) is the center of the circle + // + // |N Suv| |uc| = |b1| + // |Suv Svv| |vc| = |b2| + // + // where b1 = 0.5*(Suuu+Suvv) and b2 = 0.5*(Svvv+Svuu) + // + double detA = Suu * Svv - Suv * Suv; + if (detA == 0) + return false; + + double b1 = 0.5 * (Suuu + Suvv); + double b2 = 0.5 * (Svvv + Svuu); + + double uc = (Svv * b1 - Suv * b2) / detA; + double vc = (Suu * b2 - Suv * b1) / detA; + + double R = sqrt(uc * uc + vc * vc + (Suu + Svv) / N); + + *pxc = uc + xAvg; + *pyc = vc + yAvg; + + // Compute mean square error + double error = 0; + for (int i = 0; i < N; i++) + { + double dx = x[i] - *pxc; + double dy = y[i] - *pyc; + double d = sqrt(dx * dx + dy * dy) - R; + error += d * d; + } + + *pr = R; + *pe = sqrt(error / N); + + return true; +} + + +//------------------------------------------------------------------------------------ +// Computes the points making up a circle +// +void EdgeDrawingImpl::ComputeCirclePoints(double xc, double yc, double r, double* px, double* py, int* noPoints) +{ + int len = (int)(CV_2PI * r + 0.5); + double angleInc = CV_2PI / len; + double angle = 0; + + int count = 0; + + while (angle < CV_2PI) + { + int x = (int)(cos(angle) * r + xc + 0.5); + int y = (int)(sin(angle) * r + yc + 0.5); + + angle += angleInc; + + px[count] = x; + py[count] = y; + count++; + } + + *noPoints = count; +} + +void EdgeDrawingImpl::sortCircle(Circle* circles, int noCircles) +{ + for (int i = 0; i < noCircles - 1; i++) + { + int max = i; + for (int j = i + 1; j < noCircles; j++) + { + if (circles[j].r > circles[max].r) + max = j; + } + + if (max != i) + { + Circle t = circles[i]; + circles[i] = circles[max]; + circles[max] = t; + } + } +} + +bool EdgeDrawingImpl::EllipseFit(double* x, double* y, int noPoints, EllipseEquation* pResult, int mode) +{ + double** D = AllocateMatrix(noPoints + 1, 7); + double** S = AllocateMatrix(7, 7); + double** Const = AllocateMatrix(7, 7); + double** temp = AllocateMatrix(7, 7); + double** L = AllocateMatrix(7, 7); + double** C = AllocateMatrix(7, 7); + + double** invL = AllocateMatrix(7, 7); + double* d = new double[7]; + double** V = AllocateMatrix(7, 7); + double** sol = AllocateMatrix(7, 7); + double tx, ty; + int nrot = 0; + + memset(d, 0, sizeof(double) * 7); + + switch (mode) + { + case (FPF): + Const[1][3] = -2; + Const[2][2] = 1; + Const[3][1] = -2; + break; + case (BOOKSTEIN): + Const[1][1] = 2; + Const[2][2] = 1; + Const[3][3] = 2; + } + + if (noPoints < 6) + return false; + + // Now first fill design matrix + for (int i = 1; i <= noPoints; i++) + { + tx = x[i - 1]; + ty = y[i - 1]; + + D[i][1] = tx * tx; + D[i][2] = tx * ty; + D[i][3] = ty * ty; + D[i][4] = tx; + D[i][5] = ty; + D[i][6] = 1.0; + } + + // Now compute scatter matrix S + A_TperB(D, D, S, noPoints, 6, noPoints, 6); + + choldc(S, 6, L); + + inverse(L, invL, 6); + + AperB_T(Const, invL, temp, 6, 6, 6, 6); + AperB(invL, temp, C, 6, 6, 6, 6); + + jacobi(C, 6, d, V, nrot); + + A_TperB(invL, V, sol, 6, 6, 6, 6); + + // Now normalize them + for (int j = 1; j <= 6; j++) /* Scan columns */ + { + double mod = 0.0; + for (int i = 1; i <= 6; i++) + mod += sol[i][j] * sol[i][j]; + for (int i = 1; i <= 6; i++) + sol[i][j] /= sqrt(mod); + } + + double zero = 10e-20; + double minev = 10e+20; + int solind = 0; + int i; + switch (mode) + { + case (BOOKSTEIN): // smallest eigenvalue + for (i = 1; i <= 6; i++) + if (d[i] < minev && fabs(d[i]) > zero) + solind = i; + break; + case (FPF): + for (i = 1; i <= 6; i++) + if (d[i] < 0 && fabs(d[i]) > zero) + solind = i; + } + + bool valid = true; + if (solind == 0) + valid = false; + + if (valid) + { + // Now fetch the right solution + for (int j = 1; j <= 6; j++) + { + pResult->coeff[j] = sol[j][solind]; + } + } + + DeallocateMatrix(D, noPoints + 1); + DeallocateMatrix(S, 7); + DeallocateMatrix(Const, 7); + DeallocateMatrix(temp, 7); + DeallocateMatrix(L, 7); + DeallocateMatrix(C, 7); + DeallocateMatrix(invL, 7); + delete[] d; + DeallocateMatrix(V, 7); + DeallocateMatrix(sol, 7); + + if (valid) + { + int len = (int)computeEllipsePerimeter(pResult); + if (len <= 0 || len > 50000) + valid = false; + } + + return valid; +} + +double** EdgeDrawingImpl::AllocateMatrix(int noRows, int noColumns) +{ + double** m = new double* [noRows]; + + for (int i = 0; i < noRows; i++) + { + m[i] = new double[noColumns]; + memset(m[i], 0, sizeof(double) * noColumns); + } + + return m; +} + +void EdgeDrawingImpl::A_TperB(double** A_, double** B_, double** _res, int _righA, int _colA, int _righB, int _colB) +{ + CV_UNUSED(_righB); + int p, q, l; + for (p = 1; p <= _colA; p++) + for (q = 1; q <= _colB; q++) + { + _res[p][q] = 0.0; + for (l = 1; l <= _righA; l++) + _res[p][q] = _res[p][q] + A_[l][p] * B_[l][q]; + } +} + +//----------------------------------------------------------- +// Perform the Cholesky decomposition +// Return the lower triangular L such that L*L'=A +// +void EdgeDrawingImpl::choldc(double** a, int n, double** l) +{ + int i, j, k; + double sum; + double* p = new double[n + 1]; + memset(p, 0, sizeof(double) * (n + 1)); + + for (i = 1; i <= n; i++) + { + for (j = i; j <= n; j++) + { + for (sum = a[i][j], k = i - 1; k >= 1; k--) + sum -= a[i][k] * a[j][k]; + if (i == j) + { + if (sum <= 0.0) + { + } + else + p[i] = sqrt(sum); + } + else + { + a[j][i] = sum / p[i]; + } + } + } + + for (i = 1; i <= n; i++) + { + for (j = i; j <= n; j++) + { + if (i == j) + l[i][i] = p[i]; + else + { + l[j][i] = a[j][i]; + l[i][j] = 0.0; + } + } + } + + delete[] p; +} + +int EdgeDrawingImpl::inverse(double** TB, double** InvB, int N) +{ + int k, i, j, p, q; + double mult; + double D, temp; + double maxpivot; + int npivot; + double** B = AllocateMatrix(N + 1, N + 2); + double** A = AllocateMatrix(N + 1, 2 * N + 2); + double** C = AllocateMatrix(N + 1, N + 1); + double eps = 10e-20; + + for (k = 1; k <= N; k++) + for (j = 1; j <= N; j++) + B[k][j] = TB[k][j]; + + for (k = 1; k <= N; k++) + { + for (j = 1; j <= N + 1; j++) + A[k][j] = B[k][j]; + for (j = N + 2; j <= 2 * N + 1; j++) + A[k][j] = (double)0; + A[k][k - 1 + N + 2] = (double)1; + } + for (k = 1; k <= N; k++) + { + maxpivot = fabs((double)A[k][k]); + npivot = k; + for (i = k; i <= N; i++) + if (maxpivot < fabs((double)A[i][k])) + { + maxpivot = fabs((double)A[i][k]); + npivot = i; + } + if (maxpivot >= eps) + { + if (npivot != k) + for (j = k; j <= 2 * N + 1; j++) + { + temp = A[npivot][j]; + A[npivot][j] = A[k][j]; + A[k][j] = temp; + }; + D = A[k][k]; + for (j = 2 * N + 1; j >= k; j--) + A[k][j] = A[k][j] / D; + for (i = 1; i <= N; i++) + { + if (i != k) + { + mult = A[i][k]; + for (j = 2 * N + 1; j >= k; j--) + A[i][j] = A[i][j] - mult * A[k][j]; + } + } + } + else + { + DeallocateMatrix(B, N + 1); + DeallocateMatrix(A, N + 1); + DeallocateMatrix(C, N + 1); + + return (-1); + } + } + + for (k = 1, p = 1; k <= N; k++, p++) + for (j = N + 2, q = 1; j <= 2 * N + 1; j++, q++) + InvB[p][q] = A[k][j]; + + DeallocateMatrix(B, N + 1); + DeallocateMatrix(A, N + 1); + DeallocateMatrix(C, N + 1); + + return (0); +} + +void EdgeDrawingImpl::DeallocateMatrix(double** m, int noRows) +{ + for (int i = 0; i < noRows; i++) + delete m[i]; + delete m; +} + +void EdgeDrawingImpl::AperB_T(double** A_, double** B_, double** _res, int _righA, int _colA, int _righB, int _colB) +{ + CV_UNUSED(_righB); + int p, q, l; + for (p = 1; p <= _colA; p++) + for (q = 1; q <= _colB; q++) + { + _res[p][q] = 0.0; + for (l = 1; l <= _righA; l++) + _res[p][q] = _res[p][q] + A_[p][l] * B_[q][l]; + } +} + +void EdgeDrawingImpl::AperB(double** A_, double** B_, double** _res, int _righA, int _colA, int _righB, int _colB) +{ + CV_UNUSED(_righB); + int p, q, l; + for (p = 1; p <= _righA; p++) + for (q = 1; q <= _colB; q++) + { + _res[p][q] = 0.0; + for (l = 1; l <= _colA; l++) + _res[p][q] = _res[p][q] + A_[p][l] * B_[l][q]; + } +} + +void EdgeDrawingImpl::jacobi(double** a, int n, double d[], double** v, int nrot) +{ + int j, iq, ip, i; + double tresh, theta, tau, t, sm, s, h, g, c; + + double* b = new double[n + 1]; + double* z = new double[n + 1]; + memset(b, 0, sizeof(double) * (n + 1)); + memset(z, 0, sizeof(double) * (n + 1)); + + for (ip = 1; ip <= n; ip++) + { + for (iq = 1; iq <= n; iq++) + v[ip][iq] = 0.0; + v[ip][ip] = 1.0; + } + for (ip = 1; ip <= n; ip++) + { + b[ip] = d[ip] = a[ip][ip]; + z[ip] = 0.0; + } + nrot = 0; + for (i = 1; i <= 50; i++) + { + sm = 0.0; + for (ip = 1; ip <= n - 1; ip++) + { + for (iq = ip + 1; iq <= n; iq++) + sm += fabs(a[ip][iq]); + } + if (sm == 0.0) + { + delete[] b; + delete[] z; + return; + } + if (i < 4) + tresh = 0.2 * sm / (n * n); + else + tresh = 0.0; + for (ip = 1; ip <= n - 1; ip++) + { + for (iq = ip + 1; iq <= n; iq++) + { + g = 100.0 * fabs(a[ip][iq]); + + if (i > 4 && g == 0.0) + a[ip][iq] = 0.0; + else if (fabs(a[ip][iq]) > tresh) + { + h = d[iq] - d[ip]; + if (g == 0.0) + t = (a[ip][iq]) / h; + else + { + theta = 0.5 * h / (a[ip][iq]); + t = 1.0 / (fabs(theta) + sqrt(1.0 + theta * theta)); + if (theta < 0.0) + t = -t; + } + c = 1.0 / sqrt(1 + t * t); + s = t * c; + tau = s / (1.0 + c); + h = t * a[ip][iq]; + z[ip] -= h; + z[iq] += h; + d[ip] -= h; + d[iq] += h; + a[ip][iq] = 0.0; + for (j = 1; j <= ip - 1; j++) + { + ROTATE(a, j, ip, j, iq, tau, s); + } + for (j = ip + 1; j <= iq - 1; j++) + { + ROTATE(a, ip, j, j, iq, tau, s); + } + for (j = iq + 1; j <= n; j++) + { + ROTATE(a, ip, j, iq, j, tau, s); + } + for (j = 1; j <= n; j++) + { + ROTATE(v, j, ip, j, iq, tau, s); + } + ++nrot; + } + } + } + + for (ip = 1; ip <= n; ip++) + { + b[ip] += z[ip]; + d[ip] = b[ip]; + z[ip] = 0.0; + } + } + delete[] b; + delete[] z; +} + +void EdgeDrawingImpl::ROTATE(double** a, int i, int j, int k, int l, double tau, double s) +{ + double g, h; + g = a[i][j]; + h = a[k][l]; + a[i][j] = g - s * (h + g * tau); + a[k][l] = h + s * (g - h * tau); +} + +} // namespace cv +} // namespace ximgproc diff --git a/modules/ximgproc/src/edge_drawing_common.hpp b/modules/ximgproc/src/edge_drawing_common.hpp new file mode 100644 index 00000000000..990c1fec78f --- /dev/null +++ b/modules/ximgproc/src/edge_drawing_common.hpp @@ -0,0 +1,569 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef __OPENCV_EDGE_DRAWING_COMMON_HPP__ +#define __OPENCV_EDGE_DRAWING_COMMON_HPP__ + +#include + +#define EDGE_VERTICAL 1 +#define EDGE_HORIZONTAL 2 + +#define ANCHOR_PIXEL 254 +#define EDGE_PIXEL 255 + +#define LEFT 1 +#define RIGHT 2 +#define UP 3 +#define DOWN 4 + +#define SOUTH_SOUTH 0 +#define SOUTH_EAST 1 +#define EAST_SOUTH 2 +#define EAST_EAST 3 + +// Circular arc, circle thresholds +#define VERY_SHORT_ARC_ERROR 0.40 // Used for very short arcs (>= CANDIDATE_CIRCLE_RATIO1 && < CANDIDATE_CIRCLE_RATIO2) +#define SHORT_ARC_ERROR 1.00 // Used for short arcs (>= CANDIDATE_CIRCLE_RATIO2 && < HALF_CIRCLE_RATIO) +#define HALF_ARC_ERROR 1.25 // Used for arcs with length (>=HALF_CIRCLE_RATIO && < FULL_CIRCLE_RATIO) +#define LONG_ARC_ERROR 1.50 // Used for long arcs (>= FULL_CIRCLE_RATIO) + +#define CANDIDATE_CIRCLE_RATIO1 0.25 // 25% -- If only 25% of the circle is detected, it may be a candidate for validation +#define CANDIDATE_CIRCLE_RATIO2 0.33 // 33% -- If only 33% of the circle is detected, it may be a candidate for validation +#define HALF_CIRCLE_RATIO 0.50 // 50% -- If 50% of a circle is detected at any point during joins, we immediately make it a candidate +#define FULL_CIRCLE_RATIO 0.67 // 67% -- If 67% of the circle is detected, we assume that it is fully covered + +// Ellipse thresholds +#define CANDIDATE_ELLIPSE_RATIO 0.50 // 50% -- If 50% of the ellipse is detected, it may be candidate for validation +#define ELLIPSE_ERROR 1.50 // Used for ellipses. (used to be 1.65 for what reason?) + +using namespace std; +using namespace cv; + +class NFALUT +{ +public: + + NFALUT(int size, double _prob, int _w, int _h); + ~NFALUT(); + + int* LUT; // look up table + int LUTSize; + + double prob; + int w, h; + + bool checkValidationByNFA(int n, int k); + static double myAtan2(double yy, double xx); + +private: + double nfa(int n, int k); + static double Comb(double n, double k); +}; + +NFALUT::NFALUT(int size, double _prob, int _w, int _h) +{ + LUTSize = size; + LUT = new int[LUTSize]; + w = _w; + h = _h; + prob = _prob; + + LUT[0] = 1; + int j = 1; + for (int i = 1; i < LUTSize; i++) + { + LUT[i] = LUTSize + 1; + double ret = nfa(i, j); + if (ret >= 1.0) + { + while (j < i) + { + j++; + ret = nfa(i, j); + if (ret <= 1.0) + break; + } + + if (ret >= 1.0) + continue; + } + LUT[i] = j; + } +} + + +NFALUT::~NFALUT() +{ + delete[] LUT; +} + +bool NFALUT::checkValidationByNFA(int n, int k) +{ + if (n >= LUTSize) + return nfa(n, k) <= 1.0; + else + return k >= LUT[n]; +} + +double NFALUT::myAtan2(double yy, double xx) +{ + double angle = fastAtan2((float)yy, (float)xx); + + if (angle > 180) + angle = angle - 180; + + return angle / 180 * CV_PI; +} + +double NFALUT::Comb(double n, double k) //fast combination computation +{ + if (k > n) + return 0; + + double r = 1; + for (double d = 1; d <= k; ++d) + { + r *= n--; + r /= d; + } + return r; +} + +double NFALUT::nfa(int n, int k) +{ + double sum = 0; + double p = 0.125; + for (int i = k; i <= n; i++) + sum += Comb(n, i) * pow(p, i) * pow(1 - p, n - i); + + return sum * w * w * h * h; +} + +struct StackNode +{ + int r, c; // starting pixel + int parent; // parent chain (-1 if no parent) + int dir; // direction where you are supposed to go +}; + +// Used during Edge Linking +struct Chain +{ + int dir; // Direction of the chain + int len; // # of pixels in the chain + int parent; // Parent of this node (-1 if no parent) + int children[2]; // Children of this node (-1 if no children) + Point *pixels; // Pointer to the beginning of the pixels array +}; + +// light weight struct for Start & End coordinates of the line segment +struct LS +{ + Point2d start; + Point2d end; + + LS(Point2d _start, Point2d _end) + { + start = _start; + end = _end; + } +}; + + +struct EDLineSegment +{ + double a, b; // y = a + bx (if invert = 0) || x = a + by (if invert = 1) + int invert; + + double sx, sy; // starting x & y coordinates + double ex, ey; // ending x & y coordinates + + int segmentNo; // Edge segment that this line belongs to + int firstPixelIndex; // Index of the first pixel within the segment of pixels + int len; // No of pixels making up the line segment + + EDLineSegment(double _a, double _b, int _invert, double _sx, double _sy, double _ex, double _ey, int _segmentNo, int _firstPixelIndex, int _len) + { + a = _a; + b = _b; + invert = _invert; + sx = _sx; + sy = _sy; + ex = _ex; + ey = _ey; + segmentNo = _segmentNo; + firstPixelIndex = _firstPixelIndex; + len = _len; + } +}; + +// Circle equation: (x-xc)^2 + (y-yc)^2 = r^2 +struct mCircle { + Point2d center; + double r; + mCircle(Point2d _center, double _r) { center = _center; r = _r; } +}; + +// Ellipse equation: Ax^2 + Bxy + Cy^2 + Dx + Ey + F = 0 +struct mEllipse { + Point2d center; + Size axes; + double theta; + mEllipse(Point2d _center, Size _axes, double _theta) { center = _center; axes = _axes; theta = _theta; } +}; + +//---------------------------------------------------------- +// Ellipse Equation is of the form: +// Ax^2 + Bxy + Cy^2 + Dx + Ey + F = 0 +// +struct EllipseEquation { + double coeff[7]; // coeff[1] = A + + EllipseEquation() { + for (int i = 0; i<7; i++) coeff[i] = 0; + } + + double A() { return coeff[1]; } + double B() { return coeff[2]; } + double C() { return coeff[3]; } + double D() { return coeff[4]; } + double E() { return coeff[5]; } + double F() { return coeff[6]; } +}; + +// ================================ CIRCLES ================================ +struct Circle { + double xc, yc, r; // Center (xc, yc) & radius. + double circleFitError; // circle fit error + double coverRatio; // Percentage of the circle covered by the arcs making up this circle [0-1] + + double *x, *y; // Pointers to buffers containing the pixels making up this circle + int noPixels; // # of pixels making up this circle + + // If this circle is better approximated by an ellipse, we set isEllipse to true & eq contains the ellipse's equation + EllipseEquation eq; + double ellipseFitError; // ellipse fit error + bool isEllipse; + double majorAxisLength; // Length of the major axis + double minorAxisLength; // Length of the minor axis +}; + +// ------------------------------------------- ARCS ---------------------------------------------------- +struct MyArc { + double xc, yc, r; // center x, y and radius + double circleFitError; // Error during circle fit + + double sTheta, eTheta; // Start & end angle in radius + double coverRatio; // Ratio of the pixels covered on the covering circle [0-1] (noPixels/circumference) + + int turn; // Turn direction: 1 or -1 + + int segmentNo; // SegmentNo where this arc belongs + + int sx, sy; // Start (x, y) coordinate + int ex, ey; // End (x, y) coordinate of the arc + + double *x, *y; // Pointer to buffer containing the pixels making up this arc + int noPixels; // # of pixels making up the arc + + bool isEllipse; // Did we fit an ellipse to this arc? + EllipseEquation eq; // If an ellipse, then the ellipse's equation + double ellipseFitError; // Error during ellipse fit +}; + +// =============================== AngleSet ================================== + +// add a circular arc to the list of arcs +inline double ArcLength(double sTheta, double eTheta) +{ + if (eTheta > sTheta) + return eTheta - sTheta; + else + return CV_2PI - sTheta + eTheta; +} + +// A fast implementation of the AngleSet class. The slow implementation is really bad. About 10 times slower than this! +struct AngleSetArc { + double sTheta; + double eTheta; + int next; // Next AngleSetArc in the linked list +}; + +struct AngleSet { + AngleSetArc angles[360]; + int head; + int next; // Next AngleSetArc to be allocated + double overlapAmount; // Total overlap of the arcs in angleSet. Computed during set() function + + AngleSet() { clear(); } + void clear() { head = -1; next = 0; overlapAmount = 0; } + double overlapRatio() { return overlapAmount / CV_2PI; } + + void _set(double sTheta, double eTheta); + void set(double sTheta, double eTheta); + + double _overlap(double sTheta, double eTheta); + double overlap(double sTheta, double eTheta); + + void computeStartEndTheta(double& sTheta, double& eTheta); + double coverRatio(); +}; + +void AngleSet::_set(double sTheta, double eTheta) +{ + int arc = next++; + + angles[arc].sTheta = sTheta; + angles[arc].eTheta = eTheta; + angles[arc].next = -1; + + // Add the current arc to the linked list + int prev = -1; + int current = head; + while (1) + { + // Empty list? + if (head < 0) + { + head = arc; + break; + } + + // End of the list. Add to the end + if (current < 0) + { + CV_Assert(prev >= 0); + angles[prev].next = arc; + break; + } + + if (angles[arc].eTheta <= angles[current].sTheta) + { + // Add before current + if (prev < 0) + { + angles[arc].next = current; + head = arc; + } + else + { + angles[arc].next = current; + angles[prev].next = arc; + } + break; + } + else if (angles[arc].sTheta >= angles[current].eTheta) + { + // continue + prev = current; + current = angles[current].next; + + // End of the list? + if (current < 0) + { + angles[prev].next = arc; + break; + } + } + else + { + // overlaps with current. Join + // First delete current from the list + if (prev < 0) + head = angles[head].next; + else + angles[prev].next = angles[current].next; + + // Update overlap amount. + if (angles[arc].eTheta < angles[current].eTheta) + { + overlapAmount += angles[arc].eTheta - angles[current].sTheta; + } + else + { + overlapAmount += angles[current].eTheta - angles[arc].sTheta; + } + + // Now join current with arc + if (angles[current].sTheta < angles[arc].sTheta) + angles[arc].sTheta = angles[current].sTheta; + if (angles[current].eTheta > angles[arc].eTheta) + angles[arc].eTheta = angles[current].eTheta; + current = angles[current].next; + } + } +} + +void AngleSet::set(double sTheta, double eTheta) +{ + if (eTheta > sTheta) + { + _set(sTheta, eTheta); + } + else + { + _set(sTheta, CV_2PI); + _set(0, eTheta); + } +} + +double AngleSet::_overlap(double sTheta, double eTheta) +{ + double o = 0; + + int current = head; + while (current >= 0) + { + if (sTheta > angles[current].eTheta) + { + current = angles[current].next; + continue; + } + else if (eTheta < angles[current].sTheta) + break; + + // 3 cases. + if (sTheta < angles[current].sTheta && eTheta > angles[current].eTheta) + { + o += angles[current].eTheta - angles[current].sTheta; + + } + else if (sTheta < angles[current].sTheta) + { + o += eTheta - angles[current].sTheta; + } + else + { + o += angles[current].eTheta - sTheta; + } + + current = angles[current].next; + } + return o; +} + +double AngleSet::overlap(double sTheta, double eTheta) +{ + double o; + + if (eTheta > sTheta) + { + o = _overlap(sTheta, eTheta); + } + else + { + o = _overlap(sTheta, CV_2PI); + o += _overlap(0, eTheta); + } + return o / ArcLength(sTheta, eTheta); +} + +void AngleSet::computeStartEndTheta(double& sTheta, double& eTheta) +{ + // Special case: Just one arc + if (angles[head].next < 0) + { + sTheta = angles[head].sTheta; + eTheta = angles[head].eTheta; + + return; + } + + // OK. More than one arc. Find the biggest gap + int current = head; + int nextArc = angles[current].next; + + double biggestGapSTheta = angles[current].eTheta; + double biggestGapEtheta = angles[nextArc].sTheta; + double biggestGapLength = biggestGapEtheta - biggestGapSTheta; + + double start, end, len; + while (1) + { + current = nextArc; + nextArc = angles[nextArc].next; + if (nextArc < 0) + break; + + start = angles[current].eTheta; + end = angles[nextArc].sTheta; + len = end - start; + + if (len > biggestGapLength) + { + biggestGapSTheta = start; + biggestGapEtheta = end; + biggestGapLength = len; + } + } + + // Compute the gap between the last arc & the first arc + start = angles[current].eTheta; + end = angles[head].sTheta; + len = CV_2PI - start + end; + if (len > biggestGapLength) + { + biggestGapSTheta = start; + biggestGapEtheta = end; + } + sTheta = biggestGapEtheta; + eTheta = biggestGapSTheta; +} + +double AngleSet::coverRatio() +{ + int current = head; + + double total = 0; + while (current >= 0) + { + total += angles[current].eTheta - angles[current].sTheta; + current = angles[current].next; + } + return total / CV_2PI; +} + +struct EDArcs { + MyArc *arcs; + int noArcs; + +public: + EDArcs(int size = 10000) { + arcs = new MyArc[size]; + noArcs = 0; + } + + ~EDArcs() { + delete arcs; + } +}; + +struct BufferManager { + double *x, *y; + int index; + + BufferManager(int maxSize) { + x = new double[maxSize]; + y = new double[maxSize]; + index = 0; + } + + ~BufferManager() { + delete x; + delete y; + } + + double *getX() { return &x[index]; } + double *getY() { return &y[index]; } + void move(int size) { index += size; } +}; + +struct Info { + int sign; // -1 or 1: sign of the cross product + double angle; // angle with the next line (in radians) + bool taken; // Is this line taken during arc detection +}; + +#endif diff --git a/modules/ximgproc/test/test_fld.cpp b/modules/ximgproc/test/test_fld.cpp index 80f877648bc..71f7fd260cd 100644 --- a/modules/ximgproc/test/test_fld.cpp +++ b/modules/ximgproc/test/test_fld.cpp @@ -5,9 +5,9 @@ namespace opencv_test { namespace { -const Size img_size(640, 480); +const Size img_size(320, 240); const int FLD_TEST_SEED = 0x134679; -const int EPOCHS = 20; +const int EPOCHS = 10; class FLDBase : public testing::Test { @@ -36,6 +36,14 @@ class ximgproc_FLD: public FLDBase }; +class ximgproc_ED: public FLDBase +{ + public: + ximgproc_ED() { } + protected: + +}; + void FLDBase::GenerateWhiteNoise(Mat& image) { image = Mat(img_size, CV_8UC1); @@ -171,5 +179,75 @@ TEST_F(ximgproc_FLD, rotatedRect) ASSERT_EQ(EPOCHS, passedtests); } +//************** EDGE DRAWING ******************* + +TEST_F(ximgproc_ED, whiteNoise) +{ + for (int i = 0; i < EPOCHS; ++i) + { + GenerateWhiteNoise(test_image); + Ptr detector = createEdgeDrawing(); + detector->detectEdges(test_image); + detector->detectLines(lines); + + if(40u >= lines.size()) ++passedtests; + } + ASSERT_EQ(EPOCHS, passedtests); +} + +TEST_F(ximgproc_ED, constColor) +{ + for (int i = 0; i < EPOCHS; ++i) + { + GenerateConstColor(test_image); + Ptr detector = createEdgeDrawing(); + detector->detectEdges(test_image); + detector->detectLines(lines); + if(0u == lines.size()) ++passedtests; + } + ASSERT_EQ(EPOCHS, passedtests); +} + +TEST_F(ximgproc_ED, lines) +{ + for (int i = 0; i < EPOCHS; ++i) + { + const unsigned int numOfLines = 1; + GenerateLines(test_image, numOfLines); + Ptr detector = createEdgeDrawing(); + detector->detectEdges(test_image); + detector->detectLines(lines); + if(numOfLines * 2 == lines.size()) ++passedtests; // * 2 because of Gibbs effect + } + ASSERT_EQ(EPOCHS, passedtests); +} + +TEST_F(ximgproc_ED, mergeLines) +{ + for (int i = 0; i < EPOCHS; ++i) + { + const unsigned int numOfLines = 1; + GenerateBrokenLines(test_image, numOfLines); + Ptr detector = createEdgeDrawing(); + detector->detectEdges(test_image); + detector->detectLines(lines); + if(numOfLines * 2 == lines.size()) ++passedtests; // * 2 because of Gibbs effect + } + ASSERT_EQ(EPOCHS, passedtests); +} + +TEST_F(ximgproc_ED, rotatedRect) +{ + for (int i = 0; i < EPOCHS; ++i) + { + GenerateRotatedRect(test_image); + Ptr detector = createEdgeDrawing(); + detector->detectEdges(test_image); + detector->detectLines(lines); + + if(2u <= lines.size()) ++passedtests; + } + ASSERT_EQ(EPOCHS, passedtests); +} }} // namespace From 0b5300f74bf90424436b3cb489a975976de540bd Mon Sep 17 00:00:00 2001 From: Sun Aries <281614085@qq.com> Date: Fri, 2 Apr 2021 16:23:21 +0800 Subject: [PATCH 62/70] consider the rectangle marker in the future. It is now had been what was wanted to be here. Or we may do some asserts before e.g. `CV_ASSERT(candidateBits.rows == candidateBits.cols); `. --- modules/aruco/src/aruco.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/aruco/src/aruco.cpp b/modules/aruco/src/aruco.cpp index ca515bb3515..7a4d2537e6c 100644 --- a/modules/aruco/src/aruco.cpp +++ b/modules/aruco/src/aruco.cpp @@ -564,7 +564,7 @@ static uint8_t _identifyOneCandidate(const Ptr& dictionary, InputArr Mat onlyBits = candidateBits.rowRange(params->markerBorderBits, candidateBits.rows - params->markerBorderBits) - .colRange(params->markerBorderBits, candidateBits.rows - params->markerBorderBits); + .colRange(params->markerBorderBits, candidateBits.cols - params->markerBorderBits); // try to indentify the marker if(!dictionary->identify(onlyBits, idx, rotation, params->errorCorrectionRate)) From 5e0ff400fc587094e75c291d3cbb68121b71d0e4 Mon Sep 17 00:00:00 2001 From: LaurentBerger Date: Fri, 2 Apr 2021 12:21:37 +0200 Subject: [PATCH 63/70] Merge pull request #2882 from LaurentBerger:python_viz * python binding for viz tutorial widget pose reset test_tutorial 2 & 3 sample py 9 test sur 22 test PyWCloudNormal 16 sur 22 end? python test * oops * bug * comment but( this is not solved https://github.com/opencv/opencv_contrib/pull/2882#discussion_r591659839 * review 2 * oversight * oversight oops * viz: move out Py* classes from public C++ headers * viz: recover PyColor Co-authored-by: Alexander Alekhin --- modules/viz/include/opencv2/viz/types.hpp | 21 +- modules/viz/include/opencv2/viz/vizcore.hpp | 8 +- modules/viz/include/opencv2/viz/widgets.hpp | 13 +- modules/viz/misc/python/pyopencv_viz.impl.hpp | 114 ++ modules/viz/misc/python/python_viz.hpp | 1350 +++++++++++++++++ .../viz/misc/python/test/test_viz_simple.py | 439 ++++++ modules/viz/samples/viz_sample_01.py | 12 + modules/viz/samples/viz_sample_02.py | 33 + modules/viz/samples/viz_sample_03.py | 41 + modules/viz/samples/widget_pose.cpp | 28 +- 10 files changed, 2038 insertions(+), 21 deletions(-) create mode 100644 modules/viz/misc/python/pyopencv_viz.impl.hpp create mode 100644 modules/viz/misc/python/python_viz.hpp create mode 100644 modules/viz/misc/python/test/test_viz_simple.py create mode 100644 modules/viz/samples/viz_sample_01.py create mode 100644 modules/viz/samples/viz_sample_02.py create mode 100644 modules/viz/samples/viz_sample_03.py diff --git a/modules/viz/include/opencv2/viz/types.hpp b/modules/viz/include/opencv2/viz/types.hpp index 7a1829c1d61..c969c873eee 100644 --- a/modules/viz/include/opencv2/viz/types.hpp +++ b/modules/viz/include/opencv2/viz/types.hpp @@ -60,7 +60,7 @@ namespace cv /** @brief This class represents color in BGR order. */ - class Color : public Scalar + class Color : public Scalar // FIXIT design bug, missing CV_EXPORTS { public: Color(); @@ -117,7 +117,7 @@ namespace cv /** @brief This class wraps mesh attributes, and it can load a mesh from a ply file. : */ - class CV_EXPORTS Mesh + class CV_EXPORTS_W_SIMPLE Mesh { public: enum { @@ -126,16 +126,21 @@ namespace cv LOAD_OBJ = 2 }; - Mat cloud; //!< point coordinates of type CV_32FC3 or CV_64FC3 with only 1 row - Mat colors; //!< point color of type CV_8UC3 or CV_8UC4 with only 1 row - Mat normals; //!< point normals of type CV_32FC3, CV_32FC4, CV_64FC3 or CV_64FC4 with only 1 row + CV_PROP_RW Mat cloud; //!< point coordinates of type CV_32FC3 or CV_64FC3 with only 1 row + CV_PROP_RW Mat colors; //!< point color of type CV_8UC3 or CV_8UC4 with only 1 row + CV_PROP_RW Mat normals; //!< point normals of type CV_32FC3, CV_32FC4, CV_64FC3 or CV_64FC4 with only 1 row //! Raw integer list of the form: (n,id1,id2,...,idn, n,id1,id2,...,idn, ...) //! where n is the number of points in the polygon, and id is a zero-offset index into an associated cloud. - Mat polygons; //!< CV_32SC1 with only 1 row + CV_PROP_RW Mat polygons; //!< CV_32SC1 with only 1 row - Mat texture; - Mat tcoords; //!< CV_32FC2 or CV_64FC2 with only 1 row + CV_PROP_RW Mat texture; + CV_PROP_RW Mat tcoords; //!< CV_32FC2 or CV_64FC2 with only 1 row + + CV_WRAP Mesh() + { + // nothing + } /** @brief Loads a mesh from a ply or a obj file. diff --git a/modules/viz/include/opencv2/viz/vizcore.hpp b/modules/viz/include/opencv2/viz/vizcore.hpp index 7579ddc3ee7..29cb0c9d657 100644 --- a/modules/viz/include/opencv2/viz/vizcore.hpp +++ b/modules/viz/include/opencv2/viz/vizcore.hpp @@ -148,7 +148,7 @@ namespace cv * Supported channels: 3 and 4. * @param binary Used only for PLY format. */ - CV_EXPORTS void writeCloud(const String& file, InputArray cloud, InputArray colors = noArray(), InputArray normals = noArray(), bool binary = false); + CV_EXPORTS_W void writeCloud(const String& file, InputArray cloud, InputArray colors = noArray(), InputArray normals = noArray(), bool binary = false); /** * @param file Filename with extension. Supported formats: PLY, XYZ, OBJ and STL. @@ -157,12 +157,12 @@ namespace cv * @return A mat containing the point coordinates with depth CV_32F or CV_64F and number of * channels 3 or 4 with only 1 row. */ - CV_EXPORTS Mat readCloud (const String& file, OutputArray colors = noArray(), OutputArray normals = noArray()); + CV_EXPORTS_W Mat readCloud (const String& file, OutputArray colors = noArray(), OutputArray normals = noArray()); /////////////////////////////////////////////////////////////////////////////////////////////// /// Reads mesh. Only ply format is supported now and no texture load support - CV_EXPORTS Mesh readMesh(const String& file); + CV_EXPORTS_W Mesh readMesh(const String& file); /////////////////////////////////////////////////////////////////////////////////////////////// /// Read/write poses and trajectories @@ -211,7 +211,7 @@ namespace cv * @param mesh Input mesh. * @param normals Normals at very point in the mesh of type CV_64FC3. */ - CV_EXPORTS void computeNormals(const Mesh& mesh, OutputArray normals); + CV_EXPORTS_W void computeNormals(const Mesh& mesh, OutputArray normals); //! @} diff --git a/modules/viz/include/opencv2/viz/widgets.hpp b/modules/viz/include/opencv2/viz/widgets.hpp index 072653d039d..5f3d765aaac 100644 --- a/modules/viz/include/opencv2/viz/widgets.hpp +++ b/modules/viz/include/opencv2/viz/widgets.hpp @@ -337,7 +337,7 @@ namespace cv @param resolution Resolution of the cone. @param color Color of the cone. */ - WCone(double length, double radius, int resolution = 6.0, const Color &color = Color::white()); + WCone(double length, double radius, int resolution = 6, const Color &color = Color::white()); /** @brief Constructs repositioned planar cone. @@ -348,7 +348,7 @@ namespace cv @param color Color of the cone. */ - WCone(double radius, const Point3d& center, const Point3d& tip, int resolution = 6.0, const Color &color = Color::white()); + WCone(double radius, const Point3d& center, const Point3d& tip, int resolution = 6, const Color &color = Color::white()); }; /** @brief This 3D Widget defines a cylinder. : @@ -785,11 +785,14 @@ namespace cv @param colors Point colors. @param normals Point normals. */ - class CV_EXPORTS WMesh : public Widget3D + class CV_EXPORTS_W WMesh +#ifndef OPENCV_BINDING_PARSER + : public Widget3D +#endif { public: - WMesh(const Mesh &mesh); - WMesh(InputArray cloud, InputArray polygons, InputArray colors = noArray(), InputArray normals = noArray()); + CV_WRAP WMesh(const Mesh &mesh); + CV_WRAP WMesh(InputArray cloud, InputArray polygons, InputArray colors = noArray(), InputArray normals = noArray()); }; /** @brief This class allows to merge several widgets to single one. diff --git a/modules/viz/misc/python/pyopencv_viz.impl.hpp b/modules/viz/misc/python/pyopencv_viz.impl.hpp new file mode 100644 index 00000000000..7ef8c81ce2e --- /dev/null +++ b/modules/viz/misc/python/pyopencv_viz.impl.hpp @@ -0,0 +1,114 @@ +#ifndef OPENCV_PYTHON_VIZ_IMPL_HPP +#define OPENCV_PYTHON_VIZ_IMPL_HPP + +#include "python_viz.hpp" + +namespace cv { namespace viz { + +PyAffine3d makeTransformToGlobalPy(const Vec3d& axis_x, const Vec3d& axis_y, const Vec3d& axis_z, const Vec3d& origin) +{ + return makeTransformToGlobal(axis_x, axis_y, axis_z, origin); +} + +PyAffine3d makeCameraPosePy(const Vec3d& position, const Vec3d& focal_point, const Vec3d& y_dir) +{ + return makeCameraPose(position, focal_point, y_dir); +} + +#define WRAP_SHOW_WIDGET_1 return Viz3d::showWidget(id, *widget.widget.get()) +#define WRAP_SHOW_WIDGET_2 return Viz3d::showWidget(id, *widget.widget.get(), pose) +inline void PyViz3d::showWidget(const String &id, PyWLine &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWLine &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, PyWSphere &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWSphere &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, PyWCameraPosition &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWCameraPosition &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, PyWArrow &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWArrow &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, PyWCircle &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWCircle &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, PyWPlane &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWPlane &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, PyWCone &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWCone &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, PyWCube &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWCube &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, PyWCylinder &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWCylinder &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, PyWCoordinateSystem &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWPaintedCloud &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWPaintedCloud &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, PyWCloudCollection &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWCloudCollection &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, PyWGrid &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWGrid &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, const cv::Ptr &widget) { return Viz3d::showWidget(id, *widget.get()); } +inline void PyViz3d::showWidget(const String &id, const cv::Ptr &widget, PyAffine3d &pose) { return Viz3d::showWidget(id, *widget.get(), pose); } +inline void PyViz3d::showWidget(const String &id, PyWPolyLine &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWPolyLine &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, PyWCloud &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWCloud &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, PyWImage3D &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, PyWImage3D &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWImageOverlay &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, PyWImageOverlay &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWText &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWText &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, PyWText3D &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWText3D &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, PyWCloudNormals &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, PyWCloudNormals &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWTrajectory &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, PyWTrajectory &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWTrajectorySpheres &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, PyWTrajectorySpheres &widget) { WRAP_SHOW_WIDGET_1; } +inline void PyViz3d::showWidget(const String &id, PyWTrajectoryFrustums &widget, PyAffine3d &pose) { WRAP_SHOW_WIDGET_2; } +inline void PyViz3d::showWidget(const String &id, PyWTrajectoryFrustums &widget) { WRAP_SHOW_WIDGET_1; } +#undef WRAP_SHOW_WIDGET_1 +#undef WRAP_SHOW_WIDGET_2 + + +PyWGrid::PyWGrid(InputArray cells, InputArray cells_spacing, const PyColor& color) +{ + if (cells.kind() == _InputArray::MAT && cells_spacing.kind() == _InputArray::MAT) + { + Mat k = cells.getMat(); + Mat l = cells_spacing.getMat(); + if (k.total() == 2 && l.total() == 2 ) + { + CV_Assert(k.type() == CV_64FC1 && k.type() == CV_64FC1); + Vec2i c1((int)k.at(0), (int)k.at(1)); + Vec2d c2(l.at(0), l.at(1)); + widget = cv::makePtr(c1, c2, color); + } + else + CV_Error(cv::Error::StsVecLengthErr, "unknown size"); + } + else + CV_Error(cv::Error::StsUnsupportedFormat, "unknown type"); +} + + +PyWTrajectoryFrustums::PyWTrajectoryFrustums(InputArray path, InputArray K, double scale, const PyColor& color) +{ + if (K.kind() == _InputArray::MAT) + { + Mat k = K.getMat(); + if (k.rows == 3 && k.cols == 3) + { + Matx33d x = k; + widget = cv::makePtr(path, x, scale, color); + + } + else if (k.total() == 2) + widget = cv::makePtr(path, Vec2d(k.at(0), k.at(1)), 1.0, color); + else + CV_Error(cv::Error::StsVecLengthErr, "unknown size"); + } + else + CV_Error(cv::Error::StsVecLengthErr, "unknown size"); +} + + +}} // namespace +#endif // OPENCV_PYTHON_VIZ_IMPL_HPP diff --git a/modules/viz/misc/python/python_viz.hpp b/modules/viz/misc/python/python_viz.hpp new file mode 100644 index 00000000000..720e4f14f72 --- /dev/null +++ b/modules/viz/misc/python/python_viz.hpp @@ -0,0 +1,1350 @@ +#ifndef OPENCV_PYTHON_VIZ_HPP +#define OPENCV_PYTHON_VIZ_HPP + +#include "opencv2/viz.hpp" + +namespace cv { namespace viz { + +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(Color) PyColor +{ + CV_WRAP PyColor() {} + CV_WRAP PyColor(double gray) : c(gray) {} + CV_WRAP PyColor(double blue, double green, double red) : c(blue, green, red) {} + PyColor(const Color& v) : c(v) { } + + operator Color() const { return c; } + + CV_WRAP static PyColor black() { return PyColor(Color::black()); } + CV_WRAP static PyColor white() { return PyColor(Color::white()); } + CV_WRAP static PyColor blue() { return PyColor(Color::blue()); } + CV_WRAP static PyColor green() { return PyColor(Color::green()); } + CV_WRAP static PyColor red() { return PyColor(Color::red()); } + CV_WRAP static PyColor cyan() { return PyColor(Color::cyan()); } + CV_WRAP static PyColor yellow() { return PyColor(Color::yellow()); } + CV_WRAP static PyColor magenta() { return PyColor(Color::magenta()); } + + CV_WRAP static PyColor gray() { return PyColor(Color::gray()); } + CV_WRAP static PyColor silver() { return PyColor(Color::silver()); } + + CV_WRAP static PyColor mlab() { return PyColor(Color::mlab()); } + + CV_WRAP static PyColor navy() { return PyColor(Color::navy()); } + CV_WRAP static PyColor maroon() { return PyColor(Color::maroon()); } + CV_WRAP static PyColor teal() { return PyColor(Color::teal()); } + CV_WRAP static PyColor olive() { return PyColor(Color::olive()); } + CV_WRAP static PyColor purple() { return PyColor(Color::olive()); } + CV_WRAP static PyColor azure() { return PyColor(Color::olive()); } + CV_WRAP static PyColor chartreuse() { return PyColor(Color::olive()); } + CV_WRAP static PyColor rose() { return PyColor(Color::olive()); } + + CV_WRAP static PyColor lime() { return PyColor(Color::olive()); } + CV_WRAP static PyColor gold() { return PyColor(Color::olive()); } + CV_WRAP static PyColor orange() { return PyColor(Color::olive()); } + CV_WRAP static PyColor orange_red() { return PyColor(Color::olive()); } + CV_WRAP static PyColor indigo() { return PyColor(Color::olive()); } + + CV_WRAP static PyColor brown() { return PyColor(Color::olive()); } + CV_WRAP static PyColor apricot() { return PyColor(Color::olive()); } + CV_WRAP static PyColor pink() { return PyColor(Color::olive()); } + CV_WRAP static PyColor raspberry() { return PyColor(Color::olive()); } + CV_WRAP static PyColor cherry() { return PyColor(Color::olive()); } + CV_WRAP static PyColor violet() { return PyColor(Color::olive()); } + CV_WRAP static PyColor amethyst() { return PyColor(Color::amethyst()); } + CV_WRAP static PyColor bluberry() { return PyColor(Color::bluberry()); } + CV_WRAP static PyColor celestial_blue() { return PyColor(Color::celestial_blue()); } + CV_WRAP static PyColor turquoise() { return PyColor(Color::turquoise()); } + + static PyColor not_set() { return PyColor(Color::not_set()); } + CV_WRAP double get_blue() { return c[0]; } + CV_WRAP double get_green() { return c[1]; } + CV_WRAP double get_red() { return c[2]; } + + Color c; +}; + + +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(Affine3d) PyAffine3d +#ifndef OPENCV_BINDING_PARSER + : public Affine3d +#endif +{ + CV_WRAP PyAffine3d() + { + // nothing + } + inline PyAffine3d(const Affine3d& base) : Affine3d(base) + { + // nothing + } + CV_WRAP PyAffine3d(const Vec3d &rvec, const Vec3d &t = Vec3d::all(0)) + : Affine3d(rvec, t) + { + // nothing + } + CV_WRAP PyAffine3d(const Mat &rot, const Vec3f &t = Vec3d::all(0)) + : Affine3d(rot, t) + { + // nothing + } + CV_WRAP PyAffine3d translate(const Vec3d &t) + { + return Affine3d::translate(t); + } + CV_WRAP PyAffine3d rotate(const Vec3d &t) + { + return Affine3d::rotate(t); + } + CV_WRAP PyAffine3d product(const PyAffine3d &t) + { + return ((const Affine3d&)(*this)) * (const Affine3d&)t; + } + CV_WRAP static PyAffine3d Identity() + { + return Affine3d::Identity(); + } + CV_WRAP PyAffine3d inv() + { + return Affine3d::inv(); + } + CV_WRAP Mat mat() + { + return Mat(matrix); + } +}; + + +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WLine) PyWLine +{ + CV_WRAP PyWLine() + { + } + /** @brief Constructs a WLine. + + @param pt1 Start point of the line. + @param pt2 End point of the line. + @param color Color of the line. + */ + CV_WRAP PyWLine(const Point3d &pt1, const Point3d &pt2, const PyColor& color) + { + widget = cv::makePtr(pt1, pt2, color); + } + + CV_WRAP void setRenderingProperty(int property, double value) + { + CV_Assert(widget); + widget->setRenderingProperty(property, value); + } + + Ptr widget; +}; + +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WPlane) PyWPlane +{ +public: + /** @brief Constructs a default plane with center point at origin and normal oriented along z-axis. + + @param size Size of the plane + @param color Color of the plane. + */ + CV_WRAP PyWPlane(const Point2d& size = Point2d(1.0, 1.0), const PyColor& color = Color(255, 255,255)) + { + widget = cv::makePtr(size, color); + } + + /** @brief Constructs a repositioned plane + + @param center Center of the plane + @param normal Plane normal orientation + @param new_yaxis Up-vector. New orientation of plane y-axis. + @param size + @param color Color of the plane. + */ + CV_WRAP PyWPlane(const Point3d& center, const Vec3d& normal, const Vec3d& new_yaxis, + const Point2d& size = Point2d(1.0, 1.0), const PyColor& color = Color(255, 255, 255)) + { + widget = cv::makePtr(center, normal, new_yaxis, size, color); + } + + CV_WRAP void setRenderingProperty(int property, double value) + { + CV_Assert(widget); + widget->setRenderingProperty(property, value); + } + + Ptr widget; +}; + +/** @brief This 3D Widget defines a sphere. : +*/ +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WSphere) PyWSphere +{ +public: + CV_WRAP PyWSphere() + { + + } + /** @brief Constructs a WSphere. + + @param center Center of the sphere. + @param radius Radius of the sphere. + @param sphere_resolution Resolution of the sphere. + @param color Color of the sphere. + */ + CV_WRAP PyWSphere(const cv::Point3d ¢er, double radius, int sphere_resolution = 10, const PyColor& color = Color(255, 255,255)) + { + widget = cv::makePtr(center, radius, sphere_resolution, color); + } + + CV_WRAP void setRenderingProperty(int property, double value) + { + CV_Assert(widget); + widget->setRenderingProperty(property, value); + } + + Ptr widget; +}; + +/** @brief This 3D Widget defines an arrow. +*/ +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WArrow) PyWArrow +{ +public: + CV_WRAP PyWArrow() + { + } + /** @brief Constructs an WArrow. + + @param pt1 Start point of the arrow. + @param pt2 End point of the arrow. + @param thickness Thickness of the arrow. Thickness of arrow head is also adjusted + accordingly. + @param color Color of the arrow. + + Arrow head is located at the end point of the arrow. + */ + CV_WRAP PyWArrow(const Point3d& pt1, const Point3d& pt2, double thickness = 0.03, const PyColor& color = Color(255, 255, 255)) + { + widget = cv::makePtr(pt1, pt2, thickness, color); + } + + CV_WRAP void setRenderingProperty(int property, double value) + { + CV_Assert(widget); + widget->setRenderingProperty(property, value); + } + + Ptr widget; +}; + +/** @brief This 3D Widget defines a cube. +*/ +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WCube) PyWCube +{ +public: + /** @brief Constructs a WCube. + + @param min_point Specifies minimum (or maximum) point of the bounding box. + @param max_point Specifies maximum (or minimum) point of the bounding box, opposite to the first parameter. + @param wire_frame If true, cube is represented as wireframe. + @param color Color of the cube. + + ![Cube Widget](images/cube_widget.png) + */ + CV_WRAP PyWCube(const Point3d& min_point = Vec3d::all(-0.5), const Point3d& max_point = Vec3d::all(0.5), + bool wire_frame = true, const PyColor& color = Color(255, 255, 255)) + { + widget = cv::makePtr(min_point, max_point, wire_frame, color); + } + + CV_WRAP void setRenderingProperty(int property, double value) + { + CV_Assert(widget); + widget->setRenderingProperty(property, value); + } + + Ptr widget; +}; + +/** @brief This 3D Widget defines a PyWCircle. +*/ +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WCircle) PyWCircle +{ +public: + PyWCircle() {} + /** @brief Constructs default planar circle centered at origin with plane normal along z-axis + + @param radius Radius of the circle. + @param thickness Thickness of the circle. + @param color Color of the circle. + */ + CV_WRAP PyWCircle(double radius, double thickness = 0.01, const PyColor& color = Color::white()) + { + widget = cv::makePtr(radius, thickness, color); + } + + /** @brief Constructs repositioned planar circle. + + @param radius Radius of the circle. + @param center Center of the circle. + @param normal Normal of the plane in which the circle lies. + @param thickness Thickness of the circle. + @param color Color of the circle. + */ + CV_WRAP PyWCircle(double radius, const Point3d& center, const Vec3d& normal, double thickness = 0.01, const PyColor& color = Color::white()) + { + widget = cv::makePtr(radius, center, normal, thickness, color); + } + + CV_WRAP void setRenderingProperty(int property, double value) + { + CV_Assert(widget); + widget->setRenderingProperty(property, value); + } + + Ptr widget; +}; + +/** @brief This 3D Widget defines a cone. : +*/ +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WCone) PyWCone +{ +public: + PyWCone() {} + /** @brief Constructs default cone oriented along x-axis with center of its base located at origin + + @param length Length of the cone. + @param radius Radius of the cone. + @param resolution Resolution of the cone. + @param color Color of the cone. + */ + CV_WRAP PyWCone(double length, double radius, int resolution = 6, const PyColor& color = Color::white()) + { + widget = cv::makePtr(length, radius, resolution, color); + } + + /** @brief Constructs repositioned planar cone. + + @param radius Radius of the cone. + @param center Center of the cone base. + @param tip Tip of the cone. + @param resolution Resolution of the cone. + @param color Color of the cone. + + */ + CV_WRAP PyWCone(double radius, const Point3d& center, const Point3d& tip, int resolution = 6, const PyColor& color = Color::white()) + { + widget = cv::makePtr(radius, center, tip, resolution, color); + } + + CV_WRAP void setRenderingProperty(int property, double value) + { + CV_Assert(widget); + widget->setRenderingProperty(property, value); + } + + Ptr widget; +}; + +/** @brief This 3D Widget defines a PyWCylinder. : +*/ +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WCylinder) PyWCylinder +{ +public: + CV_WRAP PyWCylinder() {} + /** @brief Constructs a WCylinder. + + @param axis_point1 A point1 on the axis of the cylinder. + @param axis_point2 A point2 on the axis of the cylinder. + @param radius Radius of the cylinder. + @param numsides Resolution of the cylinder. + @param color Color of the cylinder. + */ + CV_WRAP PyWCylinder(const Point3d& axis_point1, const Point3d& axis_point2, double radius, int numsides = 30, const PyColor& color = Color::white()) + { + widget = cv::makePtr(axis_point1, axis_point2, radius, numsides, color); + } + Ptr widget; +}; + +/** @brief This 3D Widget represents camera position in a scene by its axes or viewing frustum. : +*/ +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WCameraPosition) PyWCameraPosition +{ +public: + /** @brief Creates camera coordinate frame at the origin. + + ![Camera coordinate frame](images/cpw1.png) + */ + CV_WRAP PyWCameraPosition(double scale = 1.0) + { + widget = cv::makePtr(scale); + } + /** @brief Display the viewing frustum + @param K Intrinsic matrix of the camera or fov Field of view of the camera (horizontal, vertical). + @param scale Scale of the frustum. + @param color Color of the frustum. + + Creates viewing frustum of the camera based on its intrinsic matrix K. + + ![Camera viewing frustum](images/cpw2.png) + */ + CV_WRAP PyWCameraPosition(InputArray K, double scale = 1.0, const PyColor& color = Color(255, 255, 255)) + { + if (K.kind() == _InputArray::MAT) + { + Mat k = K.getMat(); + if (k.rows == 3 && k.cols == 3) + { + Matx33d x = k; + widget = cv::makePtr(x, scale, color); + + } + else if (k.total() == 2) + widget = cv::makePtr(Vec2d(k.at(0), k.at(1)), scale, color); + else + CV_Error(cv::Error::StsVecLengthErr, "unknown size"); + } + else + CV_Error(cv::Error::StsUnsupportedFormat, "unknown type"); + } + + /** @brief Display image on the far plane of the viewing frustum + + @param K Intrinsic matrix of the camera. + @param image BGR or Gray-Scale image that is going to be displayed on the far plane of the frustum. + @param scale Scale of the frustum and image. + @param color Color of the frustum. + + Creates viewing frustum of the camera based on its intrinsic matrix K, and displays image on + the far end plane. + + ![Camera viewing frustum with image](images/cpw3.png) + */ + CV_WRAP PyWCameraPosition(InputArray K, InputArray image, double scale = 1.0, const PyColor& color = Color(255, 255, 255)) + { + if (K.kind() == _InputArray::MAT) + { + Mat k = K.getMat(); + if (k.rows == 3 && k.cols == 3) + { + Matx33d x = k; + widget = cv::makePtr(x, image, scale, color); + + } + else if (k.total() == 2) + widget = cv::makePtr(Vec2d(k.at(0), k.at(1)), image, scale, color); + else + CV_Error(cv::Error::StsVecLengthErr, "unknown size"); + } + else + CV_Error(cv::Error::StsUnsupportedFormat, "unknown type"); + + } + /** @brief Display image on the far plane of the viewing frustum + + @param fov Field of view of the camera (horizontal, vertical). + @param image BGR or Gray-Scale image that is going to be displayed on the far plane of the frustum. + @param scale Scale of the frustum and image. + @param color Color of the frustum. + + Creates viewing frustum of the camera based on its intrinsic matrix K, and displays image on + the far end plane. + + ![Camera viewing frustum with image](images/cpw3.png) + */ + CV_WRAP PyWCameraPosition(const Point2d &fov, InputArray image, double scale = 1.0, const PyColor& color = Color(255, 255, 255)) + { + widget = cv::makePtr(fov, image, scale, color); + } + + Ptr widget; +}; +///////////////////////////////////////////////////////////////////////////// +/// Compound widgets + +/** @brief This 3D Widget represents a coordinate system. : +*/ +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WCoordinateSystem) PyWCoordinateSystem +{ +public: + /** @brief Constructs a WCoordinateSystem. + + @param scale Determines the size of the axes. + */ + CV_WRAP PyWCoordinateSystem(double scale = 1.0) + { + widget = cv::makePtr(scale); + + } + Ptr widget; +}; + +///////////////////////////////////////////////////////////////////////////// +/// Clouds + +/** @brief This 3D Widget defines a point cloud. : + +@note In case there are four channels in the cloud, fourth channel is ignored. +*/ +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WCloud) PyWCloud +{ +public: + CV_WRAP PyWCloud() + { + // nothing + } + + /** @brief Constructs a WCloud. + @param cloud Set of points which can be of type: CV_32FC3, CV_32FC4, CV_64FC3, CV_64FC4. + @param color A single Color for the whole cloud. + + Points in the cloud belong to mask when they are set to (NaN, NaN, NaN). + */ + CV_WRAP PyWCloud(InputArray cloud, const PyColor& color = Color::white()) + { + widget = cv::makePtr(cloud, color); + } + + CV_WRAP PyWCloud(InputArray cloud, InputArray colors) + { + widget = cv::makePtr(cloud, colors); + } + + CV_WRAP PyWCloud(InputArray cloud, InputArray colors, InputArray normals) + { + widget = cv::makePtr(cloud, colors, normals); + } + + /** @brief Constructs a WCloud. + @param cloud Set of points which can be of type: CV_32FC3, CV_32FC4, CV_64FC3, CV_64FC4. + @param color A single Color for the whole cloud. + @param normals Normals for each point in cloud. + + Size and type should match with the cloud parameter. + Points in the cloud belong to mask when they are set to (NaN, NaN, NaN). + */ + CV_WRAP PyWCloud(InputArray cloud, const PyColor& color, InputArray normals) + { + widget = cv::makePtr(cloud, color, normals); + } + + CV_WRAP void setRenderingProperty(int property, double value) + { + CV_Assert(widget); + widget->setRenderingProperty(property, value); + } + + Ptr widget; +}; +/** @brief This 3D Widget defines a poly line. : +*/ +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WPolyLine) PyWPolyLine +{ +public: + CV_WRAP PyWPolyLine() + { + } + CV_WRAP PyWPolyLine(InputArray points, InputArray colors) + { + widget = cv::makePtr(points, colors); + } + /** @brief Constructs a WPolyLine. + + @param points Point set. + @param color Color of the poly line. + */ + CV_WRAP PyWPolyLine(InputArray points, const PyColor& color = Color::white()) + { + widget = cv::makePtr(points, color); + } + + CV_WRAP void setRenderingProperty(int property, double value) + { + CV_Assert(widget); + widget->setRenderingProperty(property, value); + } + + Ptr widget; +}; + +///////////////////////////////////////////////////////////////////////////// +/// Text and image widgets + +/** @brief This 2D Widget represents text overlay. +*/ +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WText) PyWText +{ +public: + CV_WRAP PyWText() + { + } + /** @brief Constructs a WText. + + @param text Text content of the widget. + @param pos Position of the text. + @param font_size Font size. + @param color Color of the text. + */ + CV_WRAP PyWText(const String &text, const Point &pos, int font_size = 20, const PyColor& color = Color::white()) + { + widget = cv::makePtr(text, pos, font_size, color); + } + + /** @brief Sets the text content of the widget. + + @param text Text content of the widget. + */ + CV_WRAP void setText(const String &text) + { + widget->setText(text); + } + /** @brief Returns the current text content of the widget. + */ + CV_WRAP String getText() const + { + return widget->getText(); + } + Ptr widget; +}; + +/** @brief This 3D Widget represents 3D text. The text always faces the camera. +*/ +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WText3D) PyWText3D +{ +public: + CV_WRAP PyWText3D() + {} + /** @brief Constructs a WText3D. + + @param text Text content of the widget. + @param position Position of the text. + @param text_scale Size of the text. + @param face_camera If true, text always faces the camera. + @param color Color of the text. + */ + CV_WRAP PyWText3D(const String &text, const Point3d &position, double text_scale = 1., bool face_camera = true, const PyColor& color = Color::white()) + { + widget = cv::makePtr(text, position, text_scale, face_camera, color); + } + /** @brief Sets the text content of the widget. + + @param text Text content of the widget. + + */ + CV_WRAP void setText(const String &text) + { + widget->setText(text); + } + /** @brief Returns the current text content of the widget. + */ + CV_WRAP String getText() const + { + return widget->getText(); + } + Ptr widget; +}; + +/** @brief This 2D Widget represents an image overlay. : +*/ +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WImageOverlay) PyWImageOverlay +{ +public: + CV_WRAP PyWImageOverlay() + { + } + /** @brief Constructs an WImageOverlay. + + @param image BGR or Gray-Scale image. + @param rect Image is scaled and positioned based on rect. + */ + CV_WRAP PyWImageOverlay(InputArray image, const Rect &rect) + { + widget = cv::makePtr(image, rect); + } + /** @brief Sets the image content of the widget. + + @param image BGR or Gray-Scale image. + */ + CV_WRAP void setImage(InputArray image) + { + widget->setImage(image); + } + + CV_WRAP void setRenderingProperty(int property, double value) + { + CV_Assert(widget); + widget->setRenderingProperty(property, value); + } + + Ptr widget; +}; + +/** @brief This 3D Widget represents an image in 3D space. : +*/ +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WImage3D) PyWImage3D +{ + CV_WRAP PyWImage3D() + { + } +public: + /** @brief Constructs an WImage3D. + + @param image BGR or Gray-Scale image. + @param size Size of the image. + */ + CV_WRAP PyWImage3D(InputArray image, const Point2d &size) + { + widget = cv::makePtr(image, size); + } + + /** @brief Constructs an WImage3D. + + @param image BGR or Gray-Scale image. + @param size Size of the image. + @param center Position of the image. + @param normal Normal of the plane that represents the image. + @param up_vector Determines orientation of the image. + */ + CV_WRAP PyWImage3D(InputArray image, const Point2d &size, const Vec3d ¢er, const Vec3d &normal, const Vec3d &up_vector) + { + widget = cv::makePtr(image, size, center, normal, up_vector); + } + + + /** @brief Sets the image content of the widget. + + @param image BGR or Gray-Scale image. + */ + CV_WRAP void setImage(InputArray image) + { + widget->setImage(image); + } + /** @brief Sets the image size of the widget. + + @param size the new size of the image. + */ + CV_WRAP void setSize(const Size& size) + { + widget->setSize(size); + } + Ptr widget; +}; + + +/** @brief This 3D Widget defines a grid. : + */ +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WGrid) PyWGrid +{ +public: + PyWGrid() {} + /** @brief Constructs a WGrid. + + @param cells Number of cell columns and rows, respectively. + @param cells_spacing Size of each cell, respectively. + @param color Color of the grid. + */ + CV_WRAP PyWGrid(InputArray cells, InputArray cells_spacing, const PyColor& color = Color::white()); + + //! Creates repositioned grid + CV_WRAP PyWGrid(const Point3d& center, const Vec3d& normal, const Vec3d& new_yaxis, + const Vec2i &cells = Vec2i::all(10), const Vec2d &cells_spacing = Vec2d::all(1.0), const PyColor& color = Color::white()) + { + widget = cv::makePtr(center, normal, new_yaxis, cells, cells_spacing, color); + } + + CV_WRAP void setRenderingProperty(int property, double value) + { + CV_Assert(widget); + widget->setRenderingProperty(property, value); + } + + Ptr widget; +}; + + +///////////////////////////////////////////////////////////////////////////// +/// Trajectories + +/** @brief This 3D Widget represents a trajectory. : +*/ +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WTrajectory) PyWTrajectory +{ +public: + enum { FRAMES = 1, PATH = 2, BOTH = FRAMES + PATH }; + PyWTrajectory() {} + /** @brief Constructs a WTrajectory. + + @param path List of poses on a trajectory. Takes std::vector\\> with T == [float | double] + @param display_mode Display mode. This can be PATH, FRAMES, and BOTH. + @param scale Scale of the frames. Polyline is not affected. + @param color Color of the polyline that represents path. + + Frames are not affected. + Displays trajectory of the given path as follows: + - PATH : Displays a poly line that represents the path. + - FRAMES : Displays coordinate frames at each pose. + - PATH & FRAMES : Displays both poly line and coordinate frames. + */ + CV_WRAP PyWTrajectory(InputArray path, int display_mode = WTrajectory::PATH, double scale = 1.0, const PyColor& color = Color::white()) + { + widget = cv::makePtr(path, display_mode, scale, color); + } + + CV_WRAP void setRenderingProperty(int property, double value) + { + CV_Assert(widget); + widget->setRenderingProperty(property, value); + } + + Ptr widget; +}; + +/** @brief This 3D Widget represents a trajectory. : +*/ +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WTrajectoryFrustums) PyWTrajectoryFrustums +{ +public: + PyWTrajectoryFrustums() {} + /** @brief Constructs a WTrajectoryFrustums. + + @param path List of poses on a trajectory. Takes std::vector\\> with T == [float | double] + @param K Intrinsic matrix of the camera or fov Field of view of the camera (horizontal, vertical). + @param scale Scale of the frustums. + @param color Color of the frustums. + + Displays frustums at each pose of the trajectory. + */ + CV_WRAP PyWTrajectoryFrustums(InputArray path, InputArray K, double scale = 1.0, const PyColor& color = Color::white()); + + CV_WRAP void setRenderingProperty(int property, double value) + { + CV_Assert(widget); + widget->setRenderingProperty(property, value); + } + + Ptr widget; +}; + +/** @brief This 3D Widget represents a trajectory using spheres and lines + +where spheres represent the positions of the camera, and lines represent the direction from +previous position to the current. : + */ +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WTrajectorySpheres) PyWTrajectorySpheres +{ +public: + PyWTrajectorySpheres() {} + /** @brief Constructs a WTrajectorySpheres. + + @param path List of poses on a trajectory. Takes std::vector\\> with T == [float | double] + @param line_length Max length of the lines which point to previous position + @param radius Radius of the spheres. + @param from Color for first sphere. + @param to Color for last sphere. Intermediate spheres will have interpolated color. + */ + CV_WRAP PyWTrajectorySpheres(InputArray path, double line_length = 0.05, double radius = 0.007, + const PyColor& from = Color::red(), const PyColor& to = Color::white()) + { + widget = cv::makePtr(path, line_length, radius, from, to); + } + + CV_WRAP void setRenderingProperty(int property, double value) + { + CV_Assert(widget); + widget->setRenderingProperty(property, value); + } + + Ptr widget; +}; + + +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WPaintedCloud) PyWPaintedCloud +{ +public: + //! Paint cloud with default gradient between cloud bounds points + PyWPaintedCloud() + { + } + + CV_WRAP PyWPaintedCloud(InputArray cloud) + { + widget = cv::makePtr(cloud); + } + + //! Paint cloud with default gradient between given points + CV_WRAP PyWPaintedCloud(InputArray cloud, const Point3d& p1, const Point3d& p2) + { + widget = cv::makePtr(cloud, p1, p2); + } + + //! Paint cloud with gradient specified by given colors between given points + CV_WRAP PyWPaintedCloud(InputArray cloud, const Point3d& p1, const Point3d& p2, const PyColor& c1, const PyColor& c2) + { + widget = cv::makePtr(cloud, p1, p2, c1, c2); + } + + CV_WRAP void setRenderingProperty(int property, double value) + { + CV_Assert(widget); + widget->setRenderingProperty(property, value); + } + + Ptr widget; +}; + +/** @brief This 3D Widget defines a collection of clouds. : +@note In case there are four channels in the cloud, fourth channel is ignored. +*/ +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WCloudCollection) PyWCloudCollection +{ +public: + CV_WRAP PyWCloudCollection() + { + widget = cv::makePtr(); + } + + /** @brief Adds a cloud to the collection. + + @param cloud Point set which can be of type: CV_32FC3, CV_32FC4, CV_64FC3, CV_64FC4. + @param colors Set of colors. It has to be of the same size with cloud. + @param pose Pose of the cloud. Points in the cloud belong to mask when they are set to (NaN, NaN, NaN). + */ + CV_WRAP void addCloud(InputArray cloud, InputArray colors, const PyAffine3d &pose = PyAffine3d::Identity()) + { + widget->addCloud(cloud, colors, pose); + } + /** @brief Adds a cloud to the collection. + + @param cloud Point set which can be of type: CV_32FC3, CV_32FC4, CV_64FC3, CV_64FC4. + @param color A single Color for the whole cloud. + @param pose Pose of the cloud. Points in the cloud belong to mask when they are set to (NaN, NaN, NaN). + */ + CV_WRAP void addCloud(InputArray cloud, const PyColor& color = Color::white(), const PyAffine3d& pose = PyAffine3d::Identity()) + { + widget->addCloud(cloud, color, pose); + } + /** @brief Finalizes cloud data by repacking to single cloud. + + Useful for large cloud collections to reduce memory usage + */ + CV_WRAP void finalize() + { + widget->finalize(); + } + + CV_WRAP void setRenderingProperty(int property, double value) + { + CV_Assert(widget); + widget->setRenderingProperty(property, value); + } + + Ptr widget; +}; + +/** @brief This 3D Widget represents normals of a point cloud. : +*/ +struct CV_EXPORTS_W_SIMPLE CV_WRAP_AS(WCloudNormals) PyWCloudNormals +{ +public: + PyWCloudNormals() + { + } + /** @brief Constructs a WCloudNormals. + + @param cloud Point set which can be of type: CV_32FC3, CV_32FC4, CV_64FC3, CV_64FC4. + @param normals A set of normals that has to be of same type with cloud. + @param level Display only every level th normal. + @param scale Scale of the arrows that represent normals. + @param color Color of the arrows that represent normals. + + @note In case there are four channels in the cloud, fourth channel is ignored. + */ + CV_WRAP PyWCloudNormals(InputArray cloud, InputArray normals, int level = 64, double scale = 0.1, const PyColor& color = Color::white()) + { + widget = cv::makePtr(cloud, normals, level, scale, color); + } + + CV_WRAP void setRenderingProperty(int property, double value) + { + CV_Assert(widget); + widget->setRenderingProperty(property, value); + } + + Ptr widget; +}; + + +CV_WRAP_AS(makeTransformToGlobal) static inline +PyAffine3d makeTransformToGlobalPy(const Vec3d& axis_x, const Vec3d& axis_y, const Vec3d& axis_z, const Vec3d& origin = Vec3d::all(0)); + +CV_WRAP_AS(makeCameraPose) static inline +PyAffine3d makeCameraPosePy(const Vec3d& position, const Vec3d& focal_point, const Vec3d& y_dir); + + +/** @brief The Viz3d class represents a 3D visualizer window. This class is implicitly shared. +*/ +class CV_EXPORTS_AS(Viz3d) PyViz3d +#ifndef OPENCV_BINDING_PARSER + : public Viz3d +#endif +{ +public: + CV_WRAP static cv::Ptr create(const std::string& window_name = std::string()) + { + return makePtr(window_name); + } + + /** @brief The constructors. + + @param window_name Name of the window. + */ + CV_WRAP PyViz3d(const std::string& window_name = std::string()) : Viz3d(window_name) {} + + CV_WRAP void showWidget(const String &id, PyWLine &widget); + CV_WRAP void showWidget(const String &id, PyWLine &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWSphere &widget); + CV_WRAP void showWidget(const String &id, PyWSphere &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWCameraPosition &widget); + CV_WRAP void showWidget(const String &id, PyWCameraPosition &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWArrow &widget); + CV_WRAP void showWidget(const String &id, PyWArrow &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWCircle &widget); + CV_WRAP void showWidget(const String &id, PyWCircle &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWPlane &widget); + CV_WRAP void showWidget(const String &id, PyWPlane &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWCone &widget); + CV_WRAP void showWidget(const String &id, PyWCone &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWCube &widget); + CV_WRAP void showWidget(const String &id, PyWCube &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWCylinder &widget); + CV_WRAP void showWidget(const String &id, PyWCylinder &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWCoordinateSystem &widget); + CV_WRAP void showWidget(const String &id, PyWPaintedCloud &widget); + CV_WRAP void showWidget(const String &id, PyWPaintedCloud &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWCloudCollection &widget); + CV_WRAP void showWidget(const String &id, PyWCloudCollection &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWGrid &widget); + CV_WRAP void showWidget(const String &id, PyWGrid &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, const cv::Ptr &widget); + CV_WRAP void showWidget(const String &id, const cv::Ptr &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWPolyLine &widget); + CV_WRAP void showWidget(const String &id, PyWPolyLine &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWCloud &widget); + CV_WRAP void showWidget(const String &id, PyWCloud &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWImage3D &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWImage3D &widget); + CV_WRAP void showWidget(const String &id, PyWImageOverlay &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWImageOverlay &widget); + CV_WRAP void showWidget(const String &id, PyWText &widget); + CV_WRAP void showWidget(const String &id, PyWText &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWText3D &widget); + CV_WRAP void showWidget(const String &id, PyWText3D &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWCloudNormals &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWCloudNormals &widget); + CV_WRAP void showWidget(const String &id, PyWTrajectory &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWTrajectory &widget); + CV_WRAP void showWidget(const String &id, PyWTrajectorySpheres &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWTrajectorySpheres &widget); + CV_WRAP void showWidget(const String &id, PyWTrajectoryFrustums &widget, PyAffine3d &pose); + CV_WRAP void showWidget(const String &id, PyWTrajectoryFrustums &widget); + + + /** @brief Removes a widget from the window. + + @param id The id of the widget that will be removed. + */ + CV_WRAP + void removeWidget(const String &id) + { return Viz3d::removeWidget(id); } + + /** @brief Removes all widgets from the window. + */ + CV_WRAP + void removeAllWidgets() + { return Viz3d::removeAllWidgets(); } + + /** @brief Removed all widgets and displays image scaled to whole window area. + + @param image Image to be displayed. + @param window_size Size of Viz3d window. Default value means no change. + */ + CV_WRAP + void showImage(InputArray image, const Size& window_size = Size(-1, -1)) + { return Viz3d::showImage(image, window_size); } + + /** @brief Sets pose of a widget in the window. + + @param id The id of the widget whose pose will be set. @param pose The new pose of the widget. + */ + CV_WRAP + void setWidgetPose(const String &id, const PyAffine3d &pose) + { return Viz3d::setWidgetPose(id, pose); } + + /** @brief Updates pose of a widget in the window by pre-multiplying its current pose. + + @param id The id of the widget whose pose will be updated. @param pose The pose that the current + pose of the widget will be pre-multiplied by. + */ + CV_WRAP + void updateWidgetPose(const String &id, const PyAffine3d &pose) + { return Viz3d::updateWidgetPose(id, pose); } + + /** @brief Returns the current pose of a widget in the window. + + @param id The id of the widget whose pose will be returned. + */ + CV_WRAP + PyAffine3d getWidgetPose(const String &id) const + { return (PyAffine3d)Viz3d::getWidgetPose(id); } + +#if 0 + /** @brief Sets the intrinsic parameters of the viewer using Camera. + + @param camera Camera object wrapping intrinsic parameters. + */ + void setCamera(const Camera &camera); + + /** @brief Returns a camera object that contains intrinsic parameters of the current viewer. + */ + Camera getCamera() const; +#endif + + /** @brief Returns the current pose of the viewer. + */ + CV_WRAP PyAffine3d getViewerPose() const + { return (PyAffine3d)Viz3d::getViewerPose(); } + + /** @brief Sets pose of the viewer. + + @param pose The new pose of the viewer. + */ + CV_WRAP + void setViewerPose(const PyAffine3d &pose) + { return Viz3d::setViewerPose(pose); } + + /** @brief Resets camera viewpoint to a 3D widget in the scene. + + @param id Id of a 3D widget. + */ + CV_WRAP + void resetCameraViewpoint(const String &id) + { return Viz3d::resetCameraViewpoint(id); } + + /** @brief Resets camera. + */ + CV_WRAP + void resetCamera() + { return Viz3d::resetCamera(); } + + /** @brief Transforms a point in world coordinate system to window coordinate system. + + @param pt Point in world coordinate system. + @param window_coord Output point in window coordinate system. + */ + CV_WRAP void convertToWindowCoordinates(const Point3d &pt, CV_OUT Point3d &window_coord) + { return Viz3d::convertToWindowCoordinates(pt, window_coord); } + +#if 0 + /** @brief Transforms a point in window coordinate system to a 3D ray in world coordinate system. + + @param window_coord Point in window coordinate system. @param origin Output origin of the ray. + @param direction Output direction of the ray. + */ + void converTo3DRay(const Point3d &window_coord, Point3d &origin, Vec3d &direction); +#endif + + /** @brief Returns the current size of the window. + */ + CV_WRAP + Size getWindowSize() const + { return Viz3d::getWindowSize(); } + + /** @brief Sets the size of the window. + + @param window_size New size of the window. + */ + CV_WRAP + void setWindowSize(const Size& window_size) + { return Viz3d::setWindowSize(window_size); } + + /** @brief Returns the name of the window which has been set in the constructor. + * `Viz - ` is prepended to the name if necessary. + */ + CV_WRAP + String getWindowName() const + { return Viz3d::getWindowName(); } + + /** @brief Returns the Mat screenshot of the current scene. + */ + CV_WRAP + cv::Mat getScreenshot() const + { return Viz3d::getScreenshot(); } + + /** @brief Saves screenshot of the current scene. + + @param file Name of the file. + */ + CV_WRAP + void saveScreenshot(const String &file) + { return Viz3d::saveScreenshot(file); } + + /** @brief Sets the position of the window in the screen. + + @param window_position coordinates of the window + */ + CV_WRAP + void setWindowPosition(const Point& window_position) + { return Viz3d::setWindowPosition(window_position); } + + /** @brief Sets or unsets full-screen rendering mode. + + @param mode If true, window will use full-screen mode. + */ + CV_WRAP + void setFullScreen(bool mode = true) + { return Viz3d::setFullScreen(mode); } + + /** @brief Sets background color. + */ + CV_WRAP + void setBackgroundColor(const PyColor& color, const PyColor& color2 = Color::not_set()) + { return Viz3d::setBackgroundColor(color, color2); } + + CV_WRAP + void setBackgroundTexture(InputArray image = noArray()) + { return Viz3d::setBackgroundTexture(image); } + + CV_WRAP + void setBackgroundMeshLab() + { return Viz3d::setBackgroundMeshLab(); } + + /** @brief The window renders and starts the event loop. + */ + CV_WRAP + void spin() + { return Viz3d::spin(); } + + /** @brief Starts the event loop for a given time. + + @param time Amount of time in milliseconds for the event loop to keep running. + @param force_redraw If true, window renders. + */ + CV_WRAP + void spinOnce(int time = 1, bool force_redraw = false) + { return Viz3d::spinOnce(time, force_redraw); } + + /** @brief Create a window in memory instead of on the screen. + */ + CV_WRAP + void setOffScreenRendering() + { return Viz3d::setOffScreenRendering(); } + + /** @brief Remove all lights from the current scene. + */ + CV_WRAP + void removeAllLights() + { return Viz3d::removeAllLights(); } + +#if 0 + /** @brief Add a light in the scene. + + @param position The position of the light. + @param focalPoint The point at which the light is shining + @param color The color of the light + @param diffuseColor The diffuse color of the light + @param ambientColor The ambient color of the light + @param specularColor The specular color of the light + */ + void addLight(const Vec3d &position, const Vec3d &focalPoint = Vec3d(0, 0, 0), const Color &color = Color::white(), + const Color &diffuseColor = Color::white(), const Color &ambientColor = Color::black(), + const Color &specularColor = Color::white()); +#endif + + /** @brief Returns whether the event loop has been stopped. + */ + CV_WRAP + bool wasStopped() const + { return Viz3d::wasStopped(); } + + CV_WRAP + void close() + { return Viz3d::close(); } + + /** @brief Sets rendering property of a widget. + + @param id Id of the widget. + @param property Property that will be modified. + @param value The new value of the property. + + Rendering property can be one of the following: + - **POINT_SIZE** + - **OPACITY** + - **LINE_WIDTH** + - **FONT_SIZE** + + REPRESENTATION: Expected values are + - **REPRESENTATION_POINTS** + - **REPRESENTATION_WIREFRAME** + - **REPRESENTATION_SURFACE** + + IMMEDIATE_RENDERING: + - Turn on immediate rendering by setting the value to 1. + - Turn off immediate rendering by setting the value to 0. + + SHADING: Expected values are + - **SHADING_FLAT** + - **SHADING_GOURAUD** + - **SHADING_PHONG** + */ + CV_WRAP + void setRenderingProperty(const String &id, int property, double value) + { return Viz3d::setRenderingProperty(id, property, value); } + + /** @brief Returns rendering property of a widget. + + @param id Id of the widget. + @param property Property. + + Rendering property can be one of the following: + - **POINT_SIZE** + - **OPACITY** + - **LINE_WIDTH** + - **FONT_SIZE** + + REPRESENTATION: Expected values are + - **REPRESENTATION_POINTS** + - **REPRESENTATION_WIREFRAME** + - **REPRESENTATION_SURFACE** + + IMMEDIATE_RENDERING: + - Turn on immediate rendering by setting the value to 1. + - Turn off immediate rendering by setting the value to 0. + + SHADING: Expected values are + - **SHADING_FLAT** + - **SHADING_GOURAUD** + - **SHADING_PHONG** + */ + CV_WRAP + double getRenderingProperty(const String &id, int property) + { return Viz3d::getRenderingProperty(id, property); } + + /** @brief Sets geometry representation of the widgets to surface, wireframe or points. + + @param representation Geometry representation which can be one of the following: + - **REPRESENTATION_POINTS** + - **REPRESENTATION_WIREFRAME** + - **REPRESENTATION_SURFACE** + */ + CV_WRAP + void setRepresentation(int representation) + { return Viz3d::setRepresentation(representation); } + + CV_WRAP + void setGlobalWarnings(bool enabled = false) + { return Viz3d::setGlobalWarnings(enabled); } +}; + + +}} // namespace +#endif // OPENCV_PYTHON_VIZ_HPP diff --git a/modules/viz/misc/python/test/test_viz_simple.py b/modules/viz/misc/python/test/test_viz_simple.py new file mode 100644 index 00000000000..973f3535a03 --- /dev/null +++ b/modules/viz/misc/python/test/test_viz_simple.py @@ -0,0 +1,439 @@ +import os +import numpy as np +import cv2 as cv + +from tests_common import NewOpenCVTests + +def generate_test_trajectory(): + result = [] + angle_i = np.arange(0, 271, 3) + angle_j = np.arange(0, 1200, 10) + for i, j in zip(angle_i, angle_j): + x = 2 * np.cos(i * 3 * np.pi/180.0) * (1.0 + 0.5 * np.cos(1.2 + i * 1.2 * np.pi/180.0)) + y = 0.25 + i/270.0 + np.sin(j * np.pi/180.0) * 0.2 * np.sin(0.6 + j * 1.5 * np.pi/180.0) + z = 2 * np.sin(i * 3 * np.pi/180.0) * (1.0 + 0.5 * np.cos(1.2 + i * np.pi/180.0)) + result.append(cv.viz.makeCameraPose((x, y, z), (0.0, 0, 0), (0.0, 1.0, 0.0))) + x = np.zeros(shape=(len(result), 1, 16 ), dtype= np.float64) + for idx, m in enumerate(result): + x[idx, 0, :] = m.mat().reshape(16) + return x, result + +def tutorial3(camera_pov, filename): + myWindow = cv.viz_Viz3d("Coordinate Frame") + myWindow.showWidget("axe",cv.viz_WCoordinateSystem()) + + cam_origin = (3.0, 3.0, 3.0) + cam_focal_point = (3.0,3.0,2.0) + cam_y_dir = (-1.0,0.0,0.0) + camera_pose = cv.viz.makeCameraPose(cam_origin, cam_focal_point, cam_y_dir) + transform = cv.viz.makeTransformToGlobal((0.0,-1.0,0.0), (-1.0,0.0,0.0), (0.0,0.0,-1.0), cam_origin) + dragon_cloud,_,_ = cv.viz.readCloud(filename) + cloud_widget = cv.viz_WCloud(dragon_cloud, cv.viz_Color().green()) + cloud_pose = cv.viz_Affine3d() + cloud_pose = cv.viz_Affine3d().rotate((0, np.pi / 2, 0)).translate((0, 0, 3)) + cloud_pose_global = transform.product(cloud_pose) + myWindow.showWidget("CPW_FRUSTUM", cv.viz_WCameraPosition((0.889484, 0.523599)), camera_pose) + if not camera_pov: + myWindow.showWidget("CPW", cv.viz_WCameraPosition(0.5), camera_pose) + myWindow.showWidget("dragon", cloud_widget, cloud_pose_global) + if camera_pov: + myWindow.setViewerPose(camera_pose) + +class viz_test(NewOpenCVTests): + def setUp(self): + super(viz_test, self).setUp() + if not bool(os.environ.get('OPENCV_PYTEST_RUN_VIZ', False)): + self.skipTest("Use OPENCV_PYTEST_RUN_VIZ=1 to enable VIZ UI tests") + + def test_viz_tutorial3_global_view(self): + tutorial3(False, self.find_file("viz/dragon.ply")) + + def test_viz_tutorial3_camera_view(self): + tutorial3(True, self.find_file("viz/dragon.ply")) + + def test_viz(self): + dragon_cloud,_,_ = cv.viz.readCloud(self.find_file("viz/dragon.ply")) + myWindow = cv.viz_Viz3d("abc") + myWindow.showWidget("coo", cv.viz_WCoordinateSystem(1)) + myWindow.showWidget("cloud", cv.viz_WPaintedCloud(dragon_cloud)) + myWindow.spinOnce(500, True) + + def test_viz_show_simple_widgets(self): + viz = cv.viz_Viz3d("show_simple_widgets") + viz.setBackgroundMeshLab() + + viz.showWidget("coos", cv.viz_WCoordinateSystem()) + viz.showWidget("cube", cv.viz_WCube()) + viz.showWidget("cub0", cv.viz_WCube((-1.0, -1, -1), (-0.5, -0.5, -0.5), False, cv.viz_Color().indigo())) + viz.showWidget("arro", cv.viz_WArrow((-0.5, -0.5, -0.5), (0.5, 0.5, 0.5), 0.009, cv.viz_Color().raspberry())) + viz.showWidget("cir1", cv.viz_WCircle(0.5, 0.01, cv.viz_Color.bluberry())) + viz.showWidget("cir2", cv.viz_WCircle(0.5, (0.5, 0.0, 0.0), (1.0, 0.0, 0.0), 0.01, cv.viz_Color().apricot())) + + viz.showWidget("cyl0", cv.viz_WCylinder((-0.5, 0.5, -0.5), (0.5, 0.5, -0.5), 0.125, 30, cv.viz_Color().brown())) + viz.showWidget("con0", cv.viz_WCone(0.25, 0.125, 6, cv.viz_Color().azure())) + viz.showWidget("con1", cv.viz_WCone(0.125, (0.5, -0.5, 0.5), (0.5, -1.0, 0.5), 6, cv.viz_Color().turquoise())) + text2d = cv.viz_WText("Different simple widgets", (20, 20), 20, cv.viz_Color().green()) + viz.showWidget("text2d", text2d) + text3d = cv.viz_WText3D("Simple 3D text", ( 0.5, 0.5, 0.5), 0.125, False, cv.viz_Color().green()) + viz.showWidget("text3d", text3d) + + viz.showWidget("plane1", cv.viz_WPlane((0.25, 0.75))) + viz.showWidget("plane2", cv.viz_WPlane((0.5, -0.5, -0.5), (0.0, 1.0, 1.0), (1.0, 1.0, 0.0), (1.0, 0.5), cv.viz_Color().gold())) + + viz.showWidget("grid1", cv.viz_WGrid((7,7), (0.75,0.75), cv.viz_Color().gray()), cv.viz_Affine3d().translate((0.0, 0.0, -1.0))) + + viz.spinOnce(500, True) + text2d.setText("Different simple widgets (updated)") + text3d.setText("Updated text 3D") + viz.spinOnce(500, True) + + def test_viz_show_overlay_image(self): + lena = cv.imread(self.find_file("viz/lena.png")) + gray = cv.cvtColor(lena, cv.COLOR_BGR2GRAY) + rows = lena.shape[0] + cols = lena.shape[1] + half_lsize = (lena.shape[1] // 2, lena.shape[0] // 2) + + viz = cv.viz_Viz3d("show_overlay_image") + viz.setBackgroundMeshLab(); + vsz = viz.getWindowSize() + + viz.showWidget("coos", cv.viz_WCoordinateSystem()) + viz.showWidget("cube", cv.viz_WCube()) + x = cv.viz_WImageOverlay(lena, (10, 10, half_lsize[1], half_lsize[0])) + viz.showWidget("img1", x) + viz.showWidget("img2", cv.viz_WImageOverlay(gray, (vsz[0] - 10 - cols // 2, 10, half_lsize[1], half_lsize[0]))) + viz.showWidget("img3", cv.viz_WImageOverlay(gray, (10, vsz[1] - 10 - rows // 2, half_lsize[1], half_lsize[0]))) + viz.showWidget("img5", cv.viz_WImageOverlay(lena, (vsz[0] - 10 - cols // 2, vsz[1] - 10 - rows // 2, half_lsize[1], half_lsize[0]))) + viz.showWidget("text2d", cv.viz_WText("Overlay images", (20, 20), 20, cv.viz_Color().green())) + + i = 0 + for num in range(50): + i = i + 1 + a = i % 360 + pose = (3 * np.sin(a * np.pi/180), 2.1, 3 * np.cos(a * np.pi/180)); + viz.setViewerPose(cv.viz.makeCameraPose(pose , (0.0, 0.5, 0.0), (0.0, 0.1, 0.0))) + img = lena * (np.sin(i * 10 * np.pi/180) * 0.5 + 0.5) + x.setImage(img.astype(np.uint8)) + viz.spinOnce(100, True) + viz.showWidget("text2d", cv.viz_WText("Overlay images (stopped)", (20, 20), 20, cv.viz_Color().green())) + viz.spinOnce(500, True) + + def test_viz_show_image_3d(self): + lena = cv.imread(self.find_file("viz/lena.png")) + lena_gray = cv.cvtColor(lena, cv.COLOR_BGR2GRAY) + + viz = cv.viz_Viz3d("show_image_3d") + viz.setBackgroundMeshLab() + viz.showWidget("coos", cv.viz_WCoordinateSystem()) + viz.showWidget("cube", cv.viz_WCube()); + viz.showWidget("arr0", cv.viz_WArrow((0.5, 0.0, 0.0), (1.5, 0.0, 0.0), 0.009, cv.viz_Color().raspberry())) + x = cv.viz_WImage3D(lena, (1.0, 1.0)) + viz.showWidget("img0", x, cv.viz_Affine3d((0.0, np.pi/2, 0.0), (.5, 0.0, 0.0))) + viz.showWidget("arr1", cv.viz_WArrow((-0.5, -0.5, 0.0), (0.2, 0.2, 0.0), 0.009, cv.viz_Color().raspberry())) + viz.showWidget("img1", cv.viz_WImage3D(lena_gray, (1.0, 1.0), (-0.5, -0.5, 0.0), (1.0, 1.0, 0.0), (0.0, 1.0, 0.0))) + + viz.showWidget("arr3", cv.viz_WArrow((-0.5, -0.5, -0.5), (0.5, 0.5, 0.5), 0.009, cv.viz_Color().raspberry())) + + viz.showWidget("text2d", cv.viz_WText("Images in 3D", (20, 20), 20, cv.viz_Color().green())) + + i = 0 + for num in range(50): + img = lena * (np.sin(i*7.5*np.pi/180) * 0.5 + 0.5) + x.setImage(img.astype(np.uint8)) + i = i + 1 + viz.spinOnce(100, True); + viz.showWidget("text2d", cv.viz_WText("Images in 3D (stopped)", (20, 20), 20, cv.viz_Color().green())) + viz.spinOnce(500, True) + + + + def test_viz_show_cloud_bluberry(self): + dragon_cloud,_,_ = cv.viz.readCloud(self.find_file("viz/dragon.ply")) + + pose = cv.viz_Affine3d() + pose = pose.rotate((0, 0.8, 0)); + viz = cv.viz_Viz3d("show_cloud_bluberry") + viz.setBackgroundColor(cv.viz_Color().black()) + viz.showWidget("coosys", cv.viz_WCoordinateSystem()) + viz.showWidget("dragon", cv.viz_WCloud(dragon_cloud, cv.viz_Color().bluberry()), pose) + + viz.showWidget("text2d", cv.viz_WText("Bluberry cloud", (20, 20), 20, cv.viz_Color().green())) + viz.spinOnce(500, True) + + def test_viz_show_cloud_random_color(self): + dragon_cloud,_,_ = cv.viz.readCloud(self.find_file("viz/dragon.ply")) + + colors = np.random.randint(0, 255, size=(dragon_cloud.shape[0],dragon_cloud.shape[1],3), dtype=np.uint8) + + pose = cv.viz_Affine3d() + pose = pose.rotate((0, 0.8, 0)); + + viz = cv.viz_Viz3d("show_cloud_random_color") + viz.setBackgroundMeshLab() + viz.showWidget("coosys", cv.viz_WCoordinateSystem()) + viz.showWidget("dragon", cv.viz_WCloud(dragon_cloud, colors), pose) + viz.showWidget("text2d", cv.viz_WText("Random color cloud", (20, 20), 20, cv.viz_Color().green())) + viz.spinOnce(500, True) + + def test_viz_show_cloud_masked(self): + dragon_cloud,_,_ = cv.viz.readCloud(self.find_file("viz/dragon.ply")) + + qnan = np.NAN + for idx in range(dragon_cloud.shape[0]): + if idx % 15 != 0: + dragon_cloud[idx,:] = qnan + + pose = cv.viz_Affine3d() + pose = pose.rotate((0, 0.8, 0)) + + + viz = cv.viz_Viz3d("show_cloud_masked"); + viz.showWidget("coosys", cv.viz_WCoordinateSystem()) + viz.showWidget("dragon", cv.viz_WCloud(dragon_cloud), pose) + viz.showWidget("text2d", cv.viz_WText("Nan masked cloud", (20, 20), 20, cv.viz_Color().green())) + viz.spinOnce(500, True) + + def test_viz_show_cloud_collection(self): + cloud,_,_ = cv.viz.readCloud(self.find_file("viz/dragon.ply")) + ccol = cv.viz_WCloudCollection() + pose = cv.viz_Affine3d() + pose1 = cv.viz_Affine3d().translate((0, 0, 0)).rotate((np.pi/2, 0, 0)) + ccol.addCloud(cloud, cv.viz_Color().white(), cv.viz_Affine3d().translate((0, 0, 0)).rotate((np.pi/2, 0, 0))) + ccol.addCloud(cloud, cv.viz_Color().blue(), cv.viz_Affine3d().translate((1, 0, 0))) + ccol.addCloud(cloud, cv.viz_Color().red(), cv.viz_Affine3d().translate((2, 0, 0))) + ccol.finalize(); + + viz = cv.viz_Viz3d("show_cloud_collection") + viz.setBackgroundColor(cv.viz_Color().mlab()) + viz.showWidget("coosys", cv.viz_WCoordinateSystem()); + viz.showWidget("ccol", ccol); + viz.showWidget("text2d", cv.viz_WText("Cloud collection", (20, 20), 20, cv.viz_Color(0, 255,0 ))) + viz.spinOnce(500, True) + + def test_viz_show_painted_clouds(self): + cloud,_,_ = cv.viz.readCloud(self.find_file("viz/dragon.ply")) + viz = cv.viz_Viz3d("show_painted_clouds") + viz.setBackgroundMeshLab() + viz.showWidget("coosys", cv.viz_WCoordinateSystem()) + pose1 = cv.viz_Affine3d((0.0, -np.pi/2, 0.0), (-1.5, 0.0, 0.0)) + pose2 = cv.viz_Affine3d((0.0, np.pi/2, 0.0), (1.5, 0.0, 0.0)) + + viz.showWidget("cloud1", cv.viz_WPaintedCloud(cloud), pose1) + viz.showWidget("cloud2", cv.viz_WPaintedCloud(cloud, (0.0, -0.75, -1.0), (0.0, 0.75, 0.0)), pose2); + viz.showWidget("cloud3", cv.viz_WPaintedCloud(cloud, (0.0, 0.0, -1.0), (0.0, 0.0, 1.0), cv.viz_Color().blue(), cv.viz_Color().red())) + viz.showWidget("arrow", cv.viz_WArrow((0.0, 1.0, -1.0), (0.0, 1.0, 1.0), 0.009, cv.viz_Color())) + viz.showWidget("text2d", cv.viz_WText("Painted clouds", (20, 20), 20, cv.viz_Color(0, 255, 0))) + viz.spinOnce(500, True) + + def test_viz_show_mesh(self): + mesh = cv.viz.readMesh(self.find_file("viz/dragon.ply")) + + viz = cv.viz_Viz3d("show_mesh") + viz.showWidget("coosys", cv.viz_WCoordinateSystem()); + viz.showWidget("mesh", cv.viz_WMesh(mesh), cv.viz_Affine3d().rotate((0, 0.8, 0))); + viz.showWidget("text2d", cv.viz_WText("Just mesh", (20, 20), 20, cv.viz_Color().green())) + viz.spinOnce(500, True) + + + def test_viz_show_mesh_random_colors(self): + mesh = cv.viz.readMesh(self.find_file("viz/dragon.ply")) + mesh.colors = np.random.randint(0, 255, size=mesh.colors.shape, dtype=np.uint8) + viz = cv.viz_Viz3d("show_mesh") + viz.showWidget("coosys", cv.viz_WCoordinateSystem()); + viz.showWidget("mesh", cv.viz_WMesh(mesh), cv.viz_Affine3d().rotate((0, 0.8, 0))) + viz.setRenderingProperty("mesh", cv.viz.SHADING, cv.viz.SHADING_PHONG) + viz.showWidget("text2d", cv.viz_WText("Random color mesh", (20, 20), 20, cv.viz_Color().green())) + viz.spinOnce(500, True) + + def test_viz_show_textured_mesh(self): + lena = cv.imread(self.find_file("viz/lena.png")) + + angle = np.arange(0,64) + points0 = np.vstack((np.zeros(shape=angle.shape, dtype=np.float32), np.cos(angle * np.pi /128), np.sin(angle* np.pi /128))) + points1 = np.vstack((1.57 * np.ones(shape=angle.shape, dtype=np.float32),np.cos(angle* np.pi /128), np.sin(angle* np.pi /128))) + tcoords0 = np.vstack((np.zeros(shape=angle.shape, dtype=np.float32), angle / 64)) + tcoords1 = np.vstack((np.ones(shape=angle.shape, dtype=np.float32), angle / 64)) + points = np.zeros(shape=(points0.shape[0], points0.shape[1] * 2 ),dtype=np.float32) + tcoords = np.zeros(shape=(tcoords0.shape[0], tcoords0.shape[1] * 2),dtype=np.float32) + tcoords[:,0::2] = tcoords0 + tcoords[:,1::2] = tcoords1 + points[:,0::2] = points0 * 0.75 + points[:,1::2] = points1 * 0.75 + polygons = np.zeros(shape=(4 * (points.shape[1]-2)+1),dtype=np.int32) + for idx in range(points.shape[1] // 2 - 1): + polygons[8 * idx: 8 * (idx + 1)] = [3, 2*idx, 2*idx+1, 2*idx+2, 3, 2*idx+1, 2*idx+2, 2*idx+3] + + mesh = cv.viz_Mesh() + mesh.cloud = points.transpose().reshape(1,points.shape[1],points.shape[0]) + mesh.tcoords = tcoords.transpose().reshape(1,tcoords.shape[1],tcoords.shape[0]) + mesh.polygons = polygons.reshape(1, 4 * (points.shape[1]-2)+1) + mesh.texture = lena + viz = cv.viz_Viz3d("show_textured_mesh") + viz.setBackgroundMeshLab(); + viz.showWidget("coosys", cv.viz_WCoordinateSystem()); + viz.showWidget("mesh", cv.viz_WMesh(mesh)) + viz.setRenderingProperty("mesh", cv.viz.SHADING, cv.viz.SHADING_PHONG) + viz.showWidget("text2d", cv.viz_WText("Textured mesh", (20, 20), 20, cv.viz_Color().green())); + viz.spinOnce(500, True) + + def test_viz_show_polyline(self): + palette = [ cv.viz_Color().red(), + cv.viz_Color().green(), + cv.viz_Color().blue(), + cv.viz_Color().gold(), + cv.viz_Color().raspberry(), + cv.viz_Color().bluberry(), + cv.viz_Color().lime()] + palette_size = len(palette) + polyline = np.zeros(shape=(1, 32, 3), dtype=np.float32) + colors = np.zeros(shape=(1, 32, 3), dtype=np.uint8) + for i in range(polyline.shape[1]): + polyline[0,i,0] = i / 16.0 + polyline[0,i,1] = np.cos(i * np.pi/6) + polyline[0,i,2] = np.sin(i * np.pi/6) + colors[0,i,0] = palette[i % palette_size].get_blue() + colors[0,i,1] = palette[i % palette_size].get_green() + colors[0,i,2] = palette[i % palette_size].get_red() + + viz = cv.viz_Viz3d("show_polyline") + viz.showWidget("polyline", cv.viz_WPolyLine(polyline, colors)) + viz.showWidget("coosys", cv.viz_WCoordinateSystem()) + viz.showWidget("text2d", cv.viz_WText("Polyline", (20, 20), 20, cv.viz_Color().green())) + viz.spinOnce(500, True) + + def test_viz_show_sampled_normals(self): + + mesh = cv.viz.readMesh(self.find_file("viz/dragon.ply")) + mesh.normals = cv.viz.computeNormals(mesh) + pose = cv.viz_Affine3d().rotate((0, 0.8, 0)) + viz = cv.viz_Viz3d("show_sampled_normals") + viz.showWidget("mesh", cv.viz_WMesh(mesh), pose) + viz.showWidget("normals", cv.viz_WCloudNormals(mesh.cloud, mesh.normals, 30, 0.1, cv.viz_Color().green()), pose) + viz.setRenderingProperty("normals", cv.viz.LINE_WIDTH, 2.0) + viz.showWidget("text2d", cv.viz_WText("Cloud or mesh normals", (20, 20), 20, cv.viz_Color().green())) + viz.spinOnce(500, True); + + + def test_viz_show_cloud_shaded_by_normals(self): + mesh = cv.viz.readMesh(self.find_file("viz/dragon.ply")) + mesh.normals = cv.viz.computeNormals(mesh) + pose = cv.viz_Affine3d().rotate((0, 0.8, 0)) + + cloud = cv.viz_WCloud(mesh.cloud, cv.viz_Color().white(), mesh.normals) + cloud.setRenderingProperty(cv.viz.SHADING, cv.viz.SHADING_GOURAUD) + viz = cv.viz_Viz3d("show_cloud_shaded_by_normals") + + viz.showWidget("cloud", cloud, pose) + viz.showWidget("text2d", cv.viz_WText("Cloud shaded by normals", (20, 20), 20, cv.viz_Color().green())) + viz.spinOnce(500, True) + + def test_viz_show_image_method(self): + lena = cv.imread(self.find_file("viz/lena.png")) + lena_gray = cv.cvtColor(lena, cv.COLOR_BGR2GRAY) + viz = cv.viz_Viz3d("show_image_method") + viz.showImage(lena) + viz.spinOnce(1500, True) + viz.showImage(lena, (lena.shape[1], lena.shape[0])) + viz.spinOnce(1500, True) + + #cv.viz.imshow("show_image_method", lena_gray).spinOnce(500, True) BUG + + def test_viz_show_follower(self): + viz = cv.viz_Viz3d("show_follower") + + viz.showWidget("coos", cv.viz_WCoordinateSystem()) + viz.showWidget("cube", cv.viz_WCube()) + text_3d = cv.viz_WText3D("Simple 3D follower", (-0.5, -0.5, 0.5), 0.125, True, cv.viz_Color().green()) + viz.showWidget("t3d_2", text_3d) + viz.showWidget("text2d", cv.viz_WText("Follower: text always facing camera", (20, 20), 20, cv.viz_Color().green())) + viz.setBackgroundMeshLab() + viz.spinOnce(500, True) + text_3d.setText("Updated follower 3D") + viz.spinOnce(500, True) + + def test_viz_show_trajectory_reposition(self): + mat, path = generate_test_trajectory() + viz = cv.viz_Viz3d("show_trajectory_reposition_to_origin") + viz.showWidget("coos", cv.viz_WCoordinateSystem()) + viz.showWidget("sub3", cv.viz_WTrajectory(mat[0: len(path) // 3,:,:], cv.viz.PyWTrajectory_BOTH, 0.2, cv.viz_Color().brown()), path[0].inv()) + viz.showWidget("text2d", cv.viz_WText("Trajectory resposition to origin", (20, 20), 20, cv.viz_Color().green())) + viz.spinOnce(500, True) + + def test_viz_show_trajectories(self): + mat, path = generate_test_trajectory() + size =len(path) + + sub0 = np.copy(mat[0: size//10+1,::]) + sub1 = np.copy(mat[size//10: size//5+1,::]) + sub2 = np.copy(mat[size//5: 11*size//12,::]) + sub3 = np.copy(mat[11 * size // 12 : size,::]) + sub4 = np.copy(mat[3 * size//4: 33*size//40,::]) + sub5 = np.copy(mat[11*size//12: size,::]) + K = np.array([[1024.0, 0.0, 320.0], [0.0, 1024.0, 240.0], [0.0, 0.0, 1.0]],dtype=np.float64) + + viz = cv.viz_Viz3d("show_trajectories") + viz.showWidget("coos", cv.viz_WCoordinateSystem()) + viz.showWidget("sub0", cv.viz_WTrajectorySpheres(sub0, 0.25, 0.07)) + viz.showWidget("sub1", cv.viz_WTrajectory(sub1, cv.viz.PyWTrajectory_PATH, 0.2, cv.viz_Color().brown())) + viz.showWidget("sub2", cv.viz_WTrajectory(sub2, cv.viz.PyWTrajectory_FRAMES, 0.2)) + viz.showWidget("sub3", cv.viz_WTrajectory(sub3, cv.viz.PyWTrajectory_BOTH, 0.2, cv.viz_Color().green())) + viz.showWidget("sub4", cv.viz_WTrajectoryFrustums(sub4, K, 0.3, cv.viz_Color().yellow())) + viz.showWidget("sub5", cv.viz_WTrajectoryFrustums(sub5, (0.78, 0.78), 0.15, cv.viz_Color().magenta())) #BUG + viz.showWidget("text2d", cv.viz_WText("Different kinds of supported trajectories", (20, 20), 20, cv.viz_Color().green())) + + i = 0 + for num in range(50): + i = i - 1 + a = i % 360 + pose = (np.sin(a * np.pi/180)* 7.5, 0.7, np.cos(a * np.pi/180)* 7.5) + viz.setViewerPose(cv.viz.makeCameraPose(pose , (0.0, 0.5, 0.0), (0.0, 0.1, 0.0))); + viz.spinOnce(100, True) + viz.resetCamera() + viz.spinOnce(500, True) + + def test_viz_show_camera_positions(self): + K = np.array([[1024.0, 0.0, 320.0], [0.0, 1024.0, 240.0], [0.0, 0.0, 1.0]],dtype=np.float64) + lena = cv.imread(self.find_file("viz/lena.png")) + lena_gray = cv.cvtColor(lena, cv.COLOR_BGR2GRAY) + + poses = [] + for i in range(2): + pose = (5 * np.sin(3.14 + 2.7 + i*60 * np.pi/180), 2 - i*1.5, 5 * np.cos(3.14 + 2.7 + i*60 * np.pi/180)) + poses.append(cv.viz.makeCameraPose(pose, (0.0, 0.0, 0.0), (0.0, -0.1, 0.0))) + viz = cv.viz_Viz3d("show_camera_positions") + viz.showWidget("sphe", cv.viz_WSphere((0,0,0), 1.0, 10, cv.viz_Color().orange_red())) + viz.showWidget("coos", cv.viz_WCoordinateSystem(1.5)) + viz.showWidget("pos1", cv.viz_WCameraPosition(0.75), poses[0]) + viz.showWidget("pos2", cv.viz_WCameraPosition((0.78, 0.78), lena, 2.2, cv.viz_Color().green()), poses[0]) + viz.showWidget("pos3", cv.viz_WCameraPosition(0.75), poses[0]) + viz.showWidget("pos4", cv.viz_WCameraPosition(K, lena_gray, 3, cv.viz_Color().indigo()), poses[1]) + viz.showWidget("text2d", cv.viz_WText("Camera positions with images", (20, 20), 20, cv.viz_Color().green())) + viz.spinOnce(500, True) +""" +TEST(Viz, show_widget_merger) +{ + WWidgetMerger merger; + merger.addWidget(WCube(Vec3d::all(0.0), Vec3d::all(1.0), true, Color::gold())); + + RNG& rng = theRNG(); + for(int i = 0; i < 77; ++i) + { + Vec3b c; + rng.fill(c, RNG::NORMAL, Scalar::all(128), Scalar::all(48), true); + merger.addWidget(WSphere(Vec3d(c)*(1.0/255.0), 7.0/255.0, 10, Color(c[2], c[1], c[0]))); + } + merger.finalize(); + + Viz3d viz("show_mesh_random_color"); + viz.showWidget("coo", WCoordinateSystem()); + viz.showWidget("merger", merger); + viz.showWidget("text2d", WText("Widget merger", Point(20, 20), 20, Color::green())); + viz.spinOnce(500, true); +} + + + +""" +if __name__ == '__main__': + NewOpenCVTests.bootstrap() diff --git a/modules/viz/samples/viz_sample_01.py b/modules/viz/samples/viz_sample_01.py new file mode 100644 index 00000000000..6da32c9d3c9 --- /dev/null +++ b/modules/viz/samples/viz_sample_01.py @@ -0,0 +1,12 @@ +import numpy as np +import cv2 as cv + +v = cv.viz.Viz3d_create("Viz Demo") + +print("First event loop is over") +v.spin() +print("Second event loop is over") +v.spinOnce(1, True) +while not v.wasStopped(): + v.spinOnce(1, True) +print("Last event loop is over") diff --git a/modules/viz/samples/viz_sample_02.py b/modules/viz/samples/viz_sample_02.py new file mode 100644 index 00000000000..e91a911560f --- /dev/null +++ b/modules/viz/samples/viz_sample_02.py @@ -0,0 +1,33 @@ +import numpy as np +import cv2 as cv + +my_window = cv.viz_Viz3d("Coordinate Frame") + +axe = cv.viz_PyWCoordinateSystem() +axis = cv.viz_PyWLine((-1.0,-1.0,-1.0), (1.0,1.0,1.0), cv.viz_PyColor().green()) +axis.setRenderingProperty(cv.viz.LINE_WIDTH, 4.0); +my_window.showWidget("axe",axis) +plan = cv.viz_PyWPlane((-1.0,-1.0,-1.0), (1.0,.0,.0), (-.0,.0,-1.0)) +#my_window.showWidget("plan", plan) +cube = cv.viz_PyWCube((0.5,0.5,0.0), (0.0,0.0,-0.5), True, cv.viz_PyColor().blue()) + +#my_window.showWidget("Cube Widget",cube) +pi = np.arccos(-1) +print("First event loop is over") +my_window.spin() +print("Second event loop is over") +my_window.spinOnce(1, True) +translation_phase = 0.0 +translation = 0.0 +rot_mat = np.zeros(shape=(3, 3), dtype=np.float32) +rot_vec = np.zeros(shape=(1,3),dtype=np.float32) +while not my_window.wasStopped(): + rot_vec[0, 0] += np.pi * 0.01 + rot_vec[0, 1] += np.pi * 0.01 + rot_vec[0, 2] += np.pi * 0.01 + translation_phase += pi * 0.01 + translation = np.sin(translation_phase) + pose = cv.viz_PyAffine3(rot_vec, (translation, translation, translation)) + my_window.setWidgetPosePy("Cube Widget", pose) + my_window.spinOnce(1, True) +print("Last event loop is over") diff --git a/modules/viz/samples/viz_sample_03.py b/modules/viz/samples/viz_sample_03.py new file mode 100644 index 00000000000..49e5722c99a --- /dev/null +++ b/modules/viz/samples/viz_sample_03.py @@ -0,0 +1,41 @@ +import numpy as np +import cv2 as cv + +def load_bunny(): + with open(cv.samples.findFile("viz/bunny.ply"), 'r') as f: + s = f.read() + ligne = s.split('\n') + if len(ligne) == 5753: + pts3d = np.zeros(shape=(1,1889,3), dtype=np.float32) + pts3d_c = 255 * np.ones(shape=(1,1889,3), dtype=np.uint8) + pts3d_n = np.ones(shape=(1,1889,3), dtype=np.float32) + for idx in range(12,1889): + d = ligne[idx].split(' ') + pts3d[0,idx-12,:] = (float(d[0]), float(d[1]), float(d[2])) + pts3d = 5 * pts3d + return cv.viz_PyWCloud(pts3d) + +myWindow = cv.viz_Viz3d("Coordinate Frame") +axe = cv.viz_PyWCoordinateSystem() +myWindow.showWidget("axe",axe) + +cam_pos = (3.0, 3.0, 3.0) +cam_focal_point = (3.0,3.0,2.0) +cam_y_dir = (-1.0,0.0,0.0) +cam_pose = cv.viz.makeCameraPosePy(cam_pos, cam_focal_point, cam_y_dir) +print("OK") +transform = cv.viz.makeTransformToGlobalPy((0.0,-1.0,0.0), (-1.0,0.0,0.0), (0.0,0.0,-1.0), cam_pos) +pw_bunny = load_bunny() +cloud_pose = cv.viz_PyAffine3() +cloud_pose = cloud_pose.translate((0, 0, 3)) +cloud_pose_global = transform.product(cloud_pose) + +cpw = cv.viz_PyWCameraPosition(0.5) +cpw_frustum = cv.viz_PyWCameraPosition(0.3) +myWindow.showWidget("CPW", cpw); +myWindow.showWidget("CPW_FRUSTUM", cpw_frustum) +myWindow.setViewerPosePy(cam_pose) +myWindow.showWidget("bunny", pw_bunny, cloud_pose_global) +#myWindow.setWidgetPosePy("bunny") +myWindow.spin(); +print("Last event loop is over") diff --git a/modules/viz/samples/widget_pose.cpp b/modules/viz/samples/widget_pose.cpp index 5de82bd3858..130044f1924 100644 --- a/modules/viz/samples/widget_pose.cpp +++ b/modules/viz/samples/widget_pose.cpp @@ -52,6 +52,26 @@ int main() /// Rodrigues vector Mat rot_vec = Mat::zeros(1,3,CV_32F); float translation_phase = 0.0, translation = 0.0; + + rot_vec.at(0, 0) += (float)CV_PI * 0.01f; + rot_vec.at(0, 1) += (float)CV_PI * 0.01f; + rot_vec.at(0, 2) += (float)CV_PI * 0.01f; + + /// Shift on (1,1,1) + translation_phase += (float)CV_PI * 0.01f; + translation = sin(translation_phase); + + Mat rot_mat; + Rodrigues(rot_vec, rot_mat); + cout << "rot_mat = " << rot_mat << endl; + /// Construct pose + Affine3f pose(rot_mat, Vec3f(translation, translation, translation)); + Affine3f pose2(pose.matrix); + cout << "pose = " << pose.matrix << endl; + cout << "pose = " << pose2.matrix << endl; + + + while(!myWindow.wasStopped()) { /* Rotation using rodrigues */ @@ -64,13 +84,13 @@ int main() translation_phase += (float)CV_PI * 0.01f; translation = sin(translation_phase); - Mat rot_mat; - Rodrigues(rot_vec, rot_mat); + Mat rot_mat1; + Rodrigues(rot_vec, rot_mat1); /// Construct pose - Affine3f pose(rot_mat, Vec3f(translation, translation, translation)); + Affine3f pose1(rot_mat1, Vec3f(translation, translation, translation)); - myWindow.setWidgetPose("Cube Widget", pose); + myWindow.setWidgetPose("Cube Widget", pose1); myWindow.spinOnce(1, true); } From e59b8f785364c6f6fc8fdb3014f75aa3d42b7202 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Mon, 5 Apr 2021 10:43:22 +0000 Subject: [PATCH 64/70] eliminate build warnings --- modules/ximgproc/src/edge_drawing.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/modules/ximgproc/src/edge_drawing.cpp b/modules/ximgproc/src/edge_drawing.cpp index 628893be513..bfabf704ce5 100644 --- a/modules/ximgproc/src/edge_drawing.cpp +++ b/modules/ximgproc/src/edge_drawing.cpp @@ -23,13 +23,13 @@ class EdgeDrawingImpl : public EdgeDrawing }; EdgeDrawingImpl(); - void detectEdges(InputArray src); - void getEdgeImage(OutputArray dst); - void getGradientImage(OutputArray dst); + void detectEdges(InputArray src) CV_OVERRIDE; + void getEdgeImage(OutputArray dst) CV_OVERRIDE; + void getGradientImage(OutputArray dst) CV_OVERRIDE; - vector > getSegments(); - void detectLines(OutputArray lines); - void detectEllipses(OutputArray ellipses); + vector > getSegments() CV_OVERRIDE; + void detectLines(OutputArray lines) CV_OVERRIDE; + void detectEllipses(OutputArray ellipses) CV_OVERRIDE; virtual void read(const FileNode& fn) CV_OVERRIDE; virtual void write(FileStorage& fs) const CV_OVERRIDE; @@ -391,6 +391,7 @@ void EdgeDrawingImpl::ComputeGradient() case SCHARR: gx = abs(3 * (com1 + com2) + 10 * (smoothImg[i * width + j + 1] - smoothImg[i * width + j - 1])); gy = abs(3 * (com1 - com2) + 10 * (smoothImg[(i + 1) * width + j] - smoothImg[(i - 1) * width + j])); + break; case LSD: // com1 and com2 differs from previous operators, because LSD has 2x2 kernel com1 = smoothImg[(i + 1) * width + j + 1] - smoothImg[i * width + j]; @@ -398,6 +399,7 @@ void EdgeDrawingImpl::ComputeGradient() gx = abs(com1 + com2); gy = abs(com1 - com2); + break; } int sum; @@ -2943,7 +2945,7 @@ void EdgeDrawingImpl::DetectArcs() bm->move(noPixels); // Try to fit a circle to the entire arc of lines - double xc, yc, radius, circleFitError; + double xc = -1, yc = -1, radius = -1, circleFitError = -1; CircleFit(x, y, noPixels, &xc, &yc, &radius, &circleFitError); double coverage = noPixels / (CV_2PI * radius); From 87ce2b7345730ac73947e128d41441d8e489bdfc Mon Sep 17 00:00:00 2001 From: pathbreak <5163416+pathbreak@users.noreply.github.com> Date: Tue, 6 Apr 2021 22:47:38 +0530 Subject: [PATCH 65/70] dnn_superres: Fix arXiv URL typos in README.md --- modules/dnn_superres/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/dnn_superres/README.md b/modules/dnn_superres/README.md index b47772b049a..b8a4250ac83 100644 --- a/modules/dnn_superres/README.md +++ b/modules/dnn_superres/README.md @@ -40,7 +40,7 @@ Trained models can be downloaded from [here](https://github.com/fannymonori/TF-E - Advantage: It is tiny and fast, and still performs well. - Disadvantage: Perform worse visually than newer, more robust models. - Speed: < 0.01 sec for every scaling factor on 256x256 images on an Intel i7-9700K CPU. -- Original paper: [Real-Time Single Image and Video Super-Resolution Using an Efficient Sub-Pixel Convolutional Neural Network](https://arxiv.org/pdf/1707.02921.pdf) [2] +- Original paper: [Real-Time Single Image and Video Super-Resolution Using an Efficient Sub-Pixel Convolutional Neural Network](https://arxiv.org/pdf/1609.05158.pdf) [2] #### FSRCNN @@ -66,7 +66,7 @@ Trained models can be downloaded from [here](https://github.com/fannymonori/TF-L - Advantage: The model can do multi-scale super-resolution with one forward pass. It can now support 2x, 4x, 8x, and [2x, 4x] and [2x, 4x, 8x] super-resolution. - Disadvantage: It is slower than ESPCN and FSRCNN, and the accuracy is worse than EDSR. - Speed: < 0.1 sec for every scaling factor on 256x256 images on an Intel i7-9700K CPU. -- Original paper: [Deep laplacian pyramid networks for fast and accurate super-resolution](https://arxiv.org/pdf/1707.02921.pdf) [4] +- Original paper: [Deep laplacian pyramid networks for fast and accurate super-resolution](https://arxiv.org/pdf/1704.03915.pdf) [4] ### Benchmarks @@ -92,4 +92,4 @@ Refer to the benchmarks located in the tutorials for more detailed benchmarking. [3] Chao Dong, Chen Change Loy, Xiaoou Tang. **"Accelerating the Super-Resolution Convolutional Neural Network"**, in Proceedings of European Conference on Computer Vision **ECCV 2016**. [[PDF](http://personal.ie.cuhk.edu.hk/~ccloy/files/eccv_2016_accelerating.pdf)] [[arXiv](https://arxiv.org/abs/1608.00367)] [[Project Page](http://mmlab.ie.cuhk.edu.hk/projects/FSRCNN.html)] -[4] Lai, W. S., Huang, J. B., Ahuja, N., and Yang, M. H., **"Deep laplacian pyramid networks for fast and accurate super-resolution"**, In Proceedings of the IEEE conference on computer vision and pattern recognition **CVPR 2017**. [[PDF](http://openaccess.thecvf.com/content_cvpr_2017/papers/Lai_Deep_Laplacian_Pyramid_CVPR_2017_paper.pdf)] [[arXiv](https://arxiv.org/abs/1710.01992)] [[Project Page](http://vllab.ucmerced.edu/wlai24/LapSRN/)] +[4] Lai, W. S., Huang, J. B., Ahuja, N., and Yang, M. H., **"Deep laplacian pyramid networks for fast and accurate super-resolution"**, In Proceedings of the IEEE conference on computer vision and pattern recognition **CVPR 2017**. [[PDF](http://openaccess.thecvf.com/content_cvpr_2017/papers/Lai_Deep_Laplacian_Pyramid_CVPR_2017_paper.pdf)] [[arXiv](https://arxiv.org/abs/1704.03915)] [[Project Page](http://vllab.ucmerced.edu/wlai24/LapSRN/)] From 930eca198d154799be57c5918ed3d13cefe8f154 Mon Sep 17 00:00:00 2001 From: TT <45615081+tsukada-cs@users.noreply.github.com> Date: Fri, 23 Oct 2020 02:03:11 +0900 Subject: [PATCH 66/70] 3.4/ximgproc: Added edge input feature to fast_line_detector backport of commit: d8197c6ad64468c8b199dcd2343184a51a366f92 --- .../opencv2/ximgproc/fast_line_detector.hpp | 5 ++-- modules/ximgproc/samples/fld_lines.cpp | 5 ++-- modules/ximgproc/src/fast_line_detector.cpp | 23 +++++++++------ modules/ximgproc/test/test_fld.cpp | 28 +++++++++++++++++++ 4 files changed, 49 insertions(+), 12 deletions(-) diff --git a/modules/ximgproc/include/opencv2/ximgproc/fast_line_detector.hpp b/modules/ximgproc/include/opencv2/ximgproc/fast_line_detector.hpp index 78ec43a61b3..7de0a8ba099 100644 --- a/modules/ximgproc/include/opencv2/ximgproc/fast_line_detector.hpp +++ b/modules/ximgproc/include/opencv2/ximgproc/fast_line_detector.hpp @@ -65,8 +65,9 @@ class CV_EXPORTS_W FastLineDetector : public Algorithm hysteresis procedure in Canny() @param _canny_th2 50 - Second threshold for hysteresis procedure in Canny() -@param _canny_aperture_size 3 - Aperturesize for the sobel - operator in Canny() +@param _canny_aperture_size 3 - Aperturesize for the sobel operator in Canny(). + If zero, Canny() is not applied and the input + image is taken as an edge image. @param _do_merge false - If true, incremental merging of segments will be performed */ diff --git a/modules/ximgproc/samples/fld_lines.cpp b/modules/ximgproc/samples/fld_lines.cpp index acb4e32ac52..87edf62420d 100644 --- a/modules/ximgproc/samples/fld_lines.cpp +++ b/modules/ximgproc/samples/fld_lines.cpp @@ -37,8 +37,9 @@ int main(int argc, char** argv) // hysteresis procedure in Canny() // canny_th2 50 - Second threshold for // hysteresis procedure in Canny() - // canny_aperture_size 3 - Aperturesize for the sobel - // operator in Canny() + // canny_aperture_size 3 - Aperturesize for the sobel operator in Canny(). + // If zero, Canny() is not applied and the input + // image is taken as an edge image. // do_merge false - If true, incremental merging of segments // will be performed int length_threshold = 10; diff --git a/modules/ximgproc/src/fast_line_detector.cpp b/modules/ximgproc/src/fast_line_detector.cpp index 33560184d2a..4217d9b33c5 100644 --- a/modules/ximgproc/src/fast_line_detector.cpp +++ b/modules/ximgproc/src/fast_line_detector.cpp @@ -29,10 +29,11 @@ class FastLineDetectorImpl : public FastLineDetector * _ hysteresis procedure in Canny() * @param _canny_th2 50 - Second threshold for * _ hysteresis procedure in Canny() - * @param _canny_aperture_size 3 - Aperturesize for the sobel - * _ operator in Canny() + * @param _canny_aperture_size 3 - Aperturesize for the sobel operator in Canny(). + * If zero, Canny() is not applied and the input + * image is taken as an edge image. * @param _do_merge false - If true, incremental merging of segments - will be perfomred + * will be performed */ FastLineDetectorImpl(int _length_threshold = 10, float _distance_threshold = 1.414213562f, double _canny_th1 = 50.0, double _canny_th2 = 50.0, int _canny_aperture_size = 3, @@ -80,7 +81,7 @@ class FastLineDetectorImpl : public FastLineDetector double distPointLine(const Mat& p, Mat& l); - void extractSegments(const std::vector& points, std::vector& segments ); + void extractSegments(const std::vector& points, std::vector& segments); void lineDetection(const Mat& src, std::vector& segments_all); @@ -113,7 +114,7 @@ FastLineDetectorImpl::FastLineDetectorImpl(int _length_threshold, float _distanc canny_th1(_canny_th1), canny_th2(_canny_th2), canny_aperture_size(_canny_aperture_size), do_merge(_do_merge) { CV_Assert(_length_threshold > 0 && _distance_threshold > 0 && - _canny_th1 > 0 && _canny_th2 > 0 && _canny_aperture_size > 0); + _canny_th1 > 0 && _canny_th2 > 0 && _canny_aperture_size >= 0); } void FastLineDetectorImpl::detect(InputArray _image, OutputArray _lines) @@ -344,7 +345,7 @@ template pt = T(pt_tmp); } -void FastLineDetectorImpl::extractSegments(const std::vector& points, std::vector& segments ) +void FastLineDetectorImpl::extractSegments(const std::vector& points, std::vector& segments) { bool is_line; @@ -544,8 +545,14 @@ void FastLineDetectorImpl::lineDetection(const Mat& src, std::vector& s std::vector points; std::vector segments, segments_tmp; Mat canny; - Canny(src, canny, canny_th1, canny_th2, canny_aperture_size); - + if (canny_aperture_size == 0) + { + canny = src; + } + else + { + Canny(src, canny, canny_th1, canny_th2, canny_aperture_size); + } canny.colRange(0,6).rowRange(0,6) = 0; canny.colRange(src.cols-5,src.cols).rowRange(src.rows-5,src.rows) = 0; diff --git a/modules/ximgproc/test/test_fld.cpp b/modules/ximgproc/test/test_fld.cpp index 71f7fd260cd..d2dc8421581 100644 --- a/modules/ximgproc/test/test_fld.cpp +++ b/modules/ximgproc/test/test_fld.cpp @@ -23,6 +23,7 @@ class FLDBase : public testing::Test void GenerateWhiteNoise(Mat& image); void GenerateConstColor(Mat& image); void GenerateLines(Mat& image, const unsigned int numLines); + void GenerateEdgeLines(Mat& image, const unsigned int numLines); void GenerateBrokenLines(Mat& image, const unsigned int numLines); void GenerateRotatedRect(Mat& image); virtual void SetUp(); @@ -55,6 +56,7 @@ void FLDBase::GenerateConstColor(Mat& image) image = Mat(img_size, CV_8UC1, Scalar::all(rng.uniform(0, 256))); } + void FLDBase::GenerateLines(Mat& image, const unsigned int numLines) { image = Mat(img_size, CV_8UC1, Scalar::all(rng.uniform(0, 128))); @@ -68,6 +70,19 @@ void FLDBase::GenerateLines(Mat& image, const unsigned int numLines) } } +void FLDBase::GenerateEdgeLines(Mat& image, const unsigned int numLines) +{ + image = Mat(img_size, CV_8UC1, Scalar::all(0)); + + for(unsigned int i = 0; i < numLines; ++i) + { + int y = rng.uniform(10, img_size.width - 10); + Point p1(y, 10); + Point p2(y, img_size.height - 10); + line(image, p1, p2, Scalar(255), 1); + } +} + void FLDBase::GenerateBrokenLines(Mat& image, const unsigned int numLines) { image = Mat(img_size, CV_8UC1, Scalar::all(rng.uniform(0, 128))); @@ -153,6 +168,19 @@ TEST_F(ximgproc_FLD, lines) ASSERT_EQ(EPOCHS, passedtests); } +TEST_F(ximgproc_FLD, edgeLines) +{ + for (int i = 0; i < EPOCHS; ++i) + { + const unsigned int numOfLines = 1; + GenerateEdgeLines(test_image, numOfLines); + Ptr detector = createFastLineDetector(10, 1.414213562f, 50, 50, 0); + detector->detect(test_image, lines); + if(numOfLines == lines.size()) ++passedtests; + } + ASSERT_EQ(EPOCHS, passedtests); +} + TEST_F(ximgproc_FLD, mergeLines) { for (int i = 0; i < EPOCHS; ++i) From ee5c1847ff61e4e932955a73c75d9a93d635f8f3 Mon Sep 17 00:00:00 2001 From: berak Date: Wed, 7 Apr 2021 11:32:24 +0200 Subject: [PATCH 67/70] wechat_qr: disable iconv dependancy for mingw --- modules/wechat_qrcode/src/zxing/zxing.hpp | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/modules/wechat_qrcode/src/zxing/zxing.hpp b/modules/wechat_qrcode/src/zxing/zxing.hpp index cae7e7dbac2..11090f74b95 100644 --- a/modules/wechat_qrcode/src/zxing/zxing.hpp +++ b/modules/wechat_qrcode/src/zxing/zxing.hpp @@ -27,7 +27,7 @@ #define USE_ONED_WRITER 1 #endif -#if defined(__ANDROID_API__) +#if defined(__ANDROID_API__) || defined(_MSC_VER) || defined(__MINGW32__) || defined(__MINGW64__) #ifndef NO_ICONV #define NO_ICONV @@ -51,15 +51,6 @@ typedef unsigned char boolean; } // namespace zxing #include - -#if defined(_MSC_VER) - -#ifndef NO_ICONV -#define NO_ICONV -#endif - -#endif - #include namespace zxing { From 14b43012da7f6e5cd98c3d6b56476445ee7a5f3f Mon Sep 17 00:00:00 2001 From: Suleyman TURKMEN Date: Wed, 7 Apr 2021 22:15:39 +0300 Subject: [PATCH 68/70] update fast_line_detector.hpp --- .../opencv2/ximgproc/fast_line_detector.hpp | 43 ++++--- modules/ximgproc/src/fast_line_detector.cpp | 108 +++++------------- 2 files changed, 46 insertions(+), 105 deletions(-) diff --git a/modules/ximgproc/include/opencv2/ximgproc/fast_line_detector.hpp b/modules/ximgproc/include/opencv2/ximgproc/fast_line_detector.hpp index 7de0a8ba099..234cc8a37c2 100644 --- a/modules/ximgproc/include/opencv2/ximgproc/fast_line_detector.hpp +++ b/modules/ximgproc/include/opencv2/ximgproc/fast_line_detector.hpp @@ -33,48 +33,45 @@ class CV_EXPORTS_W FastLineDetector : public Algorithm ![image](pics/corridor_fld.jpg) - @param _image A grayscale (CV_8UC1) input image. If only a roi needs to be + @param image A grayscale (CV_8UC1) input image. If only a roi needs to be selected, use: `fld_ptr-\>detect(image(roi), lines, ...); lines += Scalar(roi.x, roi.y, roi.x, roi.y);` - @param _lines A vector of Vec4f elements specifying the beginning + @param lines A vector of Vec4f elements specifying the beginning and ending point of a line. Where Vec4f is (x1, y1, x2, y2), point 1 is the start, point 2 - end. Returned lines are directed so that the brighter side is on their left. */ - CV_WRAP virtual void detect(InputArray _image, OutputArray _lines) = 0; + CV_WRAP virtual void detect(InputArray image, OutputArray lines) = 0; /** @brief Draws the line segments on a given image. - @param _image The image, where the lines will be drawn. Should be bigger + @param image The image, where the lines will be drawn. Should be bigger or equal to the image, where the lines were found. @param lines A vector of the lines that needed to be drawn. @param draw_arrow If true, arrow heads will be drawn. - */ - CV_WRAP virtual void drawSegments(InputOutputArray _image, InputArray lines, - bool draw_arrow = false) = 0; + @param linecolor Line color. + @param linethickness Line thickness. + */ + CV_WRAP virtual void drawSegments(InputOutputArray image, InputArray lines, + bool draw_arrow = false, Scalar linecolor = Scalar(0, 0, 255), int linethickness = 1) = 0; virtual ~FastLineDetector() { } }; /** @brief Creates a smart pointer to a FastLineDetector object and initializes it -@param _length_threshold 10 - Segment shorter than this will be discarded -@param _distance_threshold 1.41421356 - A point placed from a hypothesis line - segment farther than this will be - regarded as an outlier -@param _canny_th1 50 - First threshold for - hysteresis procedure in Canny() -@param _canny_th2 50 - Second threshold for - hysteresis procedure in Canny() -@param _canny_aperture_size 3 - Aperturesize for the sobel operator in Canny(). - If zero, Canny() is not applied and the input - image is taken as an edge image. -@param _do_merge false - If true, incremental merging of segments - will be performed +@param length_threshold Segment shorter than this will be discarded +@param distance_threshold A point placed from a hypothesis line + segment farther than this will be regarded as an outlier +@param canny_th1 First threshold for hysteresis procedure in Canny() +@param canny_th2 Second threshold for hysteresis procedure in Canny() +@param canny_aperture_size Aperturesize for the sobel operator in Canny(). + If zero, Canny() is not applied and the input image is taken as an edge image. +@param do_merge If true, incremental merging of segments will be performed */ CV_EXPORTS_W Ptr createFastLineDetector( - int _length_threshold = 10, float _distance_threshold = 1.414213562f, - double _canny_th1 = 50.0, double _canny_th2 = 50.0, int _canny_aperture_size = 3, - bool _do_merge = false); + int length_threshold = 10, float distance_threshold = 1.414213562f, + double canny_th1 = 50.0, double canny_th2 = 50.0, int canny_aperture_size = 3, + bool do_merge = false); //! @} ximgproc_fast_line_detector } diff --git a/modules/ximgproc/src/fast_line_detector.cpp b/modules/ximgproc/src/fast_line_detector.cpp index 4217d9b33c5..4e345b56ee6 100644 --- a/modules/ximgproc/src/fast_line_detector.cpp +++ b/modules/ximgproc/src/fast_line_detector.cpp @@ -21,46 +21,14 @@ namespace ximgproc{ class FastLineDetectorImpl : public FastLineDetector { public: - /** - * @param _length_threshold 10 - Segment shorter than this will be discarded - * @param _distance_threshold 1.41421356 - A point placed from a hypothesis line segment - * farther than this will be regarded as an outlier - * @param _canny_th1 50 - First threshold for - * _ hysteresis procedure in Canny() - * @param _canny_th2 50 - Second threshold for - * _ hysteresis procedure in Canny() - * @param _canny_aperture_size 3 - Aperturesize for the sobel operator in Canny(). - * If zero, Canny() is not applied and the input - * image is taken as an edge image. - * @param _do_merge false - If true, incremental merging of segments - * will be performed - */ + FastLineDetectorImpl(int _length_threshold = 10, float _distance_threshold = 1.414213562f, double _canny_th1 = 50.0, double _canny_th2 = 50.0, int _canny_aperture_size = 3, bool _do_merge = false); - /** - * Detect lines in the input image. - * - * @param _image A grayscale(CV_8UC1) input image. - * If only a roi needs to be selected, use - * lsd_ptr->detect(image(roi), ..., lines); - * lines += Scalar(roi.x, roi.y, roi.x, roi.y); - * @param _lines Return: A vector of Vec4f elements specifying the beginning and ending point of - * a line. Where Vec4f is (x1, y1, x2, y2), point 1 is the start, point 2 is the end. - * Returned lines are directed so that the brighter side is placed on left. - */ - void detect(InputArray _image, OutputArray _lines) CV_OVERRIDE; - - /** - * Draw lines on the given canvas. - * - * @param image The image, where lines will be drawn - * Should have the size of the image, where the lines were found - * @param lines The lines that need to be drawn - * @param draw_arrow If true, arrow heads will be drawn - */ - void drawSegments(InputOutputArray _image, InputArray lines, bool draw_arrow = false) CV_OVERRIDE; + void detect(InputArray image, OutputArray lines) CV_OVERRIDE; + + void drawSegments(InputOutputArray image, InputArray lines, bool draw_arrow = false, Scalar linecolor = Scalar(0, 0, 255), int linethickness = 1) CV_OVERRIDE; private: int imagewidth, imageheight, threshold_length; @@ -85,25 +53,24 @@ class FastLineDetectorImpl : public FastLineDetector void lineDetection(const Mat& src, std::vector& segments_all); - void pointInboardTest(const Mat& src, Point2i& pt); + void pointInboardTest(const Size srcSize, Point2i& pt); inline void getAngle(SEGMENT& seg); void additionalOperationsOnSegment(const Mat& src, SEGMENT& seg); - void drawSegment(Mat& mat, const SEGMENT& seg, Scalar bgr = Scalar(0,255,0), - int thickness = 1, bool directed = true); + void drawSegment(InputOutputArray image, const SEGMENT& seg, Scalar bgr = Scalar(0,255,0), int thickness = 1, bool directed = true); }; ///////////////////////////////////////////////////////////////////////////////////////// CV_EXPORTS Ptr createFastLineDetector( - int _length_threshold, float _distance_threshold, - double _canny_th1, double _canny_th2, int _canny_aperture_size, bool _do_merge) + int length_threshold, float distance_threshold, + double canny_th1, double canny_th2, int canny_aperture_size, bool do_merge) { return makePtr( - _length_threshold, _distance_threshold, - _canny_th1, _canny_th2, _canny_aperture_size, _do_merge); + length_threshold, distance_threshold, + canny_th1, canny_th2, canny_aperture_size, do_merge); } ///////////////////////////////////////////////////////////////////////////////////////// @@ -136,30 +103,23 @@ void FastLineDetectorImpl::detect(InputArray _image, OutputArray _lines) Mat(lines).copyTo(_lines); } -void FastLineDetectorImpl::drawSegments(InputOutputArray _image, InputArray lines, bool draw_arrow) +void FastLineDetectorImpl::drawSegments(InputOutputArray image, InputArray lines, bool draw_arrow, Scalar linecolor, int linethickness) { CV_INSTRUMENT_REGION(); - CV_Assert(!_image.empty() && (_image.channels() == 1 || _image.channels() == 3)); + int cn = image.channels(); + CV_Assert(!image.empty() && ( cn == 1 || cn == 3 || cn == 4)); - Mat gray; - if (_image.channels() == 1) + if (cn == 1) { - gray = _image.getMatRef(); + cvtColor(image, image, COLOR_GRAY2BGR); } - else if (_image.channels() == 3) + else { - cvtColor(_image, gray, COLOR_BGR2GRAY); + cvtColor(image, image, COLOR_BGRA2GRAY); + cvtColor(image, image, cn == 3 ? COLOR_GRAY2BGR : COLOR_GRAY2BGRA); } - // Create a 3 channel image in order to draw colored lines - std::vector planes; - planes.push_back(gray); - planes.push_back(gray); - planes.push_back(gray); - - merge(planes, _image); - double gap = 10.0; double arrow_angle = 30.0; @@ -172,7 +132,7 @@ void FastLineDetectorImpl::drawSegments(InputOutputArray _image, InputArray line const Vec4f& v = _lines.at(i); Point2f b(v[0], v[1]); Point2f e(v[2], v[3]); - line(_image.getMatRef(), b, e, Scalar(0, 0, 255), 1); + line(image, b, e, linecolor, linethickness); if(draw_arrow) { SEGMENT seg; @@ -185,8 +145,8 @@ void FastLineDetectorImpl::drawSegments(InputOutputArray _image, InputArray line Point2i p1; p1.x = cvRound(seg.x2 - gap*cos(arrow_angle * CV_PI / 180.0 + ang)); p1.y = cvRound(seg.y2 - gap*sin(arrow_angle * CV_PI / 180.0 + ang)); - pointInboardTest(_image.getMatRef(), p1); - line(_image.getMatRef(), Point(cvRound(seg.x2), cvRound(seg.y2)), p1, Scalar(0,0,255), 1); + pointInboardTest(image.size(), p1); + line(image, Point(cvRound(seg.x2), cvRound(seg.y2)), p1, linecolor, linethickness); } } } @@ -477,10 +437,10 @@ void FastLineDetectorImpl::extractSegments(const std::vector& points, s } } -void FastLineDetectorImpl::pointInboardTest(const Mat& src, Point2i& pt) +void FastLineDetectorImpl::pointInboardTest(const Size srcSize, Point2i& pt) { - pt.x = pt.x <= 5 ? 5 : pt.x >= src.cols - 5 ? src.cols - 5 : pt.x; - pt.y = pt.y <= 5 ? 5 : pt.y >= src.rows - 5 ? src.rows - 5 : pt.y; + pt.x = pt.x <= 5 ? 5 : pt.x >= srcSize.width - 5 ? srcSize.width - 5 : pt.x; + pt.y = pt.y <= 5 ? 5 : pt.y >= srcSize.height - 5 ? srcSize.height - 5 : pt.y; } bool FastLineDetectorImpl::getPointChain(const Mat& img, Point pt, @@ -692,8 +652,8 @@ void FastLineDetectorImpl::additionalOperationsOnSegment(const Mat& src, SEGMENT points_right[i].y = cvRound(points[i].y + gap*sin(90.0 * CV_PI / 180.0 + ang)); points_left[i].x = cvRound(points[i].x - gap*cos(90.0 * CV_PI / 180.0 + ang)); points_left[i].y = cvRound(points[i].y - gap*sin(90.0 * CV_PI / 180.0 + ang)); - pointInboardTest(src, points_right[i]); - pointInboardTest(src, points_left[i]); + pointInboardTest(src.size(), points_right[i]); + pointInboardTest(src.size(), points_left[i]); } int iR = 0, iL = 0; @@ -717,21 +677,5 @@ void FastLineDetectorImpl::additionalOperationsOnSegment(const Mat& src, SEGMENT return; } -void FastLineDetectorImpl::drawSegment(Mat& mat, const SEGMENT& seg, Scalar bgr, int thickness, bool directed) -{ - double gap = 10.0; - double ang = (double)seg.angle; - double arrow_angle = 30.0; - - Point2i p1; - p1.x = cvRound(seg.x2 - gap*cos(arrow_angle * CV_PI / 180.0 + ang)); - p1.y = cvRound(seg.y2 - gap*sin(arrow_angle * CV_PI / 180.0 + ang)); - pointInboardTest(mat, p1); - - line(mat, Point(cvRound(seg.x1), cvRound(seg.y1)), - Point(cvRound(seg.x2), cvRound(seg.y2)), bgr, thickness, 1); - if(directed) - line(mat, Point(cvRound(seg.x2), cvRound(seg.y2)), p1, bgr, thickness, 1); -} } // namespace cv } // namespace ximgproc From da6a95e0dbb4df268db451e0751fb5d2dd986e8f Mon Sep 17 00:00:00 2001 From: DumDereDum <46279571+DumDereDum@users.noreply.github.com> Date: Wed, 7 Apr 2021 23:49:57 +0300 Subject: [PATCH 69/70] Merge pull request #2878 from DumDereDum:colored_kinfu Colored Kinect Fusion * create dummy unworked classes * dummy class works * colored kinfu demo minor changes * add simple rgb reader and writer * add rgb info to tsdf * add new raycast * replace TsdfVoxel with RGBTsdfVoxel; integrate done * add colors info to raycast * just render colors * color processing fix * intergrate color fix * add simple shift; minor fixes * add cv_errors for not implemented functions * minor fixes * made calibration * makeColoredFrameFromDepth works * remove comments * makeColoredFrameFromDepth fix * RGBSoure remove extra code * RGBSoure remove extra code 1 * simple fix of bug with rgb size * minor fix * docs fix * docs fix * unused parameter fix * warnings fix * fix tsdf errors; rgbd_perf warnings * fix errors * minor fix * debug print * debug print 2 * debug print 3 * invoker fix * minor fix * remove debug cout * add simple tests * docs fix * warnings fix * minor fix * function warning fix * add vectorized code * minor fixes * minor speed up * minor fixes; renaming * new minor fixes * bug fix * minor fix * test update * extra code removed * bugfix; warning fix * simple interpolation * minor fix * color work minor fix * vectorized interpolation * minor fix --- modules/rgbd/include/opencv2/rgbd.hpp | 1 + .../include/opencv2/rgbd/colored_kinfu.hpp | 261 +++++ modules/rgbd/include/opencv2/rgbd/kinfu.hpp | 8 + .../rgbd/include/opencv2/rgbd/large_kinfu.hpp | 3 + modules/rgbd/include/opencv2/rgbd/volume.hpp | 10 +- modules/rgbd/perf/perf_tsdf.cpp | 2 + modules/rgbd/samples/colored_kinfu_demo.cpp | 276 +++++ modules/rgbd/samples/io_utils.hpp | 340 +++++- modules/rgbd/src/colored_kinfu.cpp | 391 +++++++ modules/rgbd/src/colored_tsdf.cpp | 1020 +++++++++++++++++ modules/rgbd/src/colored_tsdf.hpp | 61 + modules/rgbd/src/hash_tsdf.cpp | 10 +- modules/rgbd/src/kinfu.cpp | 12 + modules/rgbd/src/kinfu_frame.cpp | 239 ++++ modules/rgbd/src/kinfu_frame.hpp | 10 +- modules/rgbd/src/precomp.hpp | 1 + modules/rgbd/src/tsdf.cpp | 2 +- modules/rgbd/src/tsdf.hpp | 8 + modules/rgbd/src/tsdf_functions.cpp | 300 +++++ modules/rgbd/src/tsdf_functions.hpp | 21 + modules/rgbd/src/volume.cpp | 24 +- modules/rgbd/test/test_colored_kinfu.cpp | 479 ++++++++ 22 files changed, 3441 insertions(+), 38 deletions(-) create mode 100644 modules/rgbd/include/opencv2/rgbd/colored_kinfu.hpp create mode 100644 modules/rgbd/samples/colored_kinfu_demo.cpp create mode 100644 modules/rgbd/src/colored_kinfu.cpp create mode 100644 modules/rgbd/src/colored_tsdf.cpp create mode 100644 modules/rgbd/src/colored_tsdf.hpp create mode 100644 modules/rgbd/test/test_colored_kinfu.cpp diff --git a/modules/rgbd/include/opencv2/rgbd.hpp b/modules/rgbd/include/opencv2/rgbd.hpp index 550d0cc8398..4b40539b61e 100755 --- a/modules/rgbd/include/opencv2/rgbd.hpp +++ b/modules/rgbd/include/opencv2/rgbd.hpp @@ -15,6 +15,7 @@ #include "opencv2/rgbd/dynafu.hpp" #include "opencv2/rgbd/large_kinfu.hpp" #include "opencv2/rgbd/detail/pose_graph.hpp" +#include "opencv2/rgbd/colored_kinfu.hpp" /** @defgroup rgbd RGB-Depth Processing diff --git a/modules/rgbd/include/opencv2/rgbd/colored_kinfu.hpp b/modules/rgbd/include/opencv2/rgbd/colored_kinfu.hpp new file mode 100644 index 00000000000..427f830b972 --- /dev/null +++ b/modules/rgbd/include/opencv2/rgbd/colored_kinfu.hpp @@ -0,0 +1,261 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +// This code is also subject to the license terms in the LICENSE_KinectFusion.md file found in this module's directory + +#ifndef __OPENCV_RGBD_COLORED_KINFU_HPP__ +#define __OPENCV_RGBD_COLORED_KINFU_HPP__ + +#include "opencv2/core.hpp" +#include "opencv2/core/affine.hpp" +#include + +namespace cv { +namespace colored_kinfu { +//! @addtogroup kinect_fusion +//! @{ + +struct CV_EXPORTS_W Params +{ + + CV_WRAP Params(){} + + /** + * @brief Constructor for Params + * Sets the initial pose of the TSDF volume. + * @param volumeInitialPoseRot rotation matrix + * @param volumeInitialPoseTransl translation vector + */ + CV_WRAP Params(Matx33f volumeInitialPoseRot, Vec3f volumeInitialPoseTransl) + { + setInitialVolumePose(volumeInitialPoseRot,volumeInitialPoseTransl); + } + + /** + * @brief Constructor for Params + * Sets the initial pose of the TSDF volume. + * @param volumeInitialPose 4 by 4 Homogeneous Transform matrix to set the intial pose of TSDF volume + */ + CV_WRAP Params(Matx44f volumeInitialPose) + { + setInitialVolumePose(volumeInitialPose); + } + + /** + * @brief Set Initial Volume Pose + * Sets the initial pose of the TSDF volume. + * @param R rotation matrix + * @param t translation vector + */ + CV_WRAP void setInitialVolumePose(Matx33f R, Vec3f t); + + /** + * @brief Set Initial Volume Pose + * Sets the initial pose of the TSDF volume. + * @param homogen_tf 4 by 4 Homogeneous Transform matrix to set the intial pose of TSDF volume + */ + CV_WRAP void setInitialVolumePose(Matx44f homogen_tf); + + /** + * @brief Default parameters + * A set of parameters which provides better model quality, can be very slow. + */ + CV_WRAP static Ptr defaultParams(); + + /** @brief Coarse parameters + A set of parameters which provides better speed, can fail to match frames + in case of rapid sensor motion. + */ + CV_WRAP static Ptr coarseParams(); + + /** @brief HashTSDF parameters + A set of parameters suitable for use with HashTSDFVolume + */ + CV_WRAP static Ptr hashTSDFParams(bool isCoarse); + + /** @brief ColoredTSDF parameters + A set of parameters suitable for use with HashTSDFVolume + */ + CV_WRAP static Ptr coloredTSDFParams(bool isCoarse); + + /** @brief frame size in pixels */ + CV_PROP_RW Size frameSize; + + /** @brief rgb frame size in pixels */ + CV_PROP_RW Size rgb_frameSize; + + CV_PROP_RW kinfu::VolumeType volumeType; + + /** @brief camera intrinsics */ + CV_PROP_RW Matx33f intr; + + /** @brief rgb camera intrinsics */ + CV_PROP_RW Matx33f rgb_intr; + + /** @brief pre-scale per 1 meter for input values + + Typical values are: + * 5000 per 1 meter for the 16-bit PNG files of TUM database + * 1000 per 1 meter for Kinect 2 device + * 1 per 1 meter for the 32-bit float images in the ROS bag files + */ + CV_PROP_RW float depthFactor; + + /** @brief Depth sigma in meters for bilateral smooth */ + CV_PROP_RW float bilateral_sigma_depth; + /** @brief Spatial sigma in pixels for bilateral smooth */ + CV_PROP_RW float bilateral_sigma_spatial; + /** @brief Kernel size in pixels for bilateral smooth */ + CV_PROP_RW int bilateral_kernel_size; + + /** @brief Number of pyramid levels for ICP */ + CV_PROP_RW int pyramidLevels; + + /** @brief Resolution of voxel space + + Number of voxels in each dimension. + */ + CV_PROP_RW Vec3i volumeDims; + /** @brief Size of voxel in meters */ + CV_PROP_RW float voxelSize; + + /** @brief Minimal camera movement in meters + + Integrate new depth frame only if camera movement exceeds this value. + */ + CV_PROP_RW float tsdf_min_camera_movement; + + /** @brief initial volume pose in meters */ + Affine3f volumePose; + + /** @brief distance to truncate in meters + + Distances to surface that exceed this value will be truncated to 1.0. + */ + CV_PROP_RW float tsdf_trunc_dist; + + /** @brief max number of frames per voxel + + Each voxel keeps running average of distances no longer than this value. + */ + CV_PROP_RW int tsdf_max_weight; + + /** @brief A length of one raycast step + + How much voxel sizes we skip each raycast step + */ + CV_PROP_RW float raycast_step_factor; + + // gradient delta in voxel sizes + // fixed at 1.0f + // float gradient_delta_factor; + + /** @brief light pose for rendering in meters */ + CV_PROP_RW Vec3f lightPose; + + /** @brief distance theshold for ICP in meters */ + CV_PROP_RW float icpDistThresh; + /** angle threshold for ICP in radians */ + CV_PROP_RW float icpAngleThresh; + /** number of ICP iterations for each pyramid level */ + CV_PROP_RW std::vector icpIterations; + + /** @brief Threshold for depth truncation in meters + + All depth values beyond this threshold will be set to zero + */ + CV_PROP_RW float truncateThreshold; +}; + +/** @brief KinectFusion implementation + + This class implements a 3d reconstruction algorithm described in + @cite kinectfusion paper. + + It takes a sequence of depth images taken from depth sensor + (or any depth images source such as stereo camera matching algorithm or even raymarching renderer). + The output can be obtained as a vector of points and their normals + or can be Phong-rendered from given camera pose. + + An internal representation of a model is a voxel cuboid that keeps TSDF values + which are a sort of distances to the surface (for details read the @cite kinectfusion article about TSDF). + There is no interface to that representation yet. + + KinFu uses OpenCL acceleration automatically if available. + To enable or disable it explicitly use cv::setUseOptimized() or cv::ocl::setUseOpenCL(). + + This implementation is based on [kinfu-remake](https://github.com/Nerei/kinfu_remake). + + Note that the KinectFusion algorithm was patented and its use may be restricted by + the list of patents mentioned in README.md file in this module directory. + + That's why you need to set the OPENCV_ENABLE_NONFREE option in CMake to use KinectFusion. +*/ +class CV_EXPORTS_W ColoredKinFu +{ +public: + CV_WRAP static Ptr create(const Ptr& _params); + virtual ~ColoredKinFu(); + + /** @brief Get current parameters */ + virtual const Params& getParams() const = 0; + + /** @brief Renders a volume into an image + + Renders a 0-surface of TSDF using Phong shading into a CV_8UC4 Mat. + Light pose is fixed in KinFu params. + + @param image resulting image + @param cameraPose pose of camera to render from. If empty then render from current pose + which is a last frame camera pose. + */ + + CV_WRAP virtual void render(OutputArray image, const Matx44f& cameraPose = Matx44f::eye()) const = 0; + + /** @brief Gets points and normals of current 3d mesh + + The order of normals corresponds to order of points. + The order of points is undefined. + + @param points vector of points which are 4-float vectors + @param normals vector of normals which are 4-float vectors + */ + CV_WRAP virtual void getCloud(OutputArray points, OutputArray normals) const = 0; + + /** @brief Gets points of current 3d mesh + + The order of points is undefined. + + @param points vector of points which are 4-float vectors + */ + CV_WRAP virtual void getPoints(OutputArray points) const = 0; + + /** @brief Calculates normals for given points + @param points input vector of points which are 4-float vectors + @param normals output vector of corresponding normals which are 4-float vectors + */ + CV_WRAP virtual void getNormals(InputArray points, OutputArray normals) const = 0; + + /** @brief Resets the algorithm + + Clears current model and resets a pose. + */ + CV_WRAP virtual void reset() = 0; + + /** @brief Get current pose in voxel space */ + virtual const Affine3f getPose() const = 0; + + /** @brief Process next depth frame + @param depth input Mat of depth frame + @param rgb input Mat of rgb (colored) frame + + @return true if succeeded to align new frame with current scene, false if opposite + */ + CV_WRAP virtual bool update(InputArray depth, InputArray rgb) = 0; +}; + +//! @} +} +} +#endif diff --git a/modules/rgbd/include/opencv2/rgbd/kinfu.hpp b/modules/rgbd/include/opencv2/rgbd/kinfu.hpp index 95f732b6769..924314eddbe 100644 --- a/modules/rgbd/include/opencv2/rgbd/kinfu.hpp +++ b/modules/rgbd/include/opencv2/rgbd/kinfu.hpp @@ -74,14 +74,22 @@ struct CV_EXPORTS_W Params */ CV_WRAP static Ptr hashTSDFParams(bool isCoarse); + /** @brief ColoredTSDF parameters + A set of parameters suitable for use with ColoredTSDFVolume + */ + CV_WRAP static Ptr coloredTSDFParams(bool isCoarse); + /** @brief frame size in pixels */ CV_PROP_RW Size frameSize; + /** @brief rgb frame size in pixels */ CV_PROP_RW kinfu::VolumeType volumeType; /** @brief camera intrinsics */ CV_PROP_RW Matx33f intr; + /** @brief rgb camera intrinsics */ + CV_PROP_RW Matx33f rgb_intr; /** @brief pre-scale per 1 meter for input values Typical values are: diff --git a/modules/rgbd/include/opencv2/rgbd/large_kinfu.hpp b/modules/rgbd/include/opencv2/rgbd/large_kinfu.hpp index 7f1428c6d51..2f2d510e8e5 100644 --- a/modules/rgbd/include/opencv2/rgbd/large_kinfu.hpp +++ b/modules/rgbd/include/opencv2/rgbd/large_kinfu.hpp @@ -41,6 +41,9 @@ struct CV_EXPORTS_W Params /** @brief camera intrinsics */ CV_PROP_RW Matx33f intr; + /** @brief rgb camera intrinsics */ + CV_PROP_RW Matx33f rgb_intr; + /** @brief pre-scale per 1 meter for input values Typical values are: * 5000 per 1 meter for the 16-bit PNG files of TUM database diff --git a/modules/rgbd/include/opencv2/rgbd/volume.hpp b/modules/rgbd/include/opencv2/rgbd/volume.hpp index 33f63b1fbfb..3fb1d98ef4c 100644 --- a/modules/rgbd/include/opencv2/rgbd/volume.hpp +++ b/modules/rgbd/include/opencv2/rgbd/volume.hpp @@ -30,8 +30,13 @@ class CV_EXPORTS_W Volume virtual void integrate(InputArray _depth, float depthFactor, const Matx44f& cameraPose, const kinfu::Intr& intrinsics, const int frameId = 0) = 0; + virtual void integrate(InputArray _depth, InputArray _rgb, float depthFactor, + const Matx44f& cameraPose, const kinfu::Intr& intrinsics, + const Intr& rgb_intrinsics, const int frameId = 0) = 0; virtual void raycast(const Matx44f& cameraPose, const kinfu::Intr& intrinsics, const Size& frameSize, OutputArray points, OutputArray normals) const = 0; + virtual void raycast(const Matx44f& cameraPose, const kinfu::Intr& intrinsics, const Size& frameSize, + OutputArray points, OutputArray normals, OutputArray colors) const = 0; virtual void fetchNormals(InputArray points, OutputArray _normals) const = 0; virtual void fetchPointsNormals(OutputArray points, OutputArray normals) const = 0; virtual void reset() = 0; @@ -45,8 +50,9 @@ class CV_EXPORTS_W Volume enum class VolumeType { - TSDF = 0, - HASHTSDF = 1 + TSDF = 0, + HASHTSDF = 1, + COLOREDTSDF = 2 }; struct CV_EXPORTS_W VolumeParams diff --git a/modules/rgbd/perf/perf_tsdf.cpp b/modules/rgbd/perf/perf_tsdf.cpp index 149093665ce..da928f98f22 100644 --- a/modules/rgbd/perf/perf_tsdf.cpp +++ b/modules/rgbd/perf/perf_tsdf.cpp @@ -323,6 +323,7 @@ PERF_TEST(Perf_TSDF, integrate) startTimer(); settings.volume->integrate(depth, settings._params->depthFactor, pose, settings._params->intr); stopTimer(); + depth.release(); } SANITY_CHECK_NOTHING(); } @@ -358,6 +359,7 @@ PERF_TEST(Perf_HashTSDF, integrate) startTimer(); settings.volume->integrate(depth, settings._params->depthFactor, pose, settings._params->intr); stopTimer(); + depth.release(); } SANITY_CHECK_NOTHING(); } diff --git a/modules/rgbd/samples/colored_kinfu_demo.cpp b/modules/rgbd/samples/colored_kinfu_demo.cpp new file mode 100644 index 00000000000..5500c9dc1aa --- /dev/null +++ b/modules/rgbd/samples/colored_kinfu_demo.cpp @@ -0,0 +1,276 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +// This code is also subject to the license terms in the LICENSE_KinectFusion.md file found in this module's directory + +#include +#include +#include +#include +#include +#include + +#include "io_utils.hpp" + +using namespace cv; +using namespace cv::kinfu; +using namespace cv::colored_kinfu; +using namespace cv::io_utils; + +#ifdef HAVE_OPENCV_VIZ +#include +#endif + +#ifdef HAVE_OPENCV_VIZ +const std::string vizWindowName = "cloud"; + +struct PauseCallbackArgs +{ + PauseCallbackArgs(ColoredKinFu& _kf) : kf(_kf) + { } + + ColoredKinFu& kf; +}; + +void pauseCallback(const viz::MouseEvent& me, void* args); +void pauseCallback(const viz::MouseEvent& me, void* args) +{ + if(me.type == viz::MouseEvent::Type::MouseMove || + me.type == viz::MouseEvent::Type::MouseScrollDown || + me.type == viz::MouseEvent::Type::MouseScrollUp) + { + PauseCallbackArgs pca = *((PauseCallbackArgs*)(args)); + viz::Viz3d window(vizWindowName); + UMat rendered; + pca.kf.render(rendered, window.getViewerPose().matrix); + imshow("render", rendered); + waitKey(1); + } +} +#endif + +static const char* keys = +{ + "{help h usage ? | | print this message }" + "{depth | | Path to folder with depth.txt and rgb.txt files listing a set of depth and rgb images }" + "{camera |0| Index of depth camera to be used as a depth source }" + "{coarse | | Run on coarse settings (fast but ugly) or on default (slow but looks better)," + " in coarse mode points and normals are displayed }" + "{idle | | Do not run KinFu, just display depth frames }" + "{record | | Write depth frames to specified file list" + " (the same format as for the 'depth' key) }" +}; + +static const std::string message = + "\nThis demo uses live depth input or RGB-D dataset taken from" + "\nhttps://vision.in.tum.de/data/datasets/rgbd-dataset" + "\nto demonstrate KinectFusion implementation \n"; + + +int main(int argc, char **argv) +{ + bool coarse = false; + bool idle = false; + std::string recordPath; + + CommandLineParser parser(argc, argv, keys); + parser.about(message); + + if(!parser.check()) + { + parser.printMessage(); + parser.printErrors(); + return -1; + } + + if(parser.has("help")) + { + parser.printMessage(); + return 0; + } + if(parser.has("coarse")) + { + coarse = true; + } + if(parser.has("record")) + { + recordPath = parser.get("record"); + } + if(parser.has("idle")) + { + idle = true; + } + + Ptr ds; + Ptr rgbs; + + if (parser.has("depth")) + ds = makePtr(parser.get("depth") + "/depth.txt"); + else + ds = makePtr(parser.get("camera")); + + //TODO: intrinsics for camera + rgbs = makePtr(parser.get("depth") + "/rgb.txt"); + + if (ds->empty()) + { + std::cerr << "Failed to open depth source" << std::endl; + parser.printMessage(); + return -1; + } + + Ptr depthWriter; + Ptr rgbWriter; + + if (!recordPath.empty()) + { + depthWriter = makePtr(recordPath); + rgbWriter = makePtr(recordPath); + } + Ptr params; + Ptr kf; + + params = colored_kinfu::Params::coloredTSDFParams(coarse); + + // These params can be different for each depth sensor + ds->updateParams(*params); + + rgbs->updateParams(*params); + + // Enables OpenCL explicitly (by default can be switched-off) + cv::setUseOptimized(false); + + // Scene-specific params should be tuned for each scene individually + //float cubeSize = 1.f; + //params->voxelSize = cubeSize/params->volumeDims[0]; //meters + //params->tsdf_trunc_dist = 0.01f; //meters + //params->icpDistThresh = 0.01f; //meters + //params->volumePose = Affine3f().translate(Vec3f(-cubeSize/2.f, -cubeSize/2.f, 0.25f)); //meters + //params->tsdf_max_weight = 16; + + if(!idle) + kf = ColoredKinFu::create(params); + +#ifdef HAVE_OPENCV_VIZ + cv::viz::Viz3d window(vizWindowName); + window.setViewerPose(Affine3f::Identity()); + bool pause = false; +#endif + + UMat rendered; + UMat points; + UMat normals; + + int64 prevTime = getTickCount(); + + for(UMat frame = ds->getDepth(); !frame.empty(); frame = ds->getDepth()) + { + if(depthWriter) + depthWriter->append(frame); + UMat rgb_frame = rgbs->getRGB(); +#ifdef HAVE_OPENCV_VIZ + if(pause) + { + // doesn't happen in idle mode + kf->getCloud(points, normals); + if(!points.empty() && !normals.empty()) + { + viz::WCloud cloudWidget(points, viz::Color::white()); + viz::WCloudNormals cloudNormals(points, normals, /*level*/1, /*scale*/0.05, viz::Color::gray()); + window.showWidget("cloud", cloudWidget); + window.showWidget("normals", cloudNormals); + + Vec3d volSize = kf->getParams().voxelSize*Vec3d(kf->getParams().volumeDims); + window.showWidget("cube", viz::WCube(Vec3d::all(0), + volSize), + kf->getParams().volumePose); + PauseCallbackArgs pca(*kf); + window.registerMouseCallback(pauseCallback, (void*)&pca); + window.showWidget("text", viz::WText(cv::String("Move camera in this window. " + "Close the window or press Q to resume"), Point())); + window.spin(); + window.removeWidget("text"); + window.removeWidget("cloud"); + window.removeWidget("normals"); + window.registerMouseCallback(0); + } + + pause = false; + } + else +#endif + { + UMat cvt8; + float depthFactor = params->depthFactor; + convertScaleAbs(frame, cvt8, 0.25*256. / depthFactor); + if(!idle) + { + imshow("depth", cvt8); + imshow("rgb", rgb_frame); + if(!kf->update(frame, rgb_frame)) + { + kf->reset(); + } +#ifdef HAVE_OPENCV_VIZ + else + { + if(coarse) + { + kf->getCloud(points, normals); + if(!points.empty() && !normals.empty()) + { + viz::WCloud cloudWidget(points, viz::Color::white()); + viz::WCloudNormals cloudNormals(points, normals, /*level*/1, /*scale*/0.05, viz::Color::gray()); + window.showWidget("cloud", cloudWidget); + window.showWidget("normals", cloudNormals); + } + } + + //window.showWidget("worldAxes", viz::WCoordinateSystem()); + Vec3d volSize = kf->getParams().voxelSize*kf->getParams().volumeDims; + window.showWidget("cube", viz::WCube(Vec3d::all(0), + volSize), + kf->getParams().volumePose); + window.setViewerPose(kf->getPose()); + window.spinOnce(1, true); + } +#endif + + kf->render(rendered); + } + else + { + rendered = cvt8; + } + } + + int64 newTime = getTickCount(); + putText(rendered, cv::format("FPS: %2d press R to reset, P to pause, Q to quit", + (int)(getTickFrequency()/(newTime - prevTime))), + Point(0, rendered.rows-1), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 255)); + prevTime = newTime; + + imshow("render", rendered); + + int c = waitKey(1); + switch (c) + { + case 'r': + if(!idle) + kf->reset(); + break; + case 'q': + return 0; +#ifdef HAVE_OPENCV_VIZ + case 'p': + if(!idle) + pause = true; +#endif + default: + break; + } + } + + return 0; +} diff --git a/modules/rgbd/samples/io_utils.hpp b/modules/rgbd/samples/io_utils.hpp index f0a3a1e7372..56170b4d42b 100644 --- a/modules/rgbd/samples/io_utils.hpp +++ b/modules/rgbd/samples/io_utils.hpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace cv { @@ -79,27 +80,45 @@ struct DepthWriter namespace Kinect2Params { -static const Size frameSize = Size(512, 424); +static const Size depth_frameSize = Size(512, 424); // approximate values, no guarantee to be correct -static const float focal = 366.1f; -static const float cx = 258.2f; -static const float cy = 204.f; -static const float k1 = 0.12f; -static const float k2 = -0.34f; -static const float k3 = 0.12f; +static const float depth_focal = 366.1f; +static const float depth_cx = 258.2f; +static const float depth_cy = 204.f; +static const float depth_k1 = 0.12f; +static const float depth_k2 = -0.34f; +static const float depth_k3 = 0.12f; + +static const Size rgb_frameSize = Size(640, 480); +static const float rgb_focal = 525.0f; +static const float rgb_cx = 319.5f; +static const float rgb_cy = 239.5f; +static const float rgb_k1 = 0.0f; +static const float rgb_k2 = 0.0f; +static const float rgb_k3 = 0.0f; + }; // namespace Kinect2Params namespace AstraParams { -static const Size frameSize = Size(640, 480); +static const Size depth_frameSize = Size(640, 480); // approximate values, no guarantee to be correct -static const float fx = 535.4f; -static const float fy = 539.2f; -static const float cx = 320.1f; -static const float cy = 247.6f; -static const float k1 = 0.0f; -static const float k2 = 0.0f; -static const float k3 = 0.0f; +static const float depth_fx = 535.4f; +static const float depth_fy = 539.2f; +static const float depth_cx = 320.1f; +static const float depth_cy = 247.6f; +static const float depth_k1 = 0.0f; +static const float depth_k2 = 0.0f; +static const float depth_k3 = 0.0f; + +static const Size rgb_frameSize = Size(640, 480); +static const float rgb_focal = 525.0f; +static const float rgb_cx = 319.5f; +static const float rgb_cy = 239.5f; +static const float rgb_k1 = 0.0f; +static const float rgb_k2 = 0.0f; +static const float rgb_k3 = 0.0f; + }; // namespace Kinect2Params struct DepthSource @@ -181,7 +200,7 @@ struct DepthSource // workaround for Kinect 2 if (sourceType == Type::DEPTH_KINECT2) { - out = out(Rect(Point(), Kinect2Params::frameSize)); + out = out(Rect(Point(), Kinect2Params::depth_frameSize)); UMat outCopy; // linear remap adds gradient between valid and invalid pixels @@ -212,20 +231,20 @@ struct DepthSource Size frameSize; if (sourceType == Type::DEPTH_KINECT2) { - fx = fy = Kinect2Params::focal; - cx = Kinect2Params::cx; - cy = Kinect2Params::cy; + fx = fy = Kinect2Params::depth_focal; + cx = Kinect2Params::depth_cx; + cy = Kinect2Params::depth_cy; - frameSize = Kinect2Params::frameSize; + frameSize = Kinect2Params::depth_frameSize; } else if (sourceType == Type::DEPTH_ASTRA) { - fx = AstraParams::fx; - fy = AstraParams::fy; - cx = AstraParams::cx; - cy = AstraParams::cy; + fx = AstraParams::depth_fx; + fy = AstraParams::depth_fy; + cx = AstraParams::depth_cx; + cy = AstraParams::depth_cy; - frameSize = AstraParams::frameSize; + frameSize = AstraParams::depth_frameSize; } else { @@ -295,9 +314,9 @@ struct DepthSource if (sourceType == Type::DEPTH_KINECT2) { Matx distCoeffs; - distCoeffs(0) = Kinect2Params::k1; - distCoeffs(1) = Kinect2Params::k2; - distCoeffs(4) = Kinect2Params::k3; + distCoeffs(0) = Kinect2Params::depth_k1; + distCoeffs(1) = Kinect2Params::depth_k2; + distCoeffs(4) = Kinect2Params::depth_k3; initUndistortRectifyMap(params.intr, distCoeffs, cv::noArray(), params.intr, params.frameSize, CV_16SC2, undistortMap1, undistortMap2); @@ -317,9 +336,31 @@ struct DepthSource if (sourceType == Type::DEPTH_KINECT2) { Matx distCoeffs; - distCoeffs(0) = Kinect2Params::k1; - distCoeffs(1) = Kinect2Params::k2; - distCoeffs(4) = Kinect2Params::k3; + distCoeffs(0) = Kinect2Params::depth_k1; + distCoeffs(1) = Kinect2Params::depth_k2; + distCoeffs(4) = Kinect2Params::depth_k3; + + initUndistortRectifyMap(params.intr, distCoeffs, cv::noArray(), params.intr, + params.frameSize, CV_16SC2, undistortMap1, undistortMap2); + } + } + } + + void updateParams(colored_kinfu::Params& params) + { + if (vc.isOpened()) + { + updateIntrinsics(params.intr, params.frameSize, params.depthFactor); + updateVolumeParams(params.volumeDims, params.voxelSize, + params.tsdf_trunc_dist, params.volumePose, params.truncateThreshold); + updateICPParams(params.icpDistThresh, params.bilateral_sigma_depth); + + if (sourceType == Type::DEPTH_KINECT2) + { + Matx distCoeffs; + distCoeffs(0) = Kinect2Params::depth_k1; + distCoeffs(1) = Kinect2Params::depth_k2; + distCoeffs(4) = Kinect2Params::depth_k3; initUndistortRectifyMap(params.intr, distCoeffs, cv::noArray(), params.intr, params.frameSize, CV_16SC2, undistortMap1, undistortMap2); @@ -333,6 +374,243 @@ struct DepthSource UMat undistortMap1, undistortMap2; Type sourceType; }; + + +static std::vector readRGB(const std::string& fileList) +{ + std::vector v; + + std::fstream file(fileList); + if (!file.is_open()) + throw std::runtime_error("Failed to read rgb list"); + + std::string dir; + size_t slashIdx = fileList.rfind('/'); + slashIdx = slashIdx != std::string::npos ? slashIdx : fileList.rfind('\\'); + dir = fileList.substr(0, slashIdx); + + while (!file.eof()) + { + std::string s, imgPath; + std::getline(file, s); + if (s.empty() || s[0] == '#') + continue; + std::stringstream ss; + ss << s; + double thumb; + ss >> thumb >> imgPath; + v.push_back(dir + '/' + imgPath); + } + + return v; +} + +struct RGBWriter +{ + RGBWriter(std::string fileList) : file(fileList, std::ios::out), count(0), dir() + { + size_t slashIdx = fileList.rfind('/'); + slashIdx = slashIdx != std::string::npos ? slashIdx : fileList.rfind('\\'); + dir = fileList.substr(0, slashIdx); + + if (!file.is_open()) + throw std::runtime_error("Failed to write rgb list"); + + file << "# rgb maps saved from device" << std::endl; + file << "# useless_number filename" << std::endl; + } + + void append(InputArray _rgb) + { + Mat rgb = _rgb.getMat(); + std::string rgbFname = cv::format("%04d.png", count); + std::string fullRGBFname = dir + '/' + rgbFname; + if (!imwrite(fullRGBFname, rgb)) + throw std::runtime_error("Failed to write rgb to file " + fullRGBFname); + file << count++ << " " << rgbFname << std::endl; + } + + std::fstream file; + int count; + std::string dir; +}; + +struct RGBSource +{ + public: + enum Type + { + RGB_LIST, + RGB_KINECT2_LIST, + RGB_KINECT2, + RGB_REALSENSE, + RGB_ASTRA + }; + + RGBSource(int cam) : RGBSource("", cam) {} + + RGBSource(String fileListName) : RGBSource(fileListName, -1) {} + + RGBSource(String fileListName, int cam) + : rgbFileList(fileListName.empty() ? std::vector() + : readRGB(fileListName)), + frameIdx(0), + undistortMap1(), + undistortMap2() + { + if (cam >= 0) + { + vc = VideoCapture(VideoCaptureAPIs::CAP_OPENNI2 + cam); + if (vc.isOpened()) + { + if(cam == 20) + sourceType = Type::RGB_ASTRA; + else + sourceType = Type::RGB_KINECT2; + } + else + { + vc = VideoCapture(VideoCaptureAPIs::CAP_REALSENSE + cam); + if (vc.isOpened()) + { + sourceType = Type::RGB_REALSENSE; + } + } + } + else + { + vc = VideoCapture(); + sourceType = Type::RGB_KINECT2_LIST; + } + } + + UMat getRGB() + { + UMat out; + if (!vc.isOpened()) + { + if (frameIdx < rgbFileList.size()) + { + Mat f = cv::imread(rgbFileList[frameIdx++], IMREAD_COLOR); + f.copyTo(out); + } + else + { + return UMat(); + } + } + else + { + vc.grab(); + switch (sourceType) + { + case Type::RGB_KINECT2: vc.retrieve(out, CAP_OPENNI_BGR_IMAGE); break; + case Type::RGB_REALSENSE: vc.retrieve(out, CAP_INTELPERC_IMAGE); break; + default: + // unknown rgb source + vc.retrieve(out); + } + + // workaround for Kinect 2 + if (sourceType == Type::RGB_KINECT2) + { + out = out(Rect(Point(), Kinect2Params::rgb_frameSize)); + + UMat outCopy; + // linear remap adds gradient between valid and invalid pixels + // which causes garbage, use nearest instead + remap(out, outCopy, undistortMap1, undistortMap2, cv::INTER_NEAREST); + + cv::flip(outCopy, out, 1); + } + } + if (out.empty()) + throw std::runtime_error("Matrix is empty"); + return out; + } + + bool empty() { return rgbFileList.empty() && !(vc.isOpened()); } + + void updateIntrinsics(Matx33f& _rgb_intrinsics, Size& _rgb_frameSize) + { + if (vc.isOpened()) + { + // this should be set in according to user's rgb sensor + int w = (int)vc.get(VideoCaptureProperties::CAP_PROP_FRAME_WIDTH); + int h = (int)vc.get(VideoCaptureProperties::CAP_PROP_FRAME_HEIGHT); + + // it's recommended to calibrate sensor to obtain its intrinsics + float rgb_fx, rgb_fy, rgb_cx, rgb_cy; + Size rgb_frameSize; + if (sourceType == Type::RGB_KINECT2) + { + rgb_fx = rgb_fy = Kinect2Params::rgb_focal; + rgb_cx = Kinect2Params::rgb_cx; + rgb_cy = Kinect2Params::rgb_cy; + + rgb_frameSize = Kinect2Params::rgb_frameSize; + } + else if (sourceType == Type::RGB_ASTRA) + { + rgb_fx = rgb_fy = AstraParams::rgb_focal; + rgb_cx = AstraParams::rgb_cx; + rgb_cy = AstraParams::rgb_cy; + + rgb_frameSize = AstraParams::rgb_frameSize; + } + else + { + // TODO: replace to rgb types + rgb_fx = rgb_fy = Kinect2Params::rgb_focal; + rgb_cx = Kinect2Params::rgb_cx; + rgb_cy = Kinect2Params::rgb_cy; + rgb_frameSize = Size(w, h); + } + + Matx33f rgb_camMatrix = Matx33f(rgb_fx, 0, rgb_cx, 0, rgb_fy, rgb_cy, 0, 0, 1); + _rgb_intrinsics = rgb_camMatrix; + _rgb_frameSize = rgb_frameSize; + } + } + + void updateVolumeParams(const Vec3i&, float&, float&, Affine3f&) + { + // TODO: do this settings for rgb image + } + + void updateICPParams(float&) + { + // TODO: do this settings for rgb image icp + } + + void updateParams(colored_kinfu::Params& params) + { + if (vc.isOpened()) + { + updateIntrinsics(params.rgb_intr, params.rgb_frameSize); + updateVolumeParams(params.volumeDims, params.voxelSize, + params.tsdf_trunc_dist, params.volumePose); + updateICPParams(params.icpDistThresh); + + if (sourceType == Type::RGB_KINECT2) + { + Matx distCoeffs; + distCoeffs(0) = Kinect2Params::rgb_k1; + distCoeffs(1) = Kinect2Params::rgb_k2; + distCoeffs(4) = Kinect2Params::rgb_k3; + + initUndistortRectifyMap(params.intr, distCoeffs, cv::noArray(), params.intr, + params.frameSize, CV_16SC2, undistortMap1, undistortMap2); + } + } + } + + std::vector rgbFileList; + size_t frameIdx; + VideoCapture vc; + UMat undistortMap1, undistortMap2; + Type sourceType; +}; } // namespace io_utils } // namespace cv diff --git a/modules/rgbd/src/colored_kinfu.cpp b/modules/rgbd/src/colored_kinfu.cpp new file mode 100644 index 00000000000..dc87be6b39a --- /dev/null +++ b/modules/rgbd/src/colored_kinfu.cpp @@ -0,0 +1,391 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +// This code is also subject to the license terms in the LICENSE_KinectFusion.md file found in this module's directory + +#include "precomp.hpp" +#include "fast_icp.hpp" +#include "tsdf.hpp" +#include "hash_tsdf.hpp" +#include "colored_tsdf.hpp" +#include "kinfu_frame.hpp" + +namespace cv { +namespace colored_kinfu { +using namespace kinfu; + +void Params::setInitialVolumePose(Matx33f R, Vec3f t) +{ + setInitialVolumePose(Affine3f(R,t).matrix); +} + +void Params::setInitialVolumePose(Matx44f homogen_tf) +{ + Params::volumePose.matrix = homogen_tf; +} + +Ptr Params::defaultParams() +{ + Params p; + + p.frameSize = Size(640, 480); + + p.volumeType = VolumeType::TSDF; + + float fx, fy, cx, cy; + fx = fy = 525.f; + cx = p.frameSize.width/2 - 0.5f; + cy = p.frameSize.height/2 - 0.5f; + p.intr = Matx33f(fx, 0, cx, + 0, fy, cy, + 0, 0, 1); + + float rgb_fx, rgb_fy, rgb_cx, rgb_cy; + rgb_fx = 515.0f; + rgb_fy = 550.0f; + rgb_cx = 319.5f; + rgb_cy = 239.5f; + p.rgb_intr = Matx33f(rgb_fx, 0, rgb_cx, + 0, rgb_fy, rgb_cy, + 0, 0, 1); + + // 5000 for the 16-bit PNG files + // 1 for the 32-bit float images in the ROS bag files + p.depthFactor = 5000; + + // sigma_depth is scaled by depthFactor when calling bilateral filter + p.bilateral_sigma_depth = 0.04f; //meter + p.bilateral_sigma_spatial = 4.5; //pixels + p.bilateral_kernel_size = 7; //pixels + + p.icpAngleThresh = (float)(30. * CV_PI / 180.); // radians + p.icpDistThresh = 0.1f; // meters + + p.icpIterations = {10, 5, 4}; + p.pyramidLevels = (int)p.icpIterations.size(); + + p.tsdf_min_camera_movement = 0.f; //meters, disabled + + p.volumeDims = Vec3i::all(512); //number of voxels + + float volSize = 3.f; + p.voxelSize = volSize/512.f; //meters + + // default pose of volume cube + p.volumePose = Affine3f().translate(Vec3f(-volSize/2.f, -volSize/2.f, 0.5f)); + p.tsdf_trunc_dist = 7 * p.voxelSize; // about 0.04f in meters + p.tsdf_max_weight = 64; //frames + + p.raycast_step_factor = 0.25f; //in voxel sizes + // gradient delta factor is fixed at 1.0f and is not used + //p.gradient_delta_factor = 0.5f; //in voxel sizes + + //p.lightPose = p.volume_pose.translation()/4; //meters + p.lightPose = Vec3f::all(0.f); //meters + + // depth truncation is not used by default but can be useful in some scenes + p.truncateThreshold = 0.f; //meters + + return makePtr(p); +} + +Ptr Params::coarseParams() +{ + Ptr p = defaultParams(); + + p->icpIterations = {5, 3, 2}; + p->pyramidLevels = (int)p->icpIterations.size(); + + float volSize = 3.f; + p->volumeDims = Vec3i::all(128); //number of voxels + p->voxelSize = volSize/128.f; + p->tsdf_trunc_dist = 2 * p->voxelSize; // 0.04f in meters + + p->raycast_step_factor = 0.75f; //in voxel sizes + + return p; +} + +Ptr Params::hashTSDFParams(bool isCoarse) +{ + Ptr p; + if (isCoarse) + p = coarseParams(); + else + p = defaultParams(); + p->volumeType = VolumeType::HASHTSDF; + p->truncateThreshold = rgbd::Odometry::DEFAULT_MAX_DEPTH(); + return p; +} + +Ptr Params::coloredTSDFParams(bool isCoarse) +{ + Ptr p; + if(isCoarse) + p = coarseParams(); + else + p = defaultParams(); + p->volumeType = VolumeType::COLOREDTSDF; + + return p; +} + +// MatType should be Mat or UMat +template< typename MatType> +class ColoredKinFuImpl : public ColoredKinFu +{ +public: + ColoredKinFuImpl(const Params& _params); + virtual ~ColoredKinFuImpl(); + + const Params& getParams() const CV_OVERRIDE; + + void render(OutputArray image, const Matx44f& cameraPose) const CV_OVERRIDE; + + virtual void getCloud(OutputArray points, OutputArray normals) const CV_OVERRIDE; + void getPoints(OutputArray points) const CV_OVERRIDE; + void getNormals(InputArray points, OutputArray normals) const CV_OVERRIDE; + + void reset() CV_OVERRIDE; + + const Affine3f getPose() const CV_OVERRIDE; + + bool update(InputArray depth, InputArray rgb) CV_OVERRIDE; + + bool updateT(const MatType& depth, const MatType& rgb); + +private: + Params params; + + cv::Ptr icp; + cv::Ptr volume; + + int frameCounter; + Matx44f pose; + std::vector pyrPoints; + std::vector pyrNormals; + std::vector pyrColors; +}; + + +template< typename MatType > +ColoredKinFuImpl::ColoredKinFuImpl(const Params &_params) : + params(_params), + icp(makeICP(params.intr, params.icpIterations, params.icpAngleThresh, params.icpDistThresh)), + pyrPoints(), pyrNormals(), pyrColors() +{ + volume = makeVolume(params.volumeType, params.voxelSize, params.volumePose.matrix, params.raycast_step_factor, + params.tsdf_trunc_dist, params.tsdf_max_weight, params.truncateThreshold, params.volumeDims); + reset(); +} + +template< typename MatType > +void ColoredKinFuImpl::reset() +{ + frameCounter = 0; + pose = Affine3f::Identity().matrix; + volume->reset(); +} + +template< typename MatType > +ColoredKinFuImpl::~ColoredKinFuImpl() +{ } + +template< typename MatType > +const Params& ColoredKinFuImpl::getParams() const +{ + return params; +} + +template< typename MatType > +const Affine3f ColoredKinFuImpl::getPose() const +{ + return pose; +} + + +template<> +bool ColoredKinFuImpl::update(InputArray _depth, InputArray _rgb) +{ + CV_Assert(!_depth.empty() && _depth.size() == params.frameSize); + + Mat depth; + Mat rgb; + if(_depth.isUMat()) + { + _depth.copyTo(depth); + _rgb.copyTo(rgb); + return updateT(depth, rgb); + } + else + { + return updateT(_depth.getMat(), _rgb.getMat()); + } +} + + +template<> +bool ColoredKinFuImpl::update(InputArray _depth, InputArray _rgb) +{ + CV_Assert(!_depth.empty() && _depth.size() == params.frameSize); + + UMat depth; + UMat rgb; + if(!_depth.isUMat()) + { + _depth.copyTo(depth); + _rgb.copyTo(rgb); + return updateT(depth, rgb); + } + else + { + return updateT(_depth.getUMat(), _rgb.getUMat()); + } +} + + +template< typename MatType > +bool ColoredKinFuImpl::updateT(const MatType& _depth, const MatType& _rgb) +{ + CV_TRACE_FUNCTION(); + + MatType depth; + MatType rgb; + + if(_depth.type() != DEPTH_TYPE) + _depth.convertTo(depth, DEPTH_TYPE); + else + depth = _depth; + + if (_rgb.type() != COLOR_TYPE) + { + cv::Mat rgb_tmp, rgbchannel[3], z; + std::vector channels; + _rgb.convertTo(rgb_tmp, COLOR_TYPE); + cv::split(rgb_tmp, rgbchannel); + z = cv::Mat::zeros(rgbchannel[0].size(), CV_32F); + channels.push_back(rgbchannel[0]); channels.push_back(rgbchannel[1]); + channels.push_back(rgbchannel[2]); channels.push_back(z); + merge(channels, rgb); + } + else + rgb = _rgb; + + + std::vector newPoints, newNormals, newColors; + makeColoredFrameFromDepth(depth, rgb, + newPoints, newNormals, newColors, + params.intr, params.rgb_intr, + params.pyramidLevels, + params.depthFactor, + params.bilateral_sigma_depth, + params.bilateral_sigma_spatial, + params.bilateral_kernel_size, + params.truncateThreshold); + + if(frameCounter == 0) + { + // use depth instead of distance + volume->integrate(depth, rgb, params.depthFactor, pose, params.intr, params.rgb_intr); + pyrPoints = newPoints; + pyrNormals = newNormals; + pyrColors = newColors; + } + else + { + Affine3f affine; + bool success = icp->estimateTransform(affine, pyrPoints, pyrNormals, newPoints, newNormals); + if(!success) + return false; + + pose = (Affine3f(pose) * affine).matrix; + + float rnorm = (float)cv::norm(affine.rvec()); + float tnorm = (float)cv::norm(affine.translation()); + // We do not integrate volume if camera does not move + if((rnorm + tnorm)/2 >= params.tsdf_min_camera_movement) + { + // use depth instead of distance + volume->integrate(depth, rgb, params.depthFactor, pose, params.intr, params.rgb_intr); + } + MatType& points = pyrPoints [0]; + MatType& normals = pyrNormals[0]; + MatType& colors = pyrColors [0]; + volume->raycast(pose, params.intr, params.frameSize, points, normals, colors); + buildPyramidPointsNormals(points, normals, pyrPoints, pyrNormals, + params.pyramidLevels); + } + + frameCounter++; + return true; +} + + +template< typename MatType > +void ColoredKinFuImpl::render(OutputArray image, const Matx44f& _cameraPose) const +{ + CV_TRACE_FUNCTION(); + + Affine3f cameraPose(_cameraPose); + Affine3f _pose(pose); + + const Affine3f id = Affine3f::Identity(); + if((cameraPose.rotation() == _pose.rotation() && cameraPose.translation() == _pose.translation()) || + (cameraPose.rotation() == id.rotation() && cameraPose.translation() == id.translation())) + { + renderPointsNormalsColors(pyrPoints[0], pyrNormals[0], pyrColors[0],image, params.lightPose); + } + else + { + MatType points, normals, colors; + volume->raycast(_cameraPose, params.intr, params.frameSize, points, normals, colors); + renderPointsNormalsColors(points, normals, colors, image, params.lightPose); + } +} + + +template< typename MatType > +void ColoredKinFuImpl::getCloud(OutputArray p, OutputArray n) const +{ + volume->fetchPointsNormals(p, n); +} + + +template< typename MatType > +void ColoredKinFuImpl::getPoints(OutputArray points) const +{ + volume->fetchPointsNormals(points, noArray()); +} + + +template< typename MatType > +void ColoredKinFuImpl::getNormals(InputArray points, OutputArray normals) const +{ + volume->fetchNormals(points, normals); +} + +// importing class + +#ifdef OPENCV_ENABLE_NONFREE + +Ptr ColoredKinFu::create(const Ptr& params) +{ + CV_Assert((int)params->icpIterations.size() == params->pyramidLevels); + CV_Assert(params->intr(0,1) == 0 && params->intr(1,0) == 0 && params->intr(2,0) == 0 && params->intr(2,1) == 0 && params->intr(2,2) == 1); + return makePtr< ColoredKinFuImpl >(*params); +} + +#else +Ptr ColoredKinFu::create(const Ptr& /* params */) +{ + CV_Error(Error::StsNotImplemented, + "This algorithm is patented and is excluded in this configuration; " + "Set OPENCV_ENABLE_NONFREE CMake option and rebuild the library"); +} +#endif + +ColoredKinFu::~ColoredKinFu() {} + +} // namespace kinfu +} // namespace cv diff --git a/modules/rgbd/src/colored_tsdf.cpp b/modules/rgbd/src/colored_tsdf.cpp new file mode 100644 index 00000000000..116136a7f21 --- /dev/null +++ b/modules/rgbd/src/colored_tsdf.cpp @@ -0,0 +1,1020 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +// This code is also subject to the license terms in the LICENSE_KinectFusion.md file found in this module's directory + +#include "precomp.hpp" +#include "colored_tsdf.hpp" +#include "tsdf_functions.hpp" +#include "opencl_kernels_rgbd.hpp" + +#define USE_INTERPOLATION_IN_GETNORMAL 1 + +namespace cv { + +namespace kinfu { + +ColoredTSDFVolume::ColoredTSDFVolume(float _voxelSize, Matx44f _pose, float _raycastStepFactor, float _truncDist, + int _maxWeight, Point3i _resolution, bool zFirstMemOrder) + : Volume(_voxelSize, _pose, _raycastStepFactor), + volResolution(_resolution), + maxWeight( WeightType(_maxWeight) ) +{ + CV_Assert(_maxWeight < 255); + // Unlike original code, this should work with any volume size + // Not only when (x,y,z % 32) == 0 + volSize = Point3f(volResolution) * voxelSize; + truncDist = std::max(_truncDist, 2.1f * voxelSize); + + // (xRes*yRes*zRes) array + // Depending on zFirstMemOrder arg: + // &elem(x, y, z) = data + x*zRes*yRes + y*zRes + z; + // &elem(x, y, z) = data + x + y*xRes + z*xRes*yRes; + int xdim, ydim, zdim; + if(zFirstMemOrder) + { + xdim = volResolution.z * volResolution.y; + ydim = volResolution.z; + zdim = 1; + } + else + { + xdim = 1; + ydim = volResolution.x; + zdim = volResolution.x * volResolution.y; + } + + volDims = Vec4i(xdim, ydim, zdim); + neighbourCoords = Vec8i( + volDims.dot(Vec4i(0, 0, 0)), + volDims.dot(Vec4i(0, 0, 1)), + volDims.dot(Vec4i(0, 1, 0)), + volDims.dot(Vec4i(0, 1, 1)), + volDims.dot(Vec4i(1, 0, 0)), + volDims.dot(Vec4i(1, 0, 1)), + volDims.dot(Vec4i(1, 1, 0)), + volDims.dot(Vec4i(1, 1, 1)) + ); +} + +class ColoredTSDFVolumeCPU : public ColoredTSDFVolume +{ +public: + // dimension in voxels, size in meters + ColoredTSDFVolumeCPU(float _voxelSize, cv::Matx44f _pose, float _raycastStepFactor, float _truncDist, + int _maxWeight, Vec3i _resolution, bool zFirstMemOrder = true); + virtual void integrate(InputArray, float, const Matx44f&, const kinfu::Intr&, const int) override + { CV_Error(Error::StsNotImplemented, "Not implemented"); }; + virtual void integrate(InputArray _depth, InputArray _rgb, float depthFactor, const Matx44f& cameraPose, + const kinfu::Intr& depth_intrinsics, const Intr& rgb_intrinsics, const int frameId = 0) override; + virtual void raycast(const Matx44f& cameraPose, const kinfu::Intr& depth_intrinsics, const Size& frameSize, + OutputArray points, OutputArray normals, OutputArray colors) const override; + virtual void raycast(const Matx44f&, const kinfu::Intr&, const Size&, OutputArray, OutputArray) const override + { CV_Error(Error::StsNotImplemented, "Not implemented"); }; + + virtual void fetchNormals(InputArray points, OutputArray _normals) const override; + virtual void fetchPointsNormals(OutputArray points, OutputArray normals) const override; + + virtual void reset() override; + virtual RGBTsdfVoxel at(const Vec3i& volumeIdx) const; + + float interpolateVoxel(const cv::Point3f& p) const; + Point3f getNormalVoxel(const cv::Point3f& p) const; + float interpolateColor(float tx, float ty, float tz, float vx[8]) const; + Point3f getColorVoxel(const cv::Point3f& p) const; + +#if USE_INTRINSICS + float interpolateVoxel(const v_float32x4& p) const; + v_float32x4 getNormalVoxel(const v_float32x4& p) const; + v_float32x4 getColorVoxel(const v_float32x4& p) const; +#endif + + Vec4i volStrides; + Vec6f frameParams; + Mat pixNorms; + // See zFirstMemOrder arg of parent class constructor + // for the array layout info + // Consist of Voxel elements + Mat volume; +}; + +// dimension in voxels, size in meters +ColoredTSDFVolumeCPU::ColoredTSDFVolumeCPU(float _voxelSize, cv::Matx44f _pose, float _raycastStepFactor, + float _truncDist, int _maxWeight, Vec3i _resolution, + bool zFirstMemOrder) + : ColoredTSDFVolume(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, _resolution, + zFirstMemOrder) +{ + int xdim, ydim, zdim; + if (zFirstMemOrder) + { + xdim = volResolution.z * volResolution.y; + ydim = volResolution.z; + zdim = 1; + } + else + { + xdim = 1; + ydim = volResolution.x; + zdim = volResolution.x * volResolution.y; + } + volStrides = Vec4i(xdim, ydim, zdim); + + volume = Mat(1, volResolution.x * volResolution.y * volResolution.z, rawType()); + + reset(); +} + +// zero volume, leave rest params the same +void ColoredTSDFVolumeCPU::reset() +{ + CV_TRACE_FUNCTION(); + + volume.forEach([](VecRGBTsdfVoxel& vv, const int* /* position */) + { + RGBTsdfVoxel& v = reinterpret_cast(vv); + v.tsdf = floatToTsdf(0.0f); v.weight = 0; + }); +} + +RGBTsdfVoxel ColoredTSDFVolumeCPU::at(const Vec3i& volumeIdx) const +{ + //! Out of bounds + if ((volumeIdx[0] >= volResolution.x || volumeIdx[0] < 0) || + (volumeIdx[1] >= volResolution.y || volumeIdx[1] < 0) || + (volumeIdx[2] >= volResolution.z || volumeIdx[2] < 0)) + { + return RGBTsdfVoxel(floatToTsdf(1.f), 0, 160, 160, 160); + } + + const RGBTsdfVoxel* volData = volume.ptr(); + int coordBase = + volumeIdx[0] * volDims[0] + volumeIdx[1] * volDims[1] + volumeIdx[2] * volDims[2]; + return volData[coordBase]; +} + +// use depth instead of distance (optimization) +void ColoredTSDFVolumeCPU::integrate(InputArray _depth, InputArray _rgb, float depthFactor, const Matx44f& cameraPose, + const Intr& depth_intrinsics, const Intr& rgb_intrinsics, const int frameId) +{ + CV_TRACE_FUNCTION(); + CV_UNUSED(frameId); + CV_Assert(_depth.type() == DEPTH_TYPE); + CV_Assert(!_depth.empty()); + Depth depth = _depth.getMat(); + Colors rgb = _rgb.getMat(); + Vec6f newParams((float)depth.rows, (float)depth.cols, + depth_intrinsics.fx, depth_intrinsics.fy, + depth_intrinsics.cx, depth_intrinsics.cy); + if (!(frameParams == newParams)) + { + frameParams = newParams; + pixNorms = preCalculationPixNorm(depth, depth_intrinsics); + } + + integrateRGBVolumeUnit(truncDist, voxelSize, maxWeight, (this->pose).matrix, volResolution, volStrides, depth, rgb, + depthFactor, cameraPose, depth_intrinsics, rgb_intrinsics, pixNorms, volume); +} + +#if USE_INTRINSICS +// all coordinate checks should be done in inclosing cycle +inline float ColoredTSDFVolumeCPU::interpolateVoxel(const Point3f& _p) const +{ + v_float32x4 p(_p.x, _p.y, _p.z, 0); + return interpolateVoxel(p); +} + +inline float ColoredTSDFVolumeCPU::interpolateVoxel(const v_float32x4& p) const +{ + // tx, ty, tz = floor(p) + v_int32x4 ip = v_floor(p); + v_float32x4 t = p - v_cvt_f32(ip); + float tx = t.get0(); + t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t))); + float ty = t.get0(); + t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t))); + float tz = t.get0(); + + int xdim = volDims[0], ydim = volDims[1], zdim = volDims[2]; + const RGBTsdfVoxel* volData = volume.ptr(); + + int ix = ip.get0(); + ip = v_rotate_right<1>(ip); + int iy = ip.get0(); + ip = v_rotate_right<1>(ip); + int iz = ip.get0(); + + int coordBase = ix * xdim + iy * ydim + iz * zdim; + + TsdfType vx[8]; + for (int i = 0; i < 8; i++) + vx[i] = volData[neighbourCoords[i] + coordBase].tsdf; + + v_float32x4 v0246 = tsdfToFloat_INTR(v_int32x4(vx[0], vx[2], vx[4], vx[6])); + v_float32x4 v1357 = tsdfToFloat_INTR(v_int32x4(vx[1], vx[3], vx[5], vx[7])); + v_float32x4 vxx = v0246 + v_setall_f32(tz) * (v1357 - v0246); + + v_float32x4 v00_10 = vxx; + v_float32x4 v01_11 = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vxx))); + + v_float32x4 v0_1 = v00_10 + v_setall_f32(ty) * (v01_11 - v00_10); + float v0 = v0_1.get0(); + v0_1 = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(v0_1))); + float v1 = v0_1.get0(); + + return v0 + tx * (v1 - v0); +} +#else +inline float ColoredTSDFVolumeCPU::interpolateVoxel(const Point3f& p) const +{ + int xdim = volDims[0], ydim = volDims[1], zdim = volDims[2]; + + int ix = cvFloor(p.x); + int iy = cvFloor(p.y); + int iz = cvFloor(p.z); + + float tx = p.x - ix; + float ty = p.y - iy; + float tz = p.z - iz; + + int coordBase = ix*xdim + iy*ydim + iz*zdim; + const RGBTsdfVoxel* volData = volume.ptr(); + + float vx[8]; + for (int i = 0; i < 8; i++) + vx[i] = tsdfToFloat(volData[neighbourCoords[i] + coordBase].tsdf); + + float v00 = vx[0] + tz*(vx[1] - vx[0]); + float v01 = vx[2] + tz*(vx[3] - vx[2]); + float v10 = vx[4] + tz*(vx[5] - vx[4]); + float v11 = vx[6] + tz*(vx[7] - vx[6]); + + float v0 = v00 + ty*(v01 - v00); + float v1 = v10 + ty*(v11 - v10); + + return v0 + tx*(v1 - v0); +} +#endif + + +#if USE_INTRINSICS +//gradientDeltaFactor is fixed at 1.0 of voxel size +inline Point3f ColoredTSDFVolumeCPU::getNormalVoxel(const Point3f& _p) const +{ + v_float32x4 p(_p.x, _p.y, _p.z, 0.f); + v_float32x4 result = getNormalVoxel(p); + float CV_DECL_ALIGNED(16) ares[4]; + v_store_aligned(ares, result); + return Point3f(ares[0], ares[1], ares[2]); +} + +inline v_float32x4 ColoredTSDFVolumeCPU::getNormalVoxel(const v_float32x4& p) const +{ + if (v_check_any(p < v_float32x4(1.f, 1.f, 1.f, 0.f)) || + v_check_any(p >= v_float32x4((float)(volResolution.x - 2), + (float)(volResolution.y - 2), + (float)(volResolution.z - 2), 1.f)) + ) + return nanv; + + v_int32x4 ip = v_floor(p); + v_float32x4 t = p - v_cvt_f32(ip); + float tx = t.get0(); + t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t))); + float ty = t.get0(); + t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t))); + float tz = t.get0(); + + const int xdim = volDims[0], ydim = volDims[1], zdim = volDims[2]; + const RGBTsdfVoxel* volData = volume.ptr(); + + int ix = ip.get0(); ip = v_rotate_right<1>(ip); + int iy = ip.get0(); ip = v_rotate_right<1>(ip); + int iz = ip.get0(); + + int coordBase = ix * xdim + iy * ydim + iz * zdim; + + float CV_DECL_ALIGNED(16) an[4]; + an[0] = an[1] = an[2] = an[3] = 0.f; + for (int c = 0; c < 3; c++) + { + const int dim = volDims[c]; + float& nv = an[c]; + + float vx[8]; + for (int i = 0; i < 8; i++) + vx[i] = tsdfToFloat(volData[neighbourCoords[i] + coordBase + 1 * dim].tsdf) - + tsdfToFloat(volData[neighbourCoords[i] + coordBase - 1 * dim].tsdf); + + v_float32x4 v0246(vx[0], vx[2], vx[4], vx[6]); + v_float32x4 v1357(vx[1], vx[3], vx[5], vx[7]); + v_float32x4 vxx = v0246 + v_setall_f32(tz) * (v1357 - v0246); + + v_float32x4 v00_10 = vxx; + v_float32x4 v01_11 = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vxx))); + + v_float32x4 v0_1 = v00_10 + v_setall_f32(ty) * (v01_11 - v00_10); + float v0 = v0_1.get0(); + v0_1 = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(v0_1))); + float v1 = v0_1.get0(); + + nv = v0 + tx * (v1 - v0); + } + + v_float32x4 n = v_load_aligned(an); + v_float32x4 Norm = v_sqrt(v_setall_f32(v_reduce_sum(n * n))); + + return Norm.get0() < 0.0001f ? nanv : n / Norm; +} +#else +inline Point3f ColoredTSDFVolumeCPU::getNormalVoxel(const Point3f& p) const +{ + const int xdim = volDims[0], ydim = volDims[1], zdim = volDims[2]; + const RGBTsdfVoxel* volData = volume.ptr(); + + if(p.x < 1 || p.x >= volResolution.x - 2 || + p.y < 1 || p.y >= volResolution.y - 2 || + p.z < 1 || p.z >= volResolution.z - 2) + return nan3; + + int ix = cvFloor(p.x); + int iy = cvFloor(p.y); + int iz = cvFloor(p.z); + + float tx = p.x - ix; + float ty = p.y - iy; + float tz = p.z - iz; + + int coordBase = ix*xdim + iy*ydim + iz*zdim; + + Vec3f an; + for(int c = 0; c < 3; c++) + { + const int dim = volDims[c]; + float& nv = an[c]; + + float vx[8]; + for(int i = 0; i < 8; i++) + vx[i] = tsdfToFloat(volData[neighbourCoords[i] + coordBase + 1 * dim].tsdf) - + tsdfToFloat(volData[neighbourCoords[i] + coordBase - 1 * dim].tsdf); + + float v00 = vx[0] + tz*(vx[1] - vx[0]); + float v01 = vx[2] + tz*(vx[3] - vx[2]); + float v10 = vx[4] + tz*(vx[5] - vx[4]); + float v11 = vx[6] + tz*(vx[7] - vx[6]); + + float v0 = v00 + ty*(v01 - v00); + float v1 = v10 + ty*(v11 - v10); + + nv = v0 + tx*(v1 - v0); + } + + float nv = sqrt(an[0] * an[0] + + an[1] * an[1] + + an[2] * an[2]); + return nv < 0.0001f ? nan3 : an / nv; +} +#endif + +#if USE_INTRINSICS +inline float ColoredTSDFVolumeCPU::interpolateColor(float tx, float ty, float tz, float vx[8]) const +{ + v_float32x4 v0246, v1357; + v_load_deinterleave(vx, v0246, v1357); + + v_float32x4 vxx = v0246 + v_setall_f32(tz) * (v1357 - v0246); + + v_float32x4 v00_10 = vxx; + v_float32x4 v01_11 = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vxx))); + + v_float32x4 v0_1 = v00_10 + v_setall_f32(ty) * (v01_11 - v00_10); + float v0 = v0_1.get0(); + v0_1 = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(v0_1))); + float v1 = v0_1.get0(); + + return v0 + tx * (v1 - v0); +} +#else +inline float ColoredTSDFVolumeCPU::interpolateColor(float tx, float ty, float tz, float vx[8]) const +{ + float v00 = vx[0] + tz * (vx[1] - vx[0]); + float v01 = vx[2] + tz * (vx[3] - vx[2]); + float v10 = vx[4] + tz * (vx[5] - vx[4]); + float v11 = vx[6] + tz * (vx[7] - vx[6]); + + float v0 = v00 + ty * (v01 - v00); + float v1 = v10 + ty * (v11 - v10); + + return v0 + tx * (v1 - v0); +} +#endif + +#if USE_INTRINSICS +//gradientDeltaFactor is fixed at 1.0 of voxel size +inline Point3f ColoredTSDFVolumeCPU::getColorVoxel(const Point3f& _p) const +{ + v_float32x4 p(_p.x, _p.y, _p.z, 0.f); + v_float32x4 result = getColorVoxel(p); + float CV_DECL_ALIGNED(16) ares[4]; + v_store_aligned(ares, result); + return Point3f(ares[0], ares[1], ares[2]); +} +inline v_float32x4 ColoredTSDFVolumeCPU::getColorVoxel(const v_float32x4& p) const +{ + if (v_check_any(p < v_float32x4(1.f, 1.f, 1.f, 0.f)) || + v_check_any(p >= v_float32x4((float)(volResolution.x - 2), + (float)(volResolution.y - 2), + (float)(volResolution.z - 2), 1.f)) + ) + return nanv; + + v_int32x4 ip = v_floor(p); + + const int xdim = volDims[0], ydim = volDims[1], zdim = volDims[2]; + const RGBTsdfVoxel* volData = volume.ptr(); + + int ix = ip.get0(); ip = v_rotate_right<1>(ip); + int iy = ip.get0(); ip = v_rotate_right<1>(ip); + int iz = ip.get0(); + + int coordBase = ix * xdim + iy * ydim + iz * zdim; + float CV_DECL_ALIGNED(16) rgb[4]; + +#if USE_INTERPOLATION_IN_GETNORMAL + float r[8], g[8], b[8]; + for (int i = 0; i < 8; i++) + { + r[i] = (float)volData[neighbourCoords[i] + coordBase].r; + g[i] = (float)volData[neighbourCoords[i] + coordBase].g; + b[i] = (float)volData[neighbourCoords[i] + coordBase].b; + } + + v_float32x4 vsi(voxelSizeInv, voxelSizeInv, voxelSizeInv, voxelSizeInv); + v_float32x4 ptVox = p * vsi; + v_int32x4 iptVox = v_floor(ptVox); + v_float32x4 t = ptVox - v_cvt_f32(iptVox); + float tx = t.get0(); t = v_rotate_right<1>(t); + float ty = t.get0(); t = v_rotate_right<1>(t); + float tz = t.get0(); + rgb[0] = interpolateColor(tx, ty, tz, r); + rgb[1] = interpolateColor(tx, ty, tz, g); + rgb[2] = interpolateColor(tx, ty, tz, b); + rgb[3] = 0.f; +#else + rgb[0] = volData[coordBase].r; + rgb[1] = volData[coordBase].g; + rgb[2] = volData[coordBase].b; + rgb[3] = 0.f; +#endif + v_float32x4 res = v_load_aligned(rgb); + return res; +} +#else +inline Point3f ColoredTSDFVolumeCPU::getColorVoxel(const Point3f& p) const +{ + const int xdim = volDims[0], ydim = volDims[1], zdim = volDims[2]; + const RGBTsdfVoxel* volData = volume.ptr(); + + + + if(p.x < 1 || p.x >= volResolution.x - 2 || + p.y < 1 || p.y >= volResolution.y - 2 || + p.z < 1 || p.z >= volResolution.z - 2) + return nan3; + + int ix = cvFloor(p.x); + int iy = cvFloor(p.y); + int iz = cvFloor(p.z); + + int coordBase = ix*xdim + iy*ydim + iz*zdim; + Point3f res; + +#if USE_INTERPOLATION_IN_GETNORMAL + // TODO: create better interpolation or remove this simple version + float r[8], g[8], b[8]; + for (int i = 0; i < 8; i++) + { + r[i] = (float) volData[neighbourCoords[i] + coordBase].r; + g[i] = (float) volData[neighbourCoords[i] + coordBase].g; + b[i] = (float) volData[neighbourCoords[i] + coordBase].b; + } + + Point3f ptVox = p * voxelSizeInv; + Vec3i iptVox(cvFloor(ptVox.x), cvFloor(ptVox.y), cvFloor(ptVox.z)); + float tx = ptVox.x - iptVox[0]; + float ty = ptVox.y - iptVox[1]; + float tz = ptVox.z - iptVox[2]; + + res=Point3f(interpolateColor(tx, ty, tz, r), + interpolateColor(tx, ty, tz, g), + interpolateColor(tx, ty, tz, b)); +#else + res=Point3f(volData[coordBase].r, volData[coordBase].g, volData[coordBase].b); +#endif + colorFix(res); + return res; +} +#endif + +struct ColorRaycastInvoker : ParallelLoopBody +{ + ColorRaycastInvoker(Points& _points, Normals& _normals, Colors& _colors, const Matx44f& cameraPose, + const Intr& depth_intrinsics, const ColoredTSDFVolumeCPU& _volume) : + ParallelLoopBody(), + points(_points), + normals(_normals), + colors(_colors), + volume(_volume), + tstep(volume.truncDist * volume.raycastStepFactor), + // We do subtract voxel size to minimize checks after + // Note: origin of volume coordinate is placed + // in the center of voxel (0,0,0), not in the corner of the voxel! + boxMax(volume.volSize - Point3f(volume.voxelSize, + volume.voxelSize, + volume.voxelSize)), + boxMin(), + cam2vol(volume.pose.inv() * Affine3f(cameraPose)), + vol2cam(Affine3f(cameraPose.inv()) * volume.pose), + reprojDepth(depth_intrinsics.makeReprojector()) + { } +#if USE_INTRINSICS + virtual void operator() (const Range& range) const override + { + const v_float32x4 vfxy(reprojDepth.fxinv, reprojDepth.fyinv, 0, 0); + const v_float32x4 vcxy(reprojDepth.cx, reprojDepth.cy, 0, 0); + + const float(&cm)[16] = cam2vol.matrix.val; + const v_float32x4 camRot0(cm[0], cm[4], cm[8], 0); + const v_float32x4 camRot1(cm[1], cm[5], cm[9], 0); + const v_float32x4 camRot2(cm[2], cm[6], cm[10], 0); + const v_float32x4 camTrans(cm[3], cm[7], cm[11], 0); + + const v_float32x4 boxDown(boxMin.x, boxMin.y, boxMin.z, 0.f); + const v_float32x4 boxUp(boxMax.x, boxMax.y, boxMax.z, 0.f); + + const v_float32x4 invVoxelSize = v_float32x4(volume.voxelSizeInv, + volume.voxelSizeInv, + volume.voxelSizeInv, 1.f); + + const float(&vm)[16] = vol2cam.matrix.val; + const v_float32x4 volRot0(vm[0], vm[4], vm[8], 0); + const v_float32x4 volRot1(vm[1], vm[5], vm[9], 0); + const v_float32x4 volRot2(vm[2], vm[6], vm[10], 0); + const v_float32x4 volTrans(vm[3], vm[7], vm[11], 0); + + for (int y = range.start; y < range.end; y++) + { + ptype* ptsRow = points[y]; + ptype* nrmRow = normals[y]; + ptype* clrRow = colors[y]; + + for (int x = 0; x < points.cols; x++) + { + v_float32x4 point = nanv, normal = nanv, color = nanv; + + v_float32x4 orig = camTrans; + + // get direction through pixel in volume space: + + // 1. reproject (x, y) on projecting plane where z = 1.f + v_float32x4 planed = (v_float32x4((float)x, (float)y, 0.f, 0.f) - vcxy) * vfxy; + planed = v_combine_low(planed, v_float32x4(1.f, 0.f, 0.f, 0.f)); + + // 2. rotate to volume space + planed = v_matmuladd(planed, camRot0, camRot1, camRot2, v_setzero_f32()); + + // 3. normalize + v_float32x4 invNorm = v_invsqrt(v_setall_f32(v_reduce_sum(planed * planed))); + v_float32x4 dir = planed * invNorm; + + // compute intersection of ray with all six bbox planes + v_float32x4 rayinv = v_setall_f32(1.f) / dir; + // div by zero should be eliminated by these products + v_float32x4 tbottom = rayinv * (boxDown - orig); + v_float32x4 ttop = rayinv * (boxUp - orig); + + // re-order intersections to find smallest and largest on each axis + v_float32x4 minAx = v_min(ttop, tbottom); + v_float32x4 maxAx = v_max(ttop, tbottom); + + // near clipping plane + const float clip = 0.f; + float _minAx[4], _maxAx[4]; + v_store(_minAx, minAx); + v_store(_maxAx, maxAx); + float tmin = max({ _minAx[0], _minAx[1], _minAx[2], clip }); + float tmax = min({ _maxAx[0], _maxAx[1], _maxAx[2] }); + + // precautions against getting coordinates out of bounds + tmin = tmin + tstep; + tmax = tmax - tstep; + + if (tmin < tmax) + { + // interpolation optimized a little + orig *= invVoxelSize; + dir *= invVoxelSize; + + int xdim = volume.volDims[0]; + int ydim = volume.volDims[1]; + int zdim = volume.volDims[2]; + v_float32x4 rayStep = dir * v_setall_f32(tstep); + v_float32x4 next = (orig + dir * v_setall_f32(tmin)); + float f = volume.interpolateVoxel(next), fnext = f; + + //raymarch + int steps = 0; + int nSteps = cvFloor((tmax - tmin) / tstep); + for (; steps < nSteps; steps++) + { + next += rayStep; + v_int32x4 ip = v_round(next); + int ix = ip.get0(); ip = v_rotate_right<1>(ip); + int iy = ip.get0(); ip = v_rotate_right<1>(ip); + int iz = ip.get0(); + int coord = ix * xdim + iy * ydim + iz * zdim; + + fnext = tsdfToFloat(volume.volume.at(coord).tsdf); + if (fnext != f) + { + fnext = volume.interpolateVoxel(next); + + // when ray crosses a surface + if (std::signbit(f) != std::signbit(fnext)) + break; + + f = fnext; + } + } + + // if ray penetrates a surface from outside + // linearly interpolate t between two f values + if (f > 0.f && fnext < 0.f) + { + v_float32x4 tp = next - rayStep; + float ft = volume.interpolateVoxel(tp); + float ftdt = volume.interpolateVoxel(next); + float ts = tmin + tstep * (steps - ft / (ftdt - ft)); + + // avoid division by zero + if (!cvIsNaN(ts) && !cvIsInf(ts)) + { + v_float32x4 pv = (orig + dir * v_setall_f32(ts)); + v_float32x4 nv = volume.getNormalVoxel(pv); + v_float32x4 cv = volume.getColorVoxel(pv); + + if (!isNaN(nv)) + { + color = cv; + //convert pv and nv to camera space + normal = v_matmuladd(nv, volRot0, volRot1, volRot2, v_setzero_f32()); + // interpolation optimized a little + point = v_matmuladd(pv * v_float32x4(volume.voxelSize, + volume.voxelSize, + volume.voxelSize, 1.f), + volRot0, volRot1, volRot2, volTrans); + } + } + } + } + + v_store((float*)(&ptsRow[x]), point); + v_store((float*)(&nrmRow[x]), normal); + v_store((float*)(&clrRow[x]), color); + } + } + } +#else + virtual void operator() (const Range& range) const override + { + const Point3f camTrans = cam2vol.translation(); + const Matx33f camRot = cam2vol.rotation(); + const Matx33f volRot = vol2cam.rotation(); + + for(int y = range.start; y < range.end; y++) + { + ptype* ptsRow = points[y]; + ptype* nrmRow = normals[y]; + ptype* clrRow = colors[y]; + + for(int x = 0; x < points.cols; x++) + { + Point3f point = nan3, normal = nan3, color = nan3; + + Point3f orig = camTrans; + // direction through pixel in volume space + Point3f dir = normalize(Vec3f(camRot * reprojDepth(Point3f(float(x), float(y), 1.f)))); + + // compute intersection of ray with all six bbox planes + Vec3f rayinv(1.f/dir.x, 1.f/dir.y, 1.f/dir.z); + Point3f tbottom = rayinv.mul(boxMin - orig); + Point3f ttop = rayinv.mul(boxMax - orig); + + // re-order intersections to find smallest and largest on each axis + Point3f minAx(min(ttop.x, tbottom.x), min(ttop.y, tbottom.y), min(ttop.z, tbottom.z)); + Point3f maxAx(max(ttop.x, tbottom.x), max(ttop.y, tbottom.y), max(ttop.z, tbottom.z)); + + // near clipping plane + const float clip = 0.f; + //float tmin = max(max(max(minAx.x, minAx.y), max(minAx.x, minAx.z)), clip); + //float tmax = min(min(maxAx.x, maxAx.y), min(maxAx.x, maxAx.z)); + float tmin = max({ minAx.x, minAx.y, minAx.z, clip }); + float tmax = min({ maxAx.x, maxAx.y, maxAx.z }); + + // precautions against getting coordinates out of bounds + tmin = tmin + tstep; + tmax = tmax - tstep; + + if(tmin < tmax) + { + // interpolation optimized a little + orig = orig*volume.voxelSizeInv; + dir = dir*volume.voxelSizeInv; + + Point3f rayStep = dir * tstep; + Point3f next = (orig + dir * tmin); + float f = volume.interpolateVoxel(next), fnext = f; + + //raymarch + int steps = 0; + int nSteps = int(floor((tmax - tmin)/tstep)); + for(; steps < nSteps; steps++) + { + next += rayStep; + int xdim = volume.volDims[0]; + int ydim = volume.volDims[1]; + int zdim = volume.volDims[2]; + int ix = cvRound(next.x); + int iy = cvRound(next.y); + int iz = cvRound(next.z); + fnext = tsdfToFloat(volume.volume.at(ix*xdim + iy*ydim + iz*zdim).tsdf); + if(fnext != f) + { + fnext = volume.interpolateVoxel(next); + // when ray crosses a surface + if(std::signbit(f) != std::signbit(fnext)) + break; + + f = fnext; + } + } + // if ray penetrates a surface from outside + // linearly interpolate t between two f values + if(f > 0.f && fnext < 0.f) + { + Point3f tp = next - rayStep; + float ft = volume.interpolateVoxel(tp); + float ftdt = volume.interpolateVoxel(next); + // float t = tmin + steps*tstep; + // float ts = t - tstep*ft/(ftdt - ft); + float ts = tmin + tstep*(steps - ft/(ftdt - ft)); + + // avoid division by zero + if(!cvIsNaN(ts) && !cvIsInf(ts)) + { + Point3f pv = (orig + dir*ts); + Point3f nv = volume.getNormalVoxel(pv); + Point3f cv = volume.getColorVoxel(pv); + if(!isNaN(nv)) + { + //convert pv and nv to camera space + normal = volRot * nv; + color = cv; + // interpolation optimized a little + point = vol2cam * (pv*volume.voxelSize); + } + } + } + } + ptsRow[x] = toPtype(point); + nrmRow[x] = toPtype(normal); + clrRow[x] = toPtype(color); + } + } + } +#endif + + Points& points; + Normals& normals; + Colors& colors; + const ColoredTSDFVolumeCPU& volume; + + const float tstep; + + const Point3f boxMax; + const Point3f boxMin; + + const Affine3f cam2vol; + const Affine3f vol2cam; + const Intr::Reprojector reprojDepth; +}; + + +void ColoredTSDFVolumeCPU::raycast(const Matx44f& cameraPose, const Intr& depth_intrinsics, const Size& frameSize, + OutputArray _points, OutputArray _normals, OutputArray _colors) const +{ + CV_TRACE_FUNCTION(); + + CV_Assert(frameSize.area() > 0); + + _points.create (frameSize, POINT_TYPE); + _normals.create(frameSize, POINT_TYPE); + _colors.create(frameSize, POINT_TYPE); + + Points points = _points.getMat(); + Normals normals = _normals.getMat(); + Colors colors = _colors.getMat(); + ColorRaycastInvoker ri(points, normals, colors, cameraPose, depth_intrinsics, *this); + + const int nstripes = -1; + parallel_for_(Range(0, points.rows), ri, nstripes); +} + + +struct ColorFetchPointsNormalsInvoker : ParallelLoopBody +{ + ColorFetchPointsNormalsInvoker(const ColoredTSDFVolumeCPU& _volume, + std::vector>& _pVecs, + std::vector>& _nVecs, + bool _needNormals) : + ParallelLoopBody(), + vol(_volume), + pVecs(_pVecs), + nVecs(_nVecs), + needNormals(_needNormals) + { + volDataStart = vol.volume.ptr(); + } + + inline void coord(std::vector& points, std::vector& normals, + int x, int y, int z, Point3f V, float v0, int axis) const + { + // 0 for x, 1 for y, 2 for z + bool limits = false; + Point3i shift; + float Vc = 0.f; + if(axis == 0) + { + shift = Point3i(1, 0, 0); + limits = (x + 1 < vol.volResolution.x); + Vc = V.x; + } + if(axis == 1) + { + shift = Point3i(0, 1, 0); + limits = (y + 1 < vol.volResolution.y); + Vc = V.y; + } + if(axis == 2) + { + shift = Point3i(0, 0, 1); + limits = (z + 1 < vol.volResolution.z); + Vc = V.z; + } + + if(limits) + { + const RGBTsdfVoxel& voxeld = volDataStart[(x+shift.x)*vol.volDims[0] + + (y+shift.y)*vol.volDims[1] + + (z+shift.z)*vol.volDims[2]]; + float vd = tsdfToFloat(voxeld.tsdf); + + if(voxeld.weight != 0 && vd != 1.f) + { + if((v0 > 0 && vd < 0) || (v0 < 0 && vd > 0)) + { + //linearly interpolate coordinate + float Vn = Vc + vol.voxelSize; + float dinv = 1.f/(abs(v0)+abs(vd)); + float inter = (Vc*abs(vd) + Vn*abs(v0))*dinv; + + Point3f p(shift.x ? inter : V.x, + shift.y ? inter : V.y, + shift.z ? inter : V.z); + { + points.push_back(toPtype(vol.pose * p)); + if(needNormals) + normals.push_back(toPtype(vol.pose.rotation() * + vol.getNormalVoxel(p*vol.voxelSizeInv))); + } + } + } + } + } + + virtual void operator() (const Range& range) const override + { + std::vector points, normals; + for(int x = range.start; x < range.end; x++) + { + const RGBTsdfVoxel* volDataX = volDataStart + x*vol.volDims[0]; + for(int y = 0; y < vol.volResolution.y; y++) + { + const RGBTsdfVoxel* volDataY = volDataX + y*vol.volDims[1]; + for(int z = 0; z < vol.volResolution.z; z++) + { + const RGBTsdfVoxel& voxel0 = volDataY[z*vol.volDims[2]]; + float v0 = tsdfToFloat(voxel0.tsdf); + if(voxel0.weight != 0 && v0 != 1.f) + { + Point3f V(Point3f((float)x + 0.5f, (float)y + 0.5f, (float)z + 0.5f)*vol.voxelSize); + + coord(points, normals, x, y, z, V, v0, 0); + coord(points, normals, x, y, z, V, v0, 1); + coord(points, normals, x, y, z, V, v0, 2); + + } // if voxel is not empty + } + } + } + + AutoLock al(mutex); + pVecs.push_back(points); + nVecs.push_back(normals); + } + + const ColoredTSDFVolumeCPU& vol; + std::vector>& pVecs; + std::vector>& nVecs; + const RGBTsdfVoxel* volDataStart; + bool needNormals; + mutable Mutex mutex; +}; + +void ColoredTSDFVolumeCPU::fetchPointsNormals(OutputArray _points, OutputArray _normals) const +{ + CV_TRACE_FUNCTION(); + + if(_points.needed()) + { + std::vector> pVecs, nVecs; + ColorFetchPointsNormalsInvoker fi(*this, pVecs, nVecs, _normals.needed()); + Range range(0, volResolution.x); + const int nstripes = -1; + parallel_for_(range, fi, nstripes); + std::vector points, normals; + for(size_t i = 0; i < pVecs.size(); i++) + { + points.insert(points.end(), pVecs[i].begin(), pVecs[i].end()); + normals.insert(normals.end(), nVecs[i].begin(), nVecs[i].end()); + } + + _points.create((int)points.size(), 1, POINT_TYPE); + if(!points.empty()) + Mat((int)points.size(), 1, POINT_TYPE, &points[0]).copyTo(_points.getMat()); + + if(_normals.needed()) + { + _normals.create((int)normals.size(), 1, POINT_TYPE); + if(!normals.empty()) + Mat((int)normals.size(), 1, POINT_TYPE, &normals[0]).copyTo(_normals.getMat()); + } + } +} + +void ColoredTSDFVolumeCPU::fetchNormals(InputArray _points, OutputArray _normals) const +{ + CV_TRACE_FUNCTION(); + CV_Assert(!_points.empty()); + if(_normals.needed()) + { + Points points = _points.getMat(); + CV_Assert(points.type() == POINT_TYPE); + + _normals.createSameSize(_points, _points.type()); + Normals normals = _normals.getMat(); + + const ColoredTSDFVolumeCPU& _vol = *this; + auto PushNormals = [&](const ptype& pp, const int* position) + { + const ColoredTSDFVolumeCPU& vol(_vol); + Affine3f invPose(vol.pose.inv()); + Point3f p = fromPtype(pp); + Point3f n = nan3; + if (!isNaN(p)) + { + Point3f voxPt = (invPose * p); + voxPt = voxPt * vol.voxelSizeInv; + n = vol.pose.rotation() * vol.getNormalVoxel(voxPt); + } + normals(position[0], position[1]) = toPtype(n); + }; + points.forEach(PushNormals); + } +} + +Ptr makeColoredTSDFVolume(float _voxelSize, Matx44f _pose, float _raycastStepFactor, + float _truncDist, int _maxWeight, Point3i _resolution) +{ + return makePtr(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, _resolution); +} + +Ptr makeColoredTSDFVolume(const VolumeParams& _params) +{ + return makePtr(_params.voxelSize, _params.pose.matrix, _params.raycastStepFactor, + _params.tsdfTruncDist, _params.maxWeight, _params.resolution); +} + +} // namespace kinfu +} // namespace cv diff --git a/modules/rgbd/src/colored_tsdf.hpp b/modules/rgbd/src/colored_tsdf.hpp new file mode 100644 index 00000000000..d6b09b0156c --- /dev/null +++ b/modules/rgbd/src/colored_tsdf.hpp @@ -0,0 +1,61 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +// This code is also subject to the license terms in the LICENSE_KinectFusion.md file found in this +// module's directory + +#ifndef __OPENCV_KINFU_COLORED_TSDF_H__ +#define __OPENCV_KINFU_COLORED_TSDF_H__ + +#include + +#include "kinfu_frame.hpp" +#include "utils.hpp" + +namespace cv +{ +namespace kinfu +{ + +typedef int8_t TsdfType; +typedef uchar WeightType; +typedef short int ColorType; +struct RGBTsdfVoxel +{ + RGBTsdfVoxel(TsdfType _tsdf, WeightType _weight, ColorType _r, ColorType _g, ColorType _b) : + tsdf(_tsdf), weight(_weight), r(_r), g(_g), b(_b) + { } + TsdfType tsdf; + WeightType weight; + ColorType r, g, b; +}; + +typedef Vec VecRGBTsdfVoxel; + +class ColoredTSDFVolume : public Volume +{ + public: + // dimension in voxels, size in meters + ColoredTSDFVolume(float _voxelSize, Matx44f _pose, float _raycastStepFactor, float _truncDist, + int _maxWeight, Point3i _resolution, bool zFirstMemOrder = true); + virtual ~ColoredTSDFVolume() = default; + + public: + + Point3i volResolution; + WeightType maxWeight; + + Point3f volSize; + float truncDist; + Vec4i volDims; + Vec8i neighbourCoords; +}; + +Ptr makeColoredTSDFVolume(float _voxelSize, Matx44f _pose, float _raycastStepFactor, + float _truncDist, int _maxWeight, Point3i _resolution); +Ptr makeColoredTSDFVolume(const VolumeParams& _params); + +} // namespace kinfu +} // namespace cv +#endif diff --git a/modules/rgbd/src/hash_tsdf.cpp b/modules/rgbd/src/hash_tsdf.cpp index 08338fe83ef..4e492064a44 100644 --- a/modules/rgbd/src/hash_tsdf.cpp +++ b/modules/rgbd/src/hash_tsdf.cpp @@ -99,11 +99,14 @@ class HashTSDFVolumeCPU : public HashTSDFVolume HashTSDFVolumeCPU(const VolumeParams& _volumeParams, bool zFirstMemOrder = true); + virtual void integrate(InputArray, InputArray, float, const Matx44f&, const kinfu::Intr&, const Intr&, const int) override + { CV_Error(Error::StsNotImplemented, "Not implemented"); }; void integrate(InputArray _depth, float depthFactor, const Matx44f& cameraPose, const kinfu::Intr& intrinsics, const int frameId = 0) override; void raycast(const Matx44f& cameraPose, const kinfu::Intr& intrinsics, const Size& frameSize, OutputArray points, OutputArray normals) const override; - + void raycast(const Matx44f&, const kinfu::Intr&, const Size&, OutputArray, OutputArray, OutputArray) const override + { CV_Error(Error::StsNotImplemented, "Not implemented"); }; void fetchNormals(InputArray points, OutputArray _normals) const override; void fetchPointsNormals(OutputArray points, OutputArray normals) const override; @@ -894,11 +897,14 @@ class HashTSDFVolumeGPU : public HashTSDFVolume void markActive(const Matx44f& cameraPose, const Intr& intrinsics, const Size frameSz, const int frameId); + virtual void integrate(InputArray, InputArray, float, const Matx44f&, const kinfu::Intr&, const Intr&, const int) override + { CV_Error(Error::StsNotImplemented, "Not implemented"); }; void integrate(InputArray _depth, float depthFactor, const Matx44f& cameraPose, const kinfu::Intr& intrinsics, const int frameId = 0) override; void raycast(const Matx44f& cameraPose, const kinfu::Intr& intrinsics, const Size& frameSize, OutputArray points, OutputArray normals) const override; - + void raycast(const Matx44f&, const kinfu::Intr&, const Size&, OutputArray, OutputArray, OutputArray) const override + { CV_Error(Error::StsNotImplemented, "Not implemented"); }; void fetchNormals(InputArray points, OutputArray _normals) const override; void fetchPointsNormals(OutputArray points, OutputArray normals) const override; diff --git a/modules/rgbd/src/kinfu.cpp b/modules/rgbd/src/kinfu.cpp index 1d8314aae6f..7f9f1b32361 100644 --- a/modules/rgbd/src/kinfu.cpp +++ b/modules/rgbd/src/kinfu.cpp @@ -107,6 +107,18 @@ Ptr Params::hashTSDFParams(bool isCoarse) return p; } +Ptr Params::coloredTSDFParams(bool isCoarse) +{ + Ptr p; + if (isCoarse) + p = coarseParams(); + else + p = defaultParams(); + p->volumeType = VolumeType::COLOREDTSDF; + + return p; +} + // MatType should be Mat or UMat template< typename MatType> class KinFuImpl : public KinFu diff --git a/modules/rgbd/src/kinfu_frame.cpp b/modules/rgbd/src/kinfu_frame.cpp index 76d80902b7b..b8df7c1f5a7 100644 --- a/modules/rgbd/src/kinfu_frame.cpp +++ b/modules/rgbd/src/kinfu_frame.cpp @@ -12,6 +12,7 @@ namespace cv { namespace kinfu { static void computePointsNormals(const cv::kinfu::Intr, float depthFactor, const Depth, Points, Normals ); +void computePointsNormalsColors(const Intr, const Intr, float, const Depth, const Colors, Points, Normals, Colors); static Depth pyrDownBilateral(const Depth depth, float sigma); static void pyrDownPointsNormals(const Points p, const Normals n, Points& pdown, Normals& ndown); @@ -42,6 +43,13 @@ inline float specPow<1>(float x) return x; } +inline void colorFix(Point3f& c) +{ + if (c.x > 255) c.x = 255; + if (c.y > 255) c.y = 255; + if (c.z > 255) c.z = 255; +} + struct RenderInvoker : ParallelLoopBody { RenderInvoker(const Points& _points, const Normals& _normals, Mat_& _img, Affine3f _lightPose, Size _sz) : @@ -105,6 +113,52 @@ struct RenderInvoker : ParallelLoopBody Size sz; }; +struct RenderColorInvoker : ParallelLoopBody +{ + RenderColorInvoker(const Points& _points, const Colors& _colors, Mat_& _img, Affine3f _lightPose, Size _sz) : + ParallelLoopBody(), + points(_points), + colors(_colors), + img(_img), + lightPose(_lightPose), + sz(_sz) + { } + + virtual void operator ()(const Range& range) const override + { + for(int y = range.start; y < range.end; y++) + { + Vec4b* imgRow = img[y]; + const ptype* ptsRow = points[y]; + const ptype* clrRow = colors[y]; + + for(int x = 0; x < sz.width; x++) + { + Point3f p = fromPtype(ptsRow[x]); + Point3f c = fromPtype(clrRow[x]); + Vec4b color; + + if(isNaN(p) || isNaN(c)) + { + color = Vec4b(0, 32, 0, 0); + } + else + { + color = Vec4b((uchar)c.x, (uchar)c.y, (uchar)c.z, (uchar)0); + } + + imgRow[x] = color; + } + } + } + + const Points& points; + const Colors& colors; + Mat_& img; + Affine3f lightPose; + Size sz; +}; + void pyrDownPointsNormals(const Points p, const Normals n, Points &pdown, Normals &ndown) { @@ -275,6 +329,81 @@ struct ComputePointsNormalsInvoker : ParallelLoopBody float dfac; }; +struct ComputePointsNormalsColorsInvoker : ParallelLoopBody +{ + ComputePointsNormalsColorsInvoker(const Depth& _depth, const Colors& _rgb, Points& _points, Normals& _normals, Colors& _colors, + const Intr::Reprojector& _reproj, const Intr::Projector& _rgb_reproj, float _dfac) : + ParallelLoopBody(), + depth(_depth), + rgb(_rgb), + points(_points), + normals(_normals), + colors(_colors), + reproj(_reproj), + rgb_proj(_rgb_reproj), + dfac(_dfac) + { } + + virtual void operator ()(const Range& range) const override + { + for(int y = range.start; y < range.end; y++) + { + const depthType* depthRow0 = depth[y]; + const depthType* depthRow1 = (y < depth.rows - 1) ? depth[y + 1] : 0; + ptype *ptsRow = points[y]; + ptype *normRow = normals[y]; + ptype *clrRow = colors[y]; + + for(int x = 0; x < depth.cols; x++) + { + depthType d00 = depthRow0[x]; + depthType z00 = d00*dfac; + Point3f v00 = reproj(Point3f((float)x, (float)y, z00)); + Point2f proj = rgb_proj(v00); + int rgb_u = (int) proj.x, rgb_v = (int) proj.y; + + Point3f p = nan3, n = nan3, c = nan3; + if(x < depth.cols - 1 && y < depth.rows - 1 && + rgb_v >= 0 && rgb_v < rgb.rows && rgb_u >= 0 && rgb_u < rgb.cols) + { + depthType d01 = depthRow0[x+1]; + depthType d10 = depthRow1[x]; + + depthType z01 = d01*dfac; + depthType z10 = d10*dfac; + + // before it was + //if(z00*z01*z10 != 0) + if(z00 != 0 && z01 != 0 && z10 != 0) + { + Point3f v01 = reproj(Point3f((float)(x+1), (float)(y+0), z01)); + Point3f v10 = reproj(Point3f((float)(x+0), (float)(y+1), z10)); + + cv::Vec3f vec = (v01-v00).cross(v10-v00); + n = -normalize(vec); + p = v00; + c = fromPtype(rgb.at(rgb_v, rgb_u)); + colorFix(c); + } + } + + ptsRow[x] = toPtype(p); + normRow[x] = toPtype(n); + clrRow[x] = toPtype(c); + } + } + } + + const Depth& depth; + const Colors& rgb; + Points& points; + Normals& normals; + Colors& colors; + const Intr::Reprojector& reproj; + const Intr::Projector& rgb_proj; + float dfac; +}; + void computePointsNormals(const Intr intr, float depthFactor, const Depth depth, Points points, Normals normals) { @@ -297,6 +426,30 @@ void computePointsNormals(const Intr intr, float depthFactor, const Depth depth, parallel_for_(range, ci, nstripes); } +void computePointsNormalsColors(const Intr intr, const Intr rgb_intr, float depthFactor, + const Depth depth, const Colors rgb, Points points, Normals normals, Colors colors) +{ + CV_TRACE_FUNCTION(); + + CV_Assert(!points.empty() && !normals.empty()); + CV_Assert(depth.size() == points.size()); + CV_Assert(depth.size() == normals.size()); + CV_Assert(depth.size() == colors.size()); + + // conversion to meters + // before it was: + //float dfac = 0.001f/depthFactor; + float dfac = 1.f/depthFactor; + + Intr::Reprojector reproj = intr.makeReprojector(); + Intr::Projector projRGB = rgb_intr.makeProjector(); + + ComputePointsNormalsColorsInvoker ci(depth, rgb, points, normals, colors, reproj, projRGB, dfac); + Range range(0, depth.rows); + const int nstripes = -1; + parallel_for_(range, ci, nstripes); +} + ///////// GPU implementation ///////// #ifdef HAVE_OPENCL @@ -598,6 +751,33 @@ void renderPointsNormals(InputArray _points, InputArray _normals, OutputArray im parallel_for_(range, ri, nstripes); } +void renderPointsNormalsColors(InputArray _points, InputArray _normals, InputArray _colors, OutputArray image, Affine3f lightPose) +{ + CV_TRACE_FUNCTION(); + + CV_Assert(_points.size().area() > 0); + CV_Assert(_points.size() == _normals.size()); + + Size sz = _points.size(); + image.create(sz, CV_8UC4); + + CV_OCL_RUN(_points.isUMat() && _normals.isUMat() && image.isUMat(), + ocl_renderPointsNormals(_points.getUMat(), + _normals.getUMat(), + image.getUMat(), lightPose)) + + Points points = _points.getMat(); + Normals normals = _normals.getMat(); + Colors colors = _colors.getMat(); + + Mat_ img = image.getMat(); + + RenderColorInvoker ri(points, colors, img, lightPose, sz); + Range range(0, sz.height); + const int nstripes = -1; + parallel_for_(range, ri, nstripes); +} + void makeFrameFromDepth(InputArray _depth, OutputArray pyrPoints, OutputArray pyrNormals, @@ -659,6 +839,65 @@ void makeFrameFromDepth(InputArray _depth, } } +void makeColoredFrameFromDepth(InputArray _depth, InputArray _rgb, + OutputArray pyrPoints, OutputArray pyrNormals, OutputArray pyrColors, + const Intr intr, const Intr rgb_intr, int levels, float depthFactor, + float sigmaDepth, float sigmaSpatial, int kernelSize, + float truncateThreshold) +{ + CV_TRACE_FUNCTION(); + + CV_Assert(_depth.type() == DEPTH_TYPE); + + + int kp = pyrPoints.kind(), kn = pyrNormals.kind(), kc = pyrColors.kind(); + CV_Assert(kp == _InputArray::STD_ARRAY_MAT || kp == _InputArray::STD_VECTOR_MAT); + CV_Assert(kn == _InputArray::STD_ARRAY_MAT || kn == _InputArray::STD_VECTOR_MAT); + CV_Assert(kc == _InputArray::STD_ARRAY_MAT || kc == _InputArray::STD_VECTOR_MAT); + + Depth depth = _depth.getMat(); + Colors rgb = _rgb.getMat(); + // looks like OpenCV's bilateral filter works the same as KinFu's + Depth smooth; + Depth depthNoNans = depth.clone(); + patchNaNs(depthNoNans); + bilateralFilter(depthNoNans, smooth, kernelSize, sigmaDepth*depthFactor, sigmaSpatial); + + // depth truncation can be used in some scenes + Depth depthThreshold; + if(truncateThreshold > 0.f) + threshold(smooth, depthThreshold, truncateThreshold * depthFactor, 0.0, THRESH_TOZERO_INV); + else + depthThreshold = smooth; + + // we don't need depth pyramid outside this method + // if we do, the code is to be refactored + + Depth scaled = depthThreshold; + Size sz = smooth.size(); + pyrPoints.create(levels, 1, POINT_TYPE); + pyrNormals.create(levels, 1, POINT_TYPE); + pyrColors.create(levels, 1, POINT_TYPE); + for(int i = 0; i < levels; i++) + { + pyrPoints .create(sz, POINT_TYPE, i); + pyrNormals.create(sz, POINT_TYPE, i); + pyrColors.create(sz, POINT_TYPE, i); + + Points p = pyrPoints. getMatRef(i); + Normals n = pyrNormals.getMatRef(i); + Colors c = pyrColors. getMatRef(i); + + computePointsNormalsColors(intr.scale(i), rgb_intr.scale(i), depthFactor, scaled, rgb, p, n, c); + + if(i < levels - 1) + { + sz.width /= 2; sz.height /= 2; + scaled = pyrDownBilateral(scaled, sigmaDepth*depthFactor); + } + } +} + void buildPyramidPointsNormals(InputArray _points, InputArray _normals, OutputArrayOfArrays pyrPoints, OutputArrayOfArrays pyrNormals, diff --git a/modules/rgbd/src/kinfu_frame.hpp b/modules/rgbd/src/kinfu_frame.hpp index 16327d36aa5..199cfcf2c03 100644 --- a/modules/rgbd/src/kinfu_frame.hpp +++ b/modules/rgbd/src/kinfu_frame.hpp @@ -73,19 +73,27 @@ inline ptype toPtype(const cv::Vec3f& x) enum { DEPTH_TYPE = DataType::type, - POINT_TYPE = DataType::type + POINT_TYPE = DataType::type, + COLOR_TYPE = DataType::type }; typedef cv::Mat_< ptype > Points; typedef Points Normals; +typedef Points Colors; typedef cv::Mat_< depthType > Depth; void renderPointsNormals(InputArray _points, InputArray _normals, OutputArray image, cv::Affine3f lightPose); +void renderPointsNormalsColors(InputArray _points, InputArray _normals, InputArray _colors, OutputArray image, Affine3f lightPose); void makeFrameFromDepth(InputArray depth, OutputArray pyrPoints, OutputArray pyrNormals, const Intr intr, int levels, float depthFactor, float sigmaDepth, float sigmaSpatial, int kernelSize, float truncateThreshold); +void makeColoredFrameFromDepth(InputArray _depth, InputArray _rgb, + OutputArray pyrPoints, OutputArray pyrNormals, OutputArray pyrColors, + const Intr intr, const Intr rgb_intr, int levels, float depthFactor, + float sigmaDepth, float sigmaSpatial, int kernelSize, + float truncateThreshold); void buildPyramidPointsNormals(InputArray _points, InputArray _normals, OutputArrayOfArrays pyrPoints, OutputArrayOfArrays pyrNormals, int levels); diff --git a/modules/rgbd/src/precomp.hpp b/modules/rgbd/src/precomp.hpp index 4852274f3b0..945bc1537e9 100644 --- a/modules/rgbd/src/precomp.hpp +++ b/modules/rgbd/src/precomp.hpp @@ -21,6 +21,7 @@ #include "opencv2/core/private.hpp" #include "opencv2/core/hal/intrin.hpp" #include "opencv2/core/ocl.hpp" +#include "opencv2/core.hpp" #include "opencv2/imgproc.hpp" #include "opencv2/calib3d.hpp" #include "opencv2/rgbd.hpp" diff --git a/modules/rgbd/src/tsdf.cpp b/modules/rgbd/src/tsdf.cpp index 991d525b9d6..7b76985eb43 100644 --- a/modules/rgbd/src/tsdf.cpp +++ b/modules/rgbd/src/tsdf.cpp @@ -687,7 +687,6 @@ struct FetchPointsNormalsInvoker : ParallelLoopBody (y+shift.y)*vol.volDims[1] + (z+shift.z)*vol.volDims[2]]; float vd = tsdfToFloat(voxeld.tsdf); - if(voxeld.weight != 0 && vd != 1.f) { if((v0 > 0 && vd < 0) || (v0 < 0 && vd > 0)) @@ -761,6 +760,7 @@ void TSDFVolumeCPU::fetchPointsNormals(OutputArray _points, OutputArray _normals Range range(0, volResolution.x); const int nstripes = -1; parallel_for_(range, fi, nstripes); + std::vector points, normals; for(size_t i = 0; i < pVecs.size(); i++) { diff --git a/modules/rgbd/src/tsdf.hpp b/modules/rgbd/src/tsdf.hpp index 67361aa59d9..a3f561fbe4a 100644 --- a/modules/rgbd/src/tsdf.hpp +++ b/modules/rgbd/src/tsdf.hpp @@ -62,6 +62,10 @@ class TSDFVolumeCPU : public TSDFVolume const kinfu::Intr& intrinsics, const int frameId = 0) override; virtual void raycast(const Matx44f& cameraPose, const kinfu::Intr& intrinsics, const Size& frameSize, OutputArray points, OutputArray normals) const override; + virtual void integrate(InputArray, InputArray, float, const Matx44f&, const kinfu::Intr&, const Intr&, const int) override + { CV_Error(Error::StsNotImplemented, "Not implemented"); }; + virtual void raycast(const Matx44f&, const kinfu::Intr&, const Size&, OutputArray, OutputArray, OutputArray) const override + { CV_Error(Error::StsNotImplemented, "Not implemented"); }; virtual void fetchNormals(InputArray points, OutputArray _normals) const override; virtual void fetchPointsNormals(OutputArray points, OutputArray normals) const override; @@ -97,6 +101,10 @@ class TSDFVolumeGPU : public TSDFVolume const kinfu::Intr& intrinsics, const int frameId = 0) override; virtual void raycast(const Matx44f& cameraPose, const kinfu::Intr& intrinsics, const Size& frameSize, OutputArray _points, OutputArray _normals) const override; + virtual void integrate(InputArray, InputArray, float, const Matx44f&, const kinfu::Intr&, const Intr&, const int) override + { CV_Error(Error::StsNotImplemented, "Not implemented"); }; + virtual void raycast(const Matx44f&, const kinfu::Intr&, const Size&, OutputArray, OutputArray, OutputArray) const override + { CV_Error(Error::StsNotImplemented, "Not implemented"); }; virtual void fetchPointsNormals(OutputArray points, OutputArray normals) const override; virtual void fetchNormals(InputArray points, OutputArray normals) const override; diff --git a/modules/rgbd/src/tsdf_functions.cpp b/modules/rgbd/src/tsdf_functions.cpp index b0e5276cba7..3f8bc26f011 100644 --- a/modules/rgbd/src/tsdf_functions.cpp +++ b/modules/rgbd/src/tsdf_functions.cpp @@ -418,6 +418,306 @@ void integrateVolumeUnit( parallel_for_(integrateRange, IntegrateInvoker); } +void integrateRGBVolumeUnit( + float truncDist, float voxelSize, int maxWeight, + cv::Matx44f _pose, Point3i volResolution, Vec4i volStrides, + InputArray _depth, InputArray _rgb, float depthFactor, const cv::Matx44f& cameraPose, + const cv::kinfu::Intr& depth_intrinsics, const cv::kinfu::Intr& rgb_intrinsics, InputArray _pixNorms, InputArray _volume) +{ + CV_TRACE_FUNCTION(); + + CV_Assert(_depth.type() == DEPTH_TYPE); + CV_Assert(!_depth.empty()); + cv::Affine3f vpose(_pose); + Depth depth = _depth.getMat(); + Colors color = _rgb.getMat(); + Range integrateRange(0, volResolution.x); + + Mat volume = _volume.getMat(); + Mat pixNorms = _pixNorms.getMat(); + const Intr::Projector projDepth(depth_intrinsics.makeProjector()); + const Intr::Projector projRGB(rgb_intrinsics); + const cv::Affine3f vol2cam(Affine3f(cameraPose.inv()) * vpose); + const float truncDistInv(1.f / truncDist); + const float dfac(1.f / depthFactor); + RGBTsdfVoxel* volDataStart = volume.ptr(); + +#if USE_INTRINSICS + auto IntegrateInvoker = [&](const Range& range) + { + // zStep == vol2cam*(Point3f(x, y, 1)*voxelSize) - basePt; + Point3f zStepPt = Point3f(vol2cam.matrix(0, 2), + vol2cam.matrix(1, 2), + vol2cam.matrix(2, 2)) * voxelSize; + + v_float32x4 zStep(zStepPt.x, zStepPt.y, zStepPt.z, 0); + v_float32x4 vfxy(projDepth.fx, projDepth.fy, 0.f, 0.f), vcxy(projDepth.cx, projDepth.cy, 0.f, 0.f); + v_float32x4 rgb_vfxy(projRGB.fx, projRGB.fy, 0.f, 0.f), rgb_vcxy(projRGB.cx, projRGB.cy, 0.f, 0.f); + const v_float32x4 upLimits = v_cvt_f32(v_int32x4(depth.cols - 1, depth.rows - 1, 0, 0)); + + for (int x = range.start; x < range.end; x++) + { + RGBTsdfVoxel* volDataX = volDataStart + x * volStrides[0]; + for (int y = 0; y < volResolution.y; y++) + { + RGBTsdfVoxel* volDataY = volDataX + y * volStrides[1]; + // optimization of camSpace transformation (vector addition instead of matmul at each z) + Point3f basePt = vol2cam * (Point3f((float)x, (float)y, 0) * voxelSize); + v_float32x4 camSpacePt(basePt.x, basePt.y, basePt.z, 0); + + int startZ, endZ; + if (abs(zStepPt.z) > 1e-5) + { + int baseZ = (int)(-basePt.z / zStepPt.z); + if (zStepPt.z > 0) + { + startZ = baseZ; + endZ = volResolution.z; + } + else + { + startZ = 0; + endZ = baseZ; + } + } + else + { + if (basePt.z > 0) + { + startZ = 0; + endZ = volResolution.z; + } + else + { + // z loop shouldn't be performed + startZ = endZ = 0; + } + } + startZ = max(0, startZ); + endZ = min(int(volResolution.z), endZ); + for (int z = startZ; z < endZ; z++) + { + // optimization of the following: + //Point3f volPt = Point3f(x, y, z)*voxelSize; + //Point3f camSpacePt = vol2cam * volPt; + camSpacePt += zStep; + + float zCamSpace = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(camSpacePt))).get0(); + if (zCamSpace <= 0.f) + continue; + + v_float32x4 camPixVec = camSpacePt / v_setall_f32(zCamSpace); + v_float32x4 projected = v_muladd(camPixVec, vfxy, vcxy); + // leave only first 2 lanes + projected = v_reinterpret_as_f32(v_reinterpret_as_u32(projected) & + v_uint32x4(0xFFFFFFFF, 0xFFFFFFFF, 0, 0)); + + depthType v; + // bilinearly interpolate depth at projected + { + const v_float32x4& pt = projected; + // check coords >= 0 and < imgSize + v_uint32x4 limits = v_reinterpret_as_u32(pt < v_setzero_f32()) | + v_reinterpret_as_u32(pt >= upLimits); + limits = limits | v_rotate_right<1>(limits); + if (limits.get0()) + continue; + + // xi, yi = floor(pt) + v_int32x4 ip = v_floor(pt); + v_int32x4 ipshift = ip; + int xi = ipshift.get0(); + ipshift = v_rotate_right<1>(ipshift); + int yi = ipshift.get0(); + + const depthType* row0 = depth[yi + 0]; + const depthType* row1 = depth[yi + 1]; + + // v001 = [v(xi + 0, yi + 0), v(xi + 1, yi + 0)] + v_float32x4 v001 = v_load_low(row0 + xi); + // v101 = [v(xi + 0, yi + 1), v(xi + 1, yi + 1)] + v_float32x4 v101 = v_load_low(row1 + xi); + + v_float32x4 vall = v_combine_low(v001, v101); + + // assume correct depth is positive + // don't fix missing data + if (v_check_all(vall > v_setzero_f32())) + { + v_float32x4 t = pt - v_cvt_f32(ip); + float tx = t.get0(); + t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t))); + v_float32x4 ty = v_setall_f32(t.get0()); + // vx is y-interpolated between rows 0 and 1 + v_float32x4 vx = v001 + ty * (v101 - v001); + float v0 = vx.get0(); + vx = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vx))); + float v1 = vx.get0(); + v = v0 + tx * (v1 - v0); + } + else + continue; + } + + v_float32x4 projectedRGB = v_muladd(camPixVec, rgb_vfxy, rgb_vcxy); + // leave only first 2 lanes + projectedRGB = v_reinterpret_as_f32(v_reinterpret_as_u32(projected) & + v_uint32x4(0xFFFFFFFF, 0xFFFFFFFF, 0, 0)); + + // norm(camPixVec) produces double which is too slow + int _u = (int)projected.get0(); + int _v = (int)v_rotate_right<1>(projected).get0(); + int rgb_u = (int)projectedRGB.get0(); + int rgb_v = (int)v_rotate_right<1>(projectedRGB).get0(); + + if (!(_u >= 0 && _u < depth.cols && _v >= 0 && _v < depth.rows && + rgb_v >= 0 && rgb_v < color.rows && rgb_u >= 0 && rgb_u < color.cols)) + continue; + float pixNorm = pixNorms.at(_v, _u); + Vec4f colorRGB = color.at(rgb_v, rgb_u); + //float pixNorm = sqrt(v_reduce_sum(camPixVec*camPixVec)); + // difference between distances of point and of surface to camera + float sdf = pixNorm * (v * dfac - zCamSpace); + // possible alternative is: + // kftype sdf = norm(camSpacePt)*(v*dfac/camSpacePt.z - 1); + if (sdf >= -truncDist) + { + TsdfType tsdf = floatToTsdf(fmin(1.f, sdf * truncDistInv)); + + RGBTsdfVoxel& voxel = volDataY[z * volStrides[2]]; + WeightType& weight = voxel.weight; + TsdfType& value = voxel.tsdf; + ColorType& r = voxel.r; + ColorType& g = voxel.g; + ColorType& b = voxel.b; + + // update RGB + r = (ColorType)((float)(r * weight) + (colorRGB[0])) / (weight + 1); + g = (ColorType)((float)(g * weight) + (colorRGB[1])) / (weight + 1); + b = (ColorType)((float)(b * weight) + (colorRGB[2])) / (weight + 1); + colorFix(r, g, b); + // update TSDF + value = floatToTsdf((tsdfToFloat(value) * weight + tsdfToFloat(tsdf)) / (weight + 1)); + weight = WeightType(min(int(weight + 1), int(maxWeight))); + } + } + } + } + }; +#else + auto IntegrateInvoker = [&](const Range& range) + { + for (int x = range.start; x < range.end; x++) + { + RGBTsdfVoxel* volDataX = volDataStart + x * volStrides[0]; + for (int y = 0; y < volResolution.y; y++) + { + RGBTsdfVoxel* volDataY = volDataX + y * volStrides[1]; + // optimization of camSpace transformation (vector addition instead of matmul at each z) + Point3f basePt = vol2cam * (Point3f(float(x), float(y), 0.0f) * voxelSize); + Point3f camSpacePt = basePt; + // zStep == vol2cam*(Point3f(x, y, 1)*voxelSize) - basePt; + // zStep == vol2cam*[Point3f(x, y, 1) - Point3f(x, y, 0)]*voxelSize + Point3f zStep = Point3f(vol2cam.matrix(0, 2), + vol2cam.matrix(1, 2), + vol2cam.matrix(2, 2)) * voxelSize; + int startZ, endZ; + if (abs(zStep.z) > 1e-5) + { + int baseZ = int(-basePt.z / zStep.z); + if (zStep.z > 0) + { + startZ = baseZ; + endZ = volResolution.z; + } + else + { + startZ = 0; + endZ = baseZ; + } + } + else + { + if (basePt.z > 0) + { + startZ = 0; + endZ = volResolution.z; + } + else + { + // z loop shouldn't be performed + startZ = endZ = 0; + } + } + startZ = max(0, startZ); + endZ = min(int(volResolution.z), endZ); + + for (int z = startZ; z < endZ; z++) + { + // optimization of the following: + //Point3f volPt = Point3f(x, y, z)*volume.voxelSize; + //Point3f camSpacePt = vol2cam * volPt; + + camSpacePt += zStep; + if (camSpacePt.z <= 0) + continue; + + Point3f camPixVec; + Point2f projected = projDepth(camSpacePt, camPixVec); + Point2f projectedRGB = projRGB(camSpacePt, camPixVec); + + + depthType v = bilinearDepth(depth, projected); + if (v == 0) { + continue; + } + + int _u = (int) projected.x; + int _v = (int) projected.y; + + int rgb_u = (int) projectedRGB.x; + int rgb_v = (int) projectedRGB.y; + + if (!(_v >= 0 && _v < depth.rows && _u >= 0 && _u < depth.cols && + rgb_v >= 0 && rgb_v < color.rows && rgb_u >= 0 && rgb_u < color.cols)) + continue; + + float pixNorm = pixNorms.at(_v, _u); + Vec4f colorRGB = color.at(rgb_v, rgb_u); + // difference between distances of point and of surface to camera + float sdf = pixNorm * (v * dfac - camSpacePt.z); + // possible alternative is: + // kftype sdf = norm(camSpacePt)*(v*dfac/camSpacePt.z - 1); + if (sdf >= -truncDist) + { + TsdfType tsdf = floatToTsdf(fmin(1.f, sdf * truncDistInv)); + + RGBTsdfVoxel& voxel = volDataY[z * volStrides[2]]; + WeightType& weight = voxel.weight; + TsdfType& value = voxel.tsdf; + ColorType& r = voxel.r; + ColorType& g = voxel.g; + ColorType& b = voxel.b; + + // update RGB + if (weight < 1) + { + r = (ColorType)((float)(r * weight) + (colorRGB[0])) / (weight + 1); + g = (ColorType)((float)(g * weight) + (colorRGB[1])) / (weight + 1); + b = (ColorType)((float)(b * weight) + (colorRGB[2])) / (weight + 1); + } + + // update TSDF + value = floatToTsdf((tsdfToFloat(value) * weight + tsdfToFloat(tsdf)) / (weight + 1)); + weight = WeightType( min(int(weight + 1), int(maxWeight)) ); + } + } + } + } + }; +#endif + parallel_for_(integrateRange, IntegrateInvoker); +} } // namespace kinfu } // namespace cv diff --git a/modules/rgbd/src/tsdf_functions.hpp b/modules/rgbd/src/tsdf_functions.hpp index 09efecf1252..1031474d0b7 100644 --- a/modules/rgbd/src/tsdf_functions.hpp +++ b/modules/rgbd/src/tsdf_functions.hpp @@ -9,6 +9,7 @@ #include #include "tsdf.hpp" +#include "colored_tsdf.hpp" namespace cv { @@ -34,6 +35,20 @@ inline float tsdfToFloat(TsdfType num) return float(num) * (-1.f / 128.f); } +inline void colorFix(ColorType& r, ColorType& g, ColorType&b) +{ + if (r > 255) r = 255; + if (g > 255) g = 255; + if (b > 255) b = 255; +} + +inline void colorFix(Point3f& c) +{ + if (c.x > 255) c.x = 255; + if (c.y > 255) c.y = 255; + if (c.z > 255) c.z = 255; +} + cv::Mat preCalculationPixNorm(Depth depth, const Intr& intrinsics); cv::UMat preCalculationPixNormGPU(const UMat& depth, const Intr& intrinsics); @@ -45,6 +60,12 @@ void integrateVolumeUnit( InputArray _depth, float depthFactor, const cv::Matx44f& cameraPose, const cv::kinfu::Intr& intrinsics, InputArray _pixNorms, InputArray _volume); +void integrateRGBVolumeUnit( + float truncDist, float voxelSize, int maxWeight, + cv::Matx44f _pose, Point3i volResolution, Vec4i volStrides, + InputArray _depth, InputArray _rgb, float depthFactor, const cv::Matx44f& cameraPose, + const cv::kinfu::Intr& depth_intrinsics, const cv::kinfu::Intr& rgb_intrinsics, InputArray _pixNorms, InputArray _volume); + class CustomHashSet { diff --git a/modules/rgbd/src/volume.cpp b/modules/rgbd/src/volume.cpp index 88c46fe4b67..c05238f2602 100644 --- a/modules/rgbd/src/volume.cpp +++ b/modules/rgbd/src/volume.cpp @@ -4,10 +4,11 @@ #include -#include "hash_tsdf.hpp" #include "opencv2/core/base.hpp" #include "precomp.hpp" #include "tsdf.hpp" +#include "hash_tsdf.hpp" +#include "colored_tsdf.hpp" namespace cv { @@ -38,6 +39,14 @@ Ptr VolumeParams::defaultParams(VolumeType _volumeType) params.tsdfTruncDist = 7 * params.voxelSize; //! About 0.04f in meters return makePtr(params); } + else if (params.type == VolumeType::COLOREDTSDF) + { + params.resolution = Vec3i::all(512); + params.voxelSize = volumeSize / 512.f; + params.depthTruncThreshold = 0.f; // depthTruncThreshold not required for TSDF + params.tsdfTruncDist = 7 * params.voxelSize; //! About 0.04f in meters + return makePtr(params); + } CV_Error(Error::StsBadArg, "Invalid VolumeType does not have parameters"); } @@ -60,6 +69,13 @@ Ptr VolumeParams::coarseParams(VolumeType _volumeType) params->tsdfTruncDist = 2 * params->voxelSize; //! About 0.04f in meters return params; } + else if (params->type == VolumeType::COLOREDTSDF) + { + params->resolution = Vec3i::all(128); + params->voxelSize = volumeSize / 128.f; + params->tsdfTruncDist = 2 * params->voxelSize; //! About 0.04f in meters + return params; + } CV_Error(Error::StsBadArg, "Invalid VolumeType does not have parameters"); } @@ -69,6 +85,8 @@ Ptr makeVolume(const VolumeParams& _volumeParams) return kinfu::makeTSDFVolume(_volumeParams); else if(_volumeParams.type == VolumeType::HASHTSDF) return kinfu::makeHashTSDFVolume(_volumeParams); + else if(_volumeParams.type == VolumeType::COLOREDTSDF) + return kinfu::makeColoredTSDFVolume(_volumeParams); CV_Error(Error::StsBadArg, "Invalid VolumeType does not have parameters"); } @@ -85,6 +103,10 @@ Ptr makeVolume(VolumeType _volumeType, float _voxelSize, Matx44f _pose, { return makeHashTSDFVolume(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, _truncateThreshold); } + else if (_volumeType == VolumeType::COLOREDTSDF) + { + return makeColoredTSDFVolume(_voxelSize, _pose, _raycastStepFactor, _truncDist, _maxWeight, _presolution); + } CV_Error(Error::StsBadArg, "Invalid VolumeType does not have parameters"); } diff --git a/modules/rgbd/test/test_colored_kinfu.cpp b/modules/rgbd/test/test_colored_kinfu.cpp new file mode 100644 index 00000000000..4303b260e3e --- /dev/null +++ b/modules/rgbd/test/test_colored_kinfu.cpp @@ -0,0 +1,479 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +// This code is also subject to the license terms in the LICENSE_KinectFusion.md file found in this module's directory + +#include "test_precomp.hpp" + +// Inspired by Inigo Quilez' raymarching guide: +// http://iquilezles.org/www/articles/distfunctions/distfunctions.htm + +namespace opencv_test { +namespace { + +using namespace cv; + +/** Reprojects screen point to camera space given z coord. */ +struct Reprojector +{ + Reprojector() {} + inline Reprojector(Matx33f intr) + { + fxinv = 1.f / intr(0, 0), fyinv = 1.f / intr(1, 1); + cx = intr(0, 2), cy = intr(1, 2); + } + template + inline cv::Point3_ operator()(cv::Point3_ p) const + { + T x = p.z * (p.x - cx) * fxinv; + T y = p.z * (p.y - cy) * fyinv; + return cv::Point3_(x, y, p.z); + } + + float fxinv, fyinv, cx, cy; +}; + +template +struct RenderInvoker : ParallelLoopBody +{ + RenderInvoker(Mat_& _frame, Affine3f _pose, + Reprojector _reproj, + float _depthFactor) : ParallelLoopBody(), + frame(_frame), + pose(_pose), + reproj(_reproj), + depthFactor(_depthFactor) + { } + + virtual void operator ()(const cv::Range& r) const + { + for (int y = r.start; y < r.end; y++) + { + float* frameRow = frame[y]; + for (int x = 0; x < frame.cols; x++) + { + float pix = 0; + + Point3f orig = pose.translation(); + // direction through pixel + Point3f screenVec = reproj(Point3f((float)x, (float)y, 1.f)); + float xyt = 1.f / (screenVec.x * screenVec.x + + screenVec.y * screenVec.y + 1.f); + Point3f dir = normalize(Vec3f(pose.rotation() * screenVec)); + // screen space axis + dir.y = -dir.y; + + const float maxDepth = 20.f; + const float maxSteps = 256; + float t = 0.f; + for (int step = 0; step < maxSteps && t < maxDepth; step++) + { + Point3f p = orig + dir * t; + float d = Scene::map(p); + if (d < 0.000001f) + { + float depth = std::sqrt(t * t * xyt); + pix = depth * depthFactor; + break; + } + t += d; + } + + frameRow[x] = pix; + } + } + } + + Mat_& frame; + Affine3f pose; + Reprojector reproj; + float depthFactor; +}; + +template +struct RenderColorInvoker : ParallelLoopBody +{ + RenderColorInvoker(Mat_& _frame, Affine3f _pose, + Reprojector _reproj, + float _depthFactor) : ParallelLoopBody(), + frame(_frame), + pose(_pose), + reproj(_reproj), + depthFactor(_depthFactor) + { } + + virtual void operator ()(const cv::Range& r) const + { + for (int y = r.start; y < r.end; y++) + { + Vec3f* frameRow = frame[y]; + for (int x = 0; x < frame.cols; x++) + { + Vec3f pix = 0; + + Point3f orig = pose.translation(); + // direction through pixel + Point3f screenVec = reproj(Point3f((float)x, (float)y, 1.f)); + Point3f dir = normalize(Vec3f(pose.rotation() * screenVec)); + // screen space axis + dir.y = -dir.y; + + const float maxDepth = 20.f; + const float maxSteps = 256; + float t = 0.f; + for (int step = 0; step < maxSteps && t < maxDepth; step++) + { + Point3f p = orig + dir * t; + float d = Scene::map(p); + if (d < 0.000001f) + { + float m = 0.25f; + float p0 = float(abs(fmod(p.x, m)) > m / 2.f); + float p1 = float(abs(fmod(p.y, m)) > m / 2.f); + float p2 = float(abs(fmod(p.z, m)) > m / 2.f); + + pix[0] = p0 + p1; + pix[1] = p1 + p2; + pix[2] = p0 + p2; + + pix *= 128.f; + break; + } + t += d; + } + + frameRow[x] = pix; + } + } + } + + Mat_& frame; + Affine3f pose; + Reprojector reproj; + float depthFactor; +}; + +struct Scene +{ + virtual ~Scene() {} + static Ptr create(int nScene, Size sz, Matx33f _intr, float _depthFactor); + virtual Mat depth(Affine3f pose) = 0; + virtual Mat rgb(Affine3f pose) = 0; + virtual std::vector getPoses() = 0; +}; + +struct CubeSpheresScene : Scene +{ + const int framesPerCycle = 32; + const float nCycles = 0.25f; + const Affine3f startPose = Affine3f(Vec3f(-0.5f, 0.f, 0.f), Vec3f(2.1f, 1.4f, -2.1f)); + + CubeSpheresScene(Size sz, Matx33f _intr, float _depthFactor) : + frameSize(sz), intr(_intr), depthFactor(_depthFactor) + { } + + static float map(Point3f p) + { + float plane = p.y + 0.5f; + + Point3f boxPose = p - Point3f(-0.0f, 0.3f, 0.0f); + float boxSize = 0.5f; + float roundness = 0.08f; + Point3f boxTmp; + boxTmp.x = max(abs(boxPose.x) - boxSize, 0.0f); + boxTmp.y = max(abs(boxPose.y) - boxSize, 0.0f); + boxTmp.z = max(abs(boxPose.z) - boxSize, 0.0f); + float roundBox = (float)cv::norm(boxTmp) - roundness; + + float sphereRadius = 0.7f; + float sphere = (float)cv::norm(boxPose) - sphereRadius; + + float boxMinusSphere = max(roundBox, -sphere); + + float sphere2 = (float)cv::norm(p - Point3f(0.3f, 1.f, 0.f)) - 0.1f; + float sphere3 = (float)cv::norm(p - Point3f(0.0f, 1.f, 0.f)) - 0.2f; + float res = min(min(plane, boxMinusSphere), min(sphere2, sphere3)); + + return res; + } + + Mat depth(Affine3f pose) override + { + Mat_ frame(frameSize); + Reprojector reproj(intr); + + Range range(0, frame.rows); + parallel_for_(range, RenderInvoker(frame, pose, reproj, depthFactor)); + + return std::move(frame); + } + + Mat rgb(Affine3f pose) override + { + Mat_ frame(frameSize); + Reprojector reproj(intr); + + Range range(0, frame.rows); + parallel_for_(range, RenderColorInvoker(frame, pose, reproj, depthFactor)); + + return std::move(frame); + } + + std::vector getPoses() override + { + std::vector poses; + for (int i = 0; i < (int)(framesPerCycle * nCycles); i++) + { + float angle = (float)(CV_2PI * i / framesPerCycle); + Affine3f pose; + pose = pose.rotate(startPose.rotation()); + pose = pose.rotate(Vec3f(0.f, -1.f, 0.f) * angle); + pose = pose.translate(Vec3f(startPose.translation()[0] * sin(angle), + startPose.translation()[1], + startPose.translation()[2] * cos(angle))); + poses.push_back(pose); + } + + return poses; + } + + Size frameSize; + Matx33f intr; + float depthFactor; +}; + + +struct RotatingScene : Scene +{ + const int framesPerCycle = 32; + const float nCycles = 0.5f; + const Affine3f startPose = Affine3f(Vec3f(-1.f, 0.f, 0.f), Vec3f(1.5f, 2.f, -1.5f)); + + RotatingScene(Size sz, Matx33f _intr, float _depthFactor) : + frameSize(sz), intr(_intr), depthFactor(_depthFactor) + { + cv::RNG rng(0); + rng.fill(randTexture, cv::RNG::UNIFORM, 0.f, 1.f); + } + + static float noise(Point2f pt) + { + pt.x = abs(pt.x - (int)pt.x); + pt.y = abs(pt.y - (int)pt.y); + pt *= 256.f; + + int xi = cvFloor(pt.x), yi = cvFloor(pt.y); + + const float* row0 = randTexture[(yi + 0) % 256]; + const float* row1 = randTexture[(yi + 1) % 256]; + + float v00 = row0[(xi + 0) % 256]; + float v01 = row0[(xi + 1) % 256]; + float v10 = row1[(xi + 0) % 256]; + float v11 = row1[(xi + 1) % 256]; + + float tx = pt.x - xi, ty = pt.y - yi; + float v0 = v00 + tx * (v01 - v00); + float v1 = v10 + tx * (v11 - v10); + return v0 + ty * (v1 - v0); + } + + static float map(Point3f p) + { + const Point3f torPlace(0.f, 0.f, 0.f); + Point3f torPos(p - torPlace); + const Point2f torusParams(1.f, 0.2f); + Point2f torq(std::sqrt(torPos.x * torPos.x + torPos.z * torPos.z) - torusParams.x, torPos.y); + float torus = (float)cv::norm(torq) - torusParams.y; + + const Point3f cylShift(0.25f, 0.25f, 0.25f); + + Point3f cylPos = Point3f(abs(std::fmod(p.x - 0.1f, cylShift.x)), + p.y, + abs(std::fmod(p.z - 0.2f, cylShift.z))) - cylShift * 0.5f; + + const Point2f cylParams(0.1f, + 0.1f + 0.1f * sin(p.x * p.y * 5.f /* +std::log(1.f+abs(p.x*0.1f)) */)); + Point2f cyld = Point2f(abs(std::sqrt(cylPos.x * cylPos.x + cylPos.z * cylPos.z)), abs(cylPos.y)) - cylParams; + float pins = min(max(cyld.x, cyld.y), 0.0f) + (float)cv::norm(Point2f(max(cyld.x, 0.f), max(cyld.y, 0.f))); + + float terrain = p.y + 0.25f * noise(Point2f(p.x, p.z) * 0.01f); + + float res = min(terrain, max(-pins, torus)); + + return res; + } + + Mat depth(Affine3f pose) override + { + Mat_ frame(frameSize); + Reprojector reproj(intr); + + Range range(0, frame.rows); + parallel_for_(range, RenderInvoker(frame, pose, reproj, depthFactor)); + + return std::move(frame); + } + + Mat rgb(Affine3f pose) override + { + Mat_ frame(frameSize); + Reprojector reproj(intr); + + Range range(0, frame.rows); + parallel_for_(range, RenderColorInvoker(frame, pose, reproj, depthFactor)); + + return std::move(frame); + } + + std::vector getPoses() override + { + std::vector poses; + for (int i = 0; i < framesPerCycle * nCycles; i++) + { + float angle = (float)(CV_2PI * i / framesPerCycle); + Affine3f pose; + pose = pose.rotate(startPose.rotation()); + pose = pose.rotate(Vec3f(0.f, -1.f, 0.f) * angle); + pose = pose.translate(Vec3f(startPose.translation()[0] * sin(angle), + startPose.translation()[1], + startPose.translation()[2] * cos(angle))); + poses.push_back(pose); + } + + return poses; + } + + Size frameSize; + Matx33f intr; + float depthFactor; + static cv::Mat_ randTexture; +}; + +Mat_ RotatingScene::randTexture(256, 256); + +Ptr Scene::create(int nScene, Size sz, Matx33f _intr, float _depthFactor) +{ + if (nScene == 0) + return makePtr(sz, _intr, _depthFactor); + else + return makePtr(sz, _intr, _depthFactor); +} + +static inline void CheckFrequency(Mat image) +{ + float all = (float)image.size().height * image.size().width; + int cc1 = 0, cc2 = 0, cc3 = 0, cc4 = 0; + Vec3b c1 = Vec3b(200, 0, 0), c2 = Vec3b(0, 200, 0); + Vec3b c3 = Vec3b(0, 0, 200), c4 = Vec3b(100, 100, 100); + for (int i = 0; i < image.size().height; i++) + { + for (int j = 0; j < image.size().width; j++) + { + Vec3b color = image.at(i, j); + if (color == c1) cc1++; + if (color == c2) cc2++; + if (color == c3) cc3++; + if (color == c4) cc4++; + } + } + ASSERT_LT(float(cc1) / all, 0.2); + ASSERT_LT(float(cc2) / all, 0.2); + ASSERT_LT(float(cc3) / all, 0.2); + ASSERT_LT(float(cc4) / all, 0.2); +} + +static const bool display = false; + +void flyTest(bool hiDense, bool test_colors) +{ + Ptr params; + params = colored_kinfu::Params::coloredTSDFParams(!hiDense); + Ptr scene = Scene::create(false, params->frameSize, params->intr, params->depthFactor); + + Ptr kf = colored_kinfu::ColoredKinFu::create(params); + + std::vector poses = scene->getPoses(); + Affine3f startPoseGT = poses[0], startPoseKF; + Affine3f pose, kfPose; + for (size_t i = 0; i < poses.size(); i++) + { + pose = poses[i]; + + Mat depth = scene->depth(pose); + //DEBUG + Mat rgb = scene->rgb(pose);// creareRGBframe(depth.size()); + + ASSERT_TRUE(kf->update(depth, rgb)); + + kfPose = kf->getPose(); + if (i == 0) + startPoseKF = kfPose; + + pose = (startPoseGT.inv() * pose) * startPoseKF; + + if (display) + { + imshow("depth", depth * (1.f / params->depthFactor / 4.f)); + imshow("rgb", rgb * (1.f / 255.f)); + Mat rendered; + kf->render(rendered); + imshow("render", rendered); + waitKey(10); + } + + if (test_colors) + { + Mat rendered; + kf->render(rendered); + CheckFrequency(rendered); + return; + } + } + + double rvecThreshold = hiDense ? 0.01 : 0.02; + ASSERT_LT(cv::norm(kfPose.rvec() - pose.rvec()), rvecThreshold); + double poseThreshold = hiDense ? 0.03 : 0.1; + ASSERT_LT(cv::norm(kfPose.translation() - pose.translation()), poseThreshold); +} + + +#ifdef OPENCV_ENABLE_NONFREE +TEST(ColoredKinectFusion, lowDense) +#else +TEST(ColoredKinectFusion, DISABLED_lowDense) +#endif +{ + flyTest(false, false); +} + +#ifdef OPENCV_ENABLE_NONFREE +TEST(ColoredKinectFusion, highDense) +#else +TEST(KinectFusion, DISABLED_highDense) +#endif +{ + flyTest(true, false); +} + +#ifdef OPENCV_ENABLE_NONFREE +TEST(ColoredKinectFusion, color_lowDense) +#else +TEST(ColoredKinectFusion, DISABLED_color_lowDense) +#endif +{ + flyTest(false, true); +} + +#ifdef OPENCV_ENABLE_NONFREE +TEST(ColoredKinectFusion, color_highDense) +#else +TEST(KinectFusion, DISABLED_color_highDense) +#endif +{ + flyTest(true, true); +} + +} +} // namespace From f7c034c399ab86ca3234ad5260aa5c217a6f805e Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Thu, 8 Apr 2021 22:21:11 +0000 Subject: [PATCH 70/70] fix whitespace --- modules/rgbd/src/opencl/hash_tsdf.cl | 2 +- modules/wechat_qrcode/LICENSE | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/rgbd/src/opencl/hash_tsdf.cl b/modules/rgbd/src/opencl/hash_tsdf.cl index 0e611c1d3dd..3b5a9065185 100644 --- a/modules/rgbd/src/opencl/hash_tsdf.cl +++ b/modules/rgbd/src/opencl/hash_tsdf.cl @@ -384,7 +384,7 @@ inline float3 getNormalVoxel(float3 ptVox, __global const struct TsdfVoxel* allV int4 offsets[] = { (int4)( 1, 0, 0, 0), (int4)(-1, 0, 0, 0), (int4)( 0, 1, 0, 0), // 0-3 (int4)( 0, -1, 0, 0), (int4)( 0, 0, 1, 0), (int4)( 0, 0, -1, 0) // 4-7 }; - + const int nVals = 6; float vals[6]; #else diff --git a/modules/wechat_qrcode/LICENSE b/modules/wechat_qrcode/LICENSE index 33358269b03..303fddb1597 100644 --- a/modules/wechat_qrcode/LICENSE +++ b/modules/wechat_qrcode/LICENSE @@ -1,10 +1,10 @@ Tencent is pleased to support the open source community by making WeChat QRCode available. -Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -The below software in this distribution may have been modified by THL A29 Limited ("Tencent Modifications"). +Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +The below software in this distribution may have been modified by THL A29 Limited ("Tencent Modifications"). All Tencent Modifications are Copyright (C) THL A29 Limited. -WeChat QRCode is licensed under the Apache License Version 2.0, except for the third-party components listed below. +WeChat QRCode is licensed under the Apache License Version 2.0, except for the third-party components listed below. Terms of the Apache License Version 2.0 --------------------------------------------------------------------