Sync 1.1.0 (#5)

.gitattributes

@@ -1,2 +1,80 @@
 dependency/boost/boost_1_71_0.tar.gz filter=lfs diff=lfs merge=lfs -text
 dependency/protobuf/google-protobuf_3.7.1_bin.zip filter=lfs diff=lfs merge=lfs -text
+dependency/esdk_obs_api/huaweicloud-sdk-c-obs-3.19.9.3.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/pgxc/pgxc-v1.1.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/brotli/brotli-1.0.7.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/memcheck/euleros2.0_sp5_x86_64/debug/gcc7.3.0/lib/libasan.a filter=lfs diff=lfs merge=lfs -text
+dependency/memcheck/euleros2.0_sp8_aarch64/debug/gcc7.3.0/lib/libasan.a filter=lfs diff=lfs merge=lfs -text
+dependency/llvm/llvm-7.0.0.src.tar.xz filter=lfs diff=lfs merge=lfs -text
+buildtools/python/python_2.7.16_src.zip filter=lfs diff=lfs merge=lfs -text
+dependency/cryptography/cryptography-2.9.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/llvm/llvm-10.0.0.src.tar.xz filter=lfs diff=lfs merge=lfs -text
+dependency/kerberos/krb5-1.17.1.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/esdk_obs_api/huaweicloud-sdk-c-obs-3.1.3.tar.gz filter=lfs diff=lfs merge=lfs -text
+platform/huaweijdk8/jdk1.8.0_272/src.zip filter=lfs diff=lfs merge=lfs -text
+platform/huaweijdk8/jdk1.8.0_272/jre/lib/rt.jar filter=lfs diff=lfs merge=lfs -text
+platform/openjdk8/OpenJDK8U-jdk_x64_linux_8u222.tar.gz filter=lfs diff=lfs merge=lfs -text
+platform/openjdk8/OpenJDK8U-jdk_aarch64_linux_8u222.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/boost/boost_1_72_0.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/llvm/llvm-10.0.0.src.tar filter=lfs diff=lfs merge=lfs -text
+dependency/libevent/libevent-2.1.11-stable.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/snappy/1.1.8.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/flatbuffers/flatbuffers-1.11.0.zip filter=lfs diff=lfs merge=lfs -text
+platform/huaweijdk8/jdk1.8.0_272/jre/lib/ext/localedata.jar filter=lfs diff=lfs merge=lfs -text
+dependency/rapidjson/rapidjson-4b3d7c2f42142f10b888e580c515f60ca98e2ee9.zip filter=lfs diff=lfs merge=lfs -text
+dependency/c-ares/c-ares-1.15.0.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/jackson/jackson-databind-2.11.2.jar filter=lfs diff=lfs merge=lfs -text
+platform/huaweijdk8/jdk1.8.0_272/demo/jvmti/hprof/lib/libhprof.so filter=lfs diff=lfs merge=lfs -text
+buildtools/isl/isl-0.18.tar.bz2 filter=lfs diff=lfs merge=lfs -text
+dependency/unixodbc/unixODBC-2.3.9.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/nghttp2/nghttp2-1.41.0.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/madlib/madlib.patch filter=lfs diff=lfs merge=lfs -text
+buildtools/gmp/gmp-6.1.2.tar.xz filter=lfs diff=lfs merge=lfs -text
+dependency/zstd/zstd-1.4.4.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/flatbuffers/flatbuffers-1.11.0-src.zip filter=lfs diff=lfs merge=lfs -text
+platform/huaweijdk8/jdk1.8.0_272/jre/lib/ext/nashorn.jar filter=lfs diff=lfs merge=lfs -text
+dependency/pcre/pcre-8.42.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/pcre/pcre-8.44.tar.gz filter=lfs diff=lfs merge=lfs -text
+buildtools/mpfr/mpfr-4.0.2.tar.gz filter=lfs diff=lfs merge=lfs -text
+platform/huaweijdk8/jdk1.8.0_272/lib/sa-jdi.jar filter=lfs diff=lfs merge=lfs -text
+dependency/nghttp2/nghttp2-1.39.2.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/abseil-cpp/abseil-cpp-20200225.zip filter=lfs diff=lfs merge=lfs -text
+dependency/postgresql-hll/postgresql-hll-2.14.zip filter=lfs diff=lfs merge=lfs -text
+dependency/madlib/apache-madlib-1.17.0-src.tar.gz filter=lfs diff=lfs merge=lfs -text
+platform/huaweijdk8/jdk1.8.0_272/jre/lib/charsets.jar filter=lfs diff=lfs merge=lfs -text
+platform/huaweijdk8/jdk1.8.0_272/jre/lib/amd64/libfreetype.so.6 filter=lfs diff=lfs merge=lfs -text
+dependency/pynacl/pynacl-1.3.0.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/ncurses/ncurses-6.2.tar.gz filter=lfs diff=lfs merge=lfs -text
+platform/huaweijdk8/jdk1.8.0_272/jre/lib/resources.jar filter=lfs diff=lfs merge=lfs -text
+platform/huaweijdk8/jdk1.8.0_272/jre/lib/ext/cldrdata.jar filter=lfs diff=lfs merge=lfs -text
+dependency/libcurl/curl-7.71.1.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/libcurl/curl-7.66.0.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/libthrift/thrift-0.13.0.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/libiconv/libiconv-1.16.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/libiconv/libiconv-1.15.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/parquet/apache-arrow-0.11.1.zip filter=lfs diff=lfs merge=lfs -text
+dependency/memcheck/euleros2.0_sp5_x86_64/debug/gcc7.3.0/lib/libubsan.a filter=lfs diff=lfs merge=lfs -text
+dependency/memcheck/euleros2.0_sp8_aarch64/debug/gcc7.3.0/lib/libubsan.a filter=lfs diff=lfs merge=lfs -text
+dependency/memcheck/euleros2.0_sp5_x86_64/debug/gcc7.3.0/lib/liblsan.a filter=lfs diff=lfs merge=lfs -text
+dependency/memcheck/euleros2.0_sp8_aarch64/debug/gcc7.3.0/lib/liblsan.a filter=lfs diff=lfs merge=lfs -text
+dependency/grpc/grpc-1.28.1.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/double-conversion/v3.1.1.zip filter=lfs diff=lfs merge=lfs -text
+dependency/protobuf/protobuf-3.11.3.zip filter=lfs diff=lfs merge=lfs -text
+dependency/etcd/etcd-3.3.23.zip filter=lfs diff=lfs merge=lfs -text
+dependency/kerberos/krb5-1.17.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/openssl/openssl-1.1.1d.tar.gz filter=lfs diff=lfs merge=lfs -text
+buildtools/cmake/cmake-3.14.1.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/openssl/openssl-1.1.1g.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/orc/huawei_orc-rel-release-1.6.0.patch filter=lfs diff=lfs merge=lfs -text
+dependency/orc/orc-1.5.7.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/orc/orc-rel-release-1.6.0.tar.gz filter=lfs diff=lfs merge=lfs -text
+buildtools/python/Python-2.7.5.tgz filter=lfs diff=lfs merge=lfs -text
+dependency/orc/huawei_orc-1.5.7.patch filter=lfs diff=lfs merge=lfs -text
+dependency/grpc/grpc-1.22.0.tar.gz filter=lfs diff=lfs merge=lfs -text
+dependency/memcheck/euleros2.0_sp5_x86_64/debug/gcc7.3.0/lib/libtsan.a filter=lfs diff=lfs merge=lfs -text
+platform/huaweijdk8/jdk1.8.0_272/jre/lib/amd64/server/libjvm.so filter=lfs diff=lfs merge=lfs -text
+dependency/memcheck/euleros2.0_sp8_aarch64/debug/gcc7.3.0/lib/libtsan.a filter=lfs diff=lfs merge=lfs -text
+platform/huaweijdk8/jdk1.8.0_272/lib/ct.sym filter=lfs diff=lfs merge=lfs -text
+platform/huaweijdk8/jdk1.8.0_272/lib/tools.jar filter=lfs diff=lfs merge=lfs -text
+dependency/postgresql-hll/huawei_hll.patch filter=lfs diff=lfs merge=lfs -text
+dependency/brotli/brotli-0.6.0.tar.gz filter=lfs diff=lfs merge=lfs -text

buildtools/license_control/GaussDB_features_list

@@ -0,0 +1,21 @@
+0   Data_Storage                            Multi_Value_Column                                         yes  no  yes
+1   Data_Storage                            Json                                                       yes  yes yes
+2   Data_Storage                            XML                                                        no   no  no
+3   Data_Storage                            Data_Storage_Format                                        yes  no  yes
+4   Transaction_Characteristics          	GTM_Free                                                   no   yes no
+5   High_Availability_Disaster_Tolerance    Double_Live_Disaster_Recovery_In_The_Same_City             no   yes no
+6   High_Availability_Disaster_Tolerance    Disaster_Recovery_In_Two_Places_And_Three_Centres          no   yes no
+7   Advanced_Features                       GPU_Acceleration_In_Multidimensional_Collision_Analysis    yes  no  no
+8   Advanced_Features                       Full_Text_Index                                            yes  no  yes
+9   Advanced_Features                       EXTENSION_CONNECTOR                                        yes  no  yes
+10  Advanced_Features                       SQL_ON_HDFS                                                yes  no  yes
+11  Advanced_Features                       SQL_ON_OBS                                                 yes  yes yes
+12  Advanced_Features                       EXPRESS_CLUSTER                                            yes  no  no
+13  Advanced_Features                       Cross_DC_Collaboration                                     yes  no  no
+14  Advanced_Features                       Graph_Computing_Engine                                     yes  no  no
+15  Advanced_Features                       Sequential_Data_Engine                                     yes  no  no
+16  Advanced_Features                       PostGis_Docking                                            yes  yes yes
+17  High_Availability_Disaster_Tolerance    HA_Single_Primary_Multi_Standby                            no   yes no
+18  Advanced_Features                       Row_Level_Permission_Control                               yes  yes no
+19  Advanced_Features                       Transparent_encryption                                     yes  yes no
+20  Advanced_Features                       Private_Table                                              yes  yes no

buildtools/license_control/build.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# Copyright (c): 2012-2020, Huawei Tech. Co., Ltd.
+set -e
+export TARGET_PATH=$(pwd)/../../output/buildtools/license_control/
+
+if [ ! -d ${TARGET_PATH} ];then
+  mkdir -p ${TARGET_PATH}
+fi
+
+cp encrypted_version_file.py ${TARGET_PATH}
+cp GaussDB_features_list ${TARGET_PATH}
+

buildtools/license_control/encrypted_version_file.py

@@ -0,0 +1,197 @@
+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+
+
+import os
+import sys
+import base64
+import struct
+import binascii
+
+
+try:
+    from typing import List
+except ImportError:
+    List = list
+
+
+# Feature list file.
+PRODUCT_FEATURE_FILE = "GaussDB_features_list"
+# Gauss version file.
+VERSION_FILE = "gaussdb.version"
+LICENSE_FILE = "gaussdb.license"
+# Product version.
+PRODUCT_VERSION_GAUSS200 = "GaussDB200"
+PRODUCT_VERSION_GAUSS300 = "GaussDB300"
+LICENSE_VERSION_GAUSS200_STANDARD = "GaussDB200_Standard"
+
+
+def read_disabled_feature_list():
+    """
+    Read the disabled feature list of Gauss DB.
+
+    The directory of the feature list file is at the same level as the binary file and script directory.
+    Feature information is stored in the file line by line, such as:
+        Feature_Class Feature Gauss200_Whether_To_Support Gauss300_Whether_To_Support
+        ...
+
+    Disabled feature information will be translated as a dictionary, such as:
+        {
+            "Gauss200" : [ "Feature" ],
+            "Gauss300" : [ "Feature" ]
+        }
+
+    :return:    According to the specified Gauss DB version, return the corresponding list of disabled feature id.
+                If the feature list file does not exist, return None.
+    :rtype:     dict | None
+    """
+    sys.stdout.write("Start read the Gauss DB feature list.\n")
+
+    # Get the feature list file path.
+    feature_file = os.path.join(os.path.dirname(__file__), PRODUCT_FEATURE_FILE)
+    sys.stdout.write("The feature list file path is %s.\n" % feature_file)
+    if not os.path.exists(feature_file):
+        raise NameError("Gauss DB feature file does not exist, skip this operation.\n")
+    elif not os.path.isfile(feature_file):
+        raise NameError("Gauss DB feature file is not a file, skip this operation.\n")
+
+    # Change file permission.
+    os.chmod(feature_file, 0o600)
+    sys.stdout.write("Change the permission of the file (%s) to %o.\n" % (feature_file, 0o600))
+
+    # Load the feature list.
+    with open(feature_file, "r") as fp:
+        lines = fp.readlines()  # type: List[str]
+
+    feature_list = {PRODUCT_VERSION_GAUSS200: [], PRODUCT_VERSION_GAUSS300: [], LICENSE_VERSION_GAUSS200_STANDARD: []}
+    if lines:
+        for line in lines:
+            items = __read_items_from_line(line)
+            if items is None or len(items) == 0:
+                continue
+
+            if len(items) != 6:
+                raise ValueError("Error content %s of file %s.\n" % (line, feature_file))
+
+            feature_id, _, _, support_200, support_300, support_200_standard = items
+
+            feature_id = int(feature_id)
+            support_200 = support_200.lower()
+            support_300 = support_300.lower()
+            support_200_standard = support_200_standard.lower()
+
+            if support_200 == "no":
+                feature_list[PRODUCT_VERSION_GAUSS200].append(feature_id)
+            elif support_200 != "yes":
+                sys.stdout.write("Error content %s of file %s.\n" % (line, feature_file))
+                return None
+
+            if support_300 == "no":
+                feature_list[PRODUCT_VERSION_GAUSS300].append(feature_id)
+            elif support_300 != "yes":
+                sys.stdout.write("Error content %s of file %s.\n" % (line, feature_file))
+                return None
+
+            if support_200_standard == "no":
+                feature_list[LICENSE_VERSION_GAUSS200_STANDARD].append(feature_id)
+    else:
+        sys.stdout.write("The feature list file (%s) is exist, but is empty.\n" % feature_file)
+
+    return feature_list
+
+
+def __read_items_from_line(line, sep=" "):
+    """
+    Split the list of items from a row of data and remove empty items.
+
+    :param line:    the string which need to be split.
+    :param sep:     Separator of the string.
+
+    :type line:     str
+    :type sep:      str
+
+    :return:        Returns a list of split string.
+    :rtype:         List[int, str, str, str, str]
+    """
+    if not line:
+        return []
+
+    return [l.strip() for l in line.split(sep) if l and l.strip()]
+
+
+def encrypted_feature_info(disabled_features, product_version, feature_file):
+    """
+    Convert the disabled feature list of Gauss DB to byte stream data and save it to the specified file.
+
+    All exceptions of this method are exposed to its caller.
+
+    Finally, The format of the data which was stored in the file are as follows:
+        unsigned int crc32_code
+        unsigned int encrypted_data_len
+        unsigned int unencrypted_data_len
+        char [] base64_encode_data
+    The data before base64 encryption can be interpreted as:
+        unsigned int product_flag.
+        unsigned int disabled_features_number
+        unsigned short disabled_features[disabled_features_number]
+
+    :param disabled_features:    The disabled feature list of Gauss DB
+    :param product_version:      The version information of Gauss DB
+    :param feature_file:         Files used to save byte stream.
+
+    :type disabled_features:     list
+    :type product_version:       str
+    :type feature_file:          str
+
+    :raise ValueError:          If the value of the parameter "productVersion" is incorrect, raise this Exception.
+    """
+    # Setting product flag according to Gauss DB version.
+    if product_version.lower() == PRODUCT_VERSION_GAUSS200.lower():
+        product_flag = 2
+    elif product_version.lower() == PRODUCT_VERSION_GAUSS300.lower():
+        product_flag = 3
+    elif product_version.lower() == LICENSE_VERSION_GAUSS200_STANDARD.lower():
+        product_flag = 2
+    else:
+        raise ValueError("Illegal parameter 'product_version' values.")
+    # Combine format of "data" field converted to streaming data according to the length of the disabled feature
+    #  list.
+    data_format = "BI"
+    data_format = data_format.ljust(len(data_format) + len(disabled_features), "H")
+
+    # Stream the "data" field.
+    data = struct.pack(data_format, product_flag, len(disabled_features), *disabled_features)
+    # Base64 encode the "data" field.
+    base64_data = base64.b64encode(data)
+
+    # Calculate CRC32 value of the "data" filed which was encoded by base64.
+    crc32_code = binascii.crc32(base64_data) & 0xFFFFFFFF
+    # The format of "header" field.
+    header_format = "III"
+
+    # Stream the "header" field.
+    header_data = struct.pack(header_format, crc32_code, len(base64_data), len(data))
+
+    # Save the byte stream data to the specified file.
+    with open(feature_file, "w") as fp:
+        fp.write(header_data)
+        fp.write(base64_data)
+
+
+def gen_version_file():
+    """
+    Generate Gauss 200 and Gauss 300 license control files.
+    """
+    feature_list = read_disabled_feature_list()
+
+    for product_version in [PRODUCT_VERSION_GAUSS200, PRODUCT_VERSION_GAUSS300]:
+        version_file = os.path.join(os.path.dirname(__file__), "%s.%s" % (VERSION_FILE, product_version))
+        encrypted_feature_info(feature_list[product_version], product_version, version_file)
+
+    for product_version in [LICENSE_VERSION_GAUSS200_STANDARD]:
+        version_file = os.path.join(os.path.dirname(__file__), "%s.%s" % (LICENSE_FILE, product_version))
+        encrypted_feature_info(feature_list[product_version], product_version, version_file)
+
+
+if __name__ == "__main__":
+    gen_version_file()

buildtools/python/README.txt

@@ -0,0 +1,11 @@
+open source target name: python
+source code repository: x86(Python-2.7.5.tgz), ARM(python_2.7.16_src.zip)
+compile dependency: NUL
+upgrade open source package method:
+----|pull command: python $(pwd)../../build/pull_open_source.py "bcrypt" "bcrypt-3.1.7.tar.gz" "05833LMP"
+      |----path: the parent directory name
+      |----name: the package name in product warehouse
+      |----id: pdm version id
+the compile command: sh build.sh
+Patch Info:
+backup: 

buildtools/python/build.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+#######################################################################
+# Copyright (c): 2012-2019, Huawei Tech. Co., Ltd.
+# description: the script that make install bcrypt
+# version: 3.1.7
+# date:
+# history:
+#######################################################################
+set -e
+
+PLATFORM=$(sh $(pwd)/../../build/get_PlatForm_str.sh)
+export SSL_PATH=$(pwd)/../../dependency/openssl/install/comm/lib
+export LD_LIBRARY_PATH=${SSL_PATH}:${LD_LIBRARY_PATH}
+if [ ${PLATFORM} == "euleros2.0_sp5_x86_64" ]; then
+    PYTHON_SOURCE=Python-2.7.5
+else
+    unzip -o python_2.7.16_src.zip
+    mv ./python_2.7.16_src/Python-2.7.16.tgz ./
+    PYTHON_SOURCE=Python-2.7.16
+fi
+
+mkdir -p $(pwd)/ucs2
+tar zxvf ${PYTHON_SOURCE}.tgz -C ./ucs2
+cp openssl.patch ./ucs2/${PYTHON_SOURCE}/Modules/
+cd ./ucs2/${PYTHON_SOURCE}/Modules/
+patch  -p0 < openssl.patch
+cd ../
+./configure --enable-unicode=ucs2 --prefix=$(pwd)/../../ucs2/build
+make
+make install
+#make distclean
+
+cd ../../
+mkdir -p $(pwd)/ucs4
+tar zxvf ${PYTHON_SOURCE}.tgz -C ./ucs4
+cp openssl.patch ./ucs4/${PYTHON_SOURCE}/Modules/
+cd ./ucs4/${PYTHON_SOURCE}/Modules/
+patch  -p0 < openssl.patch
+cd ../
+./configure --enable-unicode=ucs4 --prefix=$(pwd)/../../ucs4/build 
+make 
+make install
+#make distclean
+
+cd ../../
+SETUPTOOLS_SOURCE=setuptools-44.1.0
+unzip -o ${SETUPTOOLS_SOURCE}.zip -d ./ucs2/
+cd ./ucs2/${SETUPTOOLS_SOURCE}
+./../build/bin/python setup.py build
+./../build/bin/python setup.py install
+
+cd ../../
+SETUPTOOLS_SOURCE=setuptools-44.1.0
+unzip -o ${SETUPTOOLS_SOURCE}.zip -d ./ucs4/
+cd ./ucs4/${SETUPTOOLS_SOURCE}
+./../build/bin/python setup.py build
+./../build/bin/python setup.py install