[NSE-86] Merge Arrow Data Source (#152)

* [ARROW-DATA-SOURCE-1] Reorganize the source code for the new repository organization

* [ARROW-DATA-SOURCE-1] Make compiling succeed

* [ADS-2] Fix RAM usage CI (#3)

* Update tpch.yml

* Update report_ram_log.yml

* Update report_ram_log.yml

* Delete github-ci-fix

* Delete github-ci-fix2

* [Arrow-Data-Source-4]Add mkdocs.yml and update docs

* [ADS-6]  Add utility methods to check leaked Allocators/MemoryPools (#7)

Close #6

* [NSE-51] Update ArrowWritableColumnVector

* [ADS-9][Parquet] Parquet data source not replaced by default (#10)

* [ADS-9][Parquet] Parquet data source not replaced by default

* Code style

* [ADS-13] Validate metric TaskMetrics.peakExecutionMemory for native SQL engine (#14)

Closes #13

* [ADS-11]Modify title check and automatic link to Issues for PRs (#12)

* [ADS-16] Upgrade Arrow version to 3.0.0 (#17)

Closes #16

* Initialize new repo

* Move arrow data source files to arrow-data-source directory

* move native sql files to native-sql-engine folder

* [NSE-86] Add root pom.xml; Remove native-sql-engine/core/ArrowWritableColumnVector.java (#88)

* fix github actions

Signed-off-by: Yuan Zhou <[email protected]>

* fix building & CI

Signed-off-by: Yuan Zhou <[email protected]>

Co-authored-by: Chen Haifeng <[email protected]>
Co-authored-by: zhixingheyi-tian  <[email protected]>
Co-authored-by: Hongze Zhang <[email protected]>
Co-authored-by: HongW2019 <[email protected]>
Co-authored-by: Hongze Zhang <[email protected]>
Co-authored-by: Rui Mo <[email protected]>

.github/workflows/report_ram_log.yml

@@ -61,7 +61,7 @@ jobs:
           mvn clean install -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DskipTests
       - name: Run Maven tests
         run: |
-          cd core/
+          cd native-sql-engine/core/
           mvn test -B -DmembersOnlySuites=com.intel.oap.tpc.h -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DtagsToInclude=com.intel.oap.tags.CommentOnContextPR -Dexec.skip=true
         env:
           MAVEN_OPTS: "-Xmx2048m"

.github/workflows/tpch.yml

@@ -56,7 +56,7 @@ jobs:
           mvn clean install -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn
       - name: Run Maven tests - BHJ
         run: |
-          cd core/
+          cd native-sql-engine/core/
           mvn test -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DmembersOnlySuites=com.intel.oap.tpc.h -DtagsToInclude=com.intel.oap.tags.BroadcastHashJoinMode -DargLine="-Xmx1G -XX:MaxDirectMemorySize=500M -Dio.netty.allocator.numDirectArena=1"
         env:
           MALLOC_ARENA_MAX: "4"
@@ -66,7 +66,7 @@ jobs:
           ENABLE_TPCH_TESTS: "true"
       - name: Run Maven tests - SMJ
         run: |
-          cd core/
+          cd native-sql-engine/core/
           mvn test -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DmembersOnlySuites=com.intel.oap.tpc.h -DtagsToInclude=com.intel.oap.tags.SortMergeJoinMode -DargLine="-Xmx1G -XX:MaxDirectMemorySize=500M -Dio.netty.allocator.numDirectArena=1"
         env:
           MALLOC_ARENA_MAX: "4"

.github/workflows/unittests.yml

@@ -51,7 +51,7 @@ jobs:
           sudo make install
       - name: Run unit tests
         run: |
-          cd cpp/
+          cd native-sql-engine/cpp/
           mkdir -p build
           cd build
           cmake .. -DTESTS=1

.gitignore

@@ -8,3 +8,48 @@
 # IDEA config files
 .idea/
 *.iml
+*.class
+
+# Mobile Tools for Java (J2ME)
+.mtj.tmp/
+
+# Package Files #
+*.jar
+*.war
+*.ear
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+
+.idea/
+target/
+*.iml
+
+**/target/**
+
+# logs
+*.log
+
+thirdparty/*.tar*
+CMakeFiles/
+CMakeCache.txt
+CTestTestfile.cmake
+Makefile
+cmake_install.cmake
+build/
+*-build/
+Testing/
+cmake-build-debug/
+cmake-build-release/
+
+# Editor temporary/working/backup files #
+.#*
+*\#*\#
+[#]*#
+*~
+*$
+*.bak
+*flymake*
+*.kdev4
+*.swp
+scalastyle-output.xml

arrow-data-source/.github/PULL_REQUEST_TEMPLATE

@@ -0,0 +1,12 @@
+## What changes were proposed in this pull request?
+
+(Please fill in changes proposed in this fix)
+
+
+## How was this patch tested?
+
+(Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests)
+
+
+(If this patch involves UI changes, please attach a screenshot; otherwise, remove this)
+

arrow-data-source/.github/workflows/dev_cron.yml

@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Dev PR
+
+on:
+
+  pull_request_target:
+    types:
+      - opened
+      - edited
+      - synchronize
+
+jobs:
+  process:
+    name: Process
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Comment Issues link
+        if: |
+          github.event_name == 'pull_request_target' &&
+            (github.event.action == 'opened' ||
+             github.event.action == 'edited')
+        uses: actions/github-script@v3
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const script = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_cron/issues_link.js`);
+            script({github, context});
+
+      - name: Check title
+        if: |
+          github.event_name == 'pull_request_target' &&
+            (github.event.action == 'opened' ||
+             github.event.action == 'edited')
+        uses: actions/github-script@v3
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const script = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_cron/title_check.js`);
+            script({github, context});
+

arrow-data-source/.github/workflows/dev_cron/issues_link.js

@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+function detectISSUESID(title) {
+  if (!title) {
+    return null;
+  }
+  const matched = /^\[NSE-\d+\]/.exec(title);
+  if (!matched) {
+    return null;
+  }
+  const issues_number = matched[0].replace(/[^0-9]/ig,"");
+  return issues_number;
+}
+
+async function haveComment(github, context, pullRequestNumber, body) {
+  const options = {
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    issue_number: pullRequestNumber,
+    page: 1
+  };
+  while (true) {
+    const response = await github.issues.listComments(options);
+    if (response.data.some(comment => comment.body === body)) {
+      return true;
+    }
+    if (!/;\s*rel="next"/.test(response.headers.link || "")) {
+      break;
+    }
+    options.page++;
+  }
+  return false;
+}
+
+async function commentISSUESURL(github, context, pullRequestNumber, issuesID) {
+  const issuesURL = `https://github.com/oap-project/native-sql-engine/issues/${issuesID}`;
+  if (await haveComment(github, context, pullRequestNumber, issuesURL)) {
+    return;
+  }
+  await github.issues.createComment({
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    issue_number: pullRequestNumber,
+    body: issuesURL
+  });
+}
+
+module.exports = async ({github, context}) => {
+  const pullRequestNumber = context.payload.number;
+  const title = context.payload.pull_request.title;
+  const issuesID = detectISSUESID(title);
+  if (issuesID) {
+    await commentISSUESURL(github, context, pullRequestNumber, issuesID);
+  }
+};

arrow-data-source/.github/workflows/dev_cron/title_check.js

@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const fs = require("fs");
+
+function haveISSUESID(title) {
+  if (!title) {
+    return false;
+  }
+  return /^\[NSE-\d+\]/.test(title);
+}
+
+async function commentOpenISSUESIssue(github, context, pullRequestNumber) {
+  const {data: comments} = await github.issues.listComments({
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    issue_number: pullRequestNumber,
+    per_page: 1
+  });
+  if (comments.length > 0) {
+    return;
+  }
+  const commentPath = ".github/workflows/dev_cron/title_check.md";
+  const comment = fs.readFileSync(commentPath).toString();
+  await github.issues.createComment({
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    issue_number: pullRequestNumber,
+    body: comment
+  });
+}
+
+module.exports = async ({github, context}) => {
+  const pullRequestNumber = context.payload.number;
+  const title = context.payload.pull_request.title;
+  if (!haveISSUESID(title)) {
+    await commentOpenISSUESIssue(github, context, pullRequestNumber);
+  }
+};

arrow-data-source/.github/workflows/dev_cron/title_check.md

@@ -0,0 +1,33 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+Thanks for opening a pull request!
+
+Could you open an issue for this pull request on Github Issues?
+
+https://github.com/oap-project/native-sql-engine/issues
+
+Then could you also rename ***commit message*** and ***pull request title*** in the following format?
+
+    [NSE-${ISSUES_ID}] ${detailed message}
+
+See also:
+
+  * [Other pull requests](https://github.com/oap-project/native-sql-engine/pulls/)
+