Merge pull request #964 from cauchy1988/merge-java-client

apache · May 9, 2022 · fbed9a1 · fbed9a1
2 parents 5278a66 + c1f1538
commit fbed9a1
Show file tree

Hide file tree

Showing 196 changed files with 74,360 additions and 1 deletion.
diff --git a/.github/workflows/test_java-client.yml b/.github/workflows/test_java-client.yml
@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Test - java client
+on:
+  pull_request:
+    branches:
+        - master
+        - 'v[0-9]+.*' # release branch
+        - ci-test # testing branch for github action
+        - '*dev'      # developing branch
+    paths:
+      - java-client/**
+
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  test:
+    name: test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java: [ '8', '11']
+    steps:
+      - uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+      - uses: actions/checkout@v2
+      - uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+      - name: ci
+        working-directory: ./java-client
+        run: ./scripts/ci-test.sh
diff --git a/.gitignore b/.gitignore
@@ -122,3 +122,18 @@ pegasus-server*
 admin-cli/bin/
 pegic/bin/
 golangci-*
+
+java-client/bin/
+java-client/release/
+java-client/target/
+java-client/.settings
+java-client/.project
+java-client/.classpath
+java-client/.idea
+java-client/pegasus-client.iml
+java-client/log.txt
+java-client/rolling_log/
+java-client/.vscode/
+java-client/google-java-format-*
+java-client/pegasus-*
+java-client/dependency-reduced-pom.xml
diff --git a/README.md b/README.md
@@ -18,7 +18,9 @@ under the License.
 -->
 
 [github-release]: https://github.com/apache/incubator-pegasus/releases
+<!-- markdown-link-check-disable -->
 [PacificA]: https://www.microsoft.com/en-us/research/publication/pacifica-replication-in-log-based-distributed-storage-systems/
+<!-- markdown-link-check-enable-->
 [pegasus-rocksdb]: https://github.com/xiaomi/pegasus-rocksdb
 [facebook-rocksdb]: https://github.com/facebook/rocksdb
 [hbase]: https://hbase.apache.org/
@@ -69,7 +71,9 @@ See our documentation on the [Pegasus Website][website].
 
 Pegasus has support for several languages:
 
-- [Java](https://github.com/xiaomi/pegasus-java-client)
+<!-- markdown-link-check-disable -->
+- [Java](https://github.com/apache/incubator-pegasus/blob/master/java-client)
+<!-- markdown-link-check-enable-->
 - [C++](https://github.com/apache/incubator-pegasus/blob/master/src/include/pegasus/client.h)
 - [Go](https://github.com/xiaomi/pegasus-go-client)
 - [Python](https://github.com/xiaomi/pegasus-python-client)

diff --git a/java-client/README.md b/java-client/README.md
@@ -0,0 +1,99 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Pegasus Java Client
+
+[![Test - java client](https://github.com/apache/incubator-pegasus/actions/workflows/test_java-client.yml/badge.svg)](https://github.com/apache/incubator-pegasus/actions/workflows/test_java-client.yml)
+
+## Build
+
+```
+mvn clean package -DskipTests
+```
+
+## Install
+
+```
+mvn clean install -DskipTests
+```
+
+## Test
+
+To run test, you should start pegasus onebox firstly, and run test as:
+
+```
+mvn clean package
+```
+
+or specify one test:
+
+```
+mvn clean package -Dtest=TestPing
+```
+
+## Configuration
+
+Configure client by "pegasus.properties", for example:
+
+```
+meta_servers = 127.0.0.1:34601,127.0.0.1:34602,127.0.0.1:34603
+operation_timeout = 1000
+async_workers = 4
+enable_perf_counter = true
+perf_counter_tags = k1=v1,k2=v2,k3=v3
+push_counter_interval_secs = 10
+```
+
+You can provide a parameter of 'configPath' when creating a client instance.
+
+The format of 'configPath' should be one of these:
+* zk path: zk://host1:port1,host2:port2,host3:port3/path/to/config
+* local file path: file:///path/to/config
+* resource path: resource:///path/to/config
+
+## PerfCounter(Metrics)
+
+Pegasus Java Client supports QPS and latency statistics of requests.
+
+The related configurations are:
+
+```
+enable_perf_counter = true
+perf_counter_tags = k1=v1,k2=v2,k3=v3
+push_counter_interval_secs = 10
+```
+
+For each type of request(get, set, multiset, etc.), we collect 8 metrics:
+1. cps-1sec: the request's qps
+2. cps-1min: the request's queries per 1 minute
+3. cps-5min: the request's queries per 5 minutes
+4. cps-15min: the request's queries per 15 minutes
+5. latency-p50: the moving median of request's queries
+6. latency-p99: the moving p99 of request's queries
+7. lantecy-p999: the moving p999 of request's queries
+8: latency-max: the moving max of request's queries
+
+We use io.dropwizard.metrics library to calculate the request count.
+
+Currently, metrics are integrated with open-falcon(https://open-falcon.org/),
+<!-- markdown-link-check-disable -->
+which push counters to local http agent http://127.0.0.1:1988/push/v1.
+<!-- markdown-link-check-enable-->
+
+If you'd like to integrate pegasus client with other monitor system, please let us know ASAP.
diff --git a/java-client/configuration/log4j.properties b/java-client/configuration/log4j.properties
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+log4j.rootLogger=DEBUG, rolling_file
+log4j.logger.org.apache.pegasus = INFO
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+
+log4j.appender.file=org.apache.log4j.FileAppender
+log4j.appender.file.File=log.txt
+log4j.appender.file.ImmediateFlush=true
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+log4j.appender.file.Append=true
+
+log4j.appender.rolling_file=org.apache.log4j.RollingFileAppender
+log4j.appender.rolling_file.File=rolling_log/log.txt
+log4j.appender.rolling_file.MaxFileSize=10MB
+log4j.appender.rolling_file.MaxBackupIndex=10
+log4j.appender.rolling_file.ImmediateFlush=true
+log4j.appender.rolling_file.layout=org.apache.log4j.PatternLayout
+log4j.appender.rolling_file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+log4j.appender.rolling_file.Append=true
diff --git a/java-client/configuration/pegasus.properties b/java-client/configuration/pegasus.properties
@@ -0,0 +1,31 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+meta_servers = 127.0.0.1:34601,127.0.0.1:34602,127.0.0.1:34603
+operation_timeout = 5000
+async_workers = 4
+enable_perf_counter = false
+perf_counter_tags = cluster=onebox,app=unit_test
+push_counter_interval_secs = 10
+meta_query_timeout = 5000
+auth_protocol =
+kerberos_service_name =
+kerberos_service_fqdn =
+kerberos_keytab =
+kerberos_principal =
diff --git a/java-client/dev-support/checkstyle-suppressions.xml b/java-client/dev-support/checkstyle-suppressions.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<!DOCTYPE suppressions PUBLIC
+        "-//Puppy Crawl//DTD Suppressions 1.0//EN"
+        "http://www.puppycrawl.com/dtds/suppressions_1_0.dtd">
+
+<suppressions>
+    <!--
+    /* TODO(wutao1): Temporary Suppressions
+     * The following checks should be removed from
+     * the suppression list after their checkstyle errors are fixed.
+     */
+    -->
+    <suppress checks="LineLengthCheck" files=".*\.java"/>
+    <suppress checks="IndentationCheck" files=".*\.java"/>
+    <suppress checks="NeedBracesCheck" files=".*\.java"/>
+    <suppress checks="MemberNameCheck" files=".*\.java"/>
+    <suppress checks="CustomImportOrderCheck" files=".*\.java"/>
+    <suppress checks="JavadocMethod" files=".*\.java"/>
+    <suppress checks="ArrayTypeStyle" files=".*\.java"/>
+    <suppress checks="TypeName" files=".*\.java"/>
+    <suppress checks="JavadocParagraph" files=".*\.java"/>
+    <suppress checks="SummaryJavadoc" files=".*\.java"/>
+    <suppress checks="EmptyCatchBlock" files=".*\.java"/>
+    <suppress checks="UpperEll" files=".*\.java"/>
+    <suppress checks="LocalVariableName" files=".*\.java"/>
+    <suppress checks="NoFinalizer" files=".*\.java"/>
+    <suppress checks="AbbreviationAsWordInName" files=".*\.java"/>
+    <suppress checks="MissingSwitchDefault" files=".*\.java"/>
+    <suppress checks="AvoidStarImport" files=".*\.java"/>
+    <suppress checks="SingleLineJavadoc" files=".*\.java"/>
+    <suppress checks="EmptyLineSeparator" files=".*\.java"/>
+    <suppress checks="NonEmptyAtclauseDescription" files=".*\.java"/>
+    <suppress checks="ParameterName" files=".*\.java"/>
+    <suppress checks="VariableDeclarationUsageDistance" files=".*\.java"/>
+    <suppress checks="RightCurly" files=".*\.java"/>
+</suppressions>
diff --git a/java-client/dev-support/findbugs-exclude.xml b/java-client/dev-support/findbugs-exclude.xml
@@ -0,0 +1,42 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<FindBugsFilter>
+
+    <Match>
+        <Or>
+            <!-- thrift generated files -->
+            <Package name="org.apache.pegasus.apps"/>
+            <Package name="org.apache.pegasus.base"/>
+            <Package name="org.apache.pegasus.replication"/>
+
+            <!--
+             TODO(wutao1): The following packages should be removed from
+             the exclusion list after their bugs are fixed.
+            -->
+            <Package name="org.apache.pegasus.rpc.async"/>
+            <Package name="org.apache.pegasus.client"/>
+            <Package name="org.apache.pegasus.metrics"/>
+            <Package name="org.apache.pegasus.tools"/>
+            <Package name="org.apache.pegasus.client.request"/>
+            <Package name="org.apache.pegasus.example"/>
+        </Or>
+    </Match>
+
+</FindBugsFilter>