diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml deleted file mode 100644 index 0b578f981d3..00000000000 --- a/.github/workflows/ci.yaml +++ /dev/null @@ -1,88 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# This workflow will build a Java project with Maven -# See also: -# https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven - -name: CI - -on: - push: - branches: [ '*' ] - pull_request: - branches: [ '*' ] - -jobs: - mvn: - strategy: - matrix: - profile: - - name: 'full-build-jdk8' - jdk: 8 - args: '-Pfull-build apache-rat:check verify -DskipTests spotbugs:check checkstyle:check' - - name: 'full-build-jdk11' - jdk: 11 - args: '-Pfull-build apache-rat:check verify -DskipTests spotbugs:check checkstyle:check' - - name: 'full-build-java-tests' - jdk: 11 - args: '-Pfull-build verify -Dsurefire-forkcount=1C -DskipCppUnit -Dsurefire.rerunFailingTestsCount=5' - - name: 'full-build-cppunit-tests' - jdk: 11 - args: '-Pfull-build verify -Dtest=_ -DfailIfNoTests=false' - fail-fast: false - timeout-minutes: 360 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up JDK ${{ matrix.profile.jdk }} - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.profile.jdk }} - - name: Cache local maven repository - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository/ - !~/.m2/repository/org/apache/zookeeper - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 - - name: Show the first log message - run: git log -n1 - - name: Install C Dependencies - run: sudo apt-get install libcppunit-dev libsasl2-dev - - name: Build with Maven (${{ matrix.profile.name }}) - run: mvn -B -V -e -ntp "-Dstyle.color=always" ${{ matrix.profile.args }} - env: - MAVEN_OPTS: -Djansi.force=true - - name: Upload unit test results - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: surefire-reports-${{ matrix.profile.name }} - path: ./**/target/surefire-reports/ - if-no-files-found: ignore - - name: Upload integration test results - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: failsafe-reports-${{ matrix.profile.name }} - path: ./**/target/failsafe-reports/ - if-no-files-found: ignore - diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 00000000000..418ff13f92b --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,33 @@ +name: Run Tests With Clover +on: + workflow_dispatch: +jobs: + build: + name: Build + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Cache Maven packages + uses: actions/cache@v1 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2 + # Configure credentials + - uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: eu-west-1 + # Run build steps + - run: mvn clean clover:setup + - run: mvn test + - run: mvn clover:aggregate clover:clover + if: always() + - run: aws s3 cp ./target/site/clover/clover.xml s3://codescene-on-prem/code-coverage-data/apache/zookeeper/clover.xml diff --git a/.github/workflows/manual.yaml b/.github/workflows/manual.yaml deleted file mode 100644 index 191a8b0cdd4..00000000000 --- a/.github/workflows/manual.yaml +++ /dev/null @@ -1,85 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# This workflow will build a Java project with Maven -# See also: -# https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven -# https://docs.github.com/en/actions/reference/events-that-trigger-workflows#manual-events - -name: Manual Build - -on: - workflow_dispatch: - inputs: - buildRef: - description: Ref to build (commit, branch, or refs/pull/1234/head or refs/pull/1234/merge) - required: true - default: refs/pull/1234/merge - mvnOpts: - description: Maven options - required: true - default: --fail-at-end - goals: - description: Maven goals - required: true - default: -Pfull-build apache-rat:check verify -DskipTests spotbugs:check checkstyle:check javadoc:jar -jobs: - mvn: - name: mvn (triggered by ${{ github.event.sender.login }}) - timeout-minutes: 360 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ github.event.inputs.buildRef }} - - name: Set up JDK 11 - uses: actions/setup-java@v1 - with: - java-version: 11 - - name: Cache local maven repository - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository/ - !~/.m2/repository/org/apache/zookeeper - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 - - name: Show the first log message - run: git log -n1 - - name: Install C Dependencies - run: sudo apt-get install libcppunit-dev libsasl2-dev - - name: Build with Maven - run: mvn -B -V -e -ntp "-Dstyle.color=always" ${{ github.event.inputs.mvnOpts }} ${{ github.event.inputs.goals }} - env: - MAVEN_OPTS: -Djansi.force=true - - name: Upload unit test results - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: surefire-reports - path: ./**/target/surefire-reports/ - if-no-files-found: ignore - - name: Upload integration test results - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: failsafe-reports - path: ./**/target/failsafe-reports/ - if-no-files-found: ignore - diff --git a/.github/workflows/sonar.yml b/.github/workflows/sonar.yml new file mode 100644 index 00000000000..dade8aa87b5 --- /dev/null +++ b/.github/workflows/sonar.yml @@ -0,0 +1,32 @@ +name: Run Sonar Analysis +on: + workflow_dispatch: +jobs: + build: + name: Build + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Cache SonarCloud packages + uses: actions/cache@v1 + with: + path: ~/.sonar/cache + key: ${{ runner.os }}-sonar + restore-keys: ${{ runner.os }}-sonar + - name: Cache Maven packages + uses: actions/cache@v1 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2 + # Run build steps + - run: mvn -B verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -DskipTests -e + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/Jenkinsfile b/Jenkinsfile index ed074c95617..40fcdd5bbad 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -51,7 +51,6 @@ pipeline { stages { stage('BuildAndTest') { steps { - git 'https://github.com/apache/zookeeper' sh "git clean -fxd" sh "mvn verify spotbugs:check checkstyle:check -Pfull-build -Dsurefire-forkcount=4" } diff --git a/README.md b/README.md index a616babfdff..1738f037056 100644 --- a/README.md +++ b/README.md @@ -51,3 +51,5 @@ We always welcome new contributors to the project! See [How to Contribute](https [mcLink]: https://zookeeper.apache.org/releases [trBadge]: https://travis-ci.org/apache/zookeeper.svg?branch=master [trLink]: https://travis-ci.org/apache/zookeeper +A +A diff --git a/pom.xml b/pom.xml index b0f22e8ca73..39bf7445cd3 100755 --- a/pom.xml +++ b/pom.xml @@ -448,6 +448,10 @@ + + zookeeper + knorrest + https://sonarcloud.io 1.8 1.8 @@ -467,8 +471,8 @@ 3.6.28 2.2 1.4 - 4.1.50.Final - 9.4.35.v20201120 + 4.1.59.Final + 9.4.38.v20210224 2.10.5.1 2.14.6 1.1.7.7 @@ -683,7 +687,6 @@ true - -Werror -Xlint:deprecation -Xlint:unchecked -Xlint:-options diff --git a/zookeeper-client/zookeeper-client-c/src/cli.c b/zookeeper-client/zookeeper-client-c/src/cli.c index 1864e564571..823ed72a5a0 100644 --- a/zookeeper-client/zookeeper-client-c/src/cli.c +++ b/zookeeper-client/zookeeper-client-c/src/cli.c @@ -948,6 +948,17 @@ int main(int argc, char **argv) { zoo_deterministic_conn_order(1); // enable deterministic order #ifdef HAVE_CYRUS_SASL_H + /* + * We need to disable the deprecation warnings as Apple has + * decided to deprecate all of CyrusSASL's functions with OS 10.11 + * (see MESOS-3030, ZOOKEEPER-4201). We are using GCC pragmas also + * for covering clang. + */ +#ifdef __APPLE__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + if (mechlist) { zoo_sasl_params_t sasl_params = { 0 }; int sr; @@ -977,6 +988,11 @@ int main(int argc, char **argv) { return errno; } } + +#ifdef __APPLE__ +#pragma GCC diagnostic pop +#endif + #endif /* HAVE_CYRUS_SASL_H */ if (!zh) { diff --git a/zookeeper-client/zookeeper-client-c/src/zk_sasl.c b/zookeeper-client/zookeeper-client-c/src/zk_sasl.c index e0ccfb31004..6ae7e123733 100644 --- a/zookeeper-client/zookeeper-client-c/src/zk_sasl.c +++ b/zookeeper-client/zookeeper-client-c/src/zk_sasl.c @@ -47,6 +47,17 @@ #include "zk_adaptor.h" #include "zookeeper_log.h" +/* + * We need to disable the deprecation warnings as Apple has decided to + * deprecate all of CyrusSASL's functions with OS 10.11 (see + * MESOS-3030, ZOOKEEPER-4201). We are using GCC pragmas also for + * covering clang. + */ +#ifdef __APPLE__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + /* * Store a duplicate of src, or NULL, into *target. Returns * ZSYSTEMERROR if no memory could be allocated, ZOK otherwise. @@ -539,3 +550,7 @@ sasl_callback_t *zoo_sasl_make_basic_callbacks(const char *user, return xcallbacks; } } + +#ifdef __APPLE__ +#pragma GCC diagnostic pop +#endif diff --git a/zookeeper-client/zookeeper-client-c/src/zookeeper.c b/zookeeper-client/zookeeper-client-c/src/zookeeper.c index b5d690f3515..0504a746fc7 100644 --- a/zookeeper-client/zookeeper-client-c/src/zookeeper.c +++ b/zookeeper-client/zookeeper-client-c/src/zookeeper.c @@ -279,7 +279,7 @@ static void queue_completion_nolock(completion_head_t *list, completion_list_t * int add_to_front); static void queue_completion(completion_head_t *list, completion_list_t *c, int add_to_front); -static int handle_socket_error_msg(zhandle_t *zh, int line, int rc, +static int handle_socket_error_msg(zhandle_t *zh, int line, const char *func, int rc, const char* format,...); static void cleanup_bufs(zhandle_t *zh,int callCompletion,int rc); @@ -2029,7 +2029,7 @@ static void handle_error(zhandle_t *zh,int rc) addrvec_next(&zh->addrs, &zh->addr_cur); } -static int handle_socket_error_msg(zhandle_t *zh, int line, int rc, +static int handle_socket_error_msg(zhandle_t *zh, int line, const char *func, int rc, const char* format, ...) { if(logLevel>=ZOO_LOG_LEVEL_ERROR){ @@ -2037,7 +2037,7 @@ static int handle_socket_error_msg(zhandle_t *zh, int line, int rc, char buf[1024]; va_start(va,format); vsnprintf(buf, sizeof(buf)-1,format,va); - log_message(LOGCALLBACK(zh), ZOO_LOG_LEVEL_ERROR,line,__func__, + log_message(LOGCALLBACK(zh), ZOO_LOG_LEVEL_ERROR, line, func, "Socket %s zk retcode=%d, errno=%d(%s): %s", zoo_get_current_server(zh),rc,errno,strerror(errno),buf); va_end(va); @@ -2283,7 +2283,7 @@ static int prime_connection(zhandle_t *zh) serialize_prime_connect(&req, buffer_req); rc=rc<0 ? rc : zookeeper_send(zh->fd, buffer_req, len); if (rc<0) { - return handle_socket_error_msg(zh, __LINE__, ZCONNECTIONLOSS, + return handle_socket_error_msg(zh, __LINE__, __func__, ZCONNECTIONLOSS, "failed to send a handshake packet: %s", strerror(errno)); } zh->state = ZOO_ASSOCIATING_STATE; @@ -2572,6 +2572,7 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest, if (zh->fd->sock < 0) { rc = handle_socket_error_msg(zh, __LINE__, + __func__, ZSYSTEMERROR, "socket() call failed"); return api_epilog(zh, rc); @@ -2595,6 +2596,7 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest, } else { rc = handle_socket_error_msg(zh, __LINE__, + __func__, ZCONNECTIONLOSS, "connect() call failed"); return api_epilog(zh, rc); @@ -2643,7 +2645,7 @@ int zookeeper_interest(zhandle_t *zh, socket_t *fd, int *interest, *interest=0; *tv = get_timeval(0); return api_epilog(zh,handle_socket_error_msg(zh, - __LINE__,ZOPERATIONTIMEOUT, + __LINE__, __func__, ZOPERATIONTIMEOUT, "connection to %s timed out (exceeded timeout by %dms)", format_endpoint_info(&zh->addr_cur), -recv_to)); @@ -2799,7 +2801,7 @@ static int init_ssl_for_socket(zsock_t *fd, zhandle_t *zh, int fail_on_error) { fd->ssl_sock = SSL_new(*ctx); if (fd->ssl_sock == NULL) { if (fail_on_error) { - return handle_socket_error_msg(zh,__LINE__,ZSSLCONNECTIONERROR, "error creating ssl context"); + return handle_socket_error_msg(zh, __LINE__, __func__, ZSSLCONNECTIONERROR, "error creating ssl context"); } else { LOG_ERROR(LOGCALLBACK(zh), "error creating ssl context"); return ZSSLCONNECTIONERROR; @@ -2830,7 +2832,7 @@ static int init_ssl_for_socket(zsock_t *fd, zhandle_t *zh, int fail_on_error) { FD_CLR(sock, &s_rfds); } else { if (fail_on_error) { - return handle_socket_error_msg(zh,__LINE__,ZSSLCONNECTIONERROR, "error in ssl connect"); + return handle_socket_error_msg(zh, __LINE__, __func__, ZSSLCONNECTIONERROR, "error in ssl connect"); } else { LOG_ERROR(LOGCALLBACK(zh), "error in ssl connect"); return ZSSLCONNECTIONERROR; @@ -2839,7 +2841,7 @@ static int init_ssl_for_socket(zsock_t *fd, zhandle_t *zh, int fail_on_error) { rc = select(sock + 1, &s_rfds, &s_wfds, NULL, &tv); if (rc == -1) { if (fail_on_error) { - return handle_socket_error_msg(zh,__LINE__,ZSSLCONNECTIONERROR, "error in ssl connect (after select)"); + return handle_socket_error_msg(zh, __LINE__, __func__, ZSSLCONNECTIONERROR, "error in ssl connect (after select)"); } else { LOG_ERROR(LOGCALLBACK(zh), "error in ssl connect (after select)"); return ZSSLCONNECTIONERROR; @@ -2958,7 +2960,7 @@ static int check_events(zhandle_t *zh, int events) if (rc < 0 || error) { if (rc == 0) errno = error; - return handle_socket_error_msg(zh, __LINE__,ZCONNECTIONLOSS, + return handle_socket_error_msg(zh, __LINE__, __func__, ZCONNECTIONLOSS, "server refused to accept the client"); } // We do SSL_connect() here @@ -2978,7 +2980,7 @@ static int check_events(zhandle_t *zh, int events) if (rc < 0 || error) { if (rc == 0) errno = error; - return handle_socket_error_msg(zh, __LINE__,ZCONNECTIONLOSS, + return handle_socket_error_msg(zh, __LINE__, __func__, ZCONNECTIONLOSS, "server refused to accept the client"); } @@ -2993,7 +2995,7 @@ static int check_events(zhandle_t *zh, int events) /* make the flush call non-blocking by specifying a 0 timeout */ int rc=flush_send_queue(zh,0); if (rc < 0) - return handle_socket_error_msg(zh,__LINE__,ZCONNECTIONLOSS, + return handle_socket_error_msg(zh, __LINE__, __func__, ZCONNECTIONLOSS, "failed while flushing send queue"); } if (events&ZOOKEEPER_READ) { @@ -3004,7 +3006,7 @@ static int check_events(zhandle_t *zh, int events) rc = recv_buffer(zh, zh->input_buffer); if (rc < 0) { - return handle_socket_error_msg(zh, __LINE__,ZCONNECTIONLOSS, + return handle_socket_error_msg(zh, __LINE__, __func__, ZCONNECTIONLOSS, "failed while receiving a server response"); } if (rc > 0) { @@ -3040,7 +3042,7 @@ static int check_events(zhandle_t *zh, int events) if (oldid != 0 && oldid != newid) { zh->state = ZOO_EXPIRED_SESSION_STATE; errno = ESTALE; - return handle_socket_error_msg(zh,__LINE__,ZSESSIONEXPIRED, + return handle_socket_error_msg(zh, __LINE__, __func__, ZSESSIONEXPIRED, "sessionId=%#llx has expired.",oldid); } else { zh->recv_timeout = zh->primer_storage.timeOut; @@ -3480,7 +3482,7 @@ int zookeeper_process(zhandle_t *zh, int events) // signaled and deallocated) and disconnect from the server queue_completion(&zh->sent_requests,cptr,1); return api_epilog(zh, - handle_socket_error_msg(zh, __LINE__,ZRUNTIMEINCONSISTENCY, + handle_socket_error_msg(zh, __LINE__, __func__, ZRUNTIMEINCONSISTENCY, "unexpected server response: expected %#x, but received %#x", hdr.xid,cptr->xid)); } @@ -3961,6 +3963,19 @@ static int Request_path_watch_init(zhandle_t *zh, int mode, /*---------------------------------------------------------------------------* * ASYNC API *---------------------------------------------------------------------------*/ + +/* make an attempt to send queued requests immediately without blocking */ +static int nonblocking_send(zhandle_t *zh, int rc) +{ + if (adaptor_send_queue(zh, 0) < 0) { + if (zh->fd->sock != -1) { + close_zsock(zh->fd); + zh->state = ZOO_NOTCONNECTED_STATE; + } + } + return (rc < 0) ? ZMARSHALLINGERROR : ZOK; +} + int zoo_aget(zhandle_t *zh, const char *path, int watch, data_completion_t dc, const void *data) { @@ -4000,9 +4015,8 @@ int zoo_awget(zhandle_t *zh, const char *path, LOG_DEBUG(LOGCALLBACK(zh), "Sending request xid=%#x for path [%s] to %s",h.xid,path, zoo_get_current_server(zh)); - /* make a best (non-blocking) effort to send the requests asap */ - adaptor_send_queue(zh, 0); - return (rc < 0)?ZMARSHALLINGERROR:ZOK; + + return nonblocking_send(zh, rc); } int zoo_agetconfig(zhandle_t *zh, int watch, data_completion_t dc, @@ -4044,9 +4058,8 @@ int zoo_awgetconfig(zhandle_t *zh, watcher_fn watcher, void* watcherCtx, LOG_DEBUG(LOGCALLBACK(zh), "Sending request xid=%#x for path [%s] to %s",h.xid,path, zoo_get_current_server(zh)); - /* make a best (non-blocking) effort to send the requests asap */ - adaptor_send_queue(zh, 0); - return (rc < 0)?ZMARSHALLINGERROR:ZOK; + + return nonblocking_send(zh, rc); } int zoo_areconfig(zhandle_t *zh, const char *joining, const char *leaving, @@ -4080,10 +4093,8 @@ int zoo_areconfig(zhandle_t *zh, const char *joining, const char *leaving, close_buffer_oarchive(&oa, 0); LOG_DEBUG(LOGCALLBACK(zh), "Sending Reconfig request xid=%#x to %s",h.xid, zoo_get_current_server(zh)); - /* make a best (non-blocking) effort to send the requests asap */ - adaptor_send_queue(zh, 0); - return (rc < 0)?ZMARSHALLINGERROR:ZOK; + return nonblocking_send(zh, rc); } static int SetDataRequest_init(zhandle_t *zh, struct SetDataRequest *req, @@ -4126,9 +4137,8 @@ int zoo_aset(zhandle_t *zh, const char *path, const char *buffer, int buflen, LOG_DEBUG(LOGCALLBACK(zh), "Sending request xid=%#x for path [%s] to %s",h.xid,path, zoo_get_current_server(zh)); - /* make a best (non-blocking) effort to send the requests asap */ - adaptor_send_queue(zh, 0); - return (rc < 0)?ZMARSHALLINGERROR:ZOK; + + return nonblocking_send(zh, rc); } static int CreateRequest_init(zhandle_t *zh, struct CreateRequest *req, @@ -4253,9 +4263,8 @@ int zoo_acreate_ttl(zhandle_t *zh, const char *path, const char *value, LOG_DEBUG(LOGCALLBACK(zh), "Sending request xid=%#x for path [%s] to %s",h.xid,path, zoo_get_current_server(zh)); - /* make a best (non-blocking) effort to send the requests asap */ - adaptor_send_queue(zh, 0); - return (rc < 0)?ZMARSHALLINGERROR:ZOK; + + return nonblocking_send(zh, rc); } int zoo_acreate2(zhandle_t *zh, const char *path, const char *value, @@ -4320,9 +4329,8 @@ int zoo_acreate2_ttl(zhandle_t *zh, const char *path, const char *value, LOG_DEBUG(LOGCALLBACK(zh), "Sending request xid=%#x for path [%s] to %s",h.xid,path, zoo_get_current_server(zh)); - /* make a best (non-blocking) effort to send the requests asap */ - adaptor_send_queue(zh, 0); - return (rc < 0)?ZMARSHALLINGERROR:ZOK; + + return nonblocking_send(zh, rc); } int DeleteRequest_init(zhandle_t *zh, struct DeleteRequest *req, @@ -4360,9 +4368,8 @@ int zoo_adelete(zhandle_t *zh, const char *path, int version, LOG_DEBUG(LOGCALLBACK(zh), "Sending request xid=%#x for path [%s] to %s",h.xid,path, zoo_get_current_server(zh)); - /* make a best (non-blocking) effort to send the requests asap */ - adaptor_send_queue(zh, 0); - return (rc < 0)?ZMARSHALLINGERROR:ZOK; + + return nonblocking_send(zh, rc); } int zoo_aexists(zhandle_t *zh, const char *path, int watch, @@ -4399,9 +4406,8 @@ int zoo_awexists(zhandle_t *zh, const char *path, LOG_DEBUG(LOGCALLBACK(zh), "Sending request xid=%#x for path [%s] to %s",h.xid,path, zoo_get_current_server(zh)); - /* make a best (non-blocking) effort to send the requests asap */ - adaptor_send_queue(zh, 0); - return (rc < 0)?ZMARSHALLINGERROR:ZOK; + + return nonblocking_send(zh, rc); } static int zoo_awget_children_(zhandle_t *zh, const char *path, @@ -4432,9 +4438,8 @@ static int zoo_awget_children_(zhandle_t *zh, const char *path, LOG_DEBUG(LOGCALLBACK(zh), "Sending request xid=%#x for path [%s] to %s",h.xid,path, zoo_get_current_server(zh)); - /* make a best (non-blocking) effort to send the requests asap */ - adaptor_send_queue(zh, 0); - return (rc < 0)?ZMARSHALLINGERROR:ZOK; + + return nonblocking_send(zh, rc); } int zoo_aget_children(zhandle_t *zh, const char *path, int watch, @@ -4480,9 +4485,8 @@ static int zoo_awget_children2_(zhandle_t *zh, const char *path, LOG_DEBUG(LOGCALLBACK(zh), "Sending request xid=%#x for path [%s] to %s",h.xid,path, zoo_get_current_server(zh)); - /* make a best (non-blocking) effort to send the requests asap */ - adaptor_send_queue(zh, 0); - return (rc < 0)?ZMARSHALLINGERROR:ZOK; + + return nonblocking_send(zh, rc); } int zoo_aget_children2(zhandle_t *zh, const char *path, int watch, @@ -4523,9 +4527,8 @@ int zoo_async(zhandle_t *zh, const char *path, LOG_DEBUG(LOGCALLBACK(zh), "Sending request xid=%#x for path [%s] to %s",h.xid,path, zoo_get_current_server(zh)); - /* make a best (non-blocking) effort to send the requests asap */ - adaptor_send_queue(zh, 0); - return (rc < 0)?ZMARSHALLINGERROR:ZOK; + + return nonblocking_send(zh, rc); } @@ -4553,9 +4556,8 @@ int zoo_aget_acl(zhandle_t *zh, const char *path, acl_completion_t completion, LOG_DEBUG(LOGCALLBACK(zh), "Sending request xid=%#x for path [%s] to %s",h.xid,path, zoo_get_current_server(zh)); - /* make a best (non-blocking) effort to send the requests asap */ - adaptor_send_queue(zh, 0); - return (rc < 0)?ZMARSHALLINGERROR:ZOK; + + return nonblocking_send(zh, rc); } int zoo_aset_acl(zhandle_t *zh, const char *path, int version, @@ -4584,9 +4586,8 @@ int zoo_aset_acl(zhandle_t *zh, const char *path, int version, LOG_DEBUG(LOGCALLBACK(zh), "Sending request xid=%#x for path [%s] to %s",h.xid,path, zoo_get_current_server(zh)); - /* make a best (non-blocking) effort to send the requests asap */ - adaptor_send_queue(zh, 0); - return (rc < 0)?ZMARSHALLINGERROR:ZOK; + + return nonblocking_send(zh, rc); } /* Completions for multi-op results */ @@ -4745,10 +4746,8 @@ int zoo_amulti(zhandle_t *zh, int count, const zoo_op_t *ops, LOG_DEBUG(LOGCALLBACK(zh), "Sending multi request xid=%#x with %d subrequests to %s", h.xid, index, zoo_get_current_server(zh)); - /* make a best (non-blocking) effort to send the requests asap */ - adaptor_send_queue(zh, 0); - return (rc < 0) ? ZMARSHALLINGERROR : ZOK; + return nonblocking_send(zh, rc); } typedef union WatchesRequest WatchesRequest; @@ -4831,7 +4830,6 @@ static int aremove_watches( zh, h.xid, COMPLETION_VOID, completion, data, 0, wdo, 0); rc = rc < 0 ? rc : queue_buffer_bytes(&zh->to_send, get_buffer(oa), get_buffer_len(oa)); - rc = rc < 0 ? ZMARSHALLINGERROR : ZOK; leave_critical(zh); /* We queued the buffer, so don't free it */ @@ -4840,7 +4838,7 @@ static int aremove_watches( LOG_DEBUG(LOGCALLBACK(zh), "Sending request xid=%#x for path [%s] to %s", h.xid, path, zoo_get_current_server(zh)); - adaptor_send_queue(zh, 0); + rc = nonblocking_send(zh, rc); done: free_duplicate_path(server_path, path); diff --git a/zookeeper-client/zookeeper-client-c/tests/LibCSymTable.h b/zookeeper-client/zookeeper-client-c/tests/LibCSymTable.h index 1b6f9db996d..08028abca8c 100644 --- a/zookeeper-client/zookeeper-client-c/tests/LibCSymTable.h +++ b/zookeeper-client/zookeeper-client-c/tests/LibCSymTable.h @@ -26,6 +26,7 @@ #include #include #include +#include #include // needed for _POSIX_MONOTONIC_CLOCK #ifdef THREADED diff --git a/zookeeper-client/zookeeper-client-c/tests/TestSASLAuth.cc b/zookeeper-client/zookeeper-client-c/tests/TestSASLAuth.cc index e6aa4cb3d50..c98d4bffc81 100644 --- a/zookeeper-client/zookeeper-client-c/tests/TestSASLAuth.cc +++ b/zookeeper-client/zookeeper-client-c/tests/TestSASLAuth.cc @@ -130,6 +130,16 @@ class Zookeeper_SASLAuth : public CPPUNIT_NS::TestFixture { } #ifdef HAVE_CYRUS_SASL_H + + // We need to disable the deprecation warnings as Apple has + // decided to deprecate all of CyrusSASL's functions with OS 10.11 + // (see MESOS-3030, ZOOKEEPER-4201). We are using GCC pragmas also + // for covering clang. +#ifdef __APPLE__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + void testClientSASLHelper(const char *hostPorts, const char *path) { startServer(); @@ -260,6 +270,10 @@ class Zookeeper_SASLAuth : public CPPUNIT_NS::TestFixture { stopServer(); } +#ifdef __APPLE__ +#pragma GCC diagnostic pop +#endif + #endif /* HAVE_CYRUS_SASL_H */ }; diff --git a/zookeeper-contrib/zookeeper-contrib-rest/src/main/java/org/apache/zookeeper/server/jersey/RestMain.java b/zookeeper-contrib/zookeeper-contrib-rest/src/main/java/org/apache/zookeeper/server/jersey/RestMain.java index 954ad045b1c..209207f6b14 100644 --- a/zookeeper-contrib/zookeeper-contrib-rest/src/main/java/org/apache/zookeeper/server/jersey/RestMain.java +++ b/zookeeper-contrib/zookeeper-contrib-rest/src/main/java/org/apache/zookeeper/server/jersey/RestMain.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.net.URISyntaxException; import java.net.URL; +import java.nio.file.Files; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,7 +54,8 @@ public void start() throws IOException { System.out.println("Starting grizzly ..."); boolean useSSL = cfg.useSSL(); - gws = new GrizzlyWebServer(cfg.getPort(), "/tmp/23cxv45345/2131xc2/", useSSL); + String zkRestResourcesTempPath = Files.createTempDirectory("zkRestResourcesTempPath").toFile().getCanonicalPath(); + gws = new GrizzlyWebServer(cfg.getPort(), zkRestResourcesTempPath, useSSL); // BUG: Grizzly needs a doc root if you are going to register multiple adapters for (Endpoint e : cfg.getEndpoints()) { diff --git a/zookeeper-contrib/zookeeper-contrib-zooinspector/src/main/java/org/apache/zookeeper/inspector/manager/ZooInspectorManagerImpl.java b/zookeeper-contrib/zookeeper-contrib-zooinspector/src/main/java/org/apache/zookeeper/inspector/manager/ZooInspectorManagerImpl.java index 2c5790746ad..aa9ff6e322e 100644 --- a/zookeeper-contrib/zookeeper-contrib-zooinspector/src/main/java/org/apache/zookeeper/inspector/manager/ZooInspectorManagerImpl.java +++ b/zookeeper-contrib/zookeeper-contrib-zooinspector/src/main/java/org/apache/zookeeper/inspector/manager/ZooInspectorManagerImpl.java @@ -20,9 +20,12 @@ import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; +import java.io.FileNotFoundException; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -91,6 +94,12 @@ public class ZooInspectorManagerImpl implements ZooInspectorManager { */ public static final String AUTH_DATA_KEY = "authData"; + private static final String DEFAULT_ENCRYPTION_MANAGER = + BasicDataEncryptionManager.class.getName(); + private static final int DEFAULT_TIMEOUT = 5000; + private static final String DEFAULT_HOSTS = "localhost:2181"; + private static final String DEFAULT_AUTH_SCHEME = ""; + private static final String DEFAULT_AUTH_VALUE = ""; private static final File defaultNodeViewersFile = new File( "./src/main/resources/defaultNodeViewers.cfg"); @@ -713,55 +722,47 @@ public void stop() { public List loadNodeViewersFile(File selectedFile) throws IOException { List result = new ArrayList(); - if (defaultNodeViewersFile.exists()) { - FileReader reader = new FileReader(selectedFile); - try { - BufferedReader buff = new BufferedReader(reader); - try { - while (buff.ready()) { - String line = buff.readLine(); - if (line != null && line.length() > 0 && !line.startsWith("#")) { - result.add(line); - } + + try(BufferedReader reader = getReaderForFile(selectedFile)) { + if(reader == null) { + return result; + } + + String line = ""; + while (line != null) { + line = reader.readLine(); + if(line != null) { + line = line.trim(); + if (!line.isEmpty() && !line.startsWith("#")) { + result.add(line); } - } finally { - buff.close(); } - } finally { - reader.close(); } } + return result; } private void loadDefaultConnectionFile() throws IOException { - if (defaultConnectionFile.exists()) { - Properties props = new Properties(); + Properties props = new Properties(); - FileReader reader = new FileReader(defaultConnectionFile); - try { + try(BufferedReader reader = getReaderForFile(defaultConnectionFile)) { + //If reader is null, it's OK. Default values will get set below. + if(reader != null) { props.load(reader); - } finally { - reader.close(); } - defaultEncryptionManager = props - .getProperty(DATA_ENCRYPTION_MANAGER) == null ? "org.apache.zookeeper.inspector.encryption.BasicDataEncryptionManager" - : props.getProperty(DATA_ENCRYPTION_MANAGER); - defaultTimeout = props.getProperty(SESSION_TIMEOUT) == null ? "5000" - : props.getProperty(SESSION_TIMEOUT); - defaultHosts = props.getProperty(CONNECT_STRING) == null ? "localhost:2181" - : props.getProperty(CONNECT_STRING); - defaultAuthScheme = props.getProperty(AUTH_SCHEME_KEY) == null ? "" - : props.getProperty(AUTH_SCHEME_KEY); - defaultAuthValue = props.getProperty(AUTH_DATA_KEY) == null ? "" - : props.getProperty(AUTH_DATA_KEY); - } else { - defaultEncryptionManager = "org.apache.zookeeper.inspector.encryption.BasicDataEncryptionManager"; - defaultTimeout = "5000"; - defaultHosts = "localhost:2181"; - defaultAuthScheme = ""; - defaultAuthValue = ""; } + + defaultEncryptionManager = props.getProperty(DATA_ENCRYPTION_MANAGER) == null ? + DEFAULT_ENCRYPTION_MANAGER : props.getProperty(DATA_ENCRYPTION_MANAGER); + defaultTimeout = props.getProperty(SESSION_TIMEOUT) == null ? + Integer.toString(DEFAULT_TIMEOUT) : props.getProperty(SESSION_TIMEOUT); + defaultHosts = props.getProperty(CONNECT_STRING) == null ? + DEFAULT_HOSTS : props.getProperty(CONNECT_STRING); + defaultAuthScheme = props.getProperty(AUTH_SCHEME_KEY) == null ? + DEFAULT_AUTH_SCHEME : props.getProperty(AUTH_SCHEME_KEY); + defaultAuthValue = props.getProperty(AUTH_DATA_KEY) == null ? + DEFAULT_AUTH_VALUE : props.getProperty(AUTH_DATA_KEY); } /* @@ -872,4 +873,26 @@ public Properties getLastConnectionProps() { public void setLastConnectionProps(Properties connectionProps) { this.lastConnectionProps = connectionProps; } + + private static BufferedReader getReaderForFile(File file) { + //check the filesystem first + if (file.exists()) { + try { + return new BufferedReader(new FileReader(file)); + } catch (FileNotFoundException e) { + return null; + } + } + + //fall back to checking the CLASSPATH with only the filename + //(for cases where the file exists in src/main/resources) + InputStream classpathStream = ZooInspectorManagerImpl.class.getClassLoader() + .getResourceAsStream(file.getName()); + if (classpathStream != null) { + return new BufferedReader(new InputStreamReader(classpathStream)); + } + + //couldn't find the file anywhere + return null; + } } diff --git a/zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md b/zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md index 82bcd65664d..cc34ddb40cf 100644 --- a/zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md +++ b/zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md @@ -788,7 +788,7 @@ property, when available, is noted below. must be 127 or less. Additionally, the maximum support TTL value is `1099511627775` which is smaller than what was allowed in 3.5.3 (`1152921504606846975`) -* *watchManaggerName* : +* *watchManagerName* : (Java system property only: **zookeeper.watchManagerName**) **New in 3.6.0:** Added in [ZOOKEEPER-1179](https://issues.apache.org/jira/browse/ZOOKEEPER-1179) New watcher manager WatchManagerOptimized is added to optimize the memory overhead in heavy watch use cases. This @@ -1355,7 +1355,8 @@ of servers -- that is, when deploying clusters of servers. not enable the command. By default the whitelist only contains "srvr" command which zkServer.sh uses. The rest of four letter word commands are disabled - by default. + by default: attempting to use them will gain a response + ".... is not executed because it is not in the whitelist." Here's an example of the configuration that enables stat, ruok, conf, and isro command while disabling the rest of Four Letter Words command: @@ -2242,7 +2243,7 @@ connections respectively. **New in 3.5.3:** Four Letter Words need to be explicitly white listed before using. -Please refer **4lw.commands.whitelist** +Please refer to **4lw.commands.whitelist** described in [cluster configuration section](#sc_clusterOptions) for details. Moving forward, Four Letter Words will be deprecated, please use [AdminServer](#sc_adminserver) instead. @@ -2269,9 +2270,11 @@ Moving forward, Four Letter Words will be deprecated, please use Print details about serving environment * *ruok* : - Tests if server is running in a non-error state. The server - will respond with imok if it is running. Otherwise it will not - respond at all. + Tests if the server is running in a non-error state. + When the whitelist enables ruok, the server will respond with `imok` + if it is running, otherwise it will not respond at all. + When ruok is disabled, the server responds with: + "ruok is not executed because it is not in the whitelist." A response of "imok" does not necessarily indicate that the server has joined the quorum, just that the server process is active and bound to the specified client port. Use "stat" for details on diff --git a/zookeeper-docs/src/main/resources/markdown/zookeeperCLI.md b/zookeeper-docs/src/main/resources/markdown/zookeeperCLI.md index 668e64e8ba8..7096aa0cc89 100644 --- a/zookeeper-docs/src/main/resources/markdown/zookeeperCLI.md +++ b/zookeeper-docs/src/main/resources/markdown/zookeeperCLI.md @@ -24,6 +24,8 @@ Enter into the ZooKeeper-cli bin/zkCli.sh # connect to the remote host with timeout:3s bin/zkCli.sh -timeout 3000 -server remoteIP:2181 +# connect to the remote host with -waitforconnection option to wait for connection success before executing commands +bin/zkCli.sh -waitforconnection -timeout 3000 -server remoteIP:2181 # connect with a custom client configuration properties file bin/zkCli.sh -client-configuration /path/to/client.properties ``` diff --git a/zookeeper-docs/src/main/resources/markdown/zookeeperOracleQuorums.md b/zookeeper-docs/src/main/resources/markdown/zookeeperOracleQuorums.md new file mode 100644 index 00000000000..adc547778bb --- /dev/null +++ b/zookeeper-docs/src/main/resources/markdown/zookeeperOracleQuorums.md @@ -0,0 +1,202 @@ + + +# Introduction to Oracle Quorum +The introduction to Oracle Quorum increases the availability of a cluster of 2 ZooKeeper instances with a failure detector as known as the Oracle. + The Oracle is designed to grant the permission to the instance which is the only remaining instance +in a 2-instance configuration when the other instance is identified as faulty by the fail detector, the Oracle. + +## The implementation of the Oracle +Every instance shall access to a file which contains either 0 or 1 to indicate whether this instance is authorized by the Oracle. +However, this design can be changed since the fail detector algorithms vary from each other. Therefore, ones can override the method of _askOracle()_ in _QuorumOracleMaj_ to adapt the preferred way of deciphering the message from the Oracle. + +## The deployment cotexts +The Oracle is designed to increase the availability of a cluster of 2 ZooKeeper instances; thus, the size of the voting member is **2**. +In other words, the Oracle solves the consensus problem of a possibility of faulty instance in a two-instance ensemble. + +In the case that the size of the voting members exceeds 2, the expected way to make the Oracle work correctly is to reconfigure the size of the cluster when a faulty machine is identified. +For example, with a configuration of 5 instances, when a faulty machine breaks the connection with the Leader, it is expected to have a _reconfig_ client request to the cluster, which makes the cluster to re-form as the configuration of 4 instances. +Therefore, once the size of the voting member equals to 2, the configuration falls into the problem domain which the Oracle is designed to address. + +## How to deploy the Oracle in _zoo.cfg_ +Regardless of the size of the cluster, the _oraclePath_ must be configured at the time of the initialization, which is like other static parameters. +The below shows the correct way to specify and enable the Oracle. + + oraclePath=/to/some/file + +#### An example of zoo.cfg: + + dataDir=/data + dataLogDir=/datalog + tickTime=2000 + initLimit=5 + syncLimit=2 + autopurge.snapRetainCount=3 + autopurge.purgeInterval=0 + maxClientCnxns=60 + standaloneEnabled=true + admin.enableServer=true + oraclePath=/chassis/mastership + server.1=0.0.0.0:2888:3888;2181 + server.2=hw1:2888:3888;2181 + +The QuorumOracleMaj is designed to read the result of a failure detector, which is written on a text file, the oracle file. +The configuration in the zoo.cfg like the following: + + oraclePath=/to/some/file + +Suppose you have the result of the failure detector written on /some/path/result.txt, and then the correct configuration is the following: + + oraclePath=/some/path/result.txt + +So, what is the correct content of the provided file? An example file can be created with the following command from the terminal: + + $echo 1 > /some/path/result.txt + +Any equivalent files are suitable for the current implementation of QuorumOracleMaj. +The number of oracle files should be equal to the number of ZooKeeper instances configured to enable the Oracle. +In other words, each ZooKeeper instance should have its oracle file, and the files shall not be shared; otherwise, the issues in the next section will arise. + +## What differs after the deployment of the Oracle enabled +The _QuorumPeerConfig_ will create an instance of _QuorumOracleMaj_ instead of the default QuorumVerifier, _QuorumMaj_ when it reads the _zoo.cfg_ contains _oraclePath_. +QuorumOracleMaj inheritances from QuorumMaj, and differs from its superclass by overriding the method of _containsQuorum()_. +QuorumOracleMaj is designed to execute its version of _containsQuorum_ when the Leader loses all of its followers, and fails to maintain the quorum. +In other cases, _QuorumOracleMaj_ shall execute as _QuorumMaj_. + +## What we should pay attention to the Oracle +We consider an asynchronous distributed system which consists of **2** ZooKeeper instances and an Oracle. + +### Liveness Issue: +When we consider the oracle satisfies the following property introduced by [CT]: + + Strong Completeness: There is a time after which every process that crashes is permanently suspected by every correct processes + +The liveness of the system is ensured by the Oracle. +However, when the introduced oracle fails to maintain this property, the lost of the liveness is expected as the following example, + +Suppose we have a Leader and a Follower, which are running in the broadcasting state, +The system will lose its liveness when: + + 1. The Leader fails, but the Oracle does not detect the faulty Leader, which means the Oracle will not authorize the Follower to become a new Leader. + 2. When a Follower fails, but the Oracle does not detect the faulty follower, which means the Oracle will authorize the Leader to move system forward. + +### Safety Issue: +#### Lost of Progress +The progress can lost when multiple failures occurs in the system at different time as the following example, + +Suppose we have a Leader(Ben) and a Follower(John) in the broadcasting state, + + At T1 with zxid(0x1_1): L-Ben fails, and the F-John takes over the system under the authorization from the Oracle. + At T2 with zxid(0x2_1): The F-John becomes a new Leader, L-John, and starts a new epoch. + At T3 with zxid(0x2_A): L-John fails + At T4 with zxid(0x2_A): Ben recovers up and starts its leader election. + At T5 with zxid(0x3_1): Ben becomes the new leader, L-Ben, under the authorization from the Oracle. + +In this case, the system loses its progress after the L-Ben failed. + + +However, the lost of progress can be prevented by making the Oracle is capable of referring the latest zxid. +When the Oracle could refer to the latest zxid, + + At T5 with zxid(0x2_A): Ben will not end his leader election because the Oracle would not authorize although John is down. + +Nevertheless, we exchange the liveness for the safety. +#### Split Brain Issue +We consider the Oracle satisfies the following desired property introduced by [CT], + + Accuracy: There is a time after which some correct processes is never suspected by any processes + +Nevertheless, the decisions which the Oracle gives out should be mutual exclusive. + +In other words, + +Suppose we have a Leader(Ben) and a Follower(John) in the broadcasting state, + + - At any time, the Oracle will not authorize both Ben and John even though the failure detectors think each other is faulty. + Or + - At any time, for any two values in any two Oracle files respectively, the values are not both equal to 1. + +The split brain is expected when the Oracle fails to maintain this property during the leader election phase of + + 1. Start of the system + 2. A failed instance recovers from failures. + +## Examples of Concepts for Implementation of a Failure Detector +One should consider that the failure detector's outcome is to authorize the querying ZooKeeper instance whether it has the right to move the system forward without waiting for the faulty instance, which is identified by the failure detector. + +### An Implementation of Hardware +Suppose two dedicated pieces of hardware, hw1 and hw2, can host ZooKeeper instances, zk1 and zk2, respectively, and form a cluster. +A hardware device is attached to both of the hardware, and it is capable of determining whether the hardware is power on or not. +So, when hw1 is not power on, the zk1 is undoubtedly faulty. +Therefore, the hardware device updates the oracle file on hw2 to 1, which indicates that zk1 is faulty and authorizes zk2 to move the system forwards. + +### An Implementation of Software +Suppose two dedicated pieces of hardware, hw1 and hw2, can host ZooKeeper instances, zk1 and zk2, respectively, and form a cluster. +One can have two more services, o1 and o2, on hw1 and hw2, respectively. The job of o1 and o2 are detecting the other hardware is alive or not. +For example, o1 can constantly ping hw2 to determine if hw2 is power on or not. +When o1 cannot ping hw2, o1 identifies that hw2 is faulty and then update the oracle file of zk1 to 1, which indicates that zk2 is faulty and authorizes zk1 to move the system forwards. + +### Use USB devices as Oracle to Maintain Progress +In macOS,10.15.7 (19H2), the external storage devices are mounted under `/Volumes`. +Thus, we can insert a USB device which contains the required information as the oracle. +When the device is connected, the oracle authorizes the leader to move system forward, which also means the other instance fails. +There are **SIX** steps to reproduce this stimulation. + +* Firstly, insert a USB device named `Oracle`, and then we can expect that `/Volumes/Oracle` is accessible. +* Secondly, we create a file contains `1` under `/Volumes/Oracle` named `mastership`. +Now we can access `/Volumes/Oracle/mastership`, and so does the zookeeper instances to see whether it has the right to move the system forward. +The file can easily be generated by the following command: + + + $echo 1 > mastership + +* Thirdly, you shall have a `zoo.cfg` like the example below: + + + dataDir=/data + dataLogDir=/datalog + tickTime=2000 + initLimit=5 + syncLimit=2 + autopurge.snapRetainCount=3 + autopurge.purgeInterval=0 + maxClientCnxns=60 + standaloneEnabled=true + admin.enableServer=true + oraclePath=/Volumes/Oracle/mastership + server.1=0.0.0.0:2888:3888;2181 + server.2=hw1:2888:3888;2181 + +_(NOTE) The split brain issues will not occur because there is only a SINGLE USB device in this stimulation._ +_Additionally, `mastership` should not be shared by multiple instances._ +_Thus, only one ZooKeeper instance is configured with Oracle._ +_For more, please refer to Section Safety Issue._ + +* Fourthly, start the cluster, and it is expected it forms a quorum normally. +* Fifthly, terminate the instance either without attaching to a USB device or `mastership` contains 0. +There are two scenarios to expect: + 1. A leader failure occurs, and the remained instance finishes the leader election on its own due to the oracle. + 2. The quorum is still maintained due to the oracle. + +* Lastly, when the USB device is removed, `/Volumes/Oracle/mastership` becomes unavailable. +Therefore, according to the current implementation, whenever the Leader queries the oracle, the oracle throws an exception and return `FALSE`. +Repeat the fifth step, and then it is expected that either the system cannot recover from a leader failure ,or the leader loses the quorum. +In either case, the service is interrupted. + +With these steps, we can show and practice how the oracle works with two-instance systems with ease. + +##REFERENCE +[CT] Tushar Deepak Chandra and Sam Toueg. 1991. Unreliable failure detectors for asynchronous systems (preliminary version). In Proceedings of the tenth annual ACM symposium on Principles of distributed computing (PODC '91). Association for Computing Machinery, New York, NY, USA, 325–340. DOI:https://doi.org/10.1145/112600.112627 \ No newline at end of file diff --git a/zookeeper-server/pom.xml b/zookeeper-server/pom.xml index 90ee82760ee..2636f5cd969 100755 --- a/zookeeper-server/pom.xml +++ b/zookeeper-server/pom.xml @@ -172,6 +172,12 @@ snappy-java provided + + commons-io + commons-io + 2.6 + compile + diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/ZooKeeperMain.java b/zookeeper-server/src/main/java/org/apache/zookeeper/ZooKeeperMain.java index ff8ce623286..4ee378fac05 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/ZooKeeperMain.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/ZooKeeperMain.java @@ -32,6 +32,8 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Stream; @@ -68,6 +70,7 @@ public class ZooKeeperMain { protected ZooKeeper zk; protected String host = ""; + private CountDownLatch connectLatch = null; public boolean getPrintWatches() { return printWatches; @@ -106,6 +109,13 @@ public void process(WatchedEvent event) { ZooKeeperMain.printMessage("WATCHER::"); ZooKeeperMain.printMessage(event.toString()); } + if (connectLatch != null) { + // connection success + if (event.getType() == Event.EventType.None + && event.getState() == Event.KeeperState.SyncConnected) { + connectLatch.countDown(); + } + } } } @@ -168,6 +178,8 @@ public boolean parseOptions(String[] args) { options.put("readonly", "true"); } else if (opt.equals("-client-configuration")) { options.put("client-configuration", it.next()); + } else if (opt.equals("-waitforconnection")) { + options.put("waitforconnection", "true"); } } catch (NoSuchElementException e) { System.err.println("Error: no argument found for option " + opt); @@ -261,7 +273,19 @@ protected void connectToZK(String newHost) throws InterruptedException, IOExcept } } - zk = new ZooKeeperAdmin(host, Integer.parseInt(cl.getOption("timeout")), new MyWatcher(), readOnly, clientConfig); + if (cl.getOption("waitforconnection") != null) { + connectLatch = new CountDownLatch(1); + } + + int timeout = Integer.parseInt(cl.getOption("timeout")); + zk = new ZooKeeperAdmin(host, timeout, new MyWatcher(), readOnly, clientConfig); + if (connectLatch != null) { + if (!connectLatch.await(timeout, TimeUnit.MILLISECONDS)) { + zk.close(); + throw new IOException(KeeperException.create(KeeperException.Code.CONNECTIONLOSS)); + } + } + } public static void main(String[] args) throws IOException, InterruptedException { diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/audit/AuditHelper.java b/zookeeper-server/src/main/java/org/apache/zookeeper/audit/AuditHelper.java index ec4ca136c2f..b98c42d2588 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/audit/AuditHelper.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/audit/AuditHelper.java @@ -70,14 +70,11 @@ public static void addAuditLog(Request request, ProcessTxnResult txnResult, bool case ZooDefs.OpCode.create2: case ZooDefs.OpCode.createContainer: op = AuditConstants.OP_CREATE; + CreateRequest createRequest = new CreateRequest(); + deserialize(request, createRequest); + createMode = getCreateMode(createRequest); if (failedTxn) { - CreateRequest createRequest = new CreateRequest(); - deserialize(request, createRequest); path = createRequest.getPath(); - createMode = - getCreateMode(createRequest); - } else { - createMode = getCreateMode(request); } break; case ZooDefs.OpCode.delete: @@ -99,13 +96,11 @@ public static void addAuditLog(Request request, ProcessTxnResult txnResult, bool break; case ZooDefs.OpCode.setACL: op = AuditConstants.OP_SETACL; + SetACLRequest setACLRequest = new SetACLRequest(); + deserialize(request, setACLRequest); + acls = ZKUtil.aclToString(setACLRequest.getAcl()); if (failedTxn) { - SetACLRequest setACLRequest = new SetACLRequest(); - deserialize(request, setACLRequest); path = setACLRequest.getPath(); - acls = ZKUtil.aclToString(setACLRequest.getAcl()); - } else { - acls = getACLs(request); } break; case ZooDefs.OpCode.multi: @@ -188,18 +183,6 @@ private static void log(String user, String operation, String znode, String acl, ZKAuditProvider.log(user, operation, znode, acl, createMode, session, ip, result); } - private static String getACLs(Request request) throws IOException { - SetACLRequest setACLRequest = new SetACLRequest(); - deserialize(request, setACLRequest); - return ZKUtil.aclToString(setACLRequest.getAcl()); - } - - private static String getCreateMode(Request request) throws IOException, KeeperException { - CreateRequest createRequest = new CreateRequest(); - deserialize(request, createRequest); - return getCreateMode(createRequest); - } - private static String getCreateMode(CreateRequest createRequest) throws KeeperException { return CreateMode.fromFlag(createRequest.getFlags()).toString().toLowerCase(); } diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/cli/ReconfigCommand.java b/zookeeper-server/src/main/java/org/apache/zookeeper/cli/ReconfigCommand.java index 8afc14bb517..ce490ad8039 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/cli/ReconfigCommand.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/cli/ReconfigCommand.java @@ -130,7 +130,7 @@ public CliCommand parse(String[] cmdArgs) throws CliParseException { //check that membership makes sense; leader will make these checks again //don't check for leader election ports since //client doesn't know what leader election alg is used - members = QuorumPeerConfig.parseDynamicConfig(dynamicCfg, 0, true, false).toString(); + members = QuorumPeerConfig.parseDynamicConfig(dynamicCfg, 0, true, false, null).toString(); } catch (Exception e) { throw new CliParseException("Error processing " + cl.getOptionValue("file") + e.getMessage()); } diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxn.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxn.java index 2e65da84044..02cde23a917 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxn.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/NIOServerCnxn.java @@ -545,7 +545,10 @@ private boolean readLength(SelectionKey k) throws IOException { return false; } if (len < 0 || len > BinaryInputArchive.maxBuffer) { - throw new IOException("Len error " + len); + throw new IOException("Len error. " + + "A message from " + this.getRemoteSocketAddress() + " with advertised length of " + len + + " is either a malformed message or too large to process" + + " (length is greater than jute.maxbuffer=" + BinaryInputArchive.maxBuffer + ")"); } if (!isZKServerRunning()) { throw new IOException("ZooKeeperServer not running"); diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/PrepRequestProcessor.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/PrepRequestProcessor.java index 55b070a8463..11b5ccbc8d3 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/PrepRequestProcessor.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/PrepRequestProcessor.java @@ -65,6 +65,7 @@ import org.apache.zookeeper.server.quorum.QuorumPeerConfig; import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException; import org.apache.zookeeper.server.quorum.flexible.QuorumMaj; +import org.apache.zookeeper.server.quorum.flexible.QuorumOracleMaj; import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; import org.apache.zookeeper.txn.CheckVersionTxn; import org.apache.zookeeper.txn.CloseSessionTxn; @@ -394,7 +395,7 @@ protected void pRequest2Txn(int type, long zxid, Request request, Record record, validatePath(path, request.sessionId); nodeRecord = getRecordForPath(path); zks.checkACL(request.cnxn, nodeRecord.acl, ZooDefs.Perms.WRITE, request.authInfo, path, null); - zks.checkQuota(path, setDataRequest.getData(), OpCode.setData); + zks.checkQuota(path, nodeRecord.data, setDataRequest.getData(), OpCode.setData); int newVersion = checkAndIncVersion(nodeRecord.stat.getVersion(), setDataRequest.getVersion(), path); request.setTxn(new SetDataTxn(path, setDataRequest.getData(), newVersion)); nodeRecord = nodeRecord.duplicate(request.getHdr().getZxid()); @@ -452,7 +453,7 @@ protected void pRequest2Txn(int type, long zxid, Request request, Record record, try { Properties props = new Properties(); props.load(new StringReader(newMembers)); - request.qv = QuorumPeerConfig.parseDynamicConfig(props, lzks.self.getElectionType(), true, false); + request.qv = QuorumPeerConfig.parseDynamicConfig(props, lzks.self.getElectionType(), true, false, lastSeenQV.getOraclePath()); request.qv.setVersion(request.getHdr().getZxid()); } catch (IOException | ConfigException e) { throw new KeeperException.BadArgumentsException(e.getMessage()); @@ -472,7 +473,7 @@ protected void pRequest2Txn(int type, long zxid, Request request, Record record, leavingServers = StringUtils.split(leavingServersString, ","); } - if (!(lastSeenQV instanceof QuorumMaj)) { + if (!(lastSeenQV instanceof QuorumMaj) && !(lastSeenQV instanceof QuorumOracleMaj)) { String msg = "Incremental reconfiguration requested but last configuration seen has a non-majority quorum system"; LOG.warn(msg); throw new KeeperException.BadArgumentsException(msg); @@ -514,7 +515,13 @@ protected void pRequest2Txn(int type, long zxid, Request request, Record record, } catch (ConfigException e) { throw new KeeperException.BadArgumentsException("Reconfiguration failed"); } - request.qv = new QuorumMaj(nextServers); + + if (lastSeenQV instanceof QuorumMaj) { + request.qv = new QuorumMaj(nextServers); + } else { + request.qv = new QuorumOracleMaj(nextServers, lastSeenQV.getOraclePath()); + } + request.qv.setVersion(request.getHdr().getZxid()); } if (QuorumPeerConfig.isStandaloneEnabled() && request.qv.getVotingMembers().size() < 2) { @@ -698,7 +705,7 @@ private void pRequest2TxnCreate(int type, Request request, Record record, boolea throw new KeeperException.NoChildrenForEphemeralsException(path); } int newCversion = parentRecord.stat.getCversion() + 1; - zks.checkQuota(path, data, OpCode.create); + zks.checkQuota(path, null, data, OpCode.create); if (type == OpCode.createContainer) { request.setTxn(new CreateContainerTxn(path, data, listACL, newCversion)); } else if (type == OpCode.createTTL) { diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/RequestThrottler.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/RequestThrottler.java index 32863d92a61..86923635ec7 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/RequestThrottler.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/RequestThrottler.java @@ -72,7 +72,7 @@ public class RequestThrottler extends ZooKeeperCriticalThread { static { shutdownTimeout = Integer.getInteger(SHUTDOWN_TIMEOUT, 10000); - LOG.info("{} = {}", SHUTDOWN_TIMEOUT, shutdownTimeout); + LOG.info("{} = {} ms", SHUTDOWN_TIMEOUT, shutdownTimeout); } /** diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java index d48a7bfd760..f0e3c822653 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java @@ -360,9 +360,9 @@ public ZooKeeperServer(FileTxnSnapLog txnLogFactory, int tickTime, int minSessio LOG.info( "Created server with" - + " tickTime {}" - + " minSessionTimeout {}" - + " maxSessionTimeout {}" + + " tickTime {} ms" + + " minSessionTimeout {} ms" + + " maxSessionTimeout {} ms" + " clientPortListenBacklog {}" + " datadir {}" + " snapdir {}", @@ -544,6 +544,10 @@ public void takeSnapshot(boolean syncSnap) { ServerMetrics.getMetrics().SNAPSHOT_TIME.add(elapsed); } + public boolean shouldForceWriteInitialSnapshotAfterLeaderElection() { + return txnLogFactory.shouldForceWriteInitialSnapshotAfterLeaderElection(); + } + @Override public long getDataDirSize() { if (zkDb == null) { @@ -1274,7 +1278,7 @@ public int getTickTime() { } public void setTickTime(int tickTime) { - LOG.info("tickTime set to {}", tickTime); + LOG.info("tickTime set to {} ms", tickTime); this.tickTime = tickTime; } @@ -1283,7 +1287,7 @@ public static int getThrottledOpWaitTime() { } public static void setThrottledOpWaitTime(int time) { - LOG.info("throttledOpWaitTime set to {}", time); + LOG.info("throttledOpWaitTime set to {} ms", time); throttledOpWaitTime = time; } @@ -1293,7 +1297,7 @@ public int getMinSessionTimeout() { public void setMinSessionTimeout(int min) { this.minSessionTimeout = min == -1 ? tickTime * 2 : min; - LOG.info("minSessionTimeout set to {}", this.minSessionTimeout); + LOG.info("minSessionTimeout set to {} ms", this.minSessionTimeout); } public int getMaxSessionTimeout() { @@ -1302,7 +1306,7 @@ public int getMaxSessionTimeout() { public void setMaxSessionTimeout(int max) { this.maxSessionTimeout = max == -1 ? tickTime * 20 : max; - LOG.info("maxSessionTimeout set to {}", this.maxSessionTimeout); + LOG.info("maxSessionTimeout set to {} ms", this.maxSessionTimeout); } public int getClientPortListenBacklog() { @@ -1492,7 +1496,7 @@ long getFlushDelay() { } static void setFlushDelay(long delay) { - LOG.info("{}={}", FLUSH_DELAY, delay); + LOG.info("{} = {} ms", FLUSH_DELAY, delay); flushDelay = delay; } @@ -1501,7 +1505,7 @@ long getMaxWriteQueuePollTime() { } static void setMaxWriteQueuePollTime(long maxTime) { - LOG.info("{}={}", MAX_WRITE_QUEUE_POLL_SIZE, maxTime); + LOG.info("{} = {} ms", MAX_WRITE_QUEUE_POLL_SIZE, maxTime); maxWriteQueuePollTime = maxTime; } @@ -2020,14 +2024,15 @@ public void checkACL(ServerCnxn cnxn, List acl, int perm, List ids, Str * check a path whether exceeded the quota. * * @param path - * the path of the node + * the path of the node, used for the quota prefix check + * @param lastData + * the current node data, {@code null} for none * @param data - * the data of the path + * the data to be set, or {@code null} for none * @param type * currently, create and setData need to check quota */ - - public void checkQuota(String path, byte[] data, int type) throws KeeperException.QuotaExceededException { + public void checkQuota(String path, byte[] lastData, byte[] data, int type) throws KeeperException.QuotaExceededException { if (!enforceQuota) { return; } @@ -2043,11 +2048,6 @@ public void checkQuota(String path, byte[] data, int type) throws KeeperExceptio checkQuota(lastPrefix, dataBytes, 1); break; case OpCode.setData: - DataNode node = zkDatabase.getDataTree().getNode(path); - byte[] lastData; - synchronized (node) { - lastData = node.getData(); - } checkQuota(lastPrefix, dataBytes - (lastData == null ? 0 : lastData.length), 0); break; default: diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/FileTxnSnapLog.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/FileTxnSnapLog.java index f054bc85d16..a4670ec0253 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/FileTxnSnapLog.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/persistence/FileTxnSnapLog.java @@ -229,6 +229,15 @@ public SnapshotInfo getLastSnapshotInfo() { return this.snapLog.getLastSnapshotInfo(); } + /** + * whether to force the write of an initial snapshot after a leader election, + * to address ZOOKEEPER-3781 after upgrading from Zookeeper 3.4.x. + * @return true if an initial snapshot should be written even if not otherwise required, false otherwise. + */ + public boolean shouldForceWriteInitialSnapshotAfterLeaderElection() { + return trustEmptySnapshot && getLastSnapshotInfo() == null; + } + /** * this function restores the server * database after reading from the diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FastLeaderElection.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FastLeaderElection.java index f141e854e8c..9fc9d148c39 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FastLeaderElection.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FastLeaderElection.java @@ -34,6 +34,7 @@ import org.apache.zookeeper.server.quorum.QuorumPeer.LearnerType; import org.apache.zookeeper.server.quorum.QuorumPeer.ServerState; import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException; +import org.apache.zookeeper.server.quorum.flexible.QuorumOracleMaj; import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; import org.apache.zookeeper.server.util.ZxidUtils; import org.slf4j.Logger; @@ -86,9 +87,9 @@ public class FastLeaderElection implements Election { static { minNotificationInterval = Integer.getInteger(MIN_NOTIFICATION_INTERVAL, minNotificationInterval); - LOG.info("{}={}", MIN_NOTIFICATION_INTERVAL, minNotificationInterval); + LOG.info("{} = {} ms", MIN_NOTIFICATION_INTERVAL, minNotificationInterval); maxNotificationInterval = Integer.getInteger(MAX_NOTIFICATION_INTERVAL, maxNotificationInterval); - LOG.info("{}={}", MAX_NOTIFICATION_INTERVAL, maxNotificationInterval); + LOG.info("{} = {} ms", MAX_NOTIFICATION_INTERVAL, maxNotificationInterval); } /** @@ -948,7 +949,7 @@ public Vote lookForLeader() throws InterruptedException { Long.toHexString(proposedZxid)); sendNotifications(); - SyncedLearnerTracker voteSet; + SyncedLearnerTracker voteSet = null; /* * Loop in which we exchange notifications until we find a leader @@ -977,7 +978,24 @@ public Vote lookForLeader() throws InterruptedException { */ int tmpTimeOut = notTimeout * 2; notTimeout = Math.min(tmpTimeOut, maxNotificationInterval); - LOG.info("Notification time out: {}", notTimeout); + + /* + * When a leader failure happens on a master, the backup will be supposed to receive the honour from + * Oracle and become a leader, but the honour is likely to be delay. We do a re-check once timeout happens + * + * The leader election algorithm does not provide the ability of electing a leader from a single instance + * which is in a configuration of 2 instances. + * */ + self.getQuorumVerifier().revalidateVoteset(voteSet, notTimeout != minNotificationInterval); + if (self.getQuorumVerifier() instanceof QuorumOracleMaj && voteSet != null && voteSet.hasAllQuorums() && notTimeout != minNotificationInterval) { + setPeerState(proposedLeader, voteSet); + Vote endVote = new Vote(proposedLeader, proposedZxid, logicalclock.get(), proposedEpoch); + leaveInstance(endVote); + return endVote; + } + + LOG.info("Notification time out: {} ms", notTimeout); + } else if (validVoter(n.sid) && validVoter(n.leader)) { /* * Only proceed if the vote comes from a replica in the current or next @@ -1051,43 +1069,53 @@ public Vote lookForLeader() throws InterruptedException { case OBSERVING: LOG.debug("Notification from observer: {}", n.sid); break; + + /* + * In ZOOKEEPER-3922, we separate the behaviors of FOLLOWING and LEADING. + * To avoid the duplication of codes, we create a method called followingBehavior which was used to + * shared by FOLLOWING and LEADING. This method returns a Vote. When the returned Vote is null, it follows + * the original idea to break swtich statement; otherwise, a valid returned Vote indicates, a leader + * is generated. + * + * The reason why we need to separate these behaviors is to make the algorithm runnable for 2-node + * setting. An extra condition for generating leader is needed. Due to the majority rule, only when + * there is a majority in the voteset, a leader will be generated. However, in a configuration of 2 nodes, + * the number to achieve the majority remains 2, which means a recovered node cannot generate a leader which is + * the existed leader. Therefore, we need the Oracle to kick in this situation. In a two-node configuration, the Oracle + * only grants the permission to maintain the progress to one node. The oracle either grants the permission to the + * remained node and makes it a new leader when there is a faulty machine, which is the case to maintain the progress. + * Otherwise, the oracle does not grant the permission to the remained node, which further causes a service down. + * + * In the former case, when a failed server recovers and participate in the leader election, it would not locate a + * new leader because there does not exist a majority in the voteset. It fails on the containAllQuorum() infinitely due to + * two facts. First one is the fact that it does do not have a majority in the voteset. The other fact is the fact that + * the oracle would not give the permission since the oracle already gave the permission to the existed leader, the healthy machine. + * Logically, when the oracle replies with negative, it implies the existed leader which is LEADING notification comes from is a valid leader. + * To threat this negative replies as a permission to generate the leader is the purpose to separate these two behaviors. + * + * + * */ case FOLLOWING: - case LEADING: /* - * Consider all notifications from the same epoch - * together. - */ - if (n.electionEpoch == logicalclock.get()) { - recvset.put(n.sid, new Vote(n.leader, n.zxid, n.electionEpoch, n.peerEpoch, n.state)); - voteSet = getVoteTracker(recvset, new Vote(n.version, n.leader, n.zxid, n.electionEpoch, n.peerEpoch, n.state)); - if (voteSet.hasAllQuorums() && checkLeader(recvset, n.leader, n.electionEpoch)) { - setPeerState(n.leader, voteSet); - Vote endVote = new Vote(n.leader, n.zxid, n.electionEpoch, n.peerEpoch); - leaveInstance(endVote); - return endVote; - } + * To avoid duplicate codes + * */ + Vote resultFN = receivedFollowingNotification(recvset, outofelection, voteSet, n); + if (resultFN == null) { + break; + } else { + return resultFN; } - + case LEADING: /* - * Before joining an established ensemble, verify that - * a majority are following the same leader. - * - * Note that the outofelection map also stores votes from the current leader election. - * See ZOOKEEPER-1732 for more information. - */ - outofelection.put(n.sid, new Vote(n.version, n.leader, n.zxid, n.electionEpoch, n.peerEpoch, n.state)); - voteSet = getVoteTracker(outofelection, new Vote(n.version, n.leader, n.zxid, n.electionEpoch, n.peerEpoch, n.state)); - - if (voteSet.hasAllQuorums() && checkLeader(outofelection, n.leader, n.electionEpoch)) { - synchronized (this) { - logicalclock.set(n.electionEpoch); - setPeerState(n.leader, voteSet); - } - Vote endVote = new Vote(n.leader, n.zxid, n.electionEpoch, n.peerEpoch); - leaveInstance(endVote); - return endVote; + * In leadingBehavior(), it performs followingBehvior() first. When followingBehavior() returns + * a null pointer, ask Oracle whether to follow this leader. + * */ + Vote resultLN = receivedLeadingNotification(recvset, outofelection, voteSet, n); + if (resultLN == null) { + break; + } else { + return resultLN; } - break; default: LOG.warn("Notification state unrecognized: {} (n.state), {}(n.sid)", n.state, n.sid); break; @@ -1115,6 +1143,74 @@ public Vote lookForLeader() throws InterruptedException { } } + private Vote receivedFollowingNotification(Map recvset, Map outofelection, SyncedLearnerTracker voteSet, Notification n) { + /* + * Consider all notifications from the same epoch + * together. + */ + if (n.electionEpoch == logicalclock.get()) { + recvset.put(n.sid, new Vote(n.leader, n.zxid, n.electionEpoch, n.peerEpoch, n.state)); + voteSet = getVoteTracker(recvset, new Vote(n.version, n.leader, n.zxid, n.electionEpoch, n.peerEpoch, n.state)); + if (voteSet.hasAllQuorums() && checkLeader(recvset, n.leader, n.electionEpoch)) { + setPeerState(n.leader, voteSet); + Vote endVote = new Vote(n.leader, n.zxid, n.electionEpoch, n.peerEpoch); + leaveInstance(endVote); + return endVote; + } + } + + /* + * Before joining an established ensemble, verify that + * a majority are following the same leader. + * + * Note that the outofelection map also stores votes from the current leader election. + * See ZOOKEEPER-1732 for more information. + */ + outofelection.put(n.sid, new Vote(n.version, n.leader, n.zxid, n.electionEpoch, n.peerEpoch, n.state)); + voteSet = getVoteTracker(outofelection, new Vote(n.version, n.leader, n.zxid, n.electionEpoch, n.peerEpoch, n.state)); + + if (voteSet.hasAllQuorums() && checkLeader(outofelection, n.leader, n.electionEpoch)) { + synchronized (this) { + logicalclock.set(n.electionEpoch); + setPeerState(n.leader, voteSet); + } + Vote endVote = new Vote(n.leader, n.zxid, n.electionEpoch, n.peerEpoch); + leaveInstance(endVote); + return endVote; + } + + return null; + } + + private Vote receivedLeadingNotification(Map recvset, Map outofelection, SyncedLearnerTracker voteSet, Notification n) { + /* + * + * In a two-node configuration, a recovery nodes cannot locate a leader because of the lack of the majority in the voteset. + * Therefore, it is the time for Oracle to take place as a tight breaker. + * + * */ + Vote result = receivedFollowingNotification(recvset, outofelection, voteSet, n); + if (result == null) { + /* + * Ask Oracle to see if it is okay to follow this leader. + * + * We don't need the CheckLeader() because itself cannot be a leader candidate + * */ + if (self.getQuorumVerifier().getNeedOracle() && !self.getQuorumVerifier().askOracle()) { + LOG.info("Oracle indicates to follow"); + setPeerState(n.leader, voteSet); + Vote endVote = new Vote(n.leader, n.zxid, n.electionEpoch, n.peerEpoch); + leaveInstance(endVote); + return endVote; + } else { + LOG.info("Oracle indicates not to follow"); + return null; + } + } else { + return result; + } + } + /** * Check if a given sid is represented in either the current or * the next voting view diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Leader.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Leader.java index 2de2ceeb8b0..ce8f7999c45 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Leader.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Leader.java @@ -174,6 +174,10 @@ public List getNonVotingFollowers() { void addForwardingFollower(LearnerHandler lh) { synchronized (forwardingFollowers) { forwardingFollowers.add(lh); + /* + * Any changes on forwardiongFollowers could possible affect the need of Oracle. + * */ + self.getQuorumVerifier().updateNeedOracle(new ArrayList<>(forwardingFollowers)); } } @@ -757,7 +761,27 @@ void lead() throws IOException, InterruptedException { break; } - if (!tickSkip && !syncedAckSet.hasAllQuorums()) { + /* + * + * We will need to re-validate the outstandingProposal to maintain the progress of ZooKeeper. + * It is likely a proposal is waiting for enough ACKs to be committed. The proposals are sent out, but the + * only follower goes away which makes the proposals will not be committed until the follower recovers back. + * An earlier proposal which is not committed will block any further proposals. So, We need to re-validate those + * outstanding proposal with the help from Oracle. A key point in the process of re-validation is that the proposals + * need to be processed in order. + * + * We make the whole method blocking to avoid any possible race condition on outstandingProposal and lastCommitted + * as well as to avoid nested synchronization. + * + * As a more generic approach, we pass the object of forwardingFollowers to QuorumOracleMaj to determine if we need + * the help from Oracle. + * + * + * the size of outstandingProposals can be 1. The only one outstanding proposal is the one waiting for the ACK from + * the leader itself. + * */ + if (!tickSkip && !syncedAckSet.hasAllQuorums() + && !(self.getQuorumVerifier().overrideQuorumDecision(getForwardingFollowers()) && self.getQuorumVerifier().revalidateOutstandingProp(this, new ArrayList<>(outstandingProposals.values()), lastCommitted))) { // Lost quorum of last committed and/or last proposed // config, set shutdown flag shutdownMessage = "Not sufficient followers synced, only synced with sids: [ " @@ -909,10 +933,10 @@ public synchronized boolean tryToCommit(Proposal p, long zxid, SocketAddress fol // commit proposals in order if (zxid != lastCommitted + 1) { LOG.warn( - "Commiting zxid 0x{} from {} noy first!", + "Commiting zxid 0x{} from {} not first!", Long.toHexString(zxid), followerAddr); - LOG.warn("First is {}", (lastCommitted + 1)); + LOG.warn("First is 0x{}", Long.toHexString(lastCommitted + 1)); } outstandingProposals.remove(zxid); diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Learner.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Learner.java index ef36b2f9000..8f3d00c77ea 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Learner.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Learner.java @@ -561,7 +561,13 @@ protected void syncWithLeader(long newLeaderZxid) throws Exception { if (qp.getType() == Leader.DIFF) { LOG.info("Getting a diff from the leader 0x{}", Long.toHexString(qp.getZxid())); self.setSyncMode(QuorumPeer.SyncMode.DIFF); - snapshotNeeded = false; + if (zk.shouldForceWriteInitialSnapshotAfterLeaderElection()) { + LOG.info("Forcing a snapshot write as part of upgrading from an older Zookeeper. This should only happen while upgrading."); + snapshotNeeded = true; + syncSnapshot = true; + } else { + snapshotNeeded = false; + } } else if (qp.getType() == Leader.SNAP) { self.setSyncMode(QuorumPeer.SyncMode.SNAP); LOG.info("Getting a snapshot from leader 0x{}", Long.toHexString(qp.getZxid())); diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumCnxManager.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumCnxManager.java index 0987ad30518..2f240e9bf95 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumCnxManager.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumCnxManager.java @@ -768,7 +768,7 @@ synchronized void connectOne(long sid) { if (lastSeenQV != null && lastProposedView.containsKey(sid) && (!knownId - || (lastProposedView.get(sid).electionAddr != lastCommittedView.get(sid).electionAddr))) { + || !lastProposedView.get(sid).electionAddr.equals(lastCommittedView.get(sid).electionAddr))) { knownId = true; LOG.debug("Server {} knows {} already, it is in the lastProposedView", self.getId(), sid); diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java index 3102c6379ed..19aae086027 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java @@ -78,6 +78,7 @@ import org.apache.zookeeper.server.quorum.auth.SaslQuorumAuthLearner; import org.apache.zookeeper.server.quorum.auth.SaslQuorumAuthServer; import org.apache.zookeeper.server.quorum.flexible.QuorumMaj; +import org.apache.zookeeper.server.quorum.flexible.QuorumOracleMaj; import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; import org.apache.zookeeper.server.util.ConfigUtils; import org.apache.zookeeper.server.util.JvmPauseMonitor; @@ -1254,6 +1255,22 @@ public QuorumPeer(Map quorumPeers, File snapDir, File logDir new QuorumMaj(quorumPeers)); } + public QuorumPeer(Map quorumPeers, File snapDir, File logDir, int clientPort, int electionAlg, long myid, int tickTime, int initLimit, int syncLimit, int connectToLearnerMasterLimit, String oraclePath) throws IOException { + this( + quorumPeers, + snapDir, + logDir, + electionAlg, + myid, + tickTime, + initLimit, + syncLimit, + connectToLearnerMasterLimit, + false, + ServerCnxnFactory.createFactory(getClientAddress(quorumPeers, myid, clientPort), -1), + new QuorumOracleMaj(quorumPeers, oraclePath)); + } + /** * This constructor is only used by the existing unit test code. * It defaults to FileLogProvider persistence provider. @@ -1808,7 +1825,7 @@ public int getTick() { public QuorumVerifier configFromString(String s) throws IOException, ConfigException { Properties props = new Properties(); props.load(new StringReader(s)); - return QuorumPeerConfig.parseDynamicConfig(props, electionType, false, false); + return QuorumPeerConfig.parseDynamicConfig(props, electionType, false, false, getQuorumVerifier().getOraclePath()); } /** diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java index 7a7318065e0..1b37f291f51 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeerConfig.java @@ -52,6 +52,7 @@ import org.apache.zookeeper.server.quorum.auth.QuorumAuth; import org.apache.zookeeper.server.quorum.flexible.QuorumHierarchical; import org.apache.zookeeper.server.quorum.flexible.QuorumMaj; +import org.apache.zookeeper.server.quorum.flexible.QuorumOracleMaj; import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier; import org.apache.zookeeper.server.util.JvmPauseMonitor; import org.apache.zookeeper.server.util.VerifyingFileFactory; @@ -130,6 +131,8 @@ public class QuorumPeerConfig { Integer.parseInt(System.getProperty(QuorumPeer.CONFIG_KEY_MULTI_ADDRESS_REACHABILITY_CHECK_TIMEOUT_MS, String.valueOf(MultipleAddresses.DEFAULT_TIMEOUT.toMillis()))); + protected String oraclePath; + /** * Minimum snapshot retain count. * @see org.apache.zookeeper.server.PurgeTxnLog#purge(File, File, int) @@ -180,12 +183,9 @@ public void parse(String path) throws ConfigException { .build()).create(path); Properties cfg = new Properties(); - FileInputStream in = new FileInputStream(configFile); - try { + try (FileInputStream in = new FileInputStream(configFile)) { cfg.load(in); configFileStr = path; - } finally { - in.close(); } /* Read entire config file as initial configuration */ @@ -201,8 +201,7 @@ public void parse(String path) throws ConfigException { if (dynamicConfigFileStr != null) { try { Properties dynamicCfg = new Properties(); - FileInputStream inConfig = new FileInputStream(dynamicConfigFileStr); - try { + try (FileInputStream inConfig = new FileInputStream(dynamicConfigFileStr)) { dynamicCfg.load(inConfig); if (dynamicCfg.getProperty("version") != null) { throw new ConfigException("dynamic file shouldn't have version inside"); @@ -214,8 +213,6 @@ public void parse(String path) throws ConfigException { if (version != null) { dynamicCfg.setProperty("version", version); } - } finally { - inConfig.close(); } setupQuorumPeerConfig(dynamicCfg, false); @@ -228,11 +225,8 @@ public void parse(String path) throws ConfigException { if (nextDynamicConfigFile.exists()) { try { Properties dynamicConfigNextCfg = new Properties(); - FileInputStream inConfigNext = new FileInputStream(nextDynamicConfigFile); - try { + try (FileInputStream inConfigNext = new FileInputStream(nextDynamicConfigFile)) { dynamicConfigNextCfg.load(inConfigNext); - } finally { - inConfigNext.close(); } boolean isHierarchical = false; for (Entry entry : dynamicConfigNextCfg.entrySet()) { @@ -389,6 +383,8 @@ public void parseProperties(Properties zkProp) throws IOException, ConfigExcepti multiAddressReachabilityCheckTimeoutMs = Integer.parseInt(value); } else if (key.equals("multiAddress.reachabilityCheckEnabled")) { multiAddressReachabilityCheckEnabled = parseBoolean(key, value); + } else if (key.equals("oraclePath")) { + oraclePath = value; } else { System.setProperty("zookeeper." + key, value); } @@ -529,18 +525,12 @@ private void backupOldConfig() throws IOException { new AtomicFileWritingIdiom(new File(configFileStr + ".bak"), new OutputStreamStatement() { @Override public void write(OutputStream output) throws IOException { - InputStream input = null; - try { - input = new FileInputStream(new File(configFileStr)); + try (InputStream input = new FileInputStream(new File(configFileStr))) { byte[] buf = new byte[1024]; int bytesRead; while ((bytesRead = input.read(buf)) > 0) { output.write(buf, 0, bytesRead); } - } finally { - if (input != null) { - input.close(); - } } } }); @@ -597,11 +587,8 @@ public static void editStaticConfig(final String configFileStr, final String dyn .build()).create(dynamicFileStr); final Properties cfg = new Properties(); - FileInputStream in = new FileInputStream(configFile); - try { + try (FileInputStream in = new FileInputStream(configFile)) { cfg.load(in); - } finally { - in.close(); } new AtomicFileWritingIdiom(new File(configFileStr), new WriterStatement() { @@ -647,6 +634,15 @@ public static void deleteFile(String filename) { } } + + private static QuorumVerifier createQuorumVerifier(Properties dynamicConfigProp, boolean isHierarchical, String oraclePath) throws ConfigException { + if (oraclePath == null) { + return createQuorumVerifier(dynamicConfigProp, isHierarchical); + } else { + return new QuorumOracleMaj(dynamicConfigProp, oraclePath); + } + } + private static QuorumVerifier createQuorumVerifier(Properties dynamicConfigProp, boolean isHierarchical) throws ConfigException { if (isHierarchical) { return new QuorumHierarchical(dynamicConfigProp); @@ -660,7 +656,7 @@ private static QuorumVerifier createQuorumVerifier(Properties dynamicConfigProp, } void setupQuorumPeerConfig(Properties prop, boolean configBackwardCompatibilityMode) throws IOException, ConfigException { - quorumVerifier = parseDynamicConfig(prop, electionAlg, true, configBackwardCompatibilityMode); + quorumVerifier = parseDynamicConfig(prop, electionAlg, true, configBackwardCompatibilityMode, oraclePath); setupMyId(); setupClientPort(); setupPeerType(); @@ -674,7 +670,7 @@ void setupQuorumPeerConfig(Properties prop, boolean configBackwardCompatibilityM * @throws IOException * @throws ConfigException */ - public static QuorumVerifier parseDynamicConfig(Properties dynamicConfigProp, int eAlg, boolean warnings, boolean configBackwardCompatibilityMode) throws IOException, ConfigException { + public static QuorumVerifier parseDynamicConfig(Properties dynamicConfigProp, int eAlg, boolean warnings, boolean configBackwardCompatibilityMode, String oraclePath) throws IOException, ConfigException { boolean isHierarchical = false; for (Entry entry : dynamicConfigProp.entrySet()) { String key = entry.getKey().toString().trim(); @@ -686,7 +682,7 @@ public static QuorumVerifier parseDynamicConfig(Properties dynamicConfigProp, in } } - QuorumVerifier qv = createQuorumVerifier(dynamicConfigProp, isHierarchical); + QuorumVerifier qv = createQuorumVerifier(dynamicConfigProp, isHierarchical, oraclePath); int numParticipators = qv.getVotingMembers().size(); int numObservers = qv.getObservingMembers().size(); diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/RemotePeerBean.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/RemotePeerBean.java index 2d9d7d3dd65..522ba3ad345 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/RemotePeerBean.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/RemotePeerBean.java @@ -18,7 +18,9 @@ package org.apache.zookeeper.server.quorum; +import static org.apache.zookeeper.common.NetUtils.formatInetAddr; import java.util.stream.Collectors; +import org.apache.zookeeper.common.NetUtils; import org.apache.zookeeper.jmx.ZKMBeanInfo; /** @@ -47,22 +49,20 @@ public boolean isHidden() { } public String getQuorumAddress() { - return peer.addr.getAllAddresses().stream() - .map(address -> String.format("%s:%d", address.getHostString(), address.getPort())) - .collect(Collectors.joining("|")); + return peer.addr.getAllAddresses().stream().map(NetUtils::formatInetAddr) + .collect(Collectors.joining("|")); } public String getElectionAddress() { - return peer.electionAddr.getAllAddresses().stream() - .map(address -> String.format("%s:%d", address.getHostString(), address.getPort())) - .collect(Collectors.joining("|")); + return peer.electionAddr.getAllAddresses().stream().map(NetUtils::formatInetAddr) + .collect(Collectors.joining("|")); } public String getClientAddress() { if (null == peer.clientAddr) { return ""; } - return peer.clientAddr.getHostString() + ":" + peer.clientAddr.getPort(); + return formatInetAddr(peer.clientAddr); } public String getLearnerType() { diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/flexible/QuorumHierarchical.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/flexible/QuorumHierarchical.java index ced966f58c2..5376ae6c520 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/flexible/QuorumHierarchical.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/flexible/QuorumHierarchical.java @@ -120,6 +120,7 @@ public boolean equals(Object o) { } return true; } + /** * This constructor requires the quorum configuration * to be declared in a separate file, and it takes the diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/flexible/QuorumMaj.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/flexible/QuorumMaj.java index 6e6f1c2887d..ed38533ec42 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/flexible/QuorumMaj.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/flexible/QuorumMaj.java @@ -26,6 +26,8 @@ import org.apache.zookeeper.server.quorum.QuorumPeer.LearnerType; import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer; import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class implements a validator for majority quorums. The implementation is @@ -34,11 +36,13 @@ */ public class QuorumMaj implements QuorumVerifier { + private static final Logger LOG = LoggerFactory.getLogger(QuorumMaj.class); + private Map allMembers = new HashMap(); private Map votingMembers = new HashMap(); private Map observingMembers = new HashMap(); private long version = 0; - private int half; + protected int half; public int hashCode() { assert false : "hashCode not designed"; diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/flexible/QuorumOracleMaj.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/flexible/QuorumOracleMaj.java new file mode 100644 index 00000000000..0845b802979 --- /dev/null +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/flexible/QuorumOracleMaj.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server.quorum.flexible; + +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.commons.io.FilenameUtils; +import org.apache.zookeeper.server.quorum.Leader; +import org.apache.zookeeper.server.quorum.LearnerHandler; +import org.apache.zookeeper.server.quorum.QuorumPeer; +import org.apache.zookeeper.server.quorum.QuorumPeerConfig; +import org.apache.zookeeper.server.quorum.SyncedLearnerTracker; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/* + * + * QuorumOracleMaj is a subclass of QuorumMaj. + * + * QuorumOracleMaj is designed to be functional in a 2-nodes configuration. The only method that this class overrides super + * class' method is containsQuorum(). Besides the check of oracle, it also checks the number of voting member. Whenever the + * number of voting members is greater than 2. QuorumOracleMaj shall function as hook to its super class. + * */ +public class QuorumOracleMaj extends QuorumMaj { + private static final Logger LOG = LoggerFactory.getLogger(QuorumOracleMaj.class); + + private String oracle = null; + + private final AtomicBoolean needOracle = new AtomicBoolean(true); + + public QuorumOracleMaj(Map allMembers, String oraclePath) { + super(allMembers); + setOracle(oraclePath); + } + + public QuorumOracleMaj(Properties props, String oraclePath) throws QuorumPeerConfig.ConfigException { + super(props); + setOracle(oraclePath); + } + + private void setOracle(String path) { + if (oracle == null) { + oracle = path; + LOG.info("Oracle is set to {}", path); + } else { + LOG.warn("Oracle is already set. Ignore:{}", path); + } + } + + @Override + public boolean updateNeedOracle(List forwardingFollowers) { + // Do we have the quorum + needOracle.set(forwardingFollowers.isEmpty() && super.getVotingMembers().size() == 2); + return needOracle.get(); + } + + @Override + public boolean askOracle() { + FileReader fr = null; + try { + int read; + fr = new FileReader(FilenameUtils.getFullPath(oracle) + FilenameUtils.getName(oracle)); + read = fr.read(); + LOG.debug("Oracle says:{}", (char) read); + fr.close(); + return (char) read == '1'; + } catch (Exception e) { + e.printStackTrace(); + if (oracle == null) { + LOG.error("Oracle is not set, return false"); + } + return false; + } finally { + if (fr != null) { + try { + fr.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + } + + @Override + public boolean getNeedOracle() { + return needOracle.get(); + } + + @Override + public String getOraclePath() { + return oracle; + } + + @Override + public boolean overrideQuorumDecision(List forwardingFollowers) { + if (updateNeedOracle(forwardingFollowers) && askOracle()) { + return true; + } else { + return false; + } + } + + @Override + public boolean revalidateOutstandingProp(Leader self, ArrayList outstandingProposal, long lastCommitted) { + LOG.debug("Start Revalidation outstandingProposals"); + try { + while (outstandingProposal.size() >= 1) { + outstandingProposal.sort((o1, o2) -> (int) (o1.packet.getZxid() - o2.packet.getZxid())); + + Leader.Proposal p; + int i = 0; + while (i < outstandingProposal.size()) { + p = outstandingProposal.get(i); + if (p.request.zxid > lastCommitted) { + LOG.debug("Re-validate outstanding proposal: 0x{} size:{} lastCommitted:{}", Long.toHexString(p.request.zxid), outstandingProposal.size(), Long.toHexString(lastCommitted)); + if (!self.tryToCommit(p, p.request.zxid, null)) { + break; + } else { + lastCommitted = p.request.zxid; + outstandingProposal.remove(p); + } + } + } + } + } catch (Exception e) { + e.printStackTrace(); + return false; + } + + LOG.debug("Finish Revalidation outstandingProposals"); + return true; + } + + @Override + public boolean revalidateVoteset(SyncedLearnerTracker voteSet, boolean timeout) { + return voteSet != null && voteSet.hasAllQuorums() && timeout; + } + + @Override + public boolean containsQuorum(Set ackSet) { + if (oracle == null || getVotingMembers().size() > 2) { + return super.containsQuorum(ackSet); + } else if (!super.containsQuorum(ackSet)) { + if (getNeedOracle()) { + LOG.debug("We lose the quorum, but we do not have any valid followers Oracle:{}", askOracle()); + return askOracle(); + } else { + return false; + } + } else { + return true; + } + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + QuorumOracleMaj qm = (QuorumOracleMaj) o; + if (qm.getVersion() == super.getVersion()) { + return true; + } + if (super.getAllMembers().size() != qm.getAllMembers().size()) { + return false; + } + for (QuorumPeer.QuorumServer qs : super.getAllMembers().values()) { + QuorumPeer.QuorumServer qso = qm.getAllMembers().get(qs.id); + if (qso == null || !qs.equals(qso)) { + return false; + } + } + return true; + } + + @Override + public int hashCode() { + assert false : "hashCode not designed"; + return 43; // any arbitrary constant will do + } +} + diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/flexible/QuorumVerifier.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/flexible/QuorumVerifier.java index 12d84890f06..7362313552c 100644 --- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/flexible/QuorumVerifier.java +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/flexible/QuorumVerifier.java @@ -18,9 +18,14 @@ package org.apache.zookeeper.server.quorum.flexible; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.zookeeper.server.quorum.Leader; +import org.apache.zookeeper.server.quorum.LearnerHandler; import org.apache.zookeeper.server.quorum.QuorumPeer.QuorumServer; +import org.apache.zookeeper.server.quorum.SyncedLearnerTracker; /** * All quorum validators have to implement a method called @@ -39,6 +44,37 @@ public interface QuorumVerifier { Map getVotingMembers(); Map getObservingMembers(); boolean equals(Object o); + /* + * Only QuorumOracleMaj will implement these methods. Other class will raise warning if the methods are called and + * return false always. + * */ + default boolean updateNeedOracle(List forwardingFollowers) { + return false; + } + default boolean getNeedOracle() { + return false; + } + + default boolean askOracle() { + return false; + } + + default boolean overrideQuorumDecision(List forwardingFollowers) { + return false; + } + + default boolean revalidateOutstandingProp(Leader self, ArrayList outstandingProposal, long lastCommitted) { + return false; + } + + default boolean revalidateVoteset(SyncedLearnerTracker voteSet, boolean timeout) { + return false; + } + + default String getOraclePath() { + return null; + }; + String toString(); } diff --git a/zookeeper-server/src/main/resources/lib/jetty-client-9.4.35.v20201120.LICENSE.txt b/zookeeper-server/src/main/resources/lib/jetty-http-9.4.38.v20210224.LICENSE.txt similarity index 100% rename from zookeeper-server/src/main/resources/lib/jetty-client-9.4.35.v20201120.LICENSE.txt rename to zookeeper-server/src/main/resources/lib/jetty-http-9.4.38.v20210224.LICENSE.txt diff --git a/zookeeper-server/src/main/resources/lib/jetty-http-9.4.35.v20201120.LICENSE.txt b/zookeeper-server/src/main/resources/lib/jetty-io-9.4.38.v20210224.LICENSE.txt similarity index 100% rename from zookeeper-server/src/main/resources/lib/jetty-http-9.4.35.v20201120.LICENSE.txt rename to zookeeper-server/src/main/resources/lib/jetty-io-9.4.38.v20210224.LICENSE.txt diff --git a/zookeeper-server/src/main/resources/lib/jetty-io-9.4.35.v20201120.LICENSE.txt b/zookeeper-server/src/main/resources/lib/jetty-security-9.4.38.v20210224.LICENSE.txt similarity index 100% rename from zookeeper-server/src/main/resources/lib/jetty-io-9.4.35.v20201120.LICENSE.txt rename to zookeeper-server/src/main/resources/lib/jetty-security-9.4.38.v20210224.LICENSE.txt diff --git a/zookeeper-server/src/main/resources/lib/jetty-security-9.4.35.v20201120.LICENSE.txt b/zookeeper-server/src/main/resources/lib/jetty-server-9.4.38.v20210224.LICENSE.txt similarity index 100% rename from zookeeper-server/src/main/resources/lib/jetty-security-9.4.35.v20201120.LICENSE.txt rename to zookeeper-server/src/main/resources/lib/jetty-server-9.4.38.v20210224.LICENSE.txt diff --git a/zookeeper-server/src/main/resources/lib/jetty-server-9.4.35.v20201120.LICENSE.txt b/zookeeper-server/src/main/resources/lib/jetty-servlet-9.4.38.v20210224.LICENSE.txt similarity index 100% rename from zookeeper-server/src/main/resources/lib/jetty-server-9.4.35.v20201120.LICENSE.txt rename to zookeeper-server/src/main/resources/lib/jetty-servlet-9.4.38.v20210224.LICENSE.txt diff --git a/zookeeper-server/src/main/resources/lib/jetty-servlet-9.4.35.v20201120.LICENSE.txt b/zookeeper-server/src/main/resources/lib/jetty-util-9.4.38.v20210224.LICENSE.txt similarity index 100% rename from zookeeper-server/src/main/resources/lib/jetty-servlet-9.4.35.v20201120.LICENSE.txt rename to zookeeper-server/src/main/resources/lib/jetty-util-9.4.38.v20210224.LICENSE.txt diff --git a/zookeeper-server/src/main/resources/lib/jetty-util-ajax-9.4.35.v20201120.LICENSE.txt b/zookeeper-server/src/main/resources/lib/jetty-util-ajax-9.4.35.v20201120.LICENSE.txt deleted file mode 100644 index 46f4f252464..00000000000 --- a/zookeeper-server/src/main/resources/lib/jetty-util-ajax-9.4.35.v20201120.LICENSE.txt +++ /dev/null @@ -1,414 +0,0 @@ -This program and the accompanying materials are made available under the -terms of the Eclipse Public License 2.0 which is available at -http://www.eclipse.org/legal/epl-2.0, or the Apache Software License -2.0 which is available at https://www.apache.org/licenses/LICENSE-2.0. - - - -Eclipse Public License - v 1.0 - -THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC -LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM -CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. - -1. DEFINITIONS - -"Contribution" means: - -a) in the case of the initial Contributor, the initial code and documentation - distributed under this Agreement, and -b) in the case of each subsequent Contributor: - i) changes to the Program, and - ii) additions to the Program; - - where such changes and/or additions to the Program originate from and are - distributed by that particular Contributor. A Contribution 'originates' - from a Contributor if it was added to the Program by such Contributor - itself or anyone acting on such Contributor's behalf. Contributions do not - include additions to the Program which: (i) are separate modules of - software distributed in conjunction with the Program under their own - license agreement, and (ii) are not derivative works of the Program. - -"Contributor" means any person or entity that distributes the Program. - -"Licensed Patents" mean patent claims licensable by a Contributor which are -necessarily infringed by the use or sale of its Contribution alone or when -combined with the Program. - -"Program" means the Contributions distributed in accordance with this -Agreement. - -"Recipient" means anyone who receives the Program under this Agreement, -including all Contributors. - -2. GRANT OF RIGHTS - a) Subject to the terms of this Agreement, each Contributor hereby grants - Recipient a non-exclusive, worldwide, royalty-free copyright license to - reproduce, prepare derivative works of, publicly display, publicly - perform, distribute and sublicense the Contribution of such Contributor, - if any, and such derivative works, in source code and object code form. - b) Subject to the terms of this Agreement, each Contributor hereby grants - Recipient a non-exclusive, worldwide, royalty-free patent license under - Licensed Patents to make, use, sell, offer to sell, import and otherwise - transfer the Contribution of such Contributor, if any, in source code and - object code form. This patent license shall apply to the combination of - the Contribution and the Program if, at the time the Contribution is - added by the Contributor, such addition of the Contribution causes such - combination to be covered by the Licensed Patents. The patent license - shall not apply to any other combinations which include the Contribution. - No hardware per se is licensed hereunder. - c) Recipient understands that although each Contributor grants the licenses - to its Contributions set forth herein, no assurances are provided by any - Contributor that the Program does not infringe the patent or other - intellectual property rights of any other entity. Each Contributor - disclaims any liability to Recipient for claims brought by any other - entity based on infringement of intellectual property rights or - otherwise. As a condition to exercising the rights and licenses granted - hereunder, each Recipient hereby assumes sole responsibility to secure - any other intellectual property rights needed, if any. For example, if a - third party patent license is required to allow Recipient to distribute - the Program, it is Recipient's responsibility to acquire that license - before distributing the Program. - d) Each Contributor represents that to its knowledge it has sufficient - copyright rights in its Contribution, if any, to grant the copyright - license set forth in this Agreement. - -3. REQUIREMENTS - -A Contributor may choose to distribute the Program in object code form under -its own license agreement, provided that: - - a) it complies with the terms and conditions of this Agreement; and - b) its license agreement: - i) effectively disclaims on behalf of all Contributors all warranties - and conditions, express and implied, including warranties or - conditions of title and non-infringement, and implied warranties or - conditions of merchantability and fitness for a particular purpose; - ii) effectively excludes on behalf of all Contributors all liability for - damages, including direct, indirect, special, incidental and - consequential damages, such as lost profits; - iii) states that any provisions which differ from this Agreement are - offered by that Contributor alone and not by any other party; and - iv) states that source code for the Program is available from such - Contributor, and informs licensees how to obtain it in a reasonable - manner on or through a medium customarily used for software exchange. - -When the Program is made available in source code form: - - a) it must be made available under this Agreement; and - b) a copy of this Agreement must be included with each copy of the Program. - Contributors may not remove or alter any copyright notices contained - within the Program. - -Each Contributor must identify itself as the originator of its Contribution, -if -any, in a manner that reasonably allows subsequent Recipients to identify the -originator of the Contribution. - -4. COMMERCIAL DISTRIBUTION - -Commercial distributors of software may accept certain responsibilities with -respect to end users, business partners and the like. While this license is -intended to facilitate the commercial use of the Program, the Contributor who -includes the Program in a commercial product offering should do so in a manner -which does not create potential liability for other Contributors. Therefore, -if a Contributor includes the Program in a commercial product offering, such -Contributor ("Commercial Contributor") hereby agrees to defend and indemnify -every other Contributor ("Indemnified Contributor") against any losses, -damages and costs (collectively "Losses") arising from claims, lawsuits and -other legal actions brought by a third party against the Indemnified -Contributor to the extent caused by the acts or omissions of such Commercial -Contributor in connection with its distribution of the Program in a commercial -product offering. The obligations in this section do not apply to any claims -or Losses relating to any actual or alleged intellectual property -infringement. In order to qualify, an Indemnified Contributor must: -a) promptly notify the Commercial Contributor in writing of such claim, and -b) allow the Commercial Contributor to control, and cooperate with the -Commercial Contributor in, the defense and any related settlement -negotiations. The Indemnified Contributor may participate in any such claim at -its own expense. - -For example, a Contributor might include the Program in a commercial product -offering, Product X. That Contributor is then a Commercial Contributor. If -that Commercial Contributor then makes performance claims, or offers -warranties related to Product X, those performance claims and warranties are -such Commercial Contributor's responsibility alone. Under this section, the -Commercial Contributor would have to defend claims against the other -Contributors related to those performance claims and warranties, and if a -court requires any other Contributor to pay any damages as a result, the -Commercial Contributor must pay those damages. - -5. NO WARRANTY - -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR -IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, -NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each -Recipient is solely responsible for determining the appropriateness of using -and distributing the Program and assumes all risks associated with its -exercise of rights under this Agreement , including but not limited to the -risks and costs of program errors, compliance with applicable laws, damage to -or loss of data, programs or equipment, and unavailability or interruption of -operations. - -6. DISCLAIMER OF LIABILITY - -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY -CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION -LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE -EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY -OF SUCH DAMAGES. - -7. GENERAL - -If any provision of this Agreement is invalid or unenforceable under -applicable law, it shall not affect the validity or enforceability of the -remainder of the terms of this Agreement, and without further action by the -parties hereto, such provision shall be reformed to the minimum extent -necessary to make such provision valid and enforceable. - -If Recipient institutes patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Program itself -(excluding combinations of the Program with other software or hardware) -infringes such Recipient's patent(s), then such Recipient's rights granted -under Section 2(b) shall terminate as of the date such litigation is filed. - -All Recipient's rights under this Agreement shall terminate if it fails to -comply with any of the material terms or conditions of this Agreement and does -not cure such failure in a reasonable period of time after becoming aware of -such noncompliance. If all Recipient's rights under this Agreement terminate, -Recipient agrees to cease use and distribution of the Program as soon as -reasonably practicable. However, Recipient's obligations under this Agreement -and any licenses granted by Recipient relating to the Program shall continue -and survive. - -Everyone is permitted to copy and distribute copies of this Agreement, but in -order to avoid inconsistency the Agreement is copyrighted and may only be -modified in the following manner. The Agreement Steward reserves the right to -publish new versions (including revisions) of this Agreement from time to -time. No one other than the Agreement Steward has the right to modify this -Agreement. The Eclipse Foundation is the initial Agreement Steward. The -Eclipse Foundation may assign the responsibility to serve as the Agreement -Steward to a suitable separate entity. Each new version of the Agreement will -be given a distinguishing version number. The Program (including -Contributions) may always be distributed subject to the version of the -Agreement under which it was received. In addition, after a new version of the -Agreement is published, Contributor may elect to distribute the Program -(including its Contributions) under the new version. Except as expressly -stated in Sections 2(a) and 2(b) above, Recipient receives no rights or -licenses to the intellectual property of any Contributor under this Agreement, -whether expressly, by implication, estoppel or otherwise. All rights in the -Program not expressly granted under this Agreement are reserved. - -This Agreement is governed by the laws of the State of New York and the -intellectual property laws of the United States of America. No party to this -Agreement will bring a legal action under this Agreement more than one year -after the cause of action arose. Each party waives its rights to a jury trial in -any resulting litigation. - - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/zookeeper-server/src/main/resources/lib/jetty-util-9.4.35.v20201120.LICENSE.txt b/zookeeper-server/src/main/resources/lib/jetty-util-ajax-9.4.38.v20210224.LICENSE.txt similarity index 100% rename from zookeeper-server/src/main/resources/lib/jetty-util-9.4.35.v20201120.LICENSE.txt rename to zookeeper-server/src/main/resources/lib/jetty-util-ajax-9.4.38.v20210224.LICENSE.txt diff --git a/zookeeper-server/src/main/resources/lib/netty-buffer-4.1.50.Final.LICENSE.txt b/zookeeper-server/src/main/resources/lib/netty-buffer-4.1.59.Final.LICENSE.txt similarity index 100% rename from zookeeper-server/src/main/resources/lib/netty-buffer-4.1.50.Final.LICENSE.txt rename to zookeeper-server/src/main/resources/lib/netty-buffer-4.1.59.Final.LICENSE.txt diff --git a/zookeeper-server/src/main/resources/lib/netty-codec-4.1.50.Final.LICENSE.txt b/zookeeper-server/src/main/resources/lib/netty-codec-4.1.59.Final.LICENSE.txt similarity index 100% rename from zookeeper-server/src/main/resources/lib/netty-codec-4.1.50.Final.LICENSE.txt rename to zookeeper-server/src/main/resources/lib/netty-codec-4.1.59.Final.LICENSE.txt diff --git a/zookeeper-server/src/main/resources/lib/netty-common-4.1.50.Final.LICENSE.txt b/zookeeper-server/src/main/resources/lib/netty-common-4.1.59.Final.LICENSE.txt similarity index 100% rename from zookeeper-server/src/main/resources/lib/netty-common-4.1.50.Final.LICENSE.txt rename to zookeeper-server/src/main/resources/lib/netty-common-4.1.59.Final.LICENSE.txt diff --git a/zookeeper-server/src/main/resources/lib/netty-handler-4.1.50.Final.LICENSE.txt b/zookeeper-server/src/main/resources/lib/netty-handler-4.1.59.Final.LICENSE.txt similarity index 100% rename from zookeeper-server/src/main/resources/lib/netty-handler-4.1.50.Final.LICENSE.txt rename to zookeeper-server/src/main/resources/lib/netty-handler-4.1.59.Final.LICENSE.txt diff --git a/zookeeper-server/src/main/resources/lib/netty-resolver-4.1.50.Final.LICENSE.txt b/zookeeper-server/src/main/resources/lib/netty-resolver-4.1.59.Final.LICENSE.txt similarity index 100% rename from zookeeper-server/src/main/resources/lib/netty-resolver-4.1.50.Final.LICENSE.txt rename to zookeeper-server/src/main/resources/lib/netty-resolver-4.1.59.Final.LICENSE.txt diff --git a/zookeeper-server/src/main/resources/lib/netty-transport-4.1.50.Final.LICENSE.txt b/zookeeper-server/src/main/resources/lib/netty-transport-4.1.59.Final.LICENSE.txt similarity index 100% rename from zookeeper-server/src/main/resources/lib/netty-transport-4.1.50.Final.LICENSE.txt rename to zookeeper-server/src/main/resources/lib/netty-transport-4.1.59.Final.LICENSE.txt diff --git a/zookeeper-server/src/main/resources/lib/netty-transport-native-epoll-4.1.50.Final.LICENSE.txt b/zookeeper-server/src/main/resources/lib/netty-transport-native-epoll-4.1.59.Final.LICENSE.txt similarity index 100% rename from zookeeper-server/src/main/resources/lib/netty-transport-native-epoll-4.1.50.Final.LICENSE.txt rename to zookeeper-server/src/main/resources/lib/netty-transport-native-epoll-4.1.59.Final.LICENSE.txt diff --git a/zookeeper-server/src/main/resources/lib/netty-transport-native-unix-common-4.1.50.Final.LICENSE.txt b/zookeeper-server/src/main/resources/lib/netty-transport-native-unix-common-4.1.59.Final.LICENSE.txt similarity index 100% rename from zookeeper-server/src/main/resources/lib/netty-transport-native-unix-common-4.1.50.Final.LICENSE.txt rename to zookeeper-server/src/main/resources/lib/netty-transport-native-unix-common-4.1.59.Final.LICENSE.txt diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/ZooKeeperTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/ZooKeeperTest.java index b726c47da06..f69e0c683fb 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/ZooKeeperTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/ZooKeeperTest.java @@ -774,4 +774,25 @@ private void assertClientAuthInfo(List expected, String actual) { }); } + @Test + public void testWaitForConnection() throws Exception { + // get a wrong port number + int invalidPort = PortAssignment.unique(); + long timeout = 3000L; // millisecond + String[] args1 = {"-server", "localhost:" + invalidPort, "-timeout", + Long.toString(timeout), "-waitforconnection", "ls", "/"}; + long startTime = System.currentTimeMillis(); + // try to connect to a non-existing server so as to wait until wait_timeout + try { + ZooKeeperMain zkMain = new ZooKeeperMain(args1); + fail("IOException was expected"); + } catch (IOException e) { + // do nothing + } + long endTime = System.currentTimeMillis(); + assertTrue(endTime - startTime >= timeout, + "ZooKeeeperMain does not wait until the specified timeout"); + + } + } diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/X509AuthFailureTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/X509AuthFailureTest.java index 189a40f28ab..5af4f1095dc 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/X509AuthFailureTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/X509AuthFailureTest.java @@ -20,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import org.apache.zookeeper.PortAssignment; +import org.apache.zookeeper.ZKTestCase; import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.client.ZKClientConfig; import org.apache.zookeeper.common.ClientX509Util; @@ -30,7 +31,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class X509AuthFailureTest { +public class X509AuthFailureTest extends ZKTestCase { protected static final Logger LOG = LoggerFactory.getLogger(X509AuthFailureTest.class); private static ClientX509Util clientX509Util; @@ -46,6 +47,7 @@ public void setup() throws Exception{ System.setProperty(ZKClientConfig.SECURE_CLIENT, "true"); System.setProperty(clientX509Util.getSslKeystoreLocationProperty(), testDataPath + "/ssl/testKeyStore.jks"); System.setProperty(clientX509Util.getSslKeystorePasswdProperty(), "testpass"); + System.setProperty("zookeeper.admin.serverPort", "" + PortAssignment.unique()); } @AfterEach @@ -57,6 +59,7 @@ public void teardown() throws Exception { System.clearProperty(clientX509Util.getSslKeystorePasswdProperty()); System.clearProperty(clientX509Util.getSslTruststoreLocationProperty()); System.clearProperty(clientX509Util.getSslTruststorePasswdProperty()); + System.clearProperty("zookeeper.admin.serverPort"); clientX509Util.close(); } @@ -71,6 +74,8 @@ public void testSecureStandaloneServerAuthNFailure() throws Exception { ZooKeeperServerMainTest.MainThread mt = new ZooKeeperServerMainTest.MainThread(CLIENT_PORT, SECURE_CLIENT_PORT, true, null); mt.start(); + assertTrue( + ClientBase.waitForServerUp("127.0.0.1:" + CLIENT_PORT, ClientBase.CONNECTION_TIMEOUT)); try { ZooKeeper zk = createZKClnt("127.0.0.1:" + SECURE_CLIENT_PORT); @@ -91,4 +96,4 @@ private ZooKeeper createZKClnt(String cxnString) throws Exception { return zk; } -} \ No newline at end of file +} diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/EagerACLFilterTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/EagerACLFilterTest.java index d37516bd3bf..a27d5cf9f0f 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/EagerACLFilterTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/EagerACLFilterTest.java @@ -72,7 +72,7 @@ public void setUp(ServerState serverState, boolean checkEnabled) throws Exceptio ensureCheck(checkEnabled); CountdownWatcher clientWatch = new CountdownWatcher(); CountdownWatcher clientWatchB = new CountdownWatcher(); - super.setUp(true); + super.setUp(true, true); String hostPort = getPeersMatching(serverState).split(",")[0]; int clientPort = Integer.parseInt(hostPort.split(":")[1]); diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerConfigTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerConfigTest.java index b2c350e3edd..407a9d1a58b 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerConfigTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumPeerConfigTest.java @@ -203,6 +203,7 @@ public void testParseBoolean() throws IOException, ConfigException { private Properties getDefaultZKProperties() { Properties zkProp = new Properties(); zkProp.setProperty("dataDir", new File("myDataDir").getAbsolutePath()); + zkProp.setProperty("oraclePath", new File("mastership").getAbsolutePath()); return zkProp; } diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumRequestPipelineTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumRequestPipelineTest.java index 8a463c07b18..c395a62e6be 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumRequestPipelineTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/QuorumRequestPipelineTest.java @@ -36,6 +36,7 @@ import org.apache.zookeeper.server.quorum.QuorumPeer.ServerState; import org.apache.zookeeper.test.QuorumBase; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -59,9 +60,16 @@ public static Stream data() throws Exception { Arguments.of(ServerState.OBSERVING)); } + @BeforeEach + @Override + public void setUp() { + //since parameterized test methods need a parameterized setUp method + //the inherited method has to be overridden with an empty function body + } + public void setUp(ServerState serverState) throws Exception { CountdownWatcher clientWatch = new CountdownWatcher(); - super.setUp(true); + super.setUp(true, true); zkClient = createClient(clientWatch, getPeersMatching(serverState)); zkClient.addAuthInfo(AUTH_PROVIDER, AUTH); clientWatch.waitForConnected(CONNECTION_TIMEOUT); diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/RemotePeerBeanTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/RemotePeerBeanTest.java index de145ea4f53..cc2321c10f2 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/RemotePeerBeanTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/quorum/RemotePeerBeanTest.java @@ -59,4 +59,20 @@ public void testIsLeader() { assertFalse(remotePeerBean.isLeader()); } + @Test + public void testHostPortReturnedWhenIPIsIPV6() { + QuorumPeer.QuorumServer quorumServerMock = mock(QuorumPeer.QuorumServer.class); + InetSocketAddress address = new InetSocketAddress("127::1", 2181); + MultipleAddresses multipleAddresses = new MultipleAddresses(address); + quorumServerMock.clientAddr = address; + quorumServerMock.electionAddr = multipleAddresses; + quorumServerMock.addr = multipleAddresses; + QuorumPeer peerMock = mock(QuorumPeer.class); + RemotePeerBean remotePeerBean = new RemotePeerBean(peerMock, quorumServerMock); + String expectedHostPort = "[127:0:0:0:0:0:0:1]:2181"; + assertEquals(expectedHostPort, remotePeerBean.getClientAddress()); + assertEquals(expectedHostPort, remotePeerBean.getElectionAddress()); + assertEquals(expectedHostPort, remotePeerBean.getQuorumAddress()); + } + } diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/server/watch/WatcherCleanerTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/server/watch/WatcherCleanerTest.java index ec05f14dbde..9ed145aa38d 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/server/watch/WatcherCleanerTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/server/watch/WatcherCleanerTest.java @@ -162,13 +162,13 @@ public void testDeadWatcherMetrics() { assertEquals(3L, values.get("cnt_dead_watchers_cleaner_latency")); - //Each latency should be a little over 20 ms, allow 5 ms deviation - assertEquals(20D, (Double) values.get("avg_dead_watchers_cleaner_latency"), 5); - assertEquals(20D, ((Long) values.get("min_dead_watchers_cleaner_latency")).doubleValue(), 5); - assertEquals(20D, ((Long) values.get("max_dead_watchers_cleaner_latency")).doubleValue(), 5); - assertEquals(20D, ((Long) values.get("p50_dead_watchers_cleaner_latency")).doubleValue(), 5); - assertEquals(20D, ((Long) values.get("p95_dead_watchers_cleaner_latency")).doubleValue(), 5); - assertEquals(20D, ((Long) values.get("p99_dead_watchers_cleaner_latency")).doubleValue(), 5); + //Each latency should be a little over 20 ms, allow 20 ms deviation + assertEquals(20D, (Double) values.get("avg_dead_watchers_cleaner_latency"), 20); + assertEquals(20D, ((Long) values.get("min_dead_watchers_cleaner_latency")).doubleValue(), 20); + assertEquals(20D, ((Long) values.get("max_dead_watchers_cleaner_latency")).doubleValue(), 20); + assertEquals(20D, ((Long) values.get("p50_dead_watchers_cleaner_latency")).doubleValue(), 20); + assertEquals(20D, ((Long) values.get("p95_dead_watchers_cleaner_latency")).doubleValue(), 20); + assertEquals(20D, ((Long) values.get("p99_dead_watchers_cleaner_latency")).doubleValue(), 20); } } diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/AsyncHammerTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/AsyncHammerTest.java index 2276354d15e..108b21c0f09 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/AsyncHammerTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/AsyncHammerTest.java @@ -47,7 +47,7 @@ public class AsyncHammerTest extends ZKTestCase implements StringCallback, VoidC private volatile boolean bang; public void setUp(boolean withObservers) throws Exception { - qb.setUp(withObservers); + qb.setUp(withObservers, false); } protected void restart() throws Exception { diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/EmptiedSnapshotRecoveryTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/EmptiedSnapshotRecoveryTest.java index 316d7bf2221..b9020cc9adf 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/EmptiedSnapshotRecoveryTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/EmptiedSnapshotRecoveryTest.java @@ -19,11 +19,13 @@ package org.apache.zookeeper.test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import java.io.File; import java.io.IOException; import java.io.PrintWriter; +import java.nio.file.Files; import java.util.List; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.PortAssignment; @@ -143,6 +145,54 @@ public void testRestoreWithTrustedEmptySnapFiles() throws Exception { runTest(false, true); } + @Test + public void testRestoreWithTrustedEmptySnapFilesWhenFollowing() throws Exception { + QuorumUtil qu = new QuorumUtil(1); + try { + qu.startAll(); + String connString = qu.getConnectionStringForServer(1); + try (ZooKeeper zk = new ZooKeeper(connString, CONNECTION_TIMEOUT, this)) { + for (int i = 0; i < N_TRANSACTIONS; i++) { + zk.create("/node-" + i, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + } + } + int leaderIndex = qu.getLeaderServer(); + //Shut down the cluster and delete the snapshots from the followers + for (int i = 1; i <= qu.ALL; i++) { + qu.shutdown(i); + if (i != leaderIndex) { + FileTxnSnapLog txnLogFactory = qu.getPeer(i).peer.getTxnFactory(); + List snapshots = txnLogFactory.findNRecentSnapshots(10); + assertTrue(snapshots.size() > 0, "We have a snapshot to corrupt"); + for (File file : snapshots) { + Files.delete(file.toPath()); + } + assertEquals(txnLogFactory.findNRecentSnapshots(10).size(), 0); + } + } + //Start while trusting empty snapshots, verify that the followers save snapshots + System.setProperty(FileTxnSnapLog.ZOOKEEPER_SNAPSHOT_TRUST_EMPTY, "true"); + qu.start(leaderIndex); + for (int i = 1; i <= qu.ALL; i++) { + if (i != leaderIndex) { + qu.restart(i); + FileTxnSnapLog txnLogFactory = qu.getPeer(i).peer.getTxnFactory(); + List snapshots = txnLogFactory.findNRecentSnapshots(10); + assertTrue(snapshots.size() > 0, "A snapshot should have been created on follower " + i); + } + } + //Check that the created nodes are still there + try (ZooKeeper zk = new ZooKeeper(connString, CONNECTION_TIMEOUT, this)) { + for (int i = 0; i < N_TRANSACTIONS; i++) { + assertNotNull(zk.exists("/node-" + i, false)); + } + } + } finally { + System.clearProperty(FileTxnSnapLog.ZOOKEEPER_SNAPSHOT_TRUST_EMPTY); + qu.tearDown(); + } + } + public void process(WatchedEvent event) { // do nothing } diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/InvalidSnapshotTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/InvalidSnapshotTest.java index 80267054a26..c47a60f357c 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/InvalidSnapshotTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/InvalidSnapshotTest.java @@ -20,7 +20,10 @@ import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.File; +import java.io.IOException; +import org.apache.commons.io.FileUtils; import org.apache.zookeeper.PortAssignment; import org.apache.zookeeper.ZKTestCase; import org.apache.zookeeper.ZooKeeper; @@ -61,13 +64,36 @@ public void testSnapshotFormatterWithNull() throws Exception { SnapshotFormatter.main(args); } + /** + * Verify the SnapshotFormatter fails as expected on corrupted snapshot. + */ + @Test + public void testSnapshotFormatterWithInvalidSnap() throws Exception { + File snapDir = new File(testData, "invalidsnap"); + // Broken snapshot introduced by ZOOKEEPER-367, and used to + // demonstrate recovery in testSnapshot below. + File snapfile = new File(new File(snapDir, "version-2"), "snapshot.83f"); + String[] args = {snapfile.getCanonicalFile().toString()}; + try { + SnapshotFormatter.main(args); + fail("Snapshot '" + snapfile + "' unexpectedly parsed without error."); + } catch (IOException e) { + assertTrue(e.getMessage().contains("Unreasonable length = 977468229")); + } + } + /** * test the snapshot * @throws Exception an exception could be expected */ @Test public void testSnapshot() throws Exception { - File snapDir = new File(testData, "invalidsnap"); + File origSnapDir = new File(testData, "invalidsnap"); + + // This test otherwise updates the resources directory. + File snapDir = ClientBase.createTmpDir(); + FileUtils.copyDirectory(origSnapDir, snapDir); + ZooKeeperServer zks = new ZooKeeperServer(snapDir, snapDir, 3000); SyncRequestProcessor.setSnapCount(1000); final int PORT = Integer.parseInt(HOSTPORT.split(":")[1]); diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ObserverLETest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ObserverLETest.java index 014ccb4df0c..67aa5959263 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ObserverLETest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ObserverLETest.java @@ -35,7 +35,7 @@ public class ObserverLETest extends ZKTestCase { @BeforeEach public void establishThreeParticipantOneObserverEnsemble() throws Exception { - qb.setUp(true); + qb.setUp(true, false); ct.hostPort = qb.hostPort; ct.setUpAll(); qb.s5.shutdown(); diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ObserverQuorumHammerTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ObserverQuorumHammerTest.java index 00953f792f3..45e37f2da04 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ObserverQuorumHammerTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ObserverQuorumHammerTest.java @@ -28,7 +28,7 @@ public class ObserverQuorumHammerTest extends QuorumHammerTest { @BeforeEach @Override public void setUp() throws Exception { - qb.setUp(true); + qb.setUp(true, false); cht.hostPort = qb.hostPort; cht.setUpAll(); } diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/QuorumBase.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/QuorumBase.java index 02e5e0a034c..c2396a6403d 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/QuorumBase.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/QuorumBase.java @@ -22,6 +22,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import java.io.File; +import java.io.FileWriter; import java.io.IOException; import java.net.InetSocketAddress; import java.util.ArrayList; @@ -48,6 +49,13 @@ public class QuorumBase extends ClientBase { private static final String LOCALADDR = "127.0.0.1"; + private static final String oraclePath_0 = "./tmp/oraclePath/0/mastership/"; + private static final String oraclePath_1 = "./tmp/oraclePath/1/mastership/"; + private static final String oraclePath_2 = "./tmp/oraclePath/0/mastership/"; + private static final String oraclePath_3 = "./tmp/oraclePath/1/mastership/"; + private static final String oraclePath_4 = "./tmp/oraclePath/0/mastership/"; + private static final String mastership = "value"; + File s1dir, s2dir, s3dir, s4dir, s5dir; QuorumPeer s1, s2, s3, s4, s5; protected int port1; @@ -71,13 +79,14 @@ public class QuorumBase extends ClientBase { protected boolean localSessionsEnabled = false; protected boolean localSessionsUpgradingEnabled = false; + @BeforeEach @Override public void setUp() throws Exception { - setUp(false); + setUp(false, true); } - protected void setUp(boolean withObservers) throws Exception { + protected void setUp(boolean withObservers, boolean withOracle) throws Exception { LOG.info("QuorumBase.setup {}", getTestName()); setupTestEnv(); @@ -121,21 +130,54 @@ protected void setUp(boolean withObservers) throws Exception { s4dir = ClientBase.createTmpDir(); s5dir = ClientBase.createTmpDir(); - startServers(withObservers); + startServers(withObservers, withOracle); OSMXBean osMbean = new OSMXBean(); if (osMbean.getUnix()) { LOG.info("Initial fdcount is: {}", osMbean.getOpenFileDescriptorCount()); } + if (withOracle) { + File directory = new File(oraclePath_0); + directory.mkdirs(); + FileWriter fw = new FileWriter(oraclePath_0 + mastership); + fw.write("1"); + fw.close(); + + directory = new File(oraclePath_1); + directory.mkdirs(); + fw = new FileWriter(oraclePath_1 + mastership); + fw.write("0"); + fw.close(); + + directory = new File(oraclePath_2); + directory.mkdirs(); + fw = new FileWriter(oraclePath_2 + mastership); + fw.write("0"); + fw.close(); + + directory = new File(oraclePath_3); + directory.mkdirs(); + fw = new FileWriter(oraclePath_3 + mastership); + fw.write("1"); + fw.close(); + + directory = new File(oraclePath_4); + directory.mkdirs(); + fw = new FileWriter(oraclePath_4 + mastership); + fw.write("0"); + fw.close(); + } + + LOG.info("Setup finished"); } void startServers() throws Exception { - startServers(false); + startServers(false, true); } - void startServers(boolean withObservers) throws Exception { + void startServers(boolean withObservers, boolean withOracle) throws Exception { int tickTime = 2000; int initLimit = 3; int syncLimit = 3; @@ -152,21 +194,39 @@ void startServers(boolean withObservers) throws Exception { peers.get(Long.valueOf(5)).type = LearnerType.OBSERVER; } - LOG.info("creating QuorumPeer 1 port {}", portClient1); - s1 = new QuorumPeer(peers, s1dir, s1dir, portClient1, 3, 1, tickTime, initLimit, syncLimit, connectToLearnerMasterLimit); - assertEquals(portClient1, s1.getClientPort()); - LOG.info("creating QuorumPeer 2 port {}", portClient2); - s2 = new QuorumPeer(peers, s2dir, s2dir, portClient2, 3, 2, tickTime, initLimit, syncLimit, connectToLearnerMasterLimit); - assertEquals(portClient2, s2.getClientPort()); - LOG.info("creating QuorumPeer 3 port {}", portClient3); - s3 = new QuorumPeer(peers, s3dir, s3dir, portClient3, 3, 3, tickTime, initLimit, syncLimit, connectToLearnerMasterLimit); - assertEquals(portClient3, s3.getClientPort()); - LOG.info("creating QuorumPeer 4 port {}", portClient4); - s4 = new QuorumPeer(peers, s4dir, s4dir, portClient4, 3, 4, tickTime, initLimit, syncLimit, connectToLearnerMasterLimit); - assertEquals(portClient4, s4.getClientPort()); - LOG.info("creating QuorumPeer 5 port {}", portClient5); - s5 = new QuorumPeer(peers, s5dir, s5dir, portClient5, 3, 5, tickTime, initLimit, syncLimit, connectToLearnerMasterLimit); - assertEquals(portClient5, s5.getClientPort()); + if (!withOracle) { + LOG.info("creating QuorumPeer 1 port {}", portClient1); + s1 = new QuorumPeer(peers, s1dir, s1dir, portClient1, 3, 1, tickTime, initLimit, syncLimit, connectToLearnerMasterLimit); + assertEquals(portClient1, s1.getClientPort()); + LOG.info("creating QuorumPeer 2 port {}", portClient2); + s2 = new QuorumPeer(peers, s2dir, s2dir, portClient2, 3, 2, tickTime, initLimit, syncLimit, connectToLearnerMasterLimit); + assertEquals(portClient2, s2.getClientPort()); + LOG.info("creating QuorumPeer 3 port {}", portClient3); + s3 = new QuorumPeer(peers, s3dir, s3dir, portClient3, 3, 3, tickTime, initLimit, syncLimit, connectToLearnerMasterLimit); + assertEquals(portClient3, s3.getClientPort()); + LOG.info("creating QuorumPeer 4 port {}", portClient4); + s4 = new QuorumPeer(peers, s4dir, s4dir, portClient4, 3, 4, tickTime, initLimit, syncLimit, connectToLearnerMasterLimit); + assertEquals(portClient4, s4.getClientPort()); + LOG.info("creating QuorumPeer 5 port {}", portClient5); + s5 = new QuorumPeer(peers, s5dir, s5dir, portClient5, 3, 5, tickTime, initLimit, syncLimit, connectToLearnerMasterLimit); + assertEquals(portClient5, s5.getClientPort()); + } else { + LOG.info("creating QuorumPeer 1 port {}", portClient1); + s1 = new QuorumPeer(peers, s1dir, s1dir, portClient1, 3, 1, tickTime, initLimit, syncLimit, connectToLearnerMasterLimit, oraclePath_0 + mastership); + assertEquals(portClient1, s1.getClientPort()); + LOG.info("creating QuorumPeer 2 port {}", portClient2); + s2 = new QuorumPeer(peers, s2dir, s2dir, portClient2, 3, 2, tickTime, initLimit, syncLimit, connectToLearnerMasterLimit, oraclePath_1 + mastership); + assertEquals(portClient2, s2.getClientPort()); + LOG.info("creating QuorumPeer 3 port {}", portClient3); + s3 = new QuorumPeer(peers, s3dir, s3dir, portClient3, 3, 3, tickTime, initLimit, syncLimit, connectToLearnerMasterLimit, oraclePath_2 + mastership); + assertEquals(portClient3, s3.getClientPort()); + LOG.info("creating QuorumPeer 4 port {}", portClient4); + s4 = new QuorumPeer(peers, s4dir, s4dir, portClient4, 3, 4, tickTime, initLimit, syncLimit, connectToLearnerMasterLimit, oraclePath_3 + mastership); + assertEquals(portClient4, s4.getClientPort()); + LOG.info("creating QuorumPeer 5 port {}", portClient5); + s5 = new QuorumPeer(peers, s5dir, s5dir, portClient5, 3, 5, tickTime, initLimit, syncLimit, connectToLearnerMasterLimit, oraclePath_4 + mastership); + assertEquals(portClient5, s5.getClientPort()); + } if (withObservers) { s4.setLearnerType(LearnerType.OBSERVER); @@ -230,18 +290,18 @@ void startServers(boolean withObservers) throws Exception { } public int getLeaderIndex() { - if (s1.getPeerState() == ServerState.LEADING) { - return 0; - } else if (s2.getPeerState() == ServerState.LEADING) { - return 1; - } else if (s3.getPeerState() == ServerState.LEADING) { - return 2; - } else if (s4.getPeerState() == ServerState.LEADING) { - return 3; - } else if (s5.getPeerState() == ServerState.LEADING) { - return 4; - } - return -1; + if (s1.getPeerState() == ServerState.LEADING) { + return 0; + } else if (s2.getPeerState() == ServerState.LEADING) { + return 1; + } else if (s3.getPeerState() == ServerState.LEADING) { + return 2; + } else if (s4.getPeerState() == ServerState.LEADING) { + return 3; + } else if (s5.getPeerState() == ServerState.LEADING) { + return 4; + } + return -1; } public int getLeaderClientPort() { diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/QuorumBaseOracle_2Nodes.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/QuorumBaseOracle_2Nodes.java new file mode 100644 index 00000000000..482027d1a9a --- /dev/null +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/QuorumBaseOracle_2Nodes.java @@ -0,0 +1,350 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import org.apache.zookeeper.PortAssignment; +import org.apache.zookeeper.TestableZooKeeper; +import org.apache.zookeeper.server.quorum.Election; +import org.apache.zookeeper.server.quorum.QuorumPeer; +import org.apache.zookeeper.server.util.OSMXBean; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class QuorumBaseOracle_2Nodes extends ClientBase{ + + private static final Logger LOG = LoggerFactory.getLogger(QuorumBase.class); + + private static final String LOCALADDR = "127.0.0.1"; + + private static final String oraclePath_0 = "./tmp/oraclePath/0/mastership/"; + private static final String oraclePath_1 = "./tmp/oraclePath/1/mastership/"; + + private static final String mastership = "value"; + + File s1dir, s2dir; + QuorumPeer s1, s2; + protected int port1; + protected int port2; + + protected int portLE1; + protected int portLE2; + + protected int portClient1; + protected int portClient2; + + protected boolean localSessionsEnabled = false; + protected boolean localSessionsUpgradingEnabled = false; + + + + @BeforeEach + @Override + public void setUp() throws Exception { + LOG.info("QuorumBase.setup {}", getTestName()); + setupTestEnv(); + + JMXEnv.setUp(); + + setUpAll(); + + port1 = PortAssignment.unique(); + port2 = PortAssignment.unique(); + + portLE1 = PortAssignment.unique(); + portLE2 = PortAssignment.unique(); + + portClient1 = PortAssignment.unique(); + portClient2 = PortAssignment.unique(); + + hostPort = "127.0.0.1:" + + portClient1 + + ",127.0.0.1:" + + portClient2; + LOG.info("Ports are: {}", hostPort); + + s1dir = ClientBase.createTmpDir(); + s2dir = ClientBase.createTmpDir(); + + startServers(); + + OSMXBean osMbean = new OSMXBean(); + if (osMbean.getUnix()) { + LOG.info("Initial fdcount is: {}", osMbean.getOpenFileDescriptorCount()); + } + + File directory = new File(oraclePath_0); + directory.mkdirs(); + FileWriter fw = new FileWriter(oraclePath_0 + mastership); + fw.write("0"); + fw.close(); + + directory = new File(oraclePath_1); + directory.mkdirs(); + fw = new FileWriter(oraclePath_1 + mastership); + fw.write("1"); + fw.close(); + + + LOG.info("Setup finished"); + } + + void startServers() throws Exception { + int tickTime = 2000; + int initLimit = 3; + int syncLimit = 3; + int connectToLearnerMasterLimit = 3; + Map peers = new HashMap(); + peers.put(Long.valueOf(1), new QuorumPeer.QuorumServer(1, new InetSocketAddress(LOCALADDR, port1), new InetSocketAddress(LOCALADDR, portLE1), new InetSocketAddress(LOCALADDR, portClient1), QuorumPeer.LearnerType.PARTICIPANT)); + peers.put(Long.valueOf(2), new QuorumPeer.QuorumServer(2, new InetSocketAddress(LOCALADDR, port2), new InetSocketAddress(LOCALADDR, portLE2), new InetSocketAddress(LOCALADDR, portClient2), QuorumPeer.LearnerType.PARTICIPANT)); + + LOG.info("creating QuorumPeer 1 port {}", portClient1); + s1 = new QuorumPeer(peers, s1dir, s1dir, portClient1, 3, 1, tickTime, initLimit, syncLimit, connectToLearnerMasterLimit, oraclePath_0 + mastership); + assertEquals(portClient1, s1.getClientPort()); + LOG.info("creating QuorumPeer 2 port {}", portClient2); + s2 = new QuorumPeer(peers, s2dir, s2dir, portClient2, 3, 2, tickTime, initLimit, syncLimit, connectToLearnerMasterLimit, oraclePath_1 + mastership); + assertEquals(portClient2, s2.getClientPort()); + + + LOG.info("QuorumPeer 1 voting view: {}", s1.getVotingView()); + LOG.info("QuorumPeer 2 voting view: {}", s2.getVotingView()); + + s1.enableLocalSessions(localSessionsEnabled); + s2.enableLocalSessions(localSessionsEnabled); + s1.enableLocalSessionsUpgrading(localSessionsUpgradingEnabled); + s2.enableLocalSessionsUpgrading(localSessionsUpgradingEnabled); + + LOG.info("start QuorumPeer 1"); + s1.start(); + LOG.info("start QuorumPeer 2"); + s2.start(); + + LOG.info("Checking ports {}", hostPort); + for (String hp : hostPort.split(",")) { + assertTrue(ClientBase.waitForServerUp(hp, CONNECTION_TIMEOUT), "waiting for server up"); + LOG.info("{} is accepting client connections", hp); + } + + // interesting to see what's there... + JMXEnv.dump(); + // make sure we have these 5 servers listed + Set ensureNames = new LinkedHashSet(); + for (int i = 1; i <= 2; i++) { + ensureNames.add("InMemoryDataTree"); + } + for (int i = 1; i <= 2; i++) { + ensureNames.add("name0=ReplicatedServer_id" + i + ",name1=replica." + i + ",name2="); + } + for (int i = 1; i <= 2; i++) { + for (int j = 1; j <= 2; j++) { + ensureNames.add("name0=ReplicatedServer_id" + i + ",name1=replica." + j); + } + } + for (int i = 1; i <= 2; i++) { + ensureNames.add("name0=ReplicatedServer_id" + i); + } + JMXEnv.ensureAll(ensureNames.toArray(new String[ensureNames.size()])); + } + + public int getLeaderIndex() { + if (s1.getPeerState() == QuorumPeer.ServerState.LEADING) { + return 0; + } else if (s2.getPeerState() == QuorumPeer.ServerState.LEADING) { + return 1; + } + return -1; + } + + public int getLeaderClientPort() { + if (s1.getPeerState() == QuorumPeer.ServerState.LEADING) { + return portClient1; + } else if (s2.getPeerState() == QuorumPeer.ServerState.LEADING) { + return portClient2; + } + return -1; + } + + public QuorumPeer getLeaderQuorumPeer() { + if (s1.getPeerState() == QuorumPeer.ServerState.LEADING) { + return s1; + } else if (s2.getPeerState() == QuorumPeer.ServerState.LEADING) { + return s2; + } + return null; + } + + public QuorumPeer getFirstObserver() { + if (s1.getLearnerType() == QuorumPeer.LearnerType.OBSERVER) { + return s1; + } else if (s2.getLearnerType() == QuorumPeer.LearnerType.OBSERVER) { + return s2; + } + return null; + } + + public int getFirstObserverClientPort() { + if (s1.getLearnerType() == QuorumPeer.LearnerType.OBSERVER) { + return portClient1; + } else if (s2.getLearnerType() == QuorumPeer.LearnerType.OBSERVER) { + return portClient2; + } + return -1; + } + + public String getPeersMatching(QuorumPeer.ServerState state) { + StringBuilder hosts = new StringBuilder(); + for (QuorumPeer p : getPeerList()) { + if (p.getPeerState() == state) { + hosts.append(String.format("%s:%d,", LOCALADDR, p.getClientAddress().getPort())); + } + } + LOG.info("getPeersMatching ports are {}", hosts); + return hosts.toString(); + } + + public ArrayList getPeerList() { + ArrayList peers = new ArrayList(); + peers.add(s1); + peers.add(s2); + return peers; + } + + public QuorumPeer getPeerByClientPort(int clientPort) { + for (QuorumPeer p : getPeerList()) { + if (p.getClientAddress().getPort() == clientPort) { + return p; + } + } + return null; + } + + public void setupServers() throws IOException { + setupServer(1); + setupServer(2); + } + + Map peers = null; + + public void setupServer(int i) throws IOException { + int tickTime = 2000; + int initLimit = 3; + int syncLimit = 3; + int connectToLearnerMasterLimit = 3; + + if (peers == null) { + peers = new HashMap(); + + peers.put(Long.valueOf(1), new QuorumPeer.QuorumServer(1, new InetSocketAddress(LOCALADDR, port1), new InetSocketAddress(LOCALADDR, portLE1), new InetSocketAddress(LOCALADDR, portClient1), QuorumPeer.LearnerType.PARTICIPANT)); + peers.put(Long.valueOf(2), new QuorumPeer.QuorumServer(2, new InetSocketAddress(LOCALADDR, port2), new InetSocketAddress(LOCALADDR, portLE2), new InetSocketAddress(LOCALADDR, portClient2), QuorumPeer.LearnerType.PARTICIPANT)); + } + + switch (i) { + case 1: + LOG.info("creating QuorumPeer 1 port {}", portClient1); + s1 = new QuorumPeer(peers, s1dir, s1dir, portClient1, 3, 1, tickTime, initLimit, syncLimit, connectToLearnerMasterLimit); + assertEquals(portClient1, s1.getClientPort()); + break; + case 2: + LOG.info("creating QuorumPeer 2 port {}", portClient2); + s2 = new QuorumPeer(peers, s2dir, s2dir, portClient2, 3, 2, tickTime, initLimit, syncLimit, connectToLearnerMasterLimit); + assertEquals(portClient2, s2.getClientPort()); + break; + } + } + + @AfterEach + @Override + public void tearDown() throws Exception { + LOG.info("TearDown started"); + + OSMXBean osMbean = new OSMXBean(); + if (osMbean.getUnix()) { + LOG.info("fdcount after test is: {}", osMbean.getOpenFileDescriptorCount()); + } + + shutdownServers(); + + for (String hp : hostPort.split(",")) { + assertTrue(ClientBase.waitForServerDown(hp, ClientBase.CONNECTION_TIMEOUT), "waiting for server down"); + LOG.info("{} is no longer accepting client connections", hp); + } + + JMXEnv.tearDown(); + } + public void shutdownServers() { + shutdown(s1); + shutdown(s2); + } + + public static void shutdown(QuorumPeer qp) { + if (qp == null) { + return; + } + try { + LOG.info("Shutting down quorum peer {}", qp.getName()); + qp.shutdown(); + Election e = qp.getElectionAlg(); + if (e != null) { + LOG.info("Shutting down leader election {}", qp.getName()); + e.shutdown(); + } else { + LOG.info("No election available to shutdown {}", qp.getName()); + } + LOG.info("Waiting for {} to exit thread", qp.getName()); + long readTimeout = qp.getTickTime() * qp.getInitLimit(); + long connectTimeout = qp.getTickTime() * qp.getSyncLimit(); + long maxTimeout = Math.max(readTimeout, connectTimeout); + maxTimeout = Math.max(maxTimeout, ClientBase.CONNECTION_TIMEOUT); + qp.join(maxTimeout * 2); + if (qp.isAlive()) { + fail("QP failed to shutdown in " + (maxTimeout * 2) + " seconds: " + qp.getName()); + } + } catch (InterruptedException e) { + LOG.debug("QP interrupted: {}", qp.getName(), e); + } + } + + protected TestableZooKeeper createClient() throws IOException, InterruptedException { + return createClient(hostPort); + } + + protected TestableZooKeeper createClient(String hp) throws IOException, InterruptedException { + ClientBase.CountdownWatcher watcher = new ClientBase.CountdownWatcher(); + return createClient(watcher, hp); + } + + protected TestableZooKeeper createClient(ClientBase.CountdownWatcher watcher, QuorumPeer.ServerState state) throws IOException, InterruptedException { + return createClient(watcher, getPeersMatching(state)); + } + +} diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/QuorumMajorityTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/QuorumMajorityTest.java index deaeb68d8ad..4761596736c 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/QuorumMajorityTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/QuorumMajorityTest.java @@ -57,7 +57,7 @@ public void testMajQuorums() throws Throwable { } //setup servers 1-5 to be followers - setUp(false); + setUp(false, true); Proposal p = new Proposal(); @@ -77,7 +77,7 @@ public void testMajQuorums() throws Throwable { assertEquals(true, p.hasAllQuorums()); //setup servers 1-3 to be followers and 4 and 5 to be observers - setUp(true); + setUp(true, true); p = new Proposal(); p.addQuorumVerifier(s1.getQuorumVerifier()); diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/QuorumOracleMajTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/QuorumOracleMajTest.java new file mode 100644 index 00000000000..1b1fb3125ba --- /dev/null +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/QuorumOracleMajTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zookeeper.test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.ArrayList; +import org.apache.zookeeper.jmx.MBeanRegistry; +import org.apache.zookeeper.server.quorum.Leader; +import org.apache.zookeeper.server.quorum.LearnerHandler; +import org.apache.zookeeper.server.quorum.QuorumPeer; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class QuorumOracleMajTest extends QuorumBaseOracle_2Nodes { + + protected static final Logger LOG = LoggerFactory.getLogger(QuorumMajorityTest.class); + public static final long CONNECTION_TIMEOUT = ClientTest.CONNECTION_TIMEOUT; + + /***************************************************************/ + /* Test that the majority quorum verifier only counts votes from */ + /* followers in its view */ + /***************************************************************/ + @Test + public void testMajQuorums() throws Throwable { + LOG.info("Verify QuorumPeer#electionTimeTaken jmx bean attribute"); + + ArrayList peers = getPeerList(); + for (int i = 1; i <= peers.size(); i++) { + QuorumPeer qp = peers.get(i - 1); + Long electionTimeTaken = -1L; + String bean = ""; + if (qp.getPeerState() == QuorumPeer.ServerState.FOLLOWING) { + bean = String.format("%s:name0=ReplicatedServer_id%d,name1=replica.%d,name2=Follower", MBeanRegistry.DOMAIN, i, i); + } else if (qp.getPeerState() == QuorumPeer.ServerState.LEADING) { + bean = String.format("%s:name0=ReplicatedServer_id%d,name1=replica.%d,name2=Leader", MBeanRegistry.DOMAIN, i, i); + } + electionTimeTaken = (Long) JMXEnv.ensureBeanAttribute(bean, "ElectionTimeTaken"); + assertTrue(electionTimeTaken >= 0, "Wrong electionTimeTaken value!"); + } + + //setup servers 1-2 to be followers + // id=1, oracle is false; id=2, oracle is true + setUp(); + + QuorumPeer s; + int leader; + if ((leader = getLeaderIndex()) == 1) { + s = s1; + } else { + s = s2; + } + + noDropConectionTest(s); + + dropConnectionTest(s, leader); + + } + + private void noDropConectionTest(QuorumPeer s) { + Leader.Proposal p = new Leader.Proposal(); + + + p.addQuorumVerifier(s.getQuorumVerifier()); + + // 1 followers out of 2 is not a majority + p.addAck(Long.valueOf(1)); + assertEquals(false, p.hasAllQuorums()); + + // 6 is not in the view - its vote shouldn't count + p.addAck(Long.valueOf(6)); + assertEquals(false, p.hasAllQuorums()); + + // 2 followers out of 2 is good + p.addAck(Long.valueOf(2)); + assertEquals(true, p.hasAllQuorums()); + + } + + + private void dropConnectionTest(QuorumPeer s, int leader) { + Leader.Proposal p = new Leader.Proposal(); + p.addQuorumVerifier(s.getQuorumVerifier()); + + ArrayList fake = new ArrayList<>(); + + LearnerHandler f = null; + fake.add(f); + + s.getQuorumVerifier().updateNeedOracle(fake); + // still have valid followers, the oracle should not take place + assertEquals(false, s.getQuorumVerifier().getNeedOracle()); + + fake.remove(0); + s.getQuorumVerifier().updateNeedOracle(fake); + // lose all of followers, the oracle should take place + assertEquals(true, s.getQuorumVerifier().getNeedOracle()); + + + // when leader is 1, we expect false. + // when leader is 2, we expect true. + assertEquals(leader != 1, p.hasAllQuorums()); + } +} diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/QuorumUtil.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/QuorumUtil.java index 98153b9acac..ce1cd1b3c5f 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/QuorumUtil.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/QuorumUtil.java @@ -251,22 +251,22 @@ public void shutdownAll() { public void shutdown(int id) { QuorumPeer qp = getPeer(id).peer; try { - LOG.info("Shutting down quorum peer {}", qp.getName()); + LOG.info("Shutting down quorum peer {} with id {}", qp.getName(), id); qp.shutdown(); Election e = qp.getElectionAlg(); if (e != null) { - LOG.info("Shutting down leader election {}", qp.getName()); + LOG.info("Shutting down leader election {} with id {}", qp.getName(), id); e.shutdown(); } else { - LOG.info("No election available to shutdown {}", qp.getName()); + LOG.info("No election available to shutdown {} with id {}", qp.getName(), id); } - LOG.info("Waiting for {} to exit thread", qp.getName()); + LOG.info("Waiting for {} with id {} to exit thread", qp.getName(), id); qp.join(30000); if (qp.isAlive()) { - fail("QP failed to shutdown in 30 seconds: " + qp.getName()); + fail("QP failed to shutdown in 30 seconds: " + qp.getName() + " " + id); } } catch (InterruptedException e) { - LOG.debug("QP interrupted: {}", qp.getName(), e); + LOG.debug("QP interrupted: {} {}", qp.getName(), id, e); } } diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ThrottledOpObserverTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ThrottledOpObserverTest.java index b3e1d47ee7d..52d2bd80f5b 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ThrottledOpObserverTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ThrottledOpObserverTest.java @@ -35,7 +35,7 @@ public static void applyMockUps() { @BeforeEach @Override public void setUp() throws Exception { - super.setUp(true /* withObservers */); + super.setUp(true /* withObservers */, false); } @Test diff --git a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ZooKeeperQuotaTest.java b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ZooKeeperQuotaTest.java index 67b524accef..aacd59e6719 100644 --- a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ZooKeeperQuotaTest.java +++ b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ZooKeeperQuotaTest.java @@ -20,13 +20,17 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.List; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.QuotaExceededException; +import org.apache.zookeeper.Op; import org.apache.zookeeper.Quotas; import org.apache.zookeeper.StatsTrack; import org.apache.zookeeper.ZooDefs.Ids; @@ -396,6 +400,53 @@ public void testSetQuotaWhenExceedBothBytesAndCountHardQuota() throws Exception } } + @Test + public void testMultiCreateThenSetDataShouldWork() throws Exception { + final String path = "/a"; + final String subPath = "/a/b"; + + zk.create(path, null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + + final byte[] data13b = "Hello, World!".getBytes(StandardCharsets.UTF_8); + + final StatsTrack st = new StatsTrack(); + st.setByteHardLimit(data13b.length); + SetQuotaCommand.createQuota(zk, path, st); + + final List ops = Arrays.asList( + Op.create(subPath, null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT), + Op.setData(subPath, data13b, -1)); + + zk.multi(ops); + } + + @Test + public void testMultiCreateThenSetDataShouldFail() throws Exception { + final String path = "/a"; + final String subPath = "/a/b"; + + zk.create(path, null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + + final byte[] data13b = "Hello, World!".getBytes(StandardCharsets.UTF_8); + + final StatsTrack st = new StatsTrack(); + st.setByteHardLimit(data13b.length - 1); + SetQuotaCommand.createQuota(zk, path, st); + + final List ops = Arrays.asList( + Op.create(subPath, null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT), + Op.setData(subPath, data13b, -1)); + + try { + zk.multi(ops); + fail("should fail transaction when hard quota is exceeded"); + } catch (QuotaExceededException e) { + //expected + } + + assertNull(zk.exists(subPath, null)); + } + @Test public void testDeleteBytesQuota() throws Exception { @@ -501,4 +552,4 @@ public void testListQuota() throws Exception { } } } -} \ No newline at end of file +}