[fix] [broker] Fix cursor recover ignored batch ack index

.asf.yaml

@@ -37,7 +37,7 @@ github:
   enabled_merge_buttons:
     squash:  true
     merge:   false
-    rebase:  true
+    rebase:  false
   protected_branches:
     master:
       required_status_checks:
@@ -58,6 +58,9 @@ github:
 
       required_signatures: false
 
+      # Requires all conversations on code to be resolved before a pull request can be merged.
+      required_conversation_resolution: true
+
     # The following branch protections only ensure that force pushes are not allowed
     branch-1.15: {}
     branch-1.16: {}

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -28,6 +28,8 @@ body:
         For suggestions or help, please consider:
         1. [User Mail List](mailto:[email protected]) ([subscribe](mailto:[email protected]));
         2. [Github Discussion](https://github.com/apache/pulsar/discussions).
+
+        If you are reporting a security vulnerability, please instead follow the [security policy](https://pulsar.apache.org/en/security/).
   - type: checkboxes
     attributes:
       label: Search before asking
@@ -37,11 +39,20 @@ body:
         - label: >
             I searched in the [issues](https://github.com/apache/pulsar/issues) and found nothing similar.
           required: true
+  - type: checkboxes
+    attributes:
+      label: Read release policy
+      description: >
+        Please check the [supported Pulsar versions in the release policy](https://pulsar.apache.org/contribute/release-policy/#supported-versions). 
+      options:
+        - label: >
+            I understand that unsupported versions don't get bug fixes. I will attempt to reproduce the issue on a supported version of Pulsar client and Pulsar broker.
+          required: true
   - type: textarea
     attributes:
       label: Version
       description: >
-        Please provide the OS and Pulsar version you are using. If you are using the master branch, please provide the commit id.
+        Please provide the OS, Java version and Pulsar versions (client + broker) you are using.
     validations:
       required: true
   - type: textarea

.github/ISSUE_TEMPLATE/flaky-test.yml

@@ -18,7 +18,7 @@
 name: Flaky test
 title: "Flaky-test: test_class.test_method"
 description: Report a flaky test failure
-labels: [ "component/test", "flaky-tests" ]
+labels: [ "area/test", "type/flaky-tests" ]
 body:
   - type: markdown
     attributes:

.github/actions/tune-runner-vm/action.yml

@@ -53,8 +53,8 @@ runs:
             # tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt
             # commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds)
             # nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes)
-            sudo mount -o remount,nodiscard,commit=999999,barrier=0 /
-            sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt
+            sudo mount -o remount,nodiscard,commit=999999,barrier=0 / || true
+            sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt || true
             # disable discard/trim at device level since remount with nodiscard doesn't seem to be effective
             # https://www.spinics.net/lists/linux-ide/msg52562.html
             for i in /sys/block/sd*/queue/discard_max_bytes; do
@@ -77,12 +77,6 @@ runs:
             # stop Azure Linux agent to save RAM
             sudo systemctl stop walinuxagent.service || true
 
-            # enable docker experimental mode which is
-            # required for using "docker build --squash" / "-Ddocker.squash=true"
-            daemon_json="$(sudo cat /etc/docker/daemon.json  | jq '.experimental = true')"
-            echo "$daemon_json" | sudo tee /etc/docker/daemon.json
-            # restart docker daemon
-            sudo systemctl restart docker
             echo '::endgroup::'
 
             # show memory

.github/actions/upload-coverage/action.yml

@@ -51,7 +51,7 @@ runs:
     - name: "Upload to Codecov (attempt #1)"
       id: codecov-upload-1
       if: steps.repo-check.outputs.passed == 'true'
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       continue-on-error: true
       with:
         flags: ${{ inputs.flags }}
@@ -64,7 +64,7 @@ runs:
     - name: "Upload to Codecov (attempt #2)"
       id: codecov-upload-2
       if: steps.codecov-upload-1.outcome == 'failure'
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       continue-on-error: true
       with:
         flags: ${{ inputs.flags }}
@@ -77,7 +77,7 @@ runs:
     - name: "Upload to Codecov (attempt #3)"
       id: codecov-upload-3
       if: steps.codecov-upload-2.outcome == 'failure'
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       # fail on last attempt
       continue-on-error: false
       with:

.github/changes-filter.yaml

@@ -12,6 +12,8 @@ docs:
   - 'deployment/**'
   - 'wiki/**'
   - 'pip/**'
+java_non_tests:
+  - '**/src/main/java/**/*.java'
 tests:
   - added|modified: '**/src/test/java/**/*.java'
 need_owasp:

.github/labeler.yml

@@ -16,4 +16,5 @@
 # under the License.
 
 PIP:
-- 'pip/**'
+  - changed-files:
+      - any-glob-to-any-file: 'pip/**'

.github/workflows/ci-go-functions.yaml

@@ -42,7 +42,7 @@ jobs:
       docs_only: ${{ steps.check_changes.outputs.docs_only }}
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Detect changed files
         id: changes
@@ -75,25 +75,25 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        go-version: [1.15, 1.16, 1.17]
+        go-version: ['1.21']
 
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
       - name: Set up Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
         id: go
 
       - name: InstallTool
         run: |
           cd pulsar-function-go
-          wget -O - -q https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v1.18.0
+          wget -O - -q https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v1.55.2
           ./bin/golangci-lint --version
 
       - name: Build

.github/workflows/ci-maven-cache-update.yaml

@@ -43,13 +43,14 @@ on:
 
 env:
   MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
+  JDK_DISTRIBUTION: corretto
 
 jobs:
   update-maven-dependencies-cache:
     name: Update Maven dependency cache for ${{ matrix.name }}
     env:
       JOB_NAME: Update Maven dependency cache for ${{ matrix.name }}
-      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+      DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
     runs-on: ${{ matrix.runs-on }}
     timeout-minutes: 45
 
@@ -63,7 +64,7 @@ jobs:
             mvn_arguments: ''
 
           - name: all modules - macos
-            runs-on: macos-11
+            runs-on: macos-latest
             cache_name: 'm2-dependencies-all'
 
           - name: core-modules
@@ -73,7 +74,7 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
@@ -91,7 +92,7 @@ jobs:
       - name: Cache local Maven repository
         if: ${{ github.event_name == 'schedule' || steps.changes.outputs.poms == 'true' }}
         id: cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         timeout-minutes: 5
         with:
           path: |
@@ -103,10 +104,10 @@ jobs:
           # cache would be used as the starting point for a new cache entry
 
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         if: ${{ (github.event_name == 'schedule' || steps.changes.outputs.poms == 'true') && steps.cache.outputs.cache-hit != 'true' }}
         with:
-          distribution: 'temurin'
+          distribution: ${{ env.JDK_DISTRIBUTION }}
           java-version: 17
 
       - name: Download dependencies

.github/workflows/ci-owasp-dependency-check.yaml

@@ -24,76 +24,108 @@ on:
   workflow_dispatch:
 
 env:
-  MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
+  MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
+  JDK_DISTRIBUTION: corretto
+  NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }}
 
 jobs:
   run-owasp-dependency-check:
     if: ${{ github.repository == 'apache/pulsar' || github.event_name == 'workflow_dispatch' }}
     name: Check ${{ matrix.branch }}
     env:
       JOB_NAME: Check ${{ matrix.branch }}
-      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+      DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
     runs-on: ubuntu-22.04
-    timeout-minutes: 45
+    timeout-minutes: 75
     strategy:
       fail-fast: false
+      max-parallel: 1
       matrix:
         include:
           - branch: master
-          - branch: branch-3.1
+          - branch: branch-3.3
+          - branch: branch-3.2
           - branch: branch-3.0
-          - branch: branch-2.11
-          - branch: branch-2.10
-            jdk: 11
-          - branch: branch-2.9
-            jdk: 11
-          - branch: branch-2.8
-            jdk: 11
 
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ matrix.branch }}
 
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
-      - name: Cache local Maven repository
-        uses: actions/cache@v3
+      - name: Restore Maven repository cache
+        uses: actions/cache/restore@v4
         timeout-minutes: 5
         with:
           path: |
             ~/.m2/repository/*/*/*
             !~/.m2/repository/org/apache/pulsar
-          key: ${{ runner.os }}-m2-dependencies-owasp-${{ hashFiles('**/pom.xml') }}
+          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
-            ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
             ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
             ${{ runner.os }}-m2-dependencies-core-modules-
 
       - name: Set up JDK ${{ matrix.jdk || '17' }}
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
-          distribution: 'temurin'
+          distribution: ${{ env.JDK_DISTRIBUTION }}
           java-version: ${{ matrix.jdk || '17' }}
 
       - name: run install by skip tests
-        run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true  -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true
+        run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true  -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true -DnarPluginPhase=none -pl '!distribution/io,!distribution/offloaders'
+
+      - name: OWASP cache key weeknum
+        id: get-weeknum
+        run: |
+          echo "weeknum=$(date -u +"%Y-%U")" >> $GITHUB_OUTPUT
+        shell: bash
+
+      - name: Restore OWASP Dependency Check data
+        id: restore-owasp-dependency-check-data
+        uses: actions/cache/restore@v4
+        timeout-minutes: 5
+        with:
+          path: ~/.m2/repository/org/owasp/dependency-check-data
+          key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }}
+          enableCrossOsArchive: true
+          restore-keys: |
+            owasp-dependency-check-data-
+
+      - name: Update OWASP Dependency Check data
+        id: update-owasp-dependency-check-data
+        if: ${{ matrix.branch == 'master' && (steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' || steps.restore-owasp-dependency-check-data.outputs.cache-matched-key != steps.restore-owasp-dependency-check-data.outputs.cache-primary-key) }}
+        run: mvn -B -ntp -Powasp-dependency-check initialize -pl . dependency-check:update-only
+
+      - name: Save OWASP Dependency Check data
+        if: ${{ steps.update-owasp-dependency-check-data.outcome == 'success' }}
+        uses: actions/cache/save@v4
+        timeout-minutes: 5
+        with:
+          path: ~/.m2/repository/org/owasp/dependency-check-data
+          key: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-primary-key }}
+          enableCrossOsArchive: true
 
       - name: run OWASP Dependency Check for distribution/server (-DfailBuildOnAnyVulnerability=true)
         run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/server -DfailBuildOnAnyVulnerability=true
 
-      - name: run OWASP Dependency Check for distribution/offloaders, distribution/io and pulsar-sql/presto-distribution
-        run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/offloaders,distribution/io,pulsar-sql/presto-distribution
+      - name: run OWASP Dependency Check for offloaders/tiered-storage and pulsar-io connectors (-DfailOnError=false)
+        if: ${{ !cancelled() }}
+        run: |
+          mvnprojects=$(mvn -B -ntp -Dscan=false initialize \
+            | grep -- "-< .* >-" \
+            | sed -E 's/.*-< (.*) >-.*/\1/' \
+            | grep -E 'pulsar-io-|tiered-storage-|offloader' \
+            | tr '\n' ',' | sed 's/,$/\n/' )
+          set -xe
+          mvn --fail-at-end -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -DfailOnError=false -pl "${mvnprojects}"
 
       - name: Upload OWASP Dependency Check reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: owasp-dependency-check-reports-${{ matrix.branch }}
           path: |
-            distribution/server/target/dependency-check-report.html
-            distribution/offloaders/target/dependency-check-report.html
-            distribution/io/target/dependency-check-report.html
-            pulsar-sql/presto-distribution/target/dependency-check-report.html
+            **/target/dependency-check-report.html

.github/workflows/ci-semantic-pull-request.yml

@@ -34,7 +34,7 @@ jobs:
     name: Check pull request title
     runs-on: ubuntu-latest
     steps:
-      - uses: amannn/action-semantic-pull-request@v5.0.2
+      - uses: amannn/action-semantic-pull-request@v5.5.2
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
@@ -53,7 +53,6 @@ jobs:
           # io      -> Pulsar Connectors
           # offload -> tiered storage
           # sec     -> security
-          # sql     -> Pulsar Trino Plugin
           # txn     -> transaction
           # ws      -> websocket
           # ml      -> managed ledger
@@ -79,7 +78,6 @@ jobs:
             schema
             sec
             site
-            sql
             storage
             test
             txn