diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000000..61ff625176 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,3 @@ +# Bump (c) year to 2025 +100355cecc + diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6c10bc7cd2..4c09ba720a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,11 +8,12 @@ updates: interval: "daily" open-pull-requests-limit: 20 ignore: - - dependency-name: "com.swiftmq:swiftmq-client" - - dependency-name: "org.springframework.boot:spring-boot-maven-plugin" - versions: ["[2.7,)"] + - dependency-name: "org.slf4j:slf4j-api" + versions: [ "[2.0,)" ] + - dependency-name: "ch.qos.logback:logback-classic" + versions: [ "[1.3,)" ] - package-ecosystem: "github-actions" directory: "/" schedule: interval: "daily" - target-branch: "main" \ No newline at end of file + target-branch: "main" diff --git a/.github/workflows/publish-documentation.yml b/.github/workflows/publish-documentation.yml index abdd6b93fe..33baea4572 100644 --- a/.github/workflows/publish-documentation.yml +++ b/.github/workflows/publish-documentation.yml @@ -4,14 +4,14 @@ on: workflow_dispatch jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: Set up JDK uses: actions/setup-java@v4 with: - distribution: 'zulu' + distribution: 'temurin' java-version: '21' cache: 'maven' - name: Publish Documentation diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml index b864cc19bb..8d22f67352 100644 --- a/.github/workflows/publish-snapshot.yml +++ b/.github/workflows/publish-snapshot.yml @@ -4,14 +4,14 @@ on: workflow_dispatch jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: Set up JDK uses: actions/setup-java@v4 with: - distribution: 'zulu' + distribution: 'temurin' java-version: '21' cache: 'maven' server-id: ossrh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 856252bcbd..bb1c9f9029 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,7 +5,7 @@ on: jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 @@ -15,7 +15,7 @@ jobs: uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: '8' + java-version: '11' cache: 'maven' server-id: ${{ env.maven_server_id }} server-username: MAVEN_USERNAME @@ -42,6 +42,26 @@ jobs: MAVEN_USERNAME: '' MAVEN_PASSWORD: ${{ secrets.PACKAGECLOUD_TOKEN }} MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} + - name: Checkout tls-gen + uses: actions/checkout@v4 + with: + repository: rabbitmq/tls-gen + path: './tls-gen' + - name: Start broker + run: ci/start-broker.sh + - name: Set up JDK for sanity check and documentation generation + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '21' + cache: 'maven' + - name: Sanity Check + run: | + source ./release-versions.txt + export RABBITMQ_LIBRARY_VERSION=$RELEASE_VERSION + curl -Ls https://sh.jbang.dev | bash -s - src/test/java/SanityCheck.java + - name: Stop broker + run: docker stop rabbitmq && docker rm rabbitmq - name: Publish Documentation run: | git config user.name "rabbitmq-ci" diff --git a/.github/workflows/sanity-check.yml b/.github/workflows/sanity-check.yml new file mode 100644 index 0000000000..26112a1d5f --- /dev/null +++ b/.github/workflows/sanity-check.yml @@ -0,0 +1,37 @@ +name: Library Sanity Check + +on: + workflow_dispatch: + inputs: + library_version: + description: 'Library version (e.g. 0.21.0)' + required: true + type: string + default: '0.21.0' + +jobs: + build: + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v4 + - name: Checkout tls-gen + uses: actions/checkout@v4 + with: + repository: rabbitmq/tls-gen + path: './tls-gen' + - name: Set up JDK + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Start broker + run: ci/start-broker.sh + - name: Sanity Check + run: | + curl -Ls https://sh.jbang.dev | bash -s - src/test/java/SanityCheck.java + env: + RABBITMQ_LIBRARY_VERSION: ${{ inputs.library_version }} + - name: Stop broker + run: docker stop rabbitmq && docker rm rabbitmq diff --git a/.github/workflows/test-native-image.yml b/.github/workflows/test-native-image.yml index cb813266af..0b5dcc9dc8 100644 --- a/.github/workflows/test-native-image.yml +++ b/.github/workflows/test-native-image.yml @@ -7,7 +7,7 @@ on: jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test-pr.yml b/.github/workflows/test-pr.yml index 4d491b4e33..380db0602c 100644 --- a/.github/workflows/test-pr.yml +++ b/.github/workflows/test-pr.yml @@ -1,4 +1,4 @@ -name: Test against RabbitMQ 3.12 stable (PR) +name: Test against RabbitMQ stable (PR) on: pull_request: @@ -7,7 +7,7 @@ on: jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 @@ -19,16 +19,30 @@ jobs: - name: Set up JDK uses: actions/setup-java@v4 with: - distribution: 'zulu' + distribution: 'temurin' java-version: '21' cache: 'maven' - name: Start broker run: ci/start-broker.sh - - name: Test + - name: Test (no dynamic-batch publishing) run: | ./mvnw verify -Drabbitmqctl.bin=DOCKER:rabbitmq \ + -Drabbitmq.stream.producer.dynamic.batch=false \ + -Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \ + -Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \ + -Dclient.key=./tls-gen/basic/result/client_$(hostname)_key.pem + - name: Test (dynamic-batch publishing) + run: | + ./mvnw test -Drabbitmqctl.bin=DOCKER:rabbitmq \ + -Drabbitmq.stream.producer.dynamic.batch=true \ -Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \ -Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \ -Dclient.key=./tls-gen/basic/result/client_$(hostname)_key.pem - name: Stop broker run: docker stop rabbitmq && docker rm rabbitmq + - name: Start cluster + run: ci/start-cluster.sh + - name: Test against cluster + run: ./mvnw test -Dtest="*ClusterTest" -Drabbitmqctl.bin=DOCKER:rabbitmq0 + - name: Stop cluster + run: docker compose --file ci/cluster/docker-compose.yml down diff --git a/.github/workflows/test-rabbitmq-alphas.yml b/.github/workflows/test-rabbitmq-alphas.yml index c8b2b000a7..6b8e9e32c6 100644 --- a/.github/workflows/test-rabbitmq-alphas.yml +++ b/.github/workflows/test-rabbitmq-alphas.yml @@ -10,10 +10,12 @@ on: jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: - rabbitmq-image: [ 'pivotalrabbitmq/rabbitmq:v3.12.x-otp-max-bazel', 'pivotalrabbitmq/rabbitmq:main-otp-max-bazel' ] + rabbitmq-image: + - pivotalrabbitmq/rabbitmq:v4.1.x-otp27 + - pivotalrabbitmq/rabbitmq:main-otp27 name: Test against ${{ matrix.rabbitmq-image }} steps: - uses: actions/checkout@v4 @@ -25,18 +27,34 @@ jobs: - name: Set up JDK uses: actions/setup-java@v4 with: - distribution: 'zulu' + distribution: 'temurin' java-version: '21' cache: 'maven' - name: Start broker run: ci/start-broker.sh env: RABBITMQ_IMAGE: ${{ matrix.rabbitmq-image }} - - name: Test + - name: Test (no dynamic-batch publishing) run: | ./mvnw verify -Drabbitmqctl.bin=DOCKER:rabbitmq \ + -Drabbitmq.stream.producer.dynamic.batch=false \ -Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \ -Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \ -Dclient.key=./tls-gen/basic/result/client_$(hostname)_key.pem + - name: Test (dynamic-batch publishing) + run: | + ./mvnw test -Drabbitmqctl.bin=DOCKER:rabbitmq \ + -Drabbitmq.stream.producer.dynamic.batch=true \ + -Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \ + -Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \ + -Dclient.key=./tls-gen/basic/result/client_$(hostname)_key.pem - name: Stop broker run: docker stop rabbitmq && docker rm rabbitmq + - name: Start cluster + run: ci/start-cluster.sh + env: + RABBITMQ_IMAGE: ${{ matrix.rabbitmq-image }} + - name: Test against cluster + run: ./mvnw test -Dtest="*ClusterTest" -Drabbitmqctl.bin=DOCKER:rabbitmq0 + - name: Stop cluster + run: docker compose --file ci/cluster/docker-compose.yml down diff --git a/.github/workflows/test-supported-java-versions.yml b/.github/workflows/test-supported-java-versions.yml index 7c610309b9..f2ac5a1289 100644 --- a/.github/workflows/test-supported-java-versions.yml +++ b/.github/workflows/test-supported-java-versions.yml @@ -2,16 +2,20 @@ name: Test against supported Java versions on: schedule: - - cron: '0 4 * * *' + - cron: '0 3 * * *' workflow_dispatch: jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: - java: [ '8', '11', '17', '21', '22-ea' ] - name: Test against Java ${{ matrix.java }} + distribution: [ 'temurin' ] + version: [ '11', '17', '21', '24', '25-ea' ] + include: + - distribution: 'semeru' + version: '17' + name: Test against Java ${{ matrix.distribution }} ${{ matrix.version }} steps: - uses: actions/checkout@v4 - name: Checkout tls-gen @@ -22,19 +26,34 @@ jobs: - name: Set up JDK uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: ${{ matrix.java }} + distribution: ${{ matrix.distribution }} + java-version: ${{ matrix.version }} cache: 'maven' - name: Start broker run: ci/start-broker.sh - name: Display Java version run: ./mvnw --version - - name: Test + - name: Test (no dynamic-batch publishing) run: | ./mvnw verify -Drabbitmqctl.bin=DOCKER:rabbitmq \ + -Drabbitmq.stream.producer.dynamic.batch=false \ -Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \ -Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \ -Dclient.key=./tls-gen/basic/result/client_$(hostname)_key.pem \ - -Dnet.bytebuddy.experimental=true -Djacoco.skip=true + -Dnet.bytebuddy.experimental=true -Djacoco.skip=true -Dspotbugs.skip=true + - name: Test (dynamic-batch publishing) + run: | + ./mvnw test -Drabbitmqctl.bin=DOCKER:rabbitmq \ + -Drabbitmq.stream.producer.dynamic.batch=true \ + -Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \ + -Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \ + -Dclient.key=./tls-gen/basic/result/client_$(hostname)_key.pem \ + -Dnet.bytebuddy.experimental=true -Djacoco.skip=true -Dspotbugs.skip=true - name: Stop broker run: docker stop rabbitmq && docker rm rabbitmq + - name: Start cluster + run: ci/start-cluster.sh + - name: Test against cluster + run: ./mvnw test -Dtest="*ClusterTest" -Drabbitmqctl.bin=DOCKER:rabbitmq0 + - name: Stop cluster + run: docker compose --file ci/cluster/docker-compose.yml down diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 173bac2b49..523b05cbf9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: Test against RabbitMQ 3.12 stable +name: Test against RabbitMQ stable on: push: @@ -11,7 +11,7 @@ env: jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 @@ -23,7 +23,7 @@ jobs: - name: Set up JDK uses: actions/setup-java@v4 with: - distribution: 'zulu' + distribution: 'temurin' java-version: '21' cache: 'maven' server-id: ossrh @@ -33,14 +33,28 @@ jobs: gpg-passphrase: MAVEN_GPG_PASSPHRASE - name: Start broker run: ci/start-broker.sh - - name: Test + - name: Test (no dynamic-batch publishing) run: | ./mvnw verify -Drabbitmqctl.bin=DOCKER:rabbitmq \ + -Drabbitmq.stream.producer.dynamic.batch=false \ -Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \ -Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \ -Dclient.key=./tls-gen/basic/result/client_$(hostname)_key.pem + - name: Test (dynamic-batch publishing) + run: | + ./mvnw test -Drabbitmqctl.bin=DOCKER:rabbitmq \ + -Drabbitmq.stream.producer.dynamic.batch=true \ + -Dca.certificate=./tls-gen/basic/result/ca_certificate.pem \ + -Dclient.certificate=./tls-gen/basic/result/client_$(hostname)_certificate.pem \ + -Dclient.key=./tls-gen/basic/result/client_$(hostname)_key.pem - name: Stop broker run: docker stop rabbitmq && docker rm rabbitmq + - name: Start cluster + run: ci/start-cluster.sh + - name: Test against cluster + run: ./mvnw test -Dtest="*ClusterTest" -Drabbitmqctl.bin=DOCKER:rabbitmq0 + - name: Stop cluster + run: docker compose --file ci/cluster/docker-compose.yml down - name: Upload Codecov report run: bash <(curl -s https://codecov.io/bash) - name: Publish snapshot diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties index 346d645fd0..1a60da7935 100644 --- a/.mvn/wrapper/maven-wrapper.properties +++ b/.mvn/wrapper/maven-wrapper.properties @@ -14,5 +14,5 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.6/apache-maven-3.9.6-bin.zip +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 4deefa0459..0000000000 --- a/Dockerfile +++ /dev/null @@ -1,96 +0,0 @@ -FROM ubuntu:22.04 as builder - -ARG stream_perf_test_url="set-url-here" - -RUN set -eux; \ - \ - apt-get update; \ - apt-get -y upgrade; \ - apt-get install --yes --no-install-recommends \ - ca-certificates \ - wget \ - gnupg \ - jq - -ARG JAVA_VERSION="21" - -RUN if [ "$(uname -m)" = "aarch64" ] || [ "$(uname -m)" = "arm64" ]; then echo "ARM"; ARCH="arm"; BUNDLE="jdk"; else echo "x86"; ARCH="x86"; BUNDLE="jdk"; fi \ - && wget "https://api.azul.com/zulu/download/community/v1.0/bundles/latest/?java_version=$JAVA_VERSION&ext=tar.gz&os=linux&arch=$ARCH&hw_bitness=64&release_status=ga&bundle_type=$BUNDLE" -O jdk-info.json -RUN wget --progress=bar:force:noscroll -O "jdk.tar.gz" $(cat jdk-info.json | jq --raw-output .url) -RUN echo "$(cat jdk-info.json | jq --raw-output .sha256_hash) *jdk.tar.gz" | sha256sum --check --strict - - -RUN set -eux; \ - if [ "$(uname -m)" = "x86_64" ] ; then JAVA_PATH="/usr/lib/jdk-$JAVA_VERSION"; \ - mkdir $JAVA_PATH && \ - tar --extract --file jdk.tar.gz --directory "$JAVA_PATH" --strip-components 1; \ - $JAVA_PATH/bin/jlink --compress=2 --output /jre --add-modules java.base,jdk.management,java.naming,java.xml,jdk.unsupported,jdk.crypto.cryptoki,jdk.httpserver; \ - /jre/bin/java -version; \ - fi - -RUN set -eux; \ - if [ "$(uname -m)" = "aarch64" ] || [ "$(uname -m)" = "arm64" ] ; then JAVA_PATH="/jre"; \ - mkdir $JAVA_PATH && \ - tar --extract --file jdk.tar.gz --directory "$JAVA_PATH" --strip-components 1; \ - fi - -# pgpkeys.uk is quite reliable, but allow for substitutions locally -ARG PGP_KEYSERVER=hkps://keys.openpgp.org -# If you are building this image locally and are getting `gpg: keyserver receive failed: No data` errors, -# run the build with a different PGP_KEYSERVER, e.g. docker build --tag rabbitmq:3.7 --build-arg PGP_KEYSERVER=pgpkeys.eu 3.7/ubuntu -# For context, see https://github.com/docker-library/official-images/issues/4252 - -# https://www.rabbitmq.com/signatures.html#importing-gpg -ENV RABBITMQ_PGP_KEY_ID="0x0A9AF2115F4687BD29803A206B73A36E6026DFCA" -ENV STREAM_PERF_TEST_HOME="/stream_perf_test" - -RUN set -eux; \ - \ - wget --progress dot:giga --output-document "/usr/local/src/stream-perf-test.jar.asc" "$stream_perf_test_url.asc"; \ - wget --progress dot:giga --output-document "/usr/local/src/stream-perf-test.jar" "$stream_perf_test_url"; \ - STREAM_PERF_TEST_SHA256="$(wget -qO- $stream_perf_test_url.sha256)"; \ - echo "$STREAM_PERF_TEST_SHA256 /usr/local/src/stream-perf-test.jar" | sha256sum --check --strict -; \ - \ - export GNUPGHOME="$(mktemp -d)"; \ - gpg --batch --keyserver "$PGP_KEYSERVER" --recv-keys "$RABBITMQ_PGP_KEY_ID"; \ - gpg --batch --verify "/usr/local/src/stream-perf-test.jar.asc" "/usr/local/src/stream-perf-test.jar"; \ - gpgconf --kill all; \ - rm -rf "$GNUPGHOME"; \ - \ - mkdir -p "$STREAM_PERF_TEST_HOME"; \ - cp /usr/local/src/stream-perf-test.jar $STREAM_PERF_TEST_HOME/stream-perf-test.jar - -FROM ubuntu:22.04 - -# we need locales support for characters like µ to show up correctly in the console -RUN set -eux; \ - apt-get update; \ - apt-get -y upgrade; \ - apt-get install -y --no-install-recommends \ - locales \ - wget \ - ; \ - rm -rf /var/lib/apt/lists/*; \ - locale-gen en_US.UTF-8 - -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 - -ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk/jre -RUN mkdir -p $JAVA_HOME -COPY --from=builder /jre $JAVA_HOME/ -RUN ln -svT $JAVA_HOME/bin/java /usr/local/bin/java - -RUN mkdir -p /stream_perf_test -WORKDIR /stream_perf_test -COPY --from=builder /stream_perf_test ./ -RUN set -eux; \ - if [ "$(uname -m)" = "x86_64" ] ; then java -jar stream-perf-test.jar --help ; \ - fi - -RUN groupadd --gid 1000 stream-perf-test -RUN useradd --uid 1000 --gid stream-perf-test --comment "perf-test user" stream-perf-test - -USER stream-perf-test:stream-perf-test - -ENTRYPOINT ["java", "-Dio.netty.processId=1", "-jar", "stream-perf-test.jar"] diff --git a/LICENSE-APACHE2 b/LICENSE-APACHE2 index 2cc4b7361e..0719b8dc33 100644 --- a/LICENSE-APACHE2 +++ b/LICENSE-APACHE2 @@ -187,7 +187,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2020-2023 2023 Broadcom. All Rights Reserved. + Copyright 2020-2023 2007-2025 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/LICENSE-MPL-RabbitMQ b/LICENSE-MPL-RabbitMQ index e44c13ddfd..175430d63e 100644 --- a/LICENSE-MPL-RabbitMQ +++ b/LICENSE-MPL-RabbitMQ @@ -367,5 +367,5 @@ for such a notice. The Original Code is RabbitMQ. The Initial Developer of the Original Code is Pivotal Software, Inc. -Copyright (c) 2020-2023 2023 Broadcom. All Rights Reserved. +Copyright (c) 2020-2023 2007-2025 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. \ No newline at end of file diff --git a/README.adoc b/README.adoc index 306b00e52f..2a96cd7585 100644 --- a/README.adoc +++ b/README.adoc @@ -6,8 +6,9 @@ image:https://codecov.io/gh/rabbitmq/rabbitmq-stream-java-client/branch/main/gra The RabbitMQ Stream Java Client is a Java library to communicate with the https://rabbitmq.com/stream.html[RabbitMQ Stream Plugin]. -It allows to create and delete streams, as well as to publish to and consume from -these streams. +It allows to create and delete streams, as well as to publish to and consume from these streams. +This library requires at least Java 11 but Java 21 or more is recommended. +See the https://rabbitmq.github.io/rabbitmq-stream-java-client/stable/htmlsingle/#stream-client-overview[overview] for a quick glance at the features. https://github.com/rabbitmq/rabbitmq-stream-perf-test[Stream PerfTest] is a performance testing tool based on this client library. @@ -15,19 +16,25 @@ Please refer to the https://rabbitmq.github.io/rabbitmq-stream-java-client/stabl == Project Maturity -The project is in development and stabilization phase. -Features and API are subject to change, but https://rabbitmq.github.io/rabbitmq-stream-java-client/stable/htmlsingle/#stability-of-programming-interfaces[breaking changes] will be kept to a minimum. +The library is stable and production-ready. == Support * For questions: https://groups.google.com/forum/#!forum/rabbitmq-users[RabbitMQ Users] * For bugs and feature requests: https://github.com/rabbitmq/rabbitmq-stream-java-client/issues[GitHub Issues] +See the https://www.rabbitmq.com/client-libraries/java-versions[RabbitMQ Java libraries support page] for the support timeline of this library. + == How to Use === Pre-requisites -The library requires Java 8 or later. Java 11 is recommended. +This library requires at least Java 11, but Java 21 or more is recommended. + +=== Dependencies + +* https://rabbitmq.github.io/rabbitmq-stream-java-client/stable/htmlsingle/#dependencies[Release] +* https://rabbitmq.github.io/rabbitmq-stream-java-client/snapshot/htmlsingle/#dependencies[Snapshot] === Documentation @@ -43,21 +50,11 @@ The library requires Java 8 or later. Java 11 is recommended. == Versioning -The RabbitMQ Stream Java Client is in development and stabilization phase. -When the stabilization phase ends, a 1.0.0 version will be cut, and -https://semver.org/[semantic versioning] is likely to be enforced. - -Before reaching the stable phase, the client will use a versioning scheme of `[0.MINOR.PATCH]` where: - -* `0` indicates the project is still in a stabilization phase. -* `MINOR` is a 0-based number incrementing with each new release cycle. It generally reflects significant changes like new features and potentially some programming interfaces changes. -* `PATCH` is a 0-based number incrementing with each service release, that is bux fixes. - -Breaking changes between releases can happen but will be kept to a minimum. +This library uses https://semver.org/[semantic versioning]. == Build Instructions -You need JDK 8 or later installed. +You need JDK 11 or later installed. To build the JAR file: @@ -72,7 +69,7 @@ Launch the broker: ---- docker run -it --rm --name rabbitmq -p 5552:5552 -p 5672:5672 \ -e RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS='-rabbitmq_stream advertised_host localhost' \ - rabbitmq:3.12 + rabbitmq:4.1 ---- Enable the stream plugin: @@ -106,7 +103,7 @@ Please launch the `./mvnw spotless:apply` command to format your changes before == Copyright and License -(c) 2020-2023, 2023 Broadcom. All Rights Reserved. +(c) 2020-2025 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. Double licensed under the MPL2.0 and ASL2. See link:LICENSE[LICENSE] for details. diff --git a/ci/cluster/configuration-0/enabled_plugins b/ci/cluster/configuration-0/enabled_plugins new file mode 100644 index 0000000000..244c8f60e8 --- /dev/null +++ b/ci/cluster/configuration-0/enabled_plugins @@ -0,0 +1 @@ +[rabbitmq_stream_management]. diff --git a/ci/cluster/configuration-0/rabbitmq.conf b/ci/cluster/configuration-0/rabbitmq.conf new file mode 100644 index 0000000000..9216e03142 --- /dev/null +++ b/ci/cluster/configuration-0/rabbitmq.conf @@ -0,0 +1,8 @@ +cluster_formation.peer_discovery_backend = rabbit_peer_discovery_classic_config +cluster_formation.classic_config.nodes.1 = rabbit@node0 +cluster_formation.classic_config.nodes.2 = rabbit@node1 +cluster_formation.classic_config.nodes.3 = rabbit@node2 +loopback_users = none + +stream.advertised_host = localhost +stream.advertised_port = 5552 diff --git a/ci/cluster/configuration-1/enabled_plugins b/ci/cluster/configuration-1/enabled_plugins new file mode 100644 index 0000000000..244c8f60e8 --- /dev/null +++ b/ci/cluster/configuration-1/enabled_plugins @@ -0,0 +1 @@ +[rabbitmq_stream_management]. diff --git a/ci/cluster/configuration-1/rabbitmq.conf b/ci/cluster/configuration-1/rabbitmq.conf new file mode 100644 index 0000000000..2184b5e340 --- /dev/null +++ b/ci/cluster/configuration-1/rabbitmq.conf @@ -0,0 +1,8 @@ +cluster_formation.peer_discovery_backend = rabbit_peer_discovery_classic_config +cluster_formation.classic_config.nodes.1 = rabbit@node0 +cluster_formation.classic_config.nodes.2 = rabbit@node1 +cluster_formation.classic_config.nodes.3 = rabbit@node2 +loopback_users = none + +stream.advertised_host = localhost +stream.advertised_port = 5553 diff --git a/ci/cluster/configuration-2/enabled_plugins b/ci/cluster/configuration-2/enabled_plugins new file mode 100644 index 0000000000..244c8f60e8 --- /dev/null +++ b/ci/cluster/configuration-2/enabled_plugins @@ -0,0 +1 @@ +[rabbitmq_stream_management]. diff --git a/ci/cluster/configuration-2/rabbitmq.conf b/ci/cluster/configuration-2/rabbitmq.conf new file mode 100644 index 0000000000..ff57480c50 --- /dev/null +++ b/ci/cluster/configuration-2/rabbitmq.conf @@ -0,0 +1,8 @@ +cluster_formation.peer_discovery_backend = rabbit_peer_discovery_classic_config +cluster_formation.classic_config.nodes.1 = rabbit@node0 +cluster_formation.classic_config.nodes.2 = rabbit@node1 +cluster_formation.classic_config.nodes.3 = rabbit@node2 +loopback_users = none + +stream.advertised_host = localhost +stream.advertised_port = 5554 diff --git a/ci/cluster/docker-compose.yml b/ci/cluster/docker-compose.yml new file mode 100644 index 0000000000..40a6860ee1 --- /dev/null +++ b/ci/cluster/docker-compose.yml @@ -0,0 +1,64 @@ +services: + node0: + environment: + - RABBITMQ_ERLANG_COOKIE='secret_cookie' + networks: + - rabbitmq-cluster + hostname: node0 + container_name: rabbitmq0 + image: ${RABBITMQ_IMAGE:-rabbitmq:4.1} + pull_policy: always + ports: + - "5672:5672" + - "5552:5552" + - "15672:15672" + tty: true + volumes: + - ./configuration-0/:/etc/rabbitmq/ + node1: + environment: + - RABBITMQ_ERLANG_COOKIE='secret_cookie' + networks: + - rabbitmq-cluster + hostname: node1 + container_name: rabbitmq1 + image: ${RABBITMQ_IMAGE:-rabbitmq:4.1} + pull_policy: always + ports: + - "5673:5672" + - "5553:5552" + - "15673:15672" + tty: true + volumes: + - ./configuration-1/:/etc/rabbitmq/ + node2: + environment: + - RABBITMQ_ERLANG_COOKIE='secret_cookie' + networks: + - rabbitmq-cluster + hostname: node2 + container_name: rabbitmq2 + image: ${RABBITMQ_IMAGE:-rabbitmq:4.1} + pull_policy: always + ports: + - "5674:5672" + - "5554:5552" + - "15674:15672" + tty: true + volumes: + - ./configuration-2/:/etc/rabbitmq/ + load-balander: + networks: + - rabbitmq-cluster + hostname: load-balancer + container_name: haproxy + image: haproxy:3.0 + pull_policy: always + ports: + - "5555:5555" + - "8100:8100" + tty: true + volumes: + - ./load-balancer/:/usr/local/etc/haproxy:ro +networks: + rabbitmq-cluster: diff --git a/ci/cluster/load-balancer/haproxy.cfg b/ci/cluster/load-balancer/haproxy.cfg new file mode 100644 index 0000000000..e5c628bec1 --- /dev/null +++ b/ci/cluster/load-balancer/haproxy.cfg @@ -0,0 +1,31 @@ +global + log 127.0.0.1 local0 info + maxconn 512 + +defaults + log global + mode tcp + option tcplog + option dontlognull + retries 3 + option redispatch + maxconn 512 + timeout connect 5s + timeout client 120s + timeout server 120s + +listen stream + bind :5555 + mode tcp + balance roundrobin + server rabbitmq-0 node0:5552 + server rabbitmq-1 node1:5552 + server rabbitmq-2 node2:5552 + +listen stats + bind :8100 + mode http + option httplog + stats enable + stats uri /stats + stats refresh 5s diff --git a/ci/package-stream-perf-test.sh b/ci/package-stream-perf-test.sh deleted file mode 100755 index 35f36bbc1c..0000000000 --- a/ci/package-stream-perf-test.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash - -source $(pwd)/performance-tool.txt -source $(pwd)/release-versions.txt - -if [ -z ${RELEASE+x} ] -then - CURRENT_DATE=$(date --utc '+%Y%m%d-%H%M%S') - RELEASE_VERSION="$(cat pom.xml | grep -oPm1 '(?<=)[^<]+')-$CURRENT_DATE" - FINAL_NAME="$PERF_TOOL_BASE_NAME-$RELEASE_VERSION" -fi - -FINAL_NAME="$PERF_TOOL_BASE_NAME-$RELEASE_VERSION" - -mkdir packages - -./mvnw clean package checksum:files gpg:sign -Dgpg.skip=false -Dmaven.test.skip -P performance-tool -DfinalName="$FINAL_NAME" --no-transfer-progress - -./mvnw test-compile exec:java -Dexec.mainClass="picocli.AutoComplete" -Dexec.classpathScope=test -Dexec.args="-f -n $PERF_TOOL_BASE_NAME com.rabbitmq.stream.perf.AggregatingCommandForAutoComplete" --no-transfer-progress -cp "$PERF_TOOL_BASE_NAME"_completion packages/"$PERF_TOOL_BASE_NAME"-"$RELEASE_VERSION"_completion - -rm target/*.original -cp target/"$FINAL_NAME".jar packages -cp target/"$FINAL_NAME".jar.* packages - -if [ -z ${RELEASE+x} ] -then - mkdir packages-latest - cp target/"$FINAL_NAME".jar packages-latest - cp target/"$FINAL_NAME".jar.* packages-latest - cp packages/*_completion packages-latest - - for filename in packages-latest/*; do - [ -f "$filename" ] || continue - filename_without_version=$(echo "$filename" | sed -e "s/$RELEASE_VERSION/latest/g") - mv "$filename" "$filename_without_version" - done - echo "release_name=$PERF_TOOL_BASE_NAME-$RELEASE_VERSION" >> $GITHUB_ENV - echo "tag_name=v-$PERF_TOOL_BASE_NAME-$RELEASE_VERSION" >> $GITHUB_ENV -else - echo "release_name=$RELEASE_VERSION" >> $GITHUB_ENV - echo "tag_name=v$RELEASE_VERSION" >> $GITHUB_ENV -fi - -echo "release_version=$RELEASE_VERSION" >> $GITHUB_ENV -echo "release_branch=$RELEASE_BRANCH" >> $GITHUB_ENV diff --git a/ci/publish-documentation-to-github-pages.sh b/ci/publish-documentation-to-github-pages.sh index 2afdfbed03..6f2e0a1aa5 100755 --- a/ci/publish-documentation-to-github-pages.sh +++ b/ci/publish-documentation-to-github-pages.sh @@ -24,7 +24,7 @@ git checkout gh-pages mkdir -p $RELEASE_VERSION/htmlsingle cp target/generated-docs/index.html $RELEASE_VERSION/htmlsingle mkdir -p $RELEASE_VERSION/api -cp -r target/site/apidocs/* $RELEASE_VERSION/api/ +cp -r target/reports/apidocs/* $RELEASE_VERSION/api/ git add $RELEASE_VERSION/ if [[ $LATEST == "true" ]] @@ -42,11 +42,11 @@ if [[ $LATEST == "true" ]] mkdir -p $DOC_DIR/htmlsingle cp target/generated-docs/index.html $DOC_DIR/htmlsingle mkdir -p $DOC_DIR/api - cp -r target/site/apidocs/* $DOC_DIR/api/ + cp -r target/reports/apidocs/* $DOC_DIR/api/ git add $DOC_DIR/ fi git commit -m "$MESSAGE" git push origin gh-pages -git checkout main \ No newline at end of file +git checkout main diff --git a/ci/start-broker.sh b/ci/start-broker.sh index 9e88f12a04..38c60cc1b9 100755 --- a/ci/start-broker.sh +++ b/ci/start-broker.sh @@ -2,7 +2,7 @@ LOCAL_SCRIPT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -RABBITMQ_IMAGE=${RABBITMQ_IMAGE:-rabbitmq:3.12} +RABBITMQ_IMAGE=${RABBITMQ_IMAGE:-rabbitmq:4.1} wait_for_message() { while ! docker logs "$1" | grep -q "$2"; @@ -14,6 +14,7 @@ wait_for_message() { make -C "${PWD}"/tls-gen/basic +rm -rf rabbitmq-configuration mkdir -p rabbitmq-configuration/tls cp -R "${PWD}"/tls-gen/basic/result/* rabbitmq-configuration/tls chmod o+r rabbitmq-configuration/tls/* @@ -33,7 +34,8 @@ ssl_options.fail_if_no_peer_cert = false ssl_options.depth = 1 auth_mechanisms.1 = PLAIN -auth_mechanisms.2 = EXTERNAL +auth_mechanisms.2 = ANONYMOUS +auth_mechanisms.3 = EXTERNAL stream.listeners.ssl.1 = 5551" >> rabbitmq-configuration/rabbitmq.conf @@ -47,5 +49,6 @@ docker run -d --name rabbitmq \ wait_for_message rabbitmq "completed with" +docker exec rabbitmq rabbitmqctl enable_feature_flag --opt-in khepri_db docker exec rabbitmq rabbitmq-diagnostics erlang_version docker exec rabbitmq rabbitmqctl version diff --git a/ci/start-cluster.sh b/ci/start-cluster.sh new file mode 100755 index 0000000000..b8440984fb --- /dev/null +++ b/ci/start-cluster.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +export RABBITMQ_IMAGE=${RABBITMQ_IMAGE:-rabbitmq:4.1} + +wait_for_message() { + while ! docker logs "$1" | grep -q "$2"; + do + sleep 2 + echo "Waiting 2 seconds for $1 to start..." + done +} + +docker compose --file ci/cluster/docker-compose.yml down +docker compose --file ci/cluster/docker-compose.yml up --detach + +wait_for_message rabbitmq0 "completed with" + +docker exec rabbitmq0 rabbitmqctl await_online_nodes 3 + +docker exec rabbitmq0 rabbitmqctl enable_feature_flag --opt-in khepri_db +docker exec rabbitmq1 rabbitmqctl enable_feature_flag --opt-in khepri_db +docker exec rabbitmq2 rabbitmqctl enable_feature_flag --opt-in khepri_db + +docker exec rabbitmq0 rabbitmqctl cluster_status + +docker compose --file ci/cluster/docker-compose.yml ps diff --git a/pom.xml b/pom.xml index f95f27f08d..4c8ac0194a 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.rabbitmq stream-client - 0.15.0 + 1.1.0-SNAPSHOT RabbitMQ Stream Java Client The RabbitMQ Stream Java client library allows Java applications to interface with @@ -43,62 +43,64 @@ https://github.com/rabbitmq/rabbitmq-stream-java-client scm:git:git://github.com/rabbitmq/rabbitmq-stream-java-client.git scm:git:https://github.com/rabbitmq/rabbitmq-stream-java-client.git - v0.15.0 + HEAD true 1.7.36 - 1.2.12 - 4.1.104.Final + 1.2.13 + 4.2.1.Final 0.34.1 - 4.2.23 - 1.12.1 - 12.2.2 + 4.2.30 + 1.14.6 + 13.1.2 4.7.5 - 1.25.0 - 1.5.5-11 + 1.27.1 + 1.5.7-3 1.8.0 - 1.1.10.5 - 5.10.1 - 3.24.2 - 5.8.0 - 5.20.0 - 3.14.0 - 1.16.0 - 2.10.1 - 0.10.4 + 1.1.10.7 + 5.12.2 + 3.27.3 + 5.17.0 + 5.25.0 + 3.17.0 + 1.18.0 + 2.13.1 + 0.10.6 1.2.5 - 1.2.1 - 1.0.2 - 3.12.0 - 3.2.3 - 2.7.6 + 1.4.5 + 1.0.4 + 3.14.0 + 3.5.3 + 3.8.1 1.11 - 3.1.0 - 3.2.0 + 3.2.7 + 3.2.1 3.3.1 - 3.3.0 - 3.6.3 - 3.3.0 + 3.4.1 + 3.3.1 + 3.11.2 + 3.4.2 3.4.0 - 2.2.4 - 2.5.11 - 2.2.13 - 1.4 + 3.2.0 + 3.0.0 + 2.3.2 + 3.2.1 1.37 - 2.41.1 - 1.18.1 - 0.8.11 + 2.44.4 + 1.27.0 + 0.8.13 + 4.9.3.0 + 4.9.3 - 3.12 + 4.1 6026DFCA yyyy-MM-dd'T'HH:mm:ss'Z' UTF-8 0.0.6 - 1.6.13 - + 1.7.0 true true @@ -289,6 +291,26 @@ ${jmh.version} test + + ch.qos.logback + logback-classic + ${logback.version} + test + + + + com.google.googlejavaformat + google-java-format + ${google-java-format.version} + test + + + + com.github.spotbugs + spotbugs-annotations + ${spotbugs.version} + provided + @@ -326,8 +348,7 @@ maven-compiler-plugin ${maven.compiler.plugin.version} - 1.8 - 1.8 + 11 -Xlint:deprecation -Xlint:unchecked @@ -335,6 +356,25 @@ + + org.apache.maven.plugins + maven-clean-plugin + ${maven-clean-plugin.version} + + + + org.apache.maven.plugins + maven-dependency-plugin + ${maven-dependency-plugin.version} + + + + properties + + + + + maven-surefire-plugin ${maven-surefire-plugin.version} @@ -342,7 +382,7 @@ **/*TestSuite.java - ${test-arguments} + ${argLine} ${test-arguments} true DOCKER:rabbitmq @@ -498,9 +538,10 @@ - + origin/main - // Copyright (c) $YEAR Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. + // Copyright (c) $YEAR Broadcom. All Rights Reserved. + // The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -552,6 +593,26 @@ + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} + + + com.github.spotbugs + spotbugs + ${spotbugs.version} + + + + + + check + + + + + @@ -623,12 +684,12 @@ - mockito-4-on-java-8 + jvm-test-arguments-below-java-21 - 1.8 + [11,21) - 4.11.0 + -Xshare:off @@ -637,35 +698,10 @@ [21,) - -XX:+EnableDynamicAgentLoading + -Xshare:off -javaagent:${org.mockito:mockito-core:jar} - - - - use-release-compiler-argument-on-java-9-or-more - - [9,) - - - - - maven-compiler-plugin - ${maven.compiler.plugin.version} - - 1.8 - 1.8 - 8 - - -Xlint:deprecation - -Xlint:unchecked - - - - - - diff --git a/release-versions.txt b/release-versions.txt index 977920ef86..c3aa198d6e 100644 --- a/release-versions.txt +++ b/release-versions.txt @@ -1,5 +1,5 @@ -RELEASE_VERSION="0.15.0" -DEVELOPMENT_VERSION="0.16.0-SNAPSHOT" +RELEASE_VERSION="1.1.0" +DEVELOPMENT_VERSION="1.1.0-SNAPSHOT" RELEASE_BRANCH="main" LATEST=true diff --git a/src/docs/asciidoc/api.adoc b/src/docs/asciidoc/api.adoc index 70c14b3821..f4c5d0be9a 100644 --- a/src/docs/asciidoc/api.adoc +++ b/src/docs/asciidoc/api.adoc @@ -75,7 +75,7 @@ Creating the environment to connect to a cluster node works usually seamlessly. Creating publishers and consumers can cause problems as the client uses hints from the cluster to find the nodes where stream leaders and replicas are located to connect to the appropriate nodes. These connection hints can be accurate or less appropriate depending on the infrastructure. -If you hit some connection problems at some point – like hostnames impossible to resolve for client applications - this https://blog.rabbitmq.com/posts/2021/07/connecting-to-streams/[blog post] should help you understand what is going on and fix the issues. +If you hit some connection problems at some point – like hostnames impossible to resolve for client applications - this https://www.rabbitmq.com/blog/2021/07/23/connecting-to-streams[blog post] should help you understand what is going on and fix the issues. To make the local development experience simple, the client library can choose to always use `localhost` for producers and consumers. This happens if the following conditions are met: the initial host to connect to is `localhost`, the user is `guest`, and no custom address resolver has been provided. @@ -88,10 +88,9 @@ TLS can be enabled by using the `rabbitmq-stream+tls` scheme in the URI. The default TLS port is 5551. Use the `EnvironmentBuilder#tls` method to configure TLS. -The most important setting is a `io.netty.handler.ssl.SslContext` instance, -which is created and configured with the -`io.netty.handler.ssl.SslContext#forClient` method. Note hostname verification -is enabled by default. +The most important setting is a `io.netty.handler.ssl.SslContext` instance, which is created and configured with the +`io.netty.handler.ssl.SslContext#forClient` method. +Note hostname verification is enabled by default. The following snippet shows a common configuration, whereby the client is instructed to trust servers with certificates @@ -220,6 +219,11 @@ The client retries 5 times before falling back to the stream leader node. Set to `true` only for clustered environments, not for 1-node environments, where only the stream leader is available. |`false` +|`forceLeaderForProducers` +|Force connecting to a stream leader for producers. +Set to `false` if it acceptable to stay connected to a stream replica when a load balancer is in use. +|`true` + |`id` |Informational ID for the environment instance. Used as a prefix for connection names. @@ -229,19 +233,18 @@ Used as a prefix for connection names. |Contract to change resolved node address to connect to. |Pass-through (no-op) +|`locatorConnectionCount` +|Number of locator connections to maintain (for metadata search) +|The smaller of the number of URIs and 3. + |`tls` |Configuration helper for TLS. |TLS is enabled if a `rabbitmq-stream+tls` URI is provided. -|`tls#hostnameVerification` -|Enable or disable hostname verification. -|Enabled by default. - |`tls#sslContext` |Set the `io.netty.handler.ssl.SslContext` used for the TLS connection. Use `io.netty.handler.ssl.SslContextBuilder#forClient` to configure it. -The server certificate chain and the client private key are the typical -elements that need to be configured. +The server certificate chain, the client private key, and hostname verification are the usual elements that need to be configured. |The JDK trust manager and no client private key. |`tls#trustEverything` @@ -275,7 +278,7 @@ It is the developer's responsibility to close the `EventLoopGroup` they provide. ==== When a Load Balancer is in Use A load balancer can misguide the client when it tries to connect to nodes that host stream leaders and replicas. -The https://blog.rabbitmq.com/posts/2021/07/connecting-to-streams/["Connecting to Streams"] blog post covers why client applications must connect to the appropriate nodes in a cluster and how a https://blog.rabbitmq.com/posts/2021/07/connecting-to-streams/#with-a-load-balancer[load balancer can make things complicated] for them. +The https://www.rabbitmq.com/blog/2021/07/23/connecting-to-streams["Connecting to Streams"] blog post covers why client applications must connect to the appropriate nodes in a cluster and how a https://blog.rabbitmq.com/posts/2021/07/connecting-to-streams/#with-a-load-balancer[load balancer can make things complicated] for them. The `EnvironmentBuilder#addressResolver(AddressResolver)` method allows intercepting the node resolution after metadata hints and before connection. Applications can use this hook to ignore metadata hints and always use the load balancer, as illustrated in the following snippet: @@ -288,8 +291,12 @@ include::{test-examples}/EnvironmentUsage.java[tag=address-resolver] <1> Set the load balancer address <2> Use load balancer address for initial connection <3> Ignore metadata hints, always use load balancer +<4> Set the number of locator connections to maintain -The blog post covers the https://blog.rabbitmq.com/posts/2021/07/connecting-to-streams/#client-workaround-with-a-load-balancer[underlying details of this workaround]. +Note the example above sets the number of locator connections the environment maintains. +Locator connections are used to perform infrastructure-related operations (e.g. looking up the topology of a stream to find an appropriate node to connect to). +The environment uses the number of passed-in URIs to choose an appropriate default number and will pick 1 in this case, which may be too low for a cluster deployment. +This is why it is recommended to set the value explicitly, 3 being a good default. ==== Managing Streams @@ -455,6 +462,10 @@ blocking when the limit is reached. |Period to send a batch of messages. |100 ms +|`dynamicBatch` +|Adapt batch size depending on ingress rate. +|true + |`confirmTimeout` |[[producer-confirm-timeout-configuration-entry]]Time before the client calls the confirm callback to signal outstanding unconfirmed messages timed out. @@ -464,7 +475,12 @@ outstanding unconfirmed messages timed out. |Time before enqueueing of a message fail when the maximum number of unconfirmed is reached. The callback of the message will be called with a negative status. Set the value to `Duration.ZERO` if there should be no timeout. -|10 seconds. +|10 seconds + +|`retryOnRecovery` +|Whether to republish unconfirmed messages after recovery. +Set to `false` to not republish unconfirmed messages and get a negative `ConfirmationStatus` for unconfirmed messages. +|true |=== ==== Sending Messages @@ -575,6 +591,23 @@ message will be persisted twice. Luckily RabbitMQ Stream can detect and filter out duplicated messages, based on 2 client-side elements: the _producer name_ and the _message publishing ID_. +[[deduplication-multithreading]] +[WARNING] +.Only one publisher instance with a given name and no multithreading to guarantee deduplication +==== +We'll see below that deduplication works using a strictly increasing sequence for messages. +This means messages must be published in order, so there must be only _one publisher instance with a given name_ and this instance must publish messages _within a single thread_. + +With several publisher instances with the same name, one instance can be "ahead" of the others for the sequence ID: if it publishes a message with sequence ID 100, any message from any instance with a smaller lower sequence ID will be filtered out. + +If there is only one publisher instance with a given name, it should publish messages in a single thread. +Even if messages are _created_ in order, with the proper sequence ID, they can get out of order if they are published in several threads, e.g. message 5 can be _published_ before message 2. +The deduplication mechanism will then filter out message 2 in this case. + +You have to be very careful about the way your applications publish messages when deduplication is in use: make sure publisher instances do not share the same name and use only a single thread. +If you worry about performance, note it is possible to publish hundreds of thousands of messages in a single thread with RabbitMQ Stream. +==== + [WARNING] .Deduplication is not guaranteed when using sub-entries batching ==== @@ -585,19 +618,6 @@ batching messages in a single publish frame, which can already provide very high throughput. ==== -[[deduplication-multithreading]] -[WARNING] -.Deduplication is not guaranteed when publishing on several threads -==== -We'll see below that deduplication works using a strictly increasing sequence for messages. -This means messages must be published in order and the preferred way to do this is usually _within a single thread_. -Even if messages are _created_ in order, with the proper sequence ID, if they are published in several threads, they can get out of order, e.g. message 5 can be _published_ before message 2. -The deduplication mechanism will then filter out message 2 in this case. - -So you have to be very careful about the way your applications publish messages when deduplication is in use. -If you worry about performance, note it is possible to publish hundreds of thousands of messages in a single thread with RabbitMQ Stream. -==== - ===== Setting the Name of a Producer The producer name is set when creating the producer instance, which automatically @@ -871,11 +891,11 @@ Useful when using an external store for offset tracking. |`flow#initialCredits` |Number of credits when the subscription is created. Increase for higher throughput at the expense of memory usage. -|1 +|10 |`flow#strategy` |The `ConsumerFlowStrategy` to use. -|`ConsumerFlowStrategy#creditOnChunkArrival(1)` +|`ConsumerFlowStrategy#creditOnChunkArrival(10)` |=== [NOTE] diff --git a/src/docs/asciidoc/building.adoc b/src/docs/asciidoc/building.adoc index c5feff4f79..cf1935a308 100644 --- a/src/docs/asciidoc/building.adoc +++ b/src/docs/asciidoc/building.adoc @@ -1,6 +1,6 @@ == Building the Client -You need JDK 1.8 or more installed. +You need JDK 11 or more installed. To build the JAR file: diff --git a/src/docs/asciidoc/index.adoc b/src/docs/asciidoc/index.adoc index 887c6409a6..d8f442e779 100644 --- a/src/docs/asciidoc/index.adoc +++ b/src/docs/asciidoc/index.adoc @@ -11,6 +11,8 @@ the https://rabbitmq.com/stream.html[RabbitMQ Stream Plugin]. It allows creating and deleting streams, as well as publishing to and consuming from these streams. Learn more in the <>. +This library requires at least Java 11, but Java 21 or more is recommended. + https://github.com/rabbitmq/rabbitmq-stream-perf-test[Stream PerfTest] is a performance testing tool based on this client library. diff --git a/src/docs/asciidoc/overview.adoc b/src/docs/asciidoc/overview.adoc index c37112f19f..6040e03172 100644 --- a/src/docs/asciidoc/overview.adoc +++ b/src/docs/asciidoc/overview.adoc @@ -66,37 +66,34 @@ can also be useful for development and testing. * _avoid publishing duplicate messages_ thanks to message deduplication. * _consume asynchronously from streams and resume where left off_ thanks to automatic or manual offset tracking. -* _enforce https://blog.rabbitmq.com/posts/2021/07/connecting-to-streams/[best practices] to create client connections_ – to stream leaders for publishers to minimize inter-node traffic and to stream replicas for consumers to offload leaders. +* _enforce https://www.rabbitmq.com/blog/2021/07/23/connecting-to-streams/#client-workaround-with-a-load-balancer[best practices] to create client connections_ – to stream leaders for publishers to minimize inter-node traffic and to stream replicas for consumers to offload leaders. * _optimize resources_ thanks to automatic growing and shrinking of connections depending on the number of publishers and consumers. * _let the client handle network failure_ thanks to automatic connection recovery and automatic re-subscription for consumers. +* _publish metrics_ to monitoring systems like https://prometheus.io/[Prometheus] and _ship spans_ to distributed tracing backends like https://zipkin.io/[OpenZipkin] or https://tanzu.vmware.com/observability[Wavefront] thanks to built-in support for https://micrometer.io/[Micrometer]. == Versioning -The RabbitMQ Stream Java Client is in development and stabilization phase. -When the stabilization phase ends, a 1.0.0 version will be cut, and -https://semver.org/[semantic versioning] is likely to be enforced. +This library uses https://semver.org/[semantic versioning]. -Before reaching the stable phase, the client will use a versioning scheme of `[0.MINOR.PATCH]` where: - -* `0` indicates the project is still in a stabilization phase. -* `MINOR` is a 0-based number incrementing with each new release cycle. It generally reflects significant changes like new features and potentially some programming interfaces changes. -* `PATCH` is a 0-based number incrementing with each service release, that is bux fixes. - -Breaking changes between releases can happen but will be kept to a minimum. The next section provides more details about the evolution of programming interfaces. [[stability-of-programming-interfaces]] == Stability of Programming Interfaces -The RabbitMQ Stream Java Client is in active development but its programming interfaces will remain as stable as possible. There is no guarantee though that they will remain completely stable, at least until it reaches version 1.0.0. - The client contains 2 sets of programming interfaces whose stability are of interest for application developers: -* Application Programming Interfaces (API): those are the ones used to write application logic. They include the interfaces and classes in the `com.rabbitmq.stream` package (e.g. `Producer`, `Consumer`, `Message`). These API constitute the main programming model of the client and will be kept as stable as possible. -* Service Provider Interfaces (SPI): those are interfaces to implement mainly technical behavior in the client. They are not meant to be used to implement application logic. Application developers may have to refer to them in the configuration phase and if they want to custom some internal behavior in the client. SPI include interfaces and classes in the `com.rabbitmq.stream.codec`, `com.rabbitmq.stream.compression`, `com.rabbitmq.stream.metrics` packages, among others. _These SPI are susceptible to change, but this should not impact the majority of applications_, as the changes would typically stay intern to the client. +* Application Programming Interfaces (API): those are the ones used to write application logic. +They include the interfaces and classes in the `com.rabbitmq.stream` package (e.g. `Producer`, `Consumer`, `Message`). +These API constitute the main programming model of the client and are kept as stable as possible. +New features may require to add methods to existing interfaces. +* Service Provider Interfaces (SPI): those are interfaces to implement mainly technical behavior in the client. +They are not meant to be used to implement application logic. +Application developers may have to refer to them in the configuration phase and if they want to customize some internal behavior of the client. +SPI include interfaces and classes in the `com.rabbitmq.stream.codec`, `com.rabbitmq.stream.compression`, `com.rabbitmq.stream.metrics` packages, among others. +_These SPI are susceptible to change, but this should have no impact on most applications_, as the changes are likely to be limited to the client internals. == Pre-requisites -The library requires Java 8 or later. Java 11 is recommended (CRC calculation uses methods available as of Java 9.) +This library requires at least Java 11, but Java 21 or more is recommended. diff --git a/src/docs/asciidoc/setup.adoc b/src/docs/asciidoc/setup.adoc index b19f970c9d..e9eaaa6df8 100644 --- a/src/docs/asciidoc/setup.adoc +++ b/src/docs/asciidoc/setup.adoc @@ -129,7 +129,7 @@ Snapshots require to declare the <>. === Snapshots Releases are available from Maven Central, which does not require specific declaration. -Snapshots are available from a repositoriy which must be declared in the dependency management configuration. +Snapshots are available from a repository which must be declared in the dependency management configuration. With Maven: diff --git a/src/main/java/com/rabbitmq/stream/Address.java b/src/main/java/com/rabbitmq/stream/Address.java index db0a382bbe..4d4909a96b 100644 --- a/src/main/java/com/rabbitmq/stream/Address.java +++ b/src/main/java/com/rabbitmq/stream/Address.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/AddressResolver.java b/src/main/java/com/rabbitmq/stream/AddressResolver.java index 5ebfcc8b42..361d3ebfc4 100644 --- a/src/main/java/com/rabbitmq/stream/AddressResolver.java +++ b/src/main/java/com/rabbitmq/stream/AddressResolver.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/AuthenticationFailureException.java b/src/main/java/com/rabbitmq/stream/AuthenticationFailureException.java index 3afa579b77..72a0ab806d 100644 --- a/src/main/java/com/rabbitmq/stream/AuthenticationFailureException.java +++ b/src/main/java/com/rabbitmq/stream/AuthenticationFailureException.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/BackOffDelayPolicy.java b/src/main/java/com/rabbitmq/stream/BackOffDelayPolicy.java index 3e70e140f4..4124479135 100644 --- a/src/main/java/com/rabbitmq/stream/BackOffDelayPolicy.java +++ b/src/main/java/com/rabbitmq/stream/BackOffDelayPolicy.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -72,7 +72,7 @@ static BackOffDelayPolicy fixedWithInitialDelay( */ Duration delay(int recoveryAttempt); - class FixedWithInitialDelayBackOffPolicy implements BackOffDelayPolicy { + final class FixedWithInitialDelayBackOffPolicy implements BackOffDelayPolicy { private final Duration initialDelay; private final Duration delay; @@ -98,7 +98,7 @@ public String toString() { } } - class FixedWithInitialDelayAndTimeoutBackOffPolicy implements BackOffDelayPolicy { + final class FixedWithInitialDelayAndTimeoutBackOffPolicy implements BackOffDelayPolicy { private final int attemptLimitBeforeTimeout; private final BackOffDelayPolicy delegate; diff --git a/src/main/java/com/rabbitmq/stream/ByteCapacity.java b/src/main/java/com/rabbitmq/stream/ByteCapacity.java index a879af16b3..7fbc6c0d38 100644 --- a/src/main/java/com/rabbitmq/stream/ByteCapacity.java +++ b/src/main/java/com/rabbitmq/stream/ByteCapacity.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -14,8 +14,6 @@ // info@rabbitmq.com. package com.rabbitmq.stream; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.function.BiFunction; @@ -40,15 +38,11 @@ public class ByteCapacity implements Comparable { private static final String UNIT_TB = "tb"; private static final Map> CONSTRUCTORS = - Collections.unmodifiableMap( - new HashMap>() { - { - put(UNIT_KB, (size, input) -> ByteCapacity.kB(size, input)); - put(UNIT_MB, (size, input) -> ByteCapacity.MB(size, input)); - put(UNIT_GB, (size, input) -> ByteCapacity.GB(size, input)); - put(UNIT_TB, (size, input) -> ByteCapacity.TB(size, input)); - } - }); + Map.of( + UNIT_KB, ByteCapacity::kB, + UNIT_MB, ByteCapacity::MB, + UNIT_GB, ByteCapacity::GB, + UNIT_TB, ByteCapacity::TB); private final long bytes; private final String input; @@ -105,7 +99,7 @@ public long toBytes() { public static ByteCapacity from(String value) { Matcher matcher = PATTERN.matcher(value); if (matcher.matches()) { - long size = Long.valueOf(matcher.group(GROUP_SIZE)); + long size = Long.parseLong(matcher.group(GROUP_SIZE)); String unit = matcher.group(GROUP_UNIT); ByteCapacity result; if (unit == null) { diff --git a/src/main/java/com/rabbitmq/stream/ChunkChecksum.java b/src/main/java/com/rabbitmq/stream/ChunkChecksum.java index 860f52dcd0..a576b1b5a6 100644 --- a/src/main/java/com/rabbitmq/stream/ChunkChecksum.java +++ b/src/main/java/com/rabbitmq/stream/ChunkChecksum.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/ChunkChecksumValidationException.java b/src/main/java/com/rabbitmq/stream/ChunkChecksumValidationException.java index e2b65c5ba9..b7677ce35f 100644 --- a/src/main/java/com/rabbitmq/stream/ChunkChecksumValidationException.java +++ b/src/main/java/com/rabbitmq/stream/ChunkChecksumValidationException.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/Codec.java b/src/main/java/com/rabbitmq/stream/Codec.java index d5d4ec645a..1cb3b0fb67 100644 --- a/src/main/java/com/rabbitmq/stream/Codec.java +++ b/src/main/java/com/rabbitmq/stream/Codec.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -14,6 +14,8 @@ // info@rabbitmq.com. package com.rabbitmq.stream; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + /** * Codec to encode and decode messages. * @@ -34,11 +36,13 @@ class EncodedMessage { private final int size; private final byte[] data; + @SuppressFBWarnings("EI_EXPOSE_REP2") public EncodedMessage(int size, byte[] data) { this.size = size; this.data = data; } + @SuppressFBWarnings("EI_EXPOSE_REP") public byte[] getData() { return data; } diff --git a/src/main/java/com/rabbitmq/stream/ConfirmationHandler.java b/src/main/java/com/rabbitmq/stream/ConfirmationHandler.java index d70eb7c2c5..db5d010fac 100644 --- a/src/main/java/com/rabbitmq/stream/ConfirmationHandler.java +++ b/src/main/java/com/rabbitmq/stream/ConfirmationHandler.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/ConfirmationStatus.java b/src/main/java/com/rabbitmq/stream/ConfirmationStatus.java index 9479f0ccec..2c9719966a 100644 --- a/src/main/java/com/rabbitmq/stream/ConfirmationStatus.java +++ b/src/main/java/com/rabbitmq/stream/ConfirmationStatus.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/Constants.java b/src/main/java/com/rabbitmq/stream/Constants.java index fddaf44c05..a432562d07 100644 --- a/src/main/java/com/rabbitmq/stream/Constants.java +++ b/src/main/java/com/rabbitmq/stream/Constants.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/Consumer.java b/src/main/java/com/rabbitmq/stream/Consumer.java index f51cfc935c..2b6254949b 100644 --- a/src/main/java/com/rabbitmq/stream/Consumer.java +++ b/src/main/java/com/rabbitmq/stream/Consumer.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/ConsumerBuilder.java b/src/main/java/com/rabbitmq/stream/ConsumerBuilder.java index 5deaa9adcb..29fc8646e9 100644 --- a/src/main/java/com/rabbitmq/stream/ConsumerBuilder.java +++ b/src/main/java/com/rabbitmq/stream/ConsumerBuilder.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -245,10 +245,14 @@ interface FlowConfiguration { /** * The number of initial credits for the subscription. * - *

Default is 1. + *

Default is 10. * *

This calls uses {@link ConsumerFlowStrategy#creditOnChunkArrival(int)}. * + *

Use a small value like 1 for streams with large chunks (several hundreds of messages per + * chunk) and higher values (5 or more) for streams with small chunks (1 or a few messages per + * chunk). + * * @param initialCredits the number of initial credits * @return this configuration instance * @see ConsumerFlowStrategy#creditOnChunkArrival(int) diff --git a/src/main/java/com/rabbitmq/stream/ConsumerFlowStrategy.java b/src/main/java/com/rabbitmq/stream/ConsumerFlowStrategy.java index e4af19b118..aa8863d057 100644 --- a/src/main/java/com/rabbitmq/stream/ConsumerFlowStrategy.java +++ b/src/main/java/com/rabbitmq/stream/ConsumerFlowStrategy.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2023-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -120,13 +120,14 @@ static ConsumerFlowStrategy creditOnChunkArrival() { * * @param initialCredits number of initial credits * @return flow strategy + * @see com.rabbitmq.stream.ConsumerBuilder.FlowConfiguration#initialCredits(int) */ static ConsumerFlowStrategy creditOnChunkArrival(int initialCredits) { return new CreditOnChunkArrivalConsumerFlowStrategy(initialCredits); } /** - * Strategy that provides 1 initial credit and a credit when half of the chunk messages are + * Strategy that provides 10 initial credits and a credit when half of the chunk messages are * processed. * *

Make sure to call {@link MessageHandler.Context#processed()} on every message when using @@ -135,7 +136,7 @@ static ConsumerFlowStrategy creditOnChunkArrival(int initialCredits) { * @return flow strategy */ static ConsumerFlowStrategy creditWhenHalfMessagesProcessed() { - return creditOnProcessedMessageCount(1, 0.5); + return creditOnProcessedMessageCount(10, 0.5); } /** @@ -147,6 +148,7 @@ static ConsumerFlowStrategy creditWhenHalfMessagesProcessed() { * * @param initialCredits number of initial credits * @return flow strategy + * @see com.rabbitmq.stream.ConsumerBuilder.FlowConfiguration#initialCredits(int) */ static ConsumerFlowStrategy creditWhenHalfMessagesProcessed(int initialCredits) { return creditOnProcessedMessageCount(initialCredits, 0.5); diff --git a/src/main/java/com/rabbitmq/stream/ConsumerUpdateListener.java b/src/main/java/com/rabbitmq/stream/ConsumerUpdateListener.java index e7d9c8d56e..c827855ef6 100644 --- a/src/main/java/com/rabbitmq/stream/ConsumerUpdateListener.java +++ b/src/main/java/com/rabbitmq/stream/ConsumerUpdateListener.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/Environment.java b/src/main/java/com/rabbitmq/stream/Environment.java index 2645d61807..00baf4099d 100644 --- a/src/main/java/com/rabbitmq/stream/Environment.java +++ b/src/main/java/com/rabbitmq/stream/Environment.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/EnvironmentBuilder.java b/src/main/java/com/rabbitmq/stream/EnvironmentBuilder.java index db3b6a4b2a..4c8a5239f5 100644 --- a/src/main/java/com/rabbitmq/stream/EnvironmentBuilder.java +++ b/src/main/java/com/rabbitmq/stream/EnvironmentBuilder.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -16,6 +16,7 @@ import com.rabbitmq.stream.compression.Compression; import com.rabbitmq.stream.compression.CompressionCodecFactory; +import com.rabbitmq.stream.impl.StreamEnvironmentBuilder; import com.rabbitmq.stream.metrics.MetricsCollector; import com.rabbitmq.stream.sasl.CredentialsProvider; import com.rabbitmq.stream.sasl.SaslConfiguration; @@ -62,14 +63,15 @@ public interface EnvironmentBuilder { * An {@link AddressResolver} to potentially change resolved node address to connect to. * *

Applications can use this abstraction to make sure connection attempts ignore metadata hints - * and always go to a single point like a load balancer. + * and always go to a single point like a load balancer. Consider setting {@link + * #locatorConnectionCount(int)} when using a load balancer. * *

The default implementation does not perform any logic, it just returns the passed-in * address. * *

The default implementation is overridden automatically if the following conditions are * met: the host to connect to is localhost, the user is guest, and no - * address resolver has been provided. The client will then always tries to connect to + * address resolver has been provided. The client will then always try to connect to * localhost to facilitate local development. Just provide a pass-through address resolver * to avoid this behavior, e.g.: * @@ -79,10 +81,11 @@ public interface EnvironmentBuilder { * .build(); * * - * @param addressResolver + * @param addressResolver the address resolver * @return this builder instance * @see "Connecting to * Streams" blog post + * @see #locatorConnectionCount(int) */ EnvironmentBuilder addressResolver(AddressResolver addressResolver); @@ -240,6 +243,12 @@ public interface EnvironmentBuilder { */ EnvironmentBuilder metricsCollector(MetricsCollector metricsCollector); + /** + * Set up an {@link ObservationCollector}. + * + * @param observationCollector + * @return this builder instance + */ EnvironmentBuilder observationCollector(ObservationCollector observationCollector); /** @@ -348,7 +357,7 @@ EnvironmentBuilder topologyUpdateBackOffDelayPolicy( *

Do not set this flag to true when streams have only 1 member (the leader), * e.g. for local development. * - *

Default is false. + *

Default is false. * * @param forceReplica whether to force the connection to a replica or not * @return this builder instance @@ -358,6 +367,58 @@ EnvironmentBuilder topologyUpdateBackOffDelayPolicy( */ EnvironmentBuilder forceReplicaForConsumers(boolean forceReplica); + /** + * Flag to force the connection to the stream leader for producers. + * + *

The library prefers to connect to a node that hosts a stream leader for producers (default + * behavior). + * + *

When using a load balancer, the library does not know in advance the node it connects to. It + * may have to retry to connect to the appropriate node. + * + *

It will retry until it connects to the appropriate node (flag set to true, the + * default). This provides the best data locality, but may require several attempts, delaying the + * creation or the recovery of producers. This usually suits high-throughput use cases. + * + *

The library will accept the connection to a stream replica if the flag is set to false + * . This will speed up the creation/recovery of producers, but at the cost of network hops + * between cluster nodes when publishing messages because only a stream leader accepts writes. + * This is usually acceptable for low-throughput use cases. + * + *

Changing the default value should only benefit systems where a load balancer sits between + * the client applications and the cluster nodes. + * + *

Default is true. + * + * @param forceLeader whether to force the connection to the leader or not + * @return this builder instance + * @see #recoveryBackOffDelayPolicy(BackOffDelayPolicy) + * @see #topologyUpdateBackOffDelayPolicy(BackOffDelayPolicy) + * @since 0.21.0 + */ + EnvironmentBuilder forceLeaderForProducers(boolean forceLeader); + + /** + * Set the expected number of "locator" connections to maintain. + * + *

Locator connections are used to perform infrastructure-related operations (e.g. looking up + * the topology of a stream to find an appropriate node to connect to). + * + *

It is recommended to maintain 2 to 3 locator connections. The environment uses the smaller + * of the number of passed-in URIs and 3 by default (see {@link #uris(List)}). + * + *

The number of locator connections should be explicitly set when a load balancer is used, as + * the environment cannot know the number of cluster nodes in this case (the only URI set is the + * one of the load balancer). + * + * @param locatorConnectionCount number of expected locator connections + * @return this builder instance + * @see #uris(List) + * @see #addressResolver(AddressResolver) + * @since 0.21.0 + */ + StreamEnvironmentBuilder locatorConnectionCount(int locatorConnectionCount); + /** * Create the {@link Environment} instance. * @@ -375,31 +436,12 @@ EnvironmentBuilder topologyUpdateBackOffDelayPolicy( /** Helper to configure TLS. */ interface TlsConfiguration { - /** - * Enable hostname verification. - * - *

Hostname verification is enabled by default. - * - * @return the TLS configuration helper - */ - TlsConfiguration hostnameVerification(); - - /** - * Enable or disable hostname verification. - * - *

Hostname verification is enabled by default. - * - * @param hostnameVerification - * @return the TLS configuration helper - */ - TlsConfiguration hostnameVerification(boolean hostnameVerification); - /** * Netty {@link SslContext} for TLS connections. * *

Use {@link SslContextBuilder#forClient()} to configure and create an instance. * - * @param sslContext + * @param sslContext the SSL context * @return the TLS configuration helper */ TlsConfiguration sslContext(SslContext sslContext); diff --git a/src/main/java/com/rabbitmq/stream/Message.java b/src/main/java/com/rabbitmq/stream/Message.java index 70f39ddc2a..7355b02fa2 100644 --- a/src/main/java/com/rabbitmq/stream/Message.java +++ b/src/main/java/com/rabbitmq/stream/Message.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -32,18 +32,26 @@ public interface Message { /** * Does this message has a publishing ID? * - *

Publishing IDs are used for de-duplication of outbound messages. They are not persisted. + *

Publishing IDs are used for deduplication of outbound messages. They are not persisted. * * @return true if the message has a publishing ID, false otherwise + * @see ProducerBuilder#name(String) + * @see Deduplication + * documentation */ boolean hasPublishingId(); /** * Get the publishing ID for the message. * - *

Publishing IDs are used for de-duplication of outbound messages. They are not persisted. + *

Publishing IDs are used for deduplication of outbound messages. They are not persisted. * * @return the publishing ID of the message + * @see ProducerBuilder#name(String) + * @see Deduplication + * documentation */ long getPublishingId(); diff --git a/src/main/java/com/rabbitmq/stream/MessageBuilder.java b/src/main/java/com/rabbitmq/stream/MessageBuilder.java index d9e1fcc390..02a9d3c3ee 100644 --- a/src/main/java/com/rabbitmq/stream/MessageBuilder.java +++ b/src/main/java/com/rabbitmq/stream/MessageBuilder.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -15,6 +15,8 @@ package com.rabbitmq.stream; import java.math.BigDecimal; +import java.util.List; +import java.util.Map; import java.util.UUID; /** @@ -38,12 +40,16 @@ public interface MessageBuilder { Message build(); /** - * Set the publishing ID (for de-duplication). + * Set the publishing ID (for deduplication). * *

This is value is used only for outbound messages and is not persisted. * * @param publishingId * @return this builder instance + * @see ProducerBuilder#name(String) + * @see Deduplication + * documentation */ MessageBuilder publishingId(long publishingId); @@ -182,6 +188,12 @@ interface MessageAnnotationsBuilder { MessageAnnotationsBuilder entrySymbol(String key, String value); + MessageAnnotationsBuilder entry(String key, List list); + + MessageAnnotationsBuilder entry(String key, Map map); + + MessageAnnotationsBuilder entryArray(String key, Object[] array); + /** * Go back to the message builder * diff --git a/src/main/java/com/rabbitmq/stream/MessageHandler.java b/src/main/java/com/rabbitmq/stream/MessageHandler.java index 0aca4cb5cf..f7f7f1ca90 100644 --- a/src/main/java/com/rabbitmq/stream/MessageHandler.java +++ b/src/main/java/com/rabbitmq/stream/MessageHandler.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/NoOffsetException.java b/src/main/java/com/rabbitmq/stream/NoOffsetException.java index caf43d41b6..57ca741806 100644 --- a/src/main/java/com/rabbitmq/stream/NoOffsetException.java +++ b/src/main/java/com/rabbitmq/stream/NoOffsetException.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/ObservationCollector.java b/src/main/java/com/rabbitmq/stream/ObservationCollector.java index a17744b1e9..6bbea0ed65 100644 --- a/src/main/java/com/rabbitmq/stream/ObservationCollector.java +++ b/src/main/java/com/rabbitmq/stream/ObservationCollector.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/OffsetSpecification.java b/src/main/java/com/rabbitmq/stream/OffsetSpecification.java index 55e6a8b045..64e12c8e45 100644 --- a/src/main/java/com/rabbitmq/stream/OffsetSpecification.java +++ b/src/main/java/com/rabbitmq/stream/OffsetSpecification.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/Producer.java b/src/main/java/com/rabbitmq/stream/Producer.java index 78226efdd8..035dcca74e 100644 --- a/src/main/java/com/rabbitmq/stream/Producer.java +++ b/src/main/java/com/rabbitmq/stream/Producer.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -15,7 +15,7 @@ package com.rabbitmq.stream; /** - * API to send message to a RabbitMQ Stream. + * API to send messages to a RabbitMQ Stream. * *

Instances are created and configured with a {@link ProducerBuilder}. * diff --git a/src/main/java/com/rabbitmq/stream/ProducerBuilder.java b/src/main/java/com/rabbitmq/stream/ProducerBuilder.java index 956ebb763c..b89524de96 100644 --- a/src/main/java/com/rabbitmq/stream/ProducerBuilder.java +++ b/src/main/java/com/rabbitmq/stream/ProducerBuilder.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -23,12 +23,18 @@ public interface ProducerBuilder { /** - * The logical name of the producer. + * The producer name for deduplication (read the documentation + * before use). * - *

Set a value to enable de-duplication. + *

There must be only one producer instance at the same time using a given name. * * @param name * @return this builder instance + * @see MessageBuilder#publishingId(long) + * @see Deduplication + * documentation */ ProducerBuilder name(String name); @@ -97,6 +103,32 @@ public interface ProducerBuilder { */ ProducerBuilder batchPublishingDelay(Duration batchPublishingDelay); + /** + * Adapt batch size depending on ingress rate. + * + *

A dynamic-batch approach improves latency for low ingress rates. + * + *

Set this flag to true if you want as little delay as possible between calling + * {@link Producer#send(Message, ConfirmationHandler)} and the message being sent to the broker. + * Consumers should provide enough initial credits (between 5 and 10, depending on the workload), + * see {@link ConsumerBuilder#flow()} and {@link + * ConsumerBuilder.FlowConfiguration#initialCredits(int)}. + * + *

Set this flag to false if latency is not critical for your use case and you + * want the highest throughput possible for both publishing and consuming. Consumers can provide 1 + * initial credit (depending on the workload), see {@link ConsumerBuilder#flow()} and {@link + * ConsumerBuilder.FlowConfiguration#initialCredits(int)}. + * + *

Dynamic batch is activated by default (dynamicBatch = true). + * + * @param dynamicBatch + * @return this builder instance + * @since 0.20.0 + * @see ConsumerBuilder#flow() + * @see com.rabbitmq.stream.ConsumerBuilder.FlowConfiguration#initialCredits(int) + */ + ProducerBuilder dynamicBatch(boolean dynamicBatch); + /** * The maximum number of unconfirmed outbound messages. * @@ -133,6 +165,23 @@ public interface ProducerBuilder { */ ProducerBuilder enqueueTimeout(Duration timeout); + /** + * Whether to republish unconfirmed messages after recovery. + * + *

Default is true (unconfirmed messages are republished after recovery). + * + *

Set to false to not republish unconfirmed messages and get a negative {@link + * ConfirmationStatus} for unconfirmed messages. + * + *

Note setting this flag to false translates to at-most-once semantics, that is + * published messages may be lost, unless the publishing application retries publishing them. + * + * @param retryOnRecovery retry flag + * @return this builder instance + * @since 0.19.0 + */ + ProducerBuilder retryOnRecovery(boolean retryOnRecovery); + /** * Logic to extract a filter value from a message. * diff --git a/src/main/java/com/rabbitmq/stream/Properties.java b/src/main/java/com/rabbitmq/stream/Properties.java index 4a4be68b72..5174118545 100644 --- a/src/main/java/com/rabbitmq/stream/Properties.java +++ b/src/main/java/com/rabbitmq/stream/Properties.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/RoutingStrategy.java b/src/main/java/com/rabbitmq/stream/RoutingStrategy.java index f6426d00f6..a9ad95fcc2 100644 --- a/src/main/java/com/rabbitmq/stream/RoutingStrategy.java +++ b/src/main/java/com/rabbitmq/stream/RoutingStrategy.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/StreamCreator.java b/src/main/java/com/rabbitmq/stream/StreamCreator.java index 125500ee71..659a53e728 100644 --- a/src/main/java/com/rabbitmq/stream/StreamCreator.java +++ b/src/main/java/com/rabbitmq/stream/StreamCreator.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -92,6 +92,25 @@ public interface StreamCreator { */ StreamCreator filterSize(int size); + /** + * Set the number of initial members the stream should have. + * + * @param initialMemberCount initial number of nodes + * @return this creator instance + * @see Initial Replication + * Factor + */ + StreamCreator initialMemberCount(int initialMemberCount); + + /** + * Set an argument for the stream creation. + * + * @param key argument key + * @param value argument value + * @return this creator instance + */ + StreamCreator argument(String key, String value); + /** * Configure the super stream to create. * @@ -127,23 +146,7 @@ enum LeaderLocator { * *

Default value for RabbitMQ 3.10+. */ - BALANCED("balanced"), - - /** - * The stream leader will be a random node of the cluster. - * - *

Deprecated as of RabbitMQ 3.10, same as {@link LeaderLocator#BALANCED}. - */ - RANDOM("random"), - - /** - * The stream leader will be on the node with the least number of stream leaders. - * - *

Deprecated as of RabbitMQ 3.10, same as {@link LeaderLocator#BALANCED}. - * - *

Default value for RabbitMQ 3.9. - */ - LEAST_LEADERS("least-leaders"); + BALANCED("balanced"); String value; diff --git a/src/main/java/com/rabbitmq/stream/StreamDoesNotExistException.java b/src/main/java/com/rabbitmq/stream/StreamDoesNotExistException.java index 4483a55035..494ebdf383 100644 --- a/src/main/java/com/rabbitmq/stream/StreamDoesNotExistException.java +++ b/src/main/java/com/rabbitmq/stream/StreamDoesNotExistException.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/StreamException.java b/src/main/java/com/rabbitmq/stream/StreamException.java index 41128c12f3..4830512009 100644 --- a/src/main/java/com/rabbitmq/stream/StreamException.java +++ b/src/main/java/com/rabbitmq/stream/StreamException.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/StreamNotAvailableException.java b/src/main/java/com/rabbitmq/stream/StreamNotAvailableException.java index 054ee8a141..1d59402759 100644 --- a/src/main/java/com/rabbitmq/stream/StreamNotAvailableException.java +++ b/src/main/java/com/rabbitmq/stream/StreamNotAvailableException.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/StreamStats.java b/src/main/java/com/rabbitmq/stream/StreamStats.java index 4e146d0b58..bbcdc2d39b 100644 --- a/src/main/java/com/rabbitmq/stream/StreamStats.java +++ b/src/main/java/com/rabbitmq/stream/StreamStats.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/SubscriptionListener.java b/src/main/java/com/rabbitmq/stream/SubscriptionListener.java index b2be288c2c..c63959c71a 100644 --- a/src/main/java/com/rabbitmq/stream/SubscriptionListener.java +++ b/src/main/java/com/rabbitmq/stream/SubscriptionListener.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -29,7 +29,7 @@ public interface SubscriptionListener { * *

The method is called when a {@link Consumer} is created and it registers to the broker, and * also when the subscription must be re-created (after a disconnection or when the subscription - * must moved because the stream member it was connected to becomes unavailable). + * must move because the stream member it was connected to becomes unavailable). * *

Application code can set the {@link OffsetSpecification} that will be used with the {@link * SubscriptionContext#offsetSpecification(OffsetSpecification)} method. @@ -64,5 +64,12 @@ interface SubscriptionContext { * @param offsetSpecification the offset specification to use */ void offsetSpecification(OffsetSpecification offsetSpecification); + + /** + * The stream involved. + * + * @return the stream + */ + String stream(); } } diff --git a/src/main/java/com/rabbitmq/stream/amqp/Symbol.java b/src/main/java/com/rabbitmq/stream/amqp/Symbol.java index e5a134a31a..be1100ae8a 100644 --- a/src/main/java/com/rabbitmq/stream/amqp/Symbol.java +++ b/src/main/java/com/rabbitmq/stream/amqp/Symbol.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/amqp/UnsignedByte.java b/src/main/java/com/rabbitmq/stream/amqp/UnsignedByte.java index bc4ea09f86..3b48f741a3 100644 --- a/src/main/java/com/rabbitmq/stream/amqp/UnsignedByte.java +++ b/src/main/java/com/rabbitmq/stream/amqp/UnsignedByte.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/amqp/UnsignedInteger.java b/src/main/java/com/rabbitmq/stream/amqp/UnsignedInteger.java index dd2b5ccae4..f71560af76 100644 --- a/src/main/java/com/rabbitmq/stream/amqp/UnsignedInteger.java +++ b/src/main/java/com/rabbitmq/stream/amqp/UnsignedInteger.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/amqp/UnsignedLong.java b/src/main/java/com/rabbitmq/stream/amqp/UnsignedLong.java index 2b32c62a8b..18bc4b2e8f 100644 --- a/src/main/java/com/rabbitmq/stream/amqp/UnsignedLong.java +++ b/src/main/java/com/rabbitmq/stream/amqp/UnsignedLong.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/amqp/UnsignedShort.java b/src/main/java/com/rabbitmq/stream/amqp/UnsignedShort.java index ead79747bb..002efef626 100644 --- a/src/main/java/com/rabbitmq/stream/amqp/UnsignedShort.java +++ b/src/main/java/com/rabbitmq/stream/amqp/UnsignedShort.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/amqp/package-info.java b/src/main/java/com/rabbitmq/stream/amqp/package-info.java index 88cd83e282..14121ad9d6 100644 --- a/src/main/java/com/rabbitmq/stream/amqp/package-info.java +++ b/src/main/java/com/rabbitmq/stream/amqp/package-info.java @@ -1,2 +1,2 @@ -/** Classes for AMQP 1.0 support. */ +/** Classes for AMQP 1.0 message format support. */ package com.rabbitmq.stream.amqp; diff --git a/src/main/java/com/rabbitmq/stream/codec/QpidProtonCodec.java b/src/main/java/com/rabbitmq/stream/codec/QpidProtonCodec.java index 804a1277f6..abadc2f618 100644 --- a/src/main/java/com/rabbitmq/stream/codec/QpidProtonCodec.java +++ b/src/main/java/com/rabbitmq/stream/codec/QpidProtonCodec.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -19,21 +19,17 @@ import com.rabbitmq.stream.MessageBuilder; import com.rabbitmq.stream.Properties; import java.nio.ByteBuffer; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.UUID; +import java.util.*; import java.util.function.Function; import org.apache.qpid.proton.amqp.*; -import org.apache.qpid.proton.amqp.messaging.AmqpValue; -import org.apache.qpid.proton.amqp.messaging.ApplicationProperties; -import org.apache.qpid.proton.amqp.messaging.Data; -import org.apache.qpid.proton.amqp.messaging.MessageAnnotations; +import org.apache.qpid.proton.amqp.messaging.*; import org.apache.qpid.proton.codec.ReadableBuffer; import org.apache.qpid.proton.codec.WritableBuffer; public class QpidProtonCodec implements Codec { + static final Section EMPTY_BODY = new Data(new Binary(new byte[0])); + private static final Function MESSAGE_ANNOTATIONS_STRING_KEY_EXTRACTOR = k -> k; private static final Function MESSAGE_ANNOTATIONS_SYMBOL_KEY_EXTRACTOR = Symbol::toString; @@ -64,7 +60,7 @@ private static Map createMapFromAmqpMap( if (amqpMap != null) { result = new LinkedHashMap<>(amqpMap.size()); for (Map.Entry entry : amqpMap.entrySet()) { - result.put(keyMaker.apply(entry.getKey()), convertApplicationProperty(entry.getValue())); + result.put(keyMaker.apply(entry.getKey()), fromQpidToJava(entry.getValue())); } } else { result = null; @@ -72,7 +68,7 @@ private static Map createMapFromAmqpMap( return result; } - private static Object convertApplicationProperty(Object value) { + private static Object fromQpidToJava(Object value) { if (value instanceof Boolean || value instanceof Byte || value instanceof Short @@ -82,7 +78,10 @@ private static Object convertApplicationProperty(Object value) { || value instanceof Double || value instanceof String || value instanceof Character - || value instanceof UUID) { + || value instanceof UUID + || value instanceof List + || value instanceof Map + || value instanceof Object[]) { return value; } else if (value instanceof Binary) { return ((Binary) value).getArray(); @@ -100,9 +99,10 @@ private static Object convertApplicationProperty(Object value) { return ((Symbol) value).toString(); } else if (value == null) { return null; + } else if (value.getClass().isArray()) { + return value; } else { - throw new IllegalArgumentException( - "Type not supported for an application property: " + value.getClass()); + throw new IllegalArgumentException("Type not supported: " + value.getClass()); } } @@ -233,13 +233,16 @@ public EncodedMessage encode(Message message) { qpidMessage.setMessageAnnotations(new MessageAnnotations(messageAnnotations)); } - if (message.getBodyAsBinary() != null) { + if (message.getBodyAsBinary() == null) { + qpidMessage.setBody(EMPTY_BODY); + } else { qpidMessage.setBody(new Data(new Binary(message.getBodyAsBinary()))); } } int bufferSize; if (qpidMessage.getBody() instanceof Data) { bufferSize = (int) (((Data) qpidMessage.getBody()).getValue().getLength() * 1.5); + bufferSize = bufferSize == 0 ? 128 : bufferSize; } else { bufferSize = 8192; } @@ -279,7 +282,10 @@ protected Object convertToQpidType(Object value) { || value instanceof String || value instanceof Character || value instanceof UUID - || value instanceof Date) { + || value instanceof Date + || value instanceof List + || value instanceof Map + || value instanceof Object[]) { return value; } else if (value instanceof com.rabbitmq.stream.amqp.UnsignedByte) { return UnsignedByte.valueOf(((com.rabbitmq.stream.amqp.UnsignedByte) value).byteValue()); @@ -296,8 +302,7 @@ protected Object convertToQpidType(Object value) { } else if (value == null) { return null; } else { - throw new IllegalArgumentException( - "Type not supported for an application property: " + value.getClass()); + throw new IllegalArgumentException("Type not supported: " + value.getClass()); } } @@ -632,28 +637,28 @@ public Message copy() { // from // https://github.com/apache/activemq/blob/master/activemq-amqp/src/main/java/org/apache/activemq/transport/amqp/message/AmqpWritableBuffer.java - private static class ByteArrayWritableBuffer implements WritableBuffer { + static class ByteArrayWritableBuffer implements WritableBuffer { - public static final int DEFAULT_CAPACITY = 4 * 1024; + static final int DEFAULT_CAPACITY = 4 * 1024; - byte[] buffer; - int position; + private byte[] buffer; + private int position; /** Creates a new WritableBuffer with default capacity. */ - public ByteArrayWritableBuffer() { + ByteArrayWritableBuffer() { this(DEFAULT_CAPACITY); } /** Create a new WritableBuffer with the given capacity. */ - public ByteArrayWritableBuffer(int capacity) { + ByteArrayWritableBuffer(int capacity) { this.buffer = new byte[capacity]; } - public byte[] getArray() { + byte[] getArray() { return buffer; } - public int getArrayLength() { + int getArrayLength() { return position; } diff --git a/src/main/java/com/rabbitmq/stream/codec/QpidProtonMessageBuilder.java b/src/main/java/com/rabbitmq/stream/codec/QpidProtonMessageBuilder.java index 09dcc46127..1d1851bb52 100644 --- a/src/main/java/com/rabbitmq/stream/codec/QpidProtonMessageBuilder.java +++ b/src/main/java/com/rabbitmq/stream/codec/QpidProtonMessageBuilder.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -14,13 +14,13 @@ // info@rabbitmq.com. package com.rabbitmq.stream.codec; +import static com.rabbitmq.stream.codec.QpidProtonCodec.EMPTY_BODY; + import com.rabbitmq.stream.Message; import com.rabbitmq.stream.MessageBuilder; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.math.BigDecimal; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.UUID; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.qpid.proton.amqp.Binary; import org.apache.qpid.proton.amqp.Symbol; @@ -57,6 +57,9 @@ public Message build() { message.setMessageAnnotations( new MessageAnnotations(messageAnnotationsBuilder.messageAnnotations)); } + if (message.getBody() == null) { + message.setBody(EMPTY_BODY); + } return new QpidProtonCodec.QpidProtonAmqpMessageWrapper( hasPublishingId, publishingId, message); } else { @@ -65,6 +68,7 @@ public Message build() { } @Override + @SuppressFBWarnings({"AT_NONATOMIC_64BIT_PRIMITIVE", "AT_STALE_THREAD_WRITE_OF_PRIMITIVE"}) public MessageBuilder publishingId(long publishingId) { this.publishingId = publishingId; this.hasPublishingId = true; @@ -358,6 +362,24 @@ public MessageAnnotationsBuilder entrySymbol(String key, String value) { return this; } + @Override + public MessageAnnotationsBuilder entry(String key, List list) { + messageAnnotations.put(Symbol.getSymbol(key), list); + return this; + } + + @Override + public MessageAnnotationsBuilder entry(String key, Map map) { + messageAnnotations.put(Symbol.getSymbol(key), map); + return this; + } + + @Override + public MessageAnnotationsBuilder entryArray(String key, Object[] array) { + messageAnnotations.put(Symbol.getSymbol(key), array); + return this; + } + @Override public MessageBuilder messageBuilder() { return messageBuilder; diff --git a/src/main/java/com/rabbitmq/stream/codec/SimpleCodec.java b/src/main/java/com/rabbitmq/stream/codec/SimpleCodec.java index 77f36afe4c..d7ffeb2c65 100644 --- a/src/main/java/com/rabbitmq/stream/codec/SimpleCodec.java +++ b/src/main/java/com/rabbitmq/stream/codec/SimpleCodec.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -18,14 +18,18 @@ import com.rabbitmq.stream.Message; import com.rabbitmq.stream.MessageBuilder; import com.rabbitmq.stream.Properties; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; public class SimpleCodec implements Codec { + static final byte[] EMPTY_BODY = new byte[0]; + @Override public EncodedMessage encode(Message message) { - return new EncodedMessage(message.getBodyAsBinary().length, message.getBodyAsBinary()); + byte[] body = message.getBodyAsBinary() == null ? EMPTY_BODY : message.getBodyAsBinary(); + return new EncodedMessage(body.length, body); } @Override @@ -104,6 +108,7 @@ public Message build() { } @Override + @SuppressFBWarnings({"AT_NONATOMIC_64BIT_PRIMITIVE", "AT_STALE_THREAD_WRITE_OF_PRIMITIVE"}) public MessageBuilder publishingId(long publishingId) { this.publishingId = publishingId; this.hasPublishingId = true; diff --git a/src/main/java/com/rabbitmq/stream/codec/SwiftMqCodec.java b/src/main/java/com/rabbitmq/stream/codec/SwiftMqCodec.java index 9871ec2140..76f15bd806 100644 --- a/src/main/java/com/rabbitmq/stream/codec/SwiftMqCodec.java +++ b/src/main/java/com/rabbitmq/stream/codec/SwiftMqCodec.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -25,13 +25,14 @@ import com.swiftmq.amqp.v100.types.*; import com.swiftmq.tools.util.DataByteArrayOutputStream; import java.io.IOException; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.UUID; +import java.lang.reflect.Array; +import java.nio.charset.StandardCharsets; +import java.util.*; public class SwiftMqCodec implements Codec { + static final Data EMPTY_BODY = new Data(new byte[0]); + private static Object convertAmqpMapValue(AMQPType value) { if (value instanceof AMQPBoolean) { return ((AMQPBoolean) value).getValue() ? Boolean.TRUE : Boolean.FALSE; @@ -67,13 +68,48 @@ private static Object convertAmqpMapValue(AMQPType value) { return ((AMQPUuid) value).getValue(); } else if (value instanceof AMQPSymbol) { return ((AMQPSymbol) value).getValue(); + } else if (value instanceof AMQPList) { + try { + List source = ((AMQPList) value).getValue(); + List target = new ArrayList<>(source.size()); + for (AMQPType o : source) { + target.add(convertAmqpMapValue(o)); + } + return target; + } catch (IOException e) { + throw new StreamException("Error while reading SwiftMQ list", e); + } + } else if (value instanceof AMQPMap) { + try { + Map source = ((AMQPMap) value).getValue(); + Map target = new LinkedHashMap<>(source.size()); + for (Map.Entry entry : source.entrySet()) { + target.put(convertAmqpMapValue(entry.getKey()), convertAmqpMapValue(entry.getValue())); + } + return target; + } catch (IOException e) { + throw new StreamException("Error while reading SwiftMQ map", e); + } + } else if (value instanceof AMQPArray) { + try { + AMQPType[] source = ((AMQPArray) value).getValue(); + Object target = + Array.newInstance( + source.length == 0 ? Object.class : convertAmqpMapValue(source[0]).getClass(), + source.length); + for (int i = 0; i < source.length; i++) { + Array.set(target, i, convertAmqpMapValue(source[i])); + } + return target; + } catch (IOException e) { + throw new StreamException("Error while reading SwiftMQ array", e); + } } else if (value instanceof AMQPNull) { return null; } else if (value == null) { return null; } else { - throw new IllegalArgumentException( - "Type not supported for an application property: " + value.getClass()); + throw new IllegalArgumentException("Type not supported: " + value.getClass()); } } @@ -260,7 +296,9 @@ public EncodedMessage encode(Message message) { } } - if (message.getBodyAsBinary() != null) { + if (message.getBodyAsBinary() == null) { + outboundMessage.addData(EMPTY_BODY); + } else { outboundMessage.addData(new Data(message.getBodyAsBinary())); } } @@ -316,11 +354,111 @@ protected static AMQPType convertToSwiftMqType(Object value) { return new AMQPSymbol(value.toString()); } else if (value instanceof UUID) { return new AMQPUuid((UUID) value); - } else if (value == value) { + } else if (value instanceof List) { + List source = (List) value; + List target = new ArrayList<>(source.size()); + for (Object o : source) { + target.add(convertToSwiftMqType(o)); + } + try { + return new AMQPList(target); + } catch (IOException e) { + throw new StreamException("Error while creating SwiftMQ list", e); + } + } else if (value instanceof Map) { + Map source = (Map) value; + Map target = new LinkedHashMap<>(source.size()); + for (Map.Entry entry : source.entrySet()) { + target.put(convertToSwiftMqType(entry.getKey()), convertToSwiftMqType(entry.getValue())); + } + try { + return new AMQPMap(target); + } catch (IOException e) { + throw new StreamException("Error while creating SwiftMQ map", e); + } + } else if (value instanceof Object[]) { + Object[] source = (Object[]) value; + AMQPType[] target = new AMQPType[source.length]; + for (int i = 0; i < source.length; i++) { + target[i] = convertToSwiftMqType(source[i]); + } + try { + int code = source.length == 0 ? AMQPTypeDecoder.UNKNOWN : toSwiftMqTypeCode(source[0]); + return new AMQPArray(code, target); + } catch (IOException e) { + throw new StreamException("Error while creating SwiftMQ list", e); + } + } else if (value == null) { return AMQPNull.NULL; } else { - throw new IllegalArgumentException( - "Type not supported for an application property: " + value.getClass()); + throw new IllegalArgumentException("Type not supported: " + value.getClass()); + } + } + + protected static int toSwiftMqTypeCode(Object value) { + if (value instanceof Boolean) { + return AMQPTypeDecoder.BOOLEAN; + } else if (value instanceof Byte) { + return AMQPTypeDecoder.BYTE; + } else if (value instanceof Short) { + return AMQPTypeDecoder.SHORT; + } else if (value instanceof Integer) { + int v = (Integer) value; + return (v < -128 || v > 127) ? AMQPTypeDecoder.INT : AMQPTypeDecoder.SINT; + } else if (value instanceof Long) { + long v = (Long) value; + return (v < -128 || v > 127) ? AMQPTypeDecoder.LONG : AMQPTypeDecoder.SLONG; + } else if (value instanceof UnsignedByte) { + return AMQPTypeDecoder.UBYTE; + } else if (value instanceof UnsignedShort) { + return AMQPTypeDecoder.USHORT; + } else if (value instanceof UnsignedInteger) { + return AMQPTypeDecoder.UINT; + } else if (value instanceof UnsignedLong) { + return AMQPTypeDecoder.ULONG; + } else if (value instanceof Float) { + return AMQPTypeDecoder.FLOAT; + } else if (value instanceof Double) { + return AMQPTypeDecoder.DOUBLE; + } else if (value instanceof byte[]) { + return ((byte[]) value).length > 255 ? AMQPTypeDecoder.BIN32 : AMQPTypeDecoder.BIN8; + } else if (value instanceof String) { + return value.toString().getBytes(StandardCharsets.UTF_8).length > 255 + ? AMQPTypeDecoder.STR32UTF8 + : AMQPTypeDecoder.STR8UTF8; + } else if (value instanceof Character) { + return AMQPTypeDecoder.CHAR; + } else if (value instanceof Date) { + return AMQPTypeDecoder.TIMESTAMP; + } else if (value instanceof Symbol) { + return value.toString().getBytes(StandardCharsets.US_ASCII).length > 255 + ? AMQPTypeDecoder.SYM32 + : AMQPTypeDecoder.SYM8; + } else if (value instanceof UUID) { + return AMQPTypeDecoder.UUID; + } else if (value instanceof List) { + List l = (List) value; + if (l.isEmpty()) { + return AMQPTypeDecoder.LIST0; + } else if (l.size() > 255) { + return AMQPTypeDecoder.LIST32; + } else { + return AMQPTypeDecoder.LIST8; + } + } else if (value instanceof Map) { + Map source = (Map) value; + return source.size() * 2 > 255 ? AMQPTypeDecoder.MAP32 : AMQPTypeDecoder.MAP8; + } else if (value instanceof Object[]) { + Object[] source = (Object[]) value; + if (source.length > 255) { + return AMQPTypeDecoder.ARRAY32; + } else { + return AMQPTypeDecoder.ARRAY8; + } + } else if (value == null) { + return AMQPTypeDecoder.NULL; + } else { + throw new IllegalArgumentException("Type not supported: " + value.getClass()); } } diff --git a/src/main/java/com/rabbitmq/stream/codec/SwiftMqMessageBuilder.java b/src/main/java/com/rabbitmq/stream/codec/SwiftMqMessageBuilder.java index 39e43840bb..9e4e8ea8ee 100644 --- a/src/main/java/com/rabbitmq/stream/codec/SwiftMqMessageBuilder.java +++ b/src/main/java/com/rabbitmq/stream/codec/SwiftMqMessageBuilder.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -14,6 +14,9 @@ // info@rabbitmq.com. package com.rabbitmq.stream.codec; +import static com.rabbitmq.stream.codec.SwiftMqCodec.convertToSwiftMqType; +import static com.rabbitmq.stream.codec.SwiftMqCodec.toSwiftMqTypeCode; + import com.rabbitmq.stream.Message; import com.rabbitmq.stream.MessageBuilder; import com.rabbitmq.stream.StreamException; @@ -21,11 +24,11 @@ import com.swiftmq.amqp.v100.generated.transport.definitions.SequenceNo; import com.swiftmq.amqp.v100.messaging.AMQPMessage; import com.swiftmq.amqp.v100.types.*; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.IOException; +import java.lang.reflect.Array; import java.math.BigDecimal; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.UUID; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; @@ -66,6 +69,9 @@ public Message build() { throw new StreamException("Error while setting application properties", e); } } + if (outboundMessage.getData() == null) { + outboundMessage.addData(SwiftMqCodec.EMPTY_BODY); + } return new SwiftMqCodec.SwiftMqAmqpMessageWrapper( hasPublishingId, publishingId, outboundMessage); } else { @@ -74,6 +80,7 @@ public Message build() { } @Override + @SuppressFBWarnings({"AT_NONATOMIC_64BIT_PRIMITIVE", "AT_STALE_THREAD_WRITE_OF_PRIMITIVE"}) public MessageBuilder publishingId(long publishingId) { this.publishingId = publishingId; this.hasPublishingId = true; @@ -316,6 +323,62 @@ protected void addEntry(String key, String value) { protected void addEntrySymbol(String key, String value) { map.put(keyMaker.apply(key), value == null ? AMQPNull.NULL : new AMQPSymbol(value)); } + + protected void addEntry(String key, List list) { + AMQPType amqpValue; + if (list == null) { + amqpValue = AMQPNull.NULL; + } else { + List l = new ArrayList<>(list.size()); + for (Object o : list) { + l.add(convertToSwiftMqType(o)); + } + try { + amqpValue = new AMQPList(l); + } catch (IOException e) { + throw new StreamException("Error while creating SwiftMq list", e); + } + } + map.put(keyMaker.apply(key), amqpValue); + } + + protected void addEntry(String key, Map mapEntry) { + AMQPType amqpValue; + if (mapEntry == null) { + amqpValue = AMQPNull.NULL; + } else { + Map m = new LinkedHashMap<>(mapEntry.size()); + mapEntry.forEach( + (k, v) -> { + m.put(convertToSwiftMqType(k), convertToSwiftMqType(v)); + }); + try { + amqpValue = new AMQPMap(m); + } catch (IOException e) { + throw new StreamException("Error while creating SwiftMQ map", e); + } + } + map.put(keyMaker.apply(key), amqpValue); + } + + protected void addEntry(String key, Object[] array) { + AMQPType amqpValue; + if (array == null) { + amqpValue = AMQPNull.NULL; + } else { + AMQPType[] a = new AMQPType[array.length]; + for (int i = 0; i < array.length; i++) { + a[i] = convertToSwiftMqType(Array.get(array, i)); + } + try { + int code = a.length == 0 ? AMQPTypeDecoder.UNKNOWN : toSwiftMqTypeCode(array[0]); + amqpValue = new AMQPArray(code, a); + } catch (IOException e) { + throw new StreamException("Error while creating SwiftMq list", e); + } + } + map.put(keyMaker.apply(key), amqpValue); + } } private static class SwiftMqApplicationPropertiesBuilder extends AmqpMapBuilderSupport @@ -582,6 +645,24 @@ public MessageAnnotationsBuilder entrySymbol(String key, String value) { return this; } + @Override + public MessageAnnotationsBuilder entry(String key, List list) { + addEntry(key, list); + return this; + } + + @Override + public MessageAnnotationsBuilder entry(String key, Map map) { + addEntry(key, map); + return this; + } + + @Override + public MessageAnnotationsBuilder entryArray(String key, Object[] array) { + addEntry(key, array); + return this; + } + @Override public MessageBuilder messageBuilder() { return messageBuilder; diff --git a/src/main/java/com/rabbitmq/stream/codec/WrapperMessageBuilder.java b/src/main/java/com/rabbitmq/stream/codec/WrapperMessageBuilder.java index 56b59fe501..e1c1afc850 100644 --- a/src/main/java/com/rabbitmq/stream/codec/WrapperMessageBuilder.java +++ b/src/main/java/com/rabbitmq/stream/codec/WrapperMessageBuilder.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -18,11 +18,9 @@ import com.rabbitmq.stream.MessageBuilder; import com.rabbitmq.stream.Properties; import com.rabbitmq.stream.amqp.*; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.math.BigDecimal; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.UUID; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; public class WrapperMessageBuilder implements MessageBuilder { @@ -42,7 +40,7 @@ public Message build() { return new SimpleMessage( this.hasPublishingId, this.publishingId, - body, + body == null ? SimpleCodec.EMPTY_BODY : body, this.messageAnnotationsBuilder == null ? null : this.messageAnnotationsBuilder.messageAnnotations, @@ -56,6 +54,7 @@ public Message build() { } @Override + @SuppressFBWarnings({"AT_NONATOMIC_64BIT_PRIMITIVE", "AT_STALE_THREAD_WRITE_OF_PRIMITIVE"}) public MessageBuilder publishingId(long publishingId) { this.publishingId = publishingId; this.hasPublishingId = true; @@ -218,6 +217,24 @@ public MessageAnnotationsBuilder entrySymbol(String key, String value) { return this; } + @Override + public MessageAnnotationsBuilder entry(String key, List list) { + messageAnnotations.put(key, list); + return this; + } + + @Override + public MessageAnnotationsBuilder entry(String key, Map map) { + messageAnnotations.put(key, map); + return this; + } + + @Override + public MessageAnnotationsBuilder entryArray(String key, Object[] array) { + messageAnnotations.put(key, array); + return this; + } + @Override public MessageBuilder messageBuilder() { return this.messageBuilder; diff --git a/src/main/java/com/rabbitmq/stream/compression/CommonsCompressCompressionCodecFactory.java b/src/main/java/com/rabbitmq/stream/compression/CommonsCompressCompressionCodecFactory.java index 1c5e9f1217..800120785c 100644 --- a/src/main/java/com/rabbitmq/stream/compression/CommonsCompressCompressionCodecFactory.java +++ b/src/main/java/com/rabbitmq/stream/compression/CommonsCompressCompressionCodecFactory.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/compression/Compression.java b/src/main/java/com/rabbitmq/stream/compression/Compression.java index 83c80dc1d6..0d6637fc7f 100644 --- a/src/main/java/com/rabbitmq/stream/compression/Compression.java +++ b/src/main/java/com/rabbitmq/stream/compression/Compression.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/compression/CompressionCodec.java b/src/main/java/com/rabbitmq/stream/compression/CompressionCodec.java index 137b187101..c3dbc0f0bb 100644 --- a/src/main/java/com/rabbitmq/stream/compression/CompressionCodec.java +++ b/src/main/java/com/rabbitmq/stream/compression/CompressionCodec.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/compression/CompressionCodecFactory.java b/src/main/java/com/rabbitmq/stream/compression/CompressionCodecFactory.java index 2a95ab66a5..7d5ccb6b13 100644 --- a/src/main/java/com/rabbitmq/stream/compression/CompressionCodecFactory.java +++ b/src/main/java/com/rabbitmq/stream/compression/CompressionCodecFactory.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/compression/CompressionException.java b/src/main/java/com/rabbitmq/stream/compression/CompressionException.java index c674cbdd62..46e3668831 100644 --- a/src/main/java/com/rabbitmq/stream/compression/CompressionException.java +++ b/src/main/java/com/rabbitmq/stream/compression/CompressionException.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/compression/CompressionUtils.java b/src/main/java/com/rabbitmq/stream/compression/CompressionUtils.java index 5171edc405..46090e5f55 100644 --- a/src/main/java/com/rabbitmq/stream/compression/CompressionUtils.java +++ b/src/main/java/com/rabbitmq/stream/compression/CompressionUtils.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/compression/DefaultCompressionCodecFactory.java b/src/main/java/com/rabbitmq/stream/compression/DefaultCompressionCodecFactory.java index 332d6ce0b7..5cf72e29f5 100644 --- a/src/main/java/com/rabbitmq/stream/compression/DefaultCompressionCodecFactory.java +++ b/src/main/java/com/rabbitmq/stream/compression/DefaultCompressionCodecFactory.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/impl/AsyncRetry.java b/src/main/java/com/rabbitmq/stream/impl/AsyncRetry.java index 65c46f8995..6959f83328 100644 --- a/src/main/java/com/rabbitmq/stream/impl/AsyncRetry.java +++ b/src/main/java/com/rabbitmq/stream/impl/AsyncRetry.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -14,6 +14,7 @@ // info@rabbitmq.com. package com.rabbitmq.stream.impl; +import static com.rabbitmq.stream.impl.ThreadUtils.isVirtual; import static com.rabbitmq.stream.impl.Utils.namedRunnable; import com.rabbitmq.stream.BackOffDelayPolicy; @@ -52,12 +53,25 @@ private AsyncRetry( this.completableFuture.completeExceptionally(new CancellationException()); return; } + if (this.completableFuture.isCancelled()) { + LOGGER.debug("Task '{}' cancelled", description); + return; + } try { + LOGGER.debug( + "Running task '{}' (virtual threads: {})", + description, + isVirtual(Thread.currentThread())); V result = task.call(); LOGGER.debug("Task '{}' succeeded, completing future", description); completableFuture.complete(result); } catch (Exception e) { int attemptCount = attempts.getAndIncrement(); + LOGGER.debug( + "Attempt {} for task '{}' failed with '{}', checking retry policy", + attemptCount, + description, + e.getMessage()); if (retry.test(e)) { if (delayPolicy.delay(attemptCount).equals(BackOffDelayPolicy.TIMEOUT)) { LOGGER.debug( diff --git a/src/main/java/com/rabbitmq/stream/impl/Client.java b/src/main/java/com/rabbitmq/stream/impl/Client.java index ca668fd168..8679844939 100644 --- a/src/main/java/com/rabbitmq/stream/impl/Client.java +++ b/src/main/java/com/rabbitmq/stream/impl/Client.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -15,6 +15,7 @@ package com.rabbitmq.stream.impl; import static com.rabbitmq.stream.Constants.*; +import static com.rabbitmq.stream.impl.ThreadUtils.threadFactory; import static com.rabbitmq.stream.impl.Utils.DEFAULT_USERNAME; import static com.rabbitmq.stream.impl.Utils.encodeRequestCode; import static com.rabbitmq.stream.impl.Utils.encodeResponseCode; @@ -25,7 +26,6 @@ import static java.lang.String.join; import static java.util.Arrays.asList; import static java.util.concurrent.TimeUnit.SECONDS; -import static java.util.stream.StreamSupport.stream; import com.rabbitmq.stream.AuthenticationFailureException; import com.rabbitmq.stream.ByteCapacity; @@ -46,7 +46,6 @@ import com.rabbitmq.stream.impl.Client.ShutdownContext.ShutdownReason; import com.rabbitmq.stream.impl.ServerFrameHandler.FrameHandler; import com.rabbitmq.stream.impl.ServerFrameHandler.FrameHandlerInfo; -import com.rabbitmq.stream.impl.Utils.NamedThreadFactory; import com.rabbitmq.stream.metrics.MetricsCollector; import com.rabbitmq.stream.metrics.NoOpMetricsCollector; import com.rabbitmq.stream.sasl.CredentialsProvider; @@ -55,21 +54,12 @@ import com.rabbitmq.stream.sasl.SaslConfiguration; import com.rabbitmq.stream.sasl.SaslMechanism; import com.rabbitmq.stream.sasl.UsernamePasswordCredentialsProvider; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import io.netty.bootstrap.Bootstrap; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufAllocator; import io.netty.buffer.ByteBufOutputStream; -import io.netty.channel.Channel; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.ChannelOutboundHandlerAdapter; -import io.netty.channel.ChannelPromise; -import io.netty.channel.ConnectTimeoutException; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.*; import io.netty.channel.socket.SocketChannel; import io.netty.channel.socket.nio.NioSocketChannel; import io.netty.handler.codec.DecoderException; @@ -104,10 +94,9 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import java.util.function.Supplier; import java.util.function.ToLongFunction; -import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLHandshakeException; -import javax.net.ssl.SSLParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -127,6 +116,7 @@ public class Client implements AutoCloseable { private static final Charset CHARSET = StandardCharsets.UTF_8; public static final int DEFAULT_PORT = 5552; public static final int DEFAULT_TLS_PORT = 5551; + static final int MAX_REFERENCE_SIZE = 256; static final OutboundEntityWriteCallback OUTBOUND_MESSAGE_WRITE_CALLBACK = new OutboundMessageWriteCallback(); static final OutboundEntityWriteCallback OUTBOUND_MESSAGE_BATCH_WRITE_CALLBACK = @@ -153,16 +143,21 @@ public class Client implements AutoCloseable { final ConcurrentMap> outstandingRequests = new ConcurrentHashMap<>(); final List subscriptionOffsets = new CopyOnWriteArrayList<>(); + // dispatches broker frames, except for delivery frames final ExecutorService executorService; + private final Consumer closeExecutorService; + // dispatches delivery frames only final ExecutorService dispatchingExecutorService; + private final Consumer closeDispatchingExecutorService; final TuneState tuneState; final AtomicBoolean closing = new AtomicBoolean(false); + final AtomicBoolean shuttingDownDispatching = new AtomicBoolean(false); final ChunkChecksum chunkChecksum; final MetricsCollector metricsCollector; final CompressionCodecFactory compressionCodecFactory; private final Consumer shutdownListenerCallback; private final ToLongFunction publishSequenceFunction = - new ToLongFunction() { + new ToLongFunction<>() { private final AtomicLong publishSequence = new AtomicLong(0); @Override @@ -171,12 +166,11 @@ public long applyAsLong(Object value) { } }; private final AtomicInteger correlationSequence = new AtomicInteger(0); - private final Runnable executorServiceClosing; private final SaslConfiguration saslConfiguration; private final CredentialsProvider credentialsProvider; private final Runnable nettyClosing; private final int maxFrameSize; - private final boolean frameSizeCopped; + private final boolean frameSizeCapped; private final EventLoopGroup eventLoopGroup; private final Map clientProperties; private final String NETTY_HANDLER_FLUSH_CONSOLIDATION = @@ -194,10 +188,12 @@ public long applyAsLong(Object value) { private final boolean filteringSupported; private final Runnable superStreamManagementCommandVersionsCheck; + @SuppressFBWarnings("CT_CONSTRUCTOR_THROW") public Client() { this(new ClientParameters()); } + @SuppressFBWarnings("CT_CONSTRUCTOR_THROW") public Client(ClientParameters parameters) { this.publishConfirmListener = parameters.publishConfirmListener; this.publishErrorListener = parameters.publishErrorListener; @@ -240,7 +236,7 @@ public Client(ClientParameters parameters) { if (b.config().group() == null) { EventLoopGroup eventLoopGroup; if (parameters.eventLoopGroup == null) { - this.eventLoopGroup = new NioEventLoopGroup(); + this.eventLoopGroup = Utils.eventLoopGroup(); eventLoopGroup = this.eventLoopGroup; } else { this.eventLoopGroup = null; @@ -285,26 +281,15 @@ public void initChannel(SocketChannel ch) { SslHandler sslHandler = parameters.sslContext.newHandler(ch.alloc(), parameters.host, parameters.port); - if (parameters.tlsHostnameVerification) { - SSLEngine sslEngine = sslHandler.engine(); - SSLParameters sslParameters = sslEngine.getSSLParameters(); - sslParameters.setEndpointIdentificationAlgorithm("HTTPS"); - sslEngine.setSSLParameters(sslParameters); - } - ch.pipeline().addFirst("ssl", sslHandler); } channelCustomizer.accept(ch); } }); + this.nettyClosing = Utils.makeIdempotent(this::closeNetty); ChannelFuture f; - String clientConnectionName = - parameters.clientProperties == null - ? "" - : (parameters.clientProperties.containsKey("connection_name") - ? parameters.clientProperties.get("connection_name") - : ""); + String clientConnectionName = parameters.clientProperties.getOrDefault("connection_name", ""); try { LOGGER.debug( "Trying to create stream connection to {}:{}, with client connection name '{}'", @@ -327,51 +312,78 @@ public void initChannel(SocketChannel ch) { throw new StreamException(message, e); } } - this.channel = f.channel(); - this.nettyClosing = Utils.makeIdempotent(this::closeNetty); ExecutorServiceFactory executorServiceFactory = parameters.executorServiceFactory; if (executorServiceFactory == null) { + this.closeExecutorService = + Utils.makeIdempotent( + es -> { + if (es != null) { + es.shutdownNow(); + } + }); this.executorService = - Executors.newSingleThreadExecutor(new NamedThreadFactory(clientConnectionName + "-")); + Executors.newSingleThreadExecutor(threadFactory(clientConnectionName + "-")); } else { + this.closeExecutorService = + Utils.makeIdempotent( + es -> { + if (es != null) { + executorServiceFactory.clientClosed(es); + } + }); this.executorService = executorServiceFactory.get(); } ExecutorServiceFactory dispatchingExecutorServiceFactory = parameters.dispatchingExecutorServiceFactory; if (dispatchingExecutorServiceFactory == null) { + this.closeDispatchingExecutorService = + Utils.makeIdempotent( + es -> { + if (es != null) { + List outstandingTasks = es.shutdownNow(); + this.shuttingDownDispatching.set(true); + for (Runnable outstandingTask : outstandingTasks) { + try { + outstandingTask.run(); + } catch (Exception e) { + LOGGER.info( + "Error while releasing buffer in outstanding connection tasks: {}", + e.getMessage()); + } + } + } + }); this.dispatchingExecutorService = Executors.newSingleThreadExecutor( - new NamedThreadFactory("dispatching-" + clientConnectionName + "-")); + threadFactory("dispatching-" + clientConnectionName + "-")); } else { + this.closeDispatchingExecutorService = + Utils.makeIdempotent( + es -> { + if (es != null) { + dispatchingExecutorServiceFactory.clientClosed(es); + } + }); this.dispatchingExecutorService = dispatchingExecutorServiceFactory.get(); } - this.executorServiceClosing = - Utils.makeIdempotent( - () -> { - this.dispatchingExecutorService.shutdownNow(); - if (dispatchingExecutorServiceFactory == null) { - this.dispatchingExecutorService.shutdownNow(); - } else { - dispatchingExecutorServiceFactory.clientClosed(this.dispatchingExecutorService); - } - if (executorServiceFactory == null) { - this.executorService.shutdownNow(); - } else { - executorServiceFactory.clientClosed(this.executorService); - } - }); try { this.tuneState = new TuneState( parameters.requestedMaxFrameSize, (int) parameters.requestedHeartbeat.getSeconds()); this.clientProperties = clientProperties(parameters.clientProperties); + debug(() -> "exchanging peer properties"); this.serverProperties = peerProperties(); + debug(() -> "peer properties exchanged"); + debug(() -> "starting SASL handshake"); this.saslMechanisms = getSaslMechanisms(); + debug(() -> "SASL mechanisms supported by server ({})", this.saslMechanisms); + debug(() -> "starting authentication"); authenticate(this.credentialsProvider); + debug(() -> "authenticated"); this.tuneState.await(Duration.ofSeconds(10)); this.maxFrameSize = this.tuneState.getMaxFrameSize(); - this.frameSizeCopped = this.maxFrameSize() > 0; + this.frameSizeCapped = this.maxFrameSize() > 0; LOGGER.debug( "Connection tuned with max frame size {} and heartbeat {}", this.maxFrameSize(), @@ -411,6 +423,8 @@ public void initChannel(SocketChannel ch) { started.set(true); this.metricsCollector.openConnection(); } catch (RuntimeException e) { + LOGGER.debug( + "Error while opening connection {}: {}", this.clientConnectionName, e.getMessage()); this.closingSequence(null); throw e; } @@ -434,7 +448,7 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) private static Map clientProperties(Map fromParameters) { fromParameters = fromParameters == null ? Collections.emptyMap() : fromParameters; - Map clientProperties = new HashMap<>(fromParameters); + Map clientProperties = new LinkedHashMap<>(fromParameters); clientProperties.putAll(ClientProperties.DEFAULT_CLIENT_PROPERTIES); return Collections.unmodifiableMap(clientProperties); } @@ -455,10 +469,14 @@ int maxFrameSize() { return this.maxFrameSize; } + private int nextCorrelationId() { + return this.correlationSequence.getAndIncrement(); + } + private Map peerProperties() { int clientPropertiesSize = mapSize(this.clientProperties); int length = 2 + 2 + 4 + clientPropertiesSize; - int correlationId = correlationSequence.incrementAndGet(); + int correlationId = nextCorrelationId(); try { ByteBuf bb = allocateNoCheck(length + 4); bb.writeInt(length); @@ -467,6 +485,7 @@ private Map peerProperties() { bb.writeInt(correlationId); writeMap(bb, this.clientProperties); OutstandingRequest> request = outstandingRequest(); + LOGGER.debug("Peer properties request has correlation ID {}", correlationId); outstandingRequests.put(correlationId, request); channel.writeAndFlush(bb); request.block(); @@ -498,21 +517,22 @@ void authenticate(CredentialsProvider credentialsProvider) { } else if (saslAuthenticateResponse.isChallenge()) { challenge = saslAuthenticateResponse.challenge; } else if (saslAuthenticateResponse.isAuthenticationFailure()) { - String message = - "Unexpected response code during authentication: " - + formatConstant(saslAuthenticateResponse.getResponseCode()); + StringBuilder message = + new StringBuilder( + "Unexpected response code during authentication: " + + formatConstant(saslAuthenticateResponse.getResponseCode())); if (saslAuthenticateResponse.getResponseCode() == RESPONSE_CODE_AUTHENTICATION_FAILURE_LOOPBACK) { - message += - ". The user is not authorized to connect from a remote host. " - + "If the broker is running locally, make sure the '" - + this.host - + "' hostname is resolved to " - + "the loopback interface (localhost, 127.0.0.1, ::1). " - + "See https://www.rabbitmq.com/access-control.html#loopback-users."; + message + .append(". The user is not authorized to connect from a remote host. ") + .append("If the broker is running locally, make sure the '") + .append(this.host) + .append("' hostname is resolved to ") + .append("the loopback interface (localhost, 127.0.0.1, ::1). ") + .append("See https://www.rabbitmq.com/access-control.html#loopback-users."); } throw new AuthenticationFailureException( - message, saslAuthenticateResponse.getResponseCode()); + message.toString(), saslAuthenticateResponse.getResponseCode()); } else { throw new StreamException( "Unexpected response code during authentication: " @@ -531,7 +551,7 @@ private SaslAuthenticateResponse sendSaslAuthenticate( + saslMechanism.getName().length() + 4 + (challengeResponse == null ? 0 : challengeResponse.length); - int correlationId = correlationSequence.incrementAndGet(); + int correlationId = nextCorrelationId(); try { ByteBuf bb = allocateNoCheck(length + 4); bb.writeInt(length); @@ -561,7 +581,7 @@ private SaslAuthenticateResponse sendSaslAuthenticate( private Map open(String virtualHost) { int length = 2 + 2 + 4 + 2 + virtualHost.length(); - int correlationId = correlationSequence.incrementAndGet(); + int correlationId = nextCorrelationId(); try { ByteBuf bb = allocate(length + 4); bb.writeInt(length); @@ -602,7 +622,7 @@ void send(byte[] content) { private void sendClose(short code, String reason) { int length = 2 + 2 + 4 + 2 + 2 + reason.length(); - int correlationId = correlationSequence.incrementAndGet(); + int correlationId = nextCorrelationId(); try { ByteBuf bb = allocate(length + 4); bb.writeInt(length); @@ -635,7 +655,7 @@ private void sendClose(short code, String reason) { private List getSaslMechanisms() { int length = 2 + 2 + 4; - int correlationId = correlationSequence.incrementAndGet(); + int correlationId = nextCorrelationId(); try { ByteBuf bb = allocateNoCheck(length + 4); bb.writeInt(length); @@ -662,7 +682,7 @@ public Response create(String stream) { public Response create(String stream, Map arguments) { int length = 2 + 2 + 4 + 2 + stream.length() + mapSize(arguments); - int correlationId = correlationSequence.incrementAndGet(); + int correlationId = nextCorrelationId(); try { ByteBuf bb = allocate(length + 4); bb.writeInt(length); @@ -711,7 +731,7 @@ Response createSuperStream( + collectionSize(partitions) + collectionSize(bindingKeys) + mapSize(arguments); - int correlationId = correlationSequence.incrementAndGet(); + int correlationId = nextCorrelationId(); try { ByteBuf bb = allocate(length + 4); bb.writeInt(length); @@ -740,7 +760,7 @@ Response createSuperStream( Response deleteSuperStream(String superStream) { this.superStreamManagementCommandVersionsCheck.run(); int length = 2 + 2 + 4 + 2 + superStream.length(); - int correlationId = correlationSequence.incrementAndGet(); + int correlationId = nextCorrelationId(); try { ByteBuf bb = allocate(length + 4); bb.writeInt(length); @@ -800,7 +820,7 @@ private static ByteBuf writeMap(ByteBuf bb, Map elements) { } ByteBuf allocate(ByteBufAllocator allocator, int capacity) { - if (frameSizeCopped && capacity > this.maxFrameSize()) { + if (frameSizeCapped && capacity > this.maxFrameSize()) { throw new IllegalArgumentException( "Cannot allocate " + capacity @@ -824,7 +844,7 @@ private ByteBuf allocateNoCheck(int capacity) { public Response delete(String stream) { int length = 2 + 2 + 4 + 2 + stream.length(); - int correlationId = correlationSequence.incrementAndGet(); + int correlationId = nextCorrelationId(); try { ByteBuf bb = allocate(length + 4); bb.writeInt(length); @@ -856,7 +876,7 @@ public Map metadata(String... streams) { throw new IllegalArgumentException("At least one stream must be specified"); } int length = 2 + 2 + 4 + arraySize(streams); // API code, version, correlation ID, array size - int correlationId = correlationSequence.incrementAndGet(); + int correlationId = nextCorrelationId(); try { ByteBuf bb = allocate(length + 4); bb.writeInt(length); @@ -884,12 +904,12 @@ public Response declarePublisher(byte publisherId, String publisherReference, St (publisherReference == null || publisherReference.isEmpty() ? 0 : publisherReference.length()); - if (publisherReferenceSize > 256) { + if (publisherReferenceSize >= MAX_REFERENCE_SIZE) { throw new IllegalArgumentException( "If specified, publisher reference must less than 256 characters"); } int length = 2 + 2 + 4 + 1 + 2 + publisherReferenceSize + 2 + stream.length(); - int correlationId = correlationSequence.getAndIncrement(); + int correlationId = nextCorrelationId(); try { ByteBuf bb = allocate(length + 4); bb.writeInt(length); @@ -920,7 +940,7 @@ public Response declarePublisher(byte publisherId, String publisherReference, St public Response deletePublisher(byte publisherId) { int length = 2 + 2 + 4 + 1; - int correlationId = correlationSequence.getAndIncrement(); + int correlationId = nextCorrelationId(); try { ByteBuf bb = allocate(length + 4); bb.writeInt(length); @@ -1244,7 +1264,7 @@ public Response subscribe( propertiesSize = mapSize(properties); } length += propertiesSize; - int correlationId = correlationSequence.getAndIncrement(); + int correlationId = nextCorrelationId(); try { ByteBuf bb = allocate(length + 4); bb.writeInt(length); @@ -1282,7 +1302,7 @@ public Response subscribe( } public void storeOffset(String reference, String stream, long offset) { - if (reference == null || reference.isEmpty() || reference.length() > 256) { + if (reference == null || reference.isEmpty() || reference.length() >= MAX_REFERENCE_SIZE) { throw new IllegalArgumentException( "Reference must a non-empty string of less than 256 characters"); } @@ -1312,7 +1332,7 @@ public QueryOffsetResponse queryOffset(String reference, String stream) { } int length = 2 + 2 + 4 + 2 + reference.length() + 2 + stream.length(); - int correlationId = correlationSequence.getAndIncrement(); + int correlationId = nextCorrelationId(); try { ByteBuf bb = allocate(length + 4); bb.writeInt(length); @@ -1353,7 +1373,7 @@ public long queryPublisherSequence(String publisherReference, String stream) { } int length = 2 + 2 + 4 + 2 + publisherReference.length() + 2 + stream.length(); - int correlationId = correlationSequence.getAndIncrement(); + int correlationId = nextCorrelationId(); try { ByteBuf bb = allocate(length + 4); bb.writeInt(length); @@ -1390,7 +1410,7 @@ public long queryPublisherSequence(String publisherReference, String stream) { public Response unsubscribe(byte subscriptionId) { int length = 2 + 2 + 4 + 1; - int correlationId = correlationSequence.getAndIncrement(); + int correlationId = nextCorrelationId(); try { ByteBuf bb = allocate(length + 4); bb.writeInt(length); @@ -1429,14 +1449,21 @@ void closingSequence(ShutdownContext.ShutdownReason reason) { this.shutdownListenerCallback.accept(reason); } this.nettyClosing.run(); - this.executorServiceClosing.run(); + if (this.closeDispatchingExecutorService != null) { + this.closeDispatchingExecutorService.accept(this.dispatchingExecutorService); + } + if (this.closeExecutorService != null) { + this.closeExecutorService.accept(this.executorService); + } } private void closeNetty() { try { - if (this.channel.isOpen()) { + if (this.channel != null && this.channel.isOpen()) { LOGGER.debug("Closing Netty channel"); this.channel.close().get(10, TimeUnit.SECONDS); + } else { + LOGGER.debug("No Netty channel to close"); } } catch (InterruptedException e) { LOGGER.info("Channel closing has been interrupted"); @@ -1492,6 +1519,10 @@ String connectionName() { return builder.append(serverAddress()).toString(); } + String clientConnectionName() { + return this.clientConnectionName; + } + private String serverAddress() { SocketAddress remoteAddress = remoteAddress(); if (remoteAddress instanceof InetSocketAddress) { @@ -1518,7 +1549,7 @@ public List route(String routingKey, String superStream) { + routingKey.length() + 2 + superStream.length(); // API code, version, correlation ID, 2 strings - int correlationId = correlationSequence.incrementAndGet(); + int correlationId = nextCorrelationId(); try { ByteBuf bb = allocate(length + 4); bb.writeInt(length); @@ -1553,7 +1584,7 @@ public List partitions(String superStream) { } int length = 2 + 2 + 4 + 2 + superStream.length(); // API code, version, correlation ID, 1 string - int correlationId = correlationSequence.incrementAndGet(); + int correlationId = nextCorrelationId(); try { ByteBuf bb = allocate(length + 4); bb.writeInt(length); @@ -1581,7 +1612,7 @@ List exchangeCommandVersions() { List commandVersions = ServerFrameHandler.commandVersions(); int length = 2 + 2 + 4 + 4; // API code, version, correlation ID, array size length += commandVersions.size() * (2 + 2 + 2); - int correlationId = correlationSequence.incrementAndGet(); + int correlationId = nextCorrelationId(); try { ByteBuf bb = allocate(length + 4); bb.writeInt(length); @@ -1614,7 +1645,7 @@ StreamStatsResponse streamStats(String stream) { throw new IllegalArgumentException("stream must not be null"); } int length = 2 + 2 + 4 + 2 + stream.length(); // API code, version, correlation ID, 1 string - int correlationId = correlationSequence.incrementAndGet(); + int correlationId = nextCorrelationId(); try { ByteBuf bb = allocate(length + 4); bb.writeInt(length); @@ -1678,7 +1709,7 @@ String serverAdvertisedHost() { } int serverAdvertisedPort() { - return Integer.valueOf(this.connectionProperties("advertised_port")); + return Integer.parseInt(this.connectionProperties("advertised_port")); } public String brokerVersion() { @@ -2211,7 +2242,7 @@ static class StreamStatsResponse extends Response { StreamStatsResponse(short responseCode, Map info) { super(responseCode); - this.info = Collections.unmodifiableMap(new HashMap<>(info)); + this.info = Map.copyOf(info); } public Map getInfo() { @@ -2233,7 +2264,10 @@ public StreamMetadata(String stream, short responseCode, Broker leader, List getReplicas() { - return replicas; + return this.replicas; + } + + boolean hasReplicas() { + return !this.replicas.isEmpty(); } public String getStream() { @@ -2349,7 +2388,6 @@ public static class ClientParameters { private ChunkChecksum chunkChecksum = JdkChunkChecksum.CRC32_SINGLETON; private MetricsCollector metricsCollector = NoOpMetricsCollector.SINGLETON; private SslContext sslContext; - private boolean tlsHostnameVerification = true; private ByteBufAllocator byteBufAllocator; private Duration rpcTimeout; private Consumer channelCustomizer = noOpConsumer(); @@ -2506,11 +2544,6 @@ public ClientParameters sslContext(SslContext sslContext) { return this; } - public ClientParameters tlsHostnameVerification(boolean tlsHostnameVerification) { - this.tlsHostnameVerification = tlsHostnameVerification; - return this; - } - public ClientParameters compressionCodecFactory( CompressionCodecFactory compressionCodecFactory) { this.compressionCodecFactory = compressionCodecFactory; @@ -2542,7 +2575,7 @@ int port() { } Map clientProperties() { - return Collections.unmodifiableMap(this.clientProperties); + return Map.copyOf(this.clientProperties); } Codec codec() { @@ -2563,6 +2596,10 @@ public ClientParameters bootstrapCustomizer(Consumer bootstrapCustomi return this; } + Duration rpcTimeout() { + return this.rpcTimeout; + } + ClientParameters duplicate() { ClientParameters duplicate = new ClientParameters(); for (Field field : ClientParameters.class.getDeclaredFields()) { @@ -2679,12 +2716,16 @@ public StreamParametersBuilder maxLengthTb(long teraBytes) { } public StreamParametersBuilder maxSegmentSizeBytes(long bytes) { - this.parameters.put("stream-max-segment-size-bytes", String.valueOf(bytes)); + if (bytes <= 0) { + this.parameters.remove("stream-max-segment-size-bytes"); + } else { + this.parameters.put("stream-max-segment-size-bytes", String.valueOf(bytes)); + } return this; } public StreamParametersBuilder maxSegmentSizeBytes(ByteCapacity bytes) { - return maxSegmentSizeBytes(bytes.toBytes()); + return maxSegmentSizeBytes(bytes == null ? 0L : bytes.toBytes()); } public StreamParametersBuilder maxSegmentSizeKb(long kiloBytes) { @@ -2727,13 +2768,25 @@ public StreamParametersBuilder filterSize(int size) { return this; } + public StreamParametersBuilder initialMemberCount(int initialMemberCount) { + if (initialMemberCount <= 0) { + throw new IllegalArgumentException("The initial member count must be greater than 0"); + } + this.parameters.put("initial-cluster-size", String.valueOf(initialMemberCount)); + return this; + } + public StreamParametersBuilder put(String key, String value) { - parameters.put(key, value); + if (value == null) { + parameters.remove(key); + } else { + parameters.put(key, value); + } return this; } public Map build() { - return parameters; + return new HashMap<>(parameters); } } @@ -2763,7 +2816,9 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) { } } else { FrameHandler frameHandler = ServerFrameHandler.lookup(commandId, version, m); - task = () -> frameHandler.handle(Client.this, frameSize, ctx, m); + task = + new FrameHandlerTask( + frameHandler, Client.this, frameSize, ctx, m, shuttingDownDispatching); } if (task != null) { @@ -2785,7 +2840,17 @@ public void channelInactive(ChannelHandlerContext ctx) { // because it will be handled later anyway. if (shutdownReason == null) { if (closing.compareAndSet(false, true)) { - executorService.submit(() -> closingSequence(ShutdownReason.UNKNOWN)); + if (executorService == null) { + // the TCP connection is closed before the state is initialized + // we do our best the execute the closing sequence + new Thread( + () -> { + closingSequence(ShutdownReason.UNKNOWN); + }) + .start(); + } else { + executorService.submit(() -> closingSequence(ShutdownReason.UNKNOWN)); + } } } } @@ -2832,6 +2897,44 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { } } + private static class FrameHandlerTask implements Runnable { + + private final FrameHandler frameHandler; + private final Client client; + private final int frameSize; + private final ChannelHandlerContext ctx; + private final ByteBuf message; + private final AtomicBoolean shouldRelease; + + private FrameHandlerTask( + FrameHandler frameHandler, + Client client, + int frameSize, + ChannelHandlerContext ctx, + ByteBuf message, + AtomicBoolean shouldRelease) { + this.frameHandler = frameHandler; + this.client = client; + this.frameSize = frameSize; + this.ctx = ctx; + this.message = message; + this.shouldRelease = shouldRelease; + } + + @Override + public void run() { + if (this.shouldRelease.get()) { + try { + this.message.release(); + } catch (Exception e) { + LOGGER.info("Error while releasing buffer after connection closing: {}", e.getMessage()); + } + } else { + this.frameHandler.handle(this.client, this.frameSize, this.ctx, this.message); + } + } + } + private OutstandingRequest outstandingRequest() { return new OutstandingRequest<>(this.rpcTimeout, this.host + ":" + this.port); } @@ -2840,4 +2943,10 @@ private OutstandingRequest outstandingRequest() { public String toString() { return "Client{connectionName='" + connectionName() + "'}"; } + + private void debug(Supplier format, Object... args) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Connection '" + this.clientConnectionName + "': " + format.get(), args); + } + } } diff --git a/src/main/java/com/rabbitmq/stream/impl/ClientProperties.java b/src/main/java/com/rabbitmq/stream/impl/ClientProperties.java index 2a6c206585..933f22021d 100644 --- a/src/main/java/com/rabbitmq/stream/impl/ClientProperties.java +++ b/src/main/java/com/rabbitmq/stream/impl/ClientProperties.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -15,10 +15,7 @@ package com.rabbitmq.stream.impl; import java.io.InputStream; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; +import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,12 +37,12 @@ public final class ClientProperties { public static final Map DEFAULT_CLIENT_PROPERTIES = Collections.unmodifiableMap( - new HashMap() { + new LinkedHashMap<>() { { put("product", "RabbitMQ Stream"); put("version", ClientProperties.VERSION); put("platform", "Java"); - put("copyright", "Copyright (c) 2020-2023 Broadcom Inc. and/or its subsidiaries."); + put("copyright", "Copyright (c) 2020-2025 Broadcom Inc. and/or its subsidiaries."); put("information", "Licensed under the MPL 2.0. See https://www.rabbitmq.com/"); } }); diff --git a/src/main/java/com/rabbitmq/stream/impl/Clock.java b/src/main/java/com/rabbitmq/stream/impl/Clock.java index f2a2f615df..224f0881d3 100644 --- a/src/main/java/com/rabbitmq/stream/impl/Clock.java +++ b/src/main/java/com/rabbitmq/stream/impl/Clock.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/impl/Codecs.java b/src/main/java/com/rabbitmq/stream/impl/Codecs.java index ab9fa008ef..401a5c128e 100644 --- a/src/main/java/com/rabbitmq/stream/impl/Codecs.java +++ b/src/main/java/com/rabbitmq/stream/impl/Codecs.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/impl/CompressionCodecs.java b/src/main/java/com/rabbitmq/stream/impl/CompressionCodecs.java index 1928205ad6..85c2a5f0e3 100644 --- a/src/main/java/com/rabbitmq/stream/impl/CompressionCodecs.java +++ b/src/main/java/com/rabbitmq/stream/impl/CompressionCodecs.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/impl/ConcurrencyUtils.java b/src/main/java/com/rabbitmq/stream/impl/ConcurrencyUtils.java new file mode 100644 index 0000000000..f83d2abf29 --- /dev/null +++ b/src/main/java/com/rabbitmq/stream/impl/ConcurrencyUtils.java @@ -0,0 +1,61 @@ +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// +// This software, the RabbitMQ Stream Java client library, is dual-licensed under the +// Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). +// For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL, +// please see LICENSE-APACHE2. +// +// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, +// either express or implied. See the LICENSE file for specific language governing +// rights and limitations of this software. +// +// If you have any questions regarding licensing, please contact us at +// info@rabbitmq.com. +package com.rabbitmq.stream.impl; + +import java.lang.reflect.InvocationTargetException; +import java.util.Arrays; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +final class ConcurrencyUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(ConcurrencyUtils.class); + + private static final ThreadFactory THREAD_FACTORY; + + static { + if (isJava21OrMore()) { + LOGGER.debug("Running Java 21 or more, using virtual threads"); + Class builderClass = + Arrays.stream(Thread.class.getDeclaredClasses()) + .filter(c -> "Builder".equals(c.getSimpleName())) + .findFirst() + .get(); + // Reflection code is the same as: + // Thread.ofVirtual().factory(); + try { + Object builder = Thread.class.getDeclaredMethod("ofVirtual").invoke(null); + THREAD_FACTORY = (ThreadFactory) builderClass.getDeclaredMethod("factory").invoke(builder); + } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + throw new RuntimeException(e); + } + } else { + THREAD_FACTORY = Executors.defaultThreadFactory(); + } + } + + private ConcurrencyUtils() {} + + static ThreadFactory defaultThreadFactory() { + return THREAD_FACTORY; + } + + private static boolean isJava21OrMore() { + String version = System.getProperty("java.version").replace("-beta", ""); + return Utils.versionCompare(version, "21.0") >= 0; + } +} diff --git a/src/main/java/com/rabbitmq/stream/impl/ConnectionStreamException.java b/src/main/java/com/rabbitmq/stream/impl/ConnectionStreamException.java index 9bb278aad7..675cce3359 100644 --- a/src/main/java/com/rabbitmq/stream/impl/ConnectionStreamException.java +++ b/src/main/java/com/rabbitmq/stream/impl/ConnectionStreamException.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/impl/ConsumersCoordinator.java b/src/main/java/com/rabbitmq/stream/impl/ConsumersCoordinator.java index 99191cee2a..05a9ae00c1 100644 --- a/src/main/java/com/rabbitmq/stream/impl/ConsumersCoordinator.java +++ b/src/main/java/com/rabbitmq/stream/impl/ConsumersCoordinator.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -14,14 +14,10 @@ // info@rabbitmq.com. package com.rabbitmq.stream.impl; -import static com.rabbitmq.stream.impl.Utils.convertCodeToException; -import static com.rabbitmq.stream.impl.Utils.formatConstant; -import static com.rabbitmq.stream.impl.Utils.isSac; -import static com.rabbitmq.stream.impl.Utils.jsonField; -import static com.rabbitmq.stream.impl.Utils.namedFunction; -import static com.rabbitmq.stream.impl.Utils.namedRunnable; -import static com.rabbitmq.stream.impl.Utils.quote; +import static com.rabbitmq.stream.Constants.RESPONSE_CODE_SUBSCRIPTION_ID_ALREADY_EXISTS; +import static com.rabbitmq.stream.impl.Utils.*; import static java.lang.String.format; +import static java.util.stream.Collectors.toList; import com.rabbitmq.stream.*; import com.rabbitmq.stream.Consumer; @@ -41,7 +37,6 @@ import java.util.Map.Entry; import java.util.NavigableSet; import java.util.Objects; -import java.util.Random; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; @@ -51,21 +46,23 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.*; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -class ConsumersCoordinator { +final class ConsumersCoordinator implements AutoCloseable { static final int MAX_SUBSCRIPTIONS_PER_CLIENT = 256; static final int MAX_ATTEMPT_BEFORE_FALLING_BACK_TO_LEADER = 5; + private static final boolean DEBUG = false; static final OffsetSpecification DEFAULT_OFFSET_SPECIFICATION = OffsetSpecification.next(); private static final Logger LOGGER = LoggerFactory.getLogger(ConsumersCoordinator.class); - private final Random random = new Random(); private final StreamEnvironment environment; private final ClientFactory clientFactory; private final int maxConsumersByConnection; @@ -73,29 +70,28 @@ class ConsumersCoordinator { private final AtomicLong managerIdSequence = new AtomicLong(0); private final NavigableSet managers = new ConcurrentSkipListSet<>(); private final AtomicLong trackerIdSequence = new AtomicLong(0); + private final Function, Broker> brokerPicker; - private final boolean debug = false; private final List trackers = new CopyOnWriteArrayList<>(); private final ExecutorServiceFactory executorServiceFactory = new DefaultExecutorServiceFactory( - Runtime.getRuntime().availableProcessors(), 10, "rabbitmq-stream-consumer-connection-"); + AVAILABLE_PROCESSORS, 10, "rabbitmq-stream-consumer-connection-"); private final boolean forceReplica; + private final Lock coordinatorLock = new ReentrantLock(); ConsumersCoordinator( StreamEnvironment environment, int maxConsumersByConnection, Function connectionNamingStrategy, ClientFactory clientFactory, - boolean forceReplica) { + boolean forceReplica, + Function, Broker> brokerPicker) { this.environment = environment; this.clientFactory = clientFactory; this.maxConsumersByConnection = maxConsumersByConnection; this.connectionNamingStrategy = connectionNamingStrategy; this.forceReplica = forceReplica; - } - - private static String keyForClientSubscription(Client.Broker broker) { - return broker.getHost() + ":" + broker.getPort(); + this.brokerPicker = brokerPicker; } private BackOffDelayPolicy recoveryBackOffDelayPolicy() { @@ -116,51 +112,56 @@ Runnable subscribe( MessageHandler messageHandler, Map subscriptionProperties, ConsumerFlowStrategy flowStrategy) { - List candidates = findBrokersForStream(stream, forceReplica); - Client.Broker newNode = pickBroker(candidates); - if (newNode == null) { - throw new IllegalStateException("No available node to subscribe to"); - } - - // create stream subscription to track final and changing state of this very subscription - // we keep this instance when we move the subscription from a client to another one - SubscriptionTracker subscriptionTracker = - new SubscriptionTracker( - this.trackerIdSequence.getAndIncrement(), - consumer, - stream, - offsetSpecification, - trackingReference, - subscriptionListener, - trackingClosingCallback, - messageHandler, - subscriptionProperties, - flowStrategy); + return lock( + this.coordinatorLock, + () -> { + List candidates = findCandidateNodes(stream, forceReplica); + Broker newNode = pickBroker(this.brokerPicker, candidates); + if (newNode == null) { + throw new IllegalStateException("No available node to subscribe to"); + } - try { - addToManager(newNode, subscriptionTracker, offsetSpecification, true); - } catch (ConnectionStreamException e) { - // these exceptions are not public - throw new StreamException(e.getMessage()); - } + // create stream subscription to track final and changing state of this very subscription + // we keep this instance when we move the subscription from a client to another one + SubscriptionTracker subscriptionTracker = + new SubscriptionTracker( + this.trackerIdSequence.getAndIncrement(), + consumer, + stream, + offsetSpecification, + trackingReference, + subscriptionListener, + trackingClosingCallback, + messageHandler, + subscriptionProperties, + flowStrategy); + + try { + addToManager(newNode, candidates, subscriptionTracker, offsetSpecification, true); + } catch (ConnectionStreamException e) { + // these exceptions are not public + throw new StreamException(e.getMessage()); + } - if (debug) { - this.trackers.add(subscriptionTracker); - return () -> { - try { - this.trackers.remove(subscriptionTracker); - } catch (Exception e) { - LOGGER.debug("Error while removing subscription tracker from list"); - } - subscriptionTracker.cancel(); - }; - } else { - return subscriptionTracker::cancel; - } + if (DEBUG) { + this.trackers.add(subscriptionTracker); + return () -> { + try { + this.trackers.remove(subscriptionTracker); + } catch (Exception e) { + LOGGER.debug("Error while removing subscription tracker from list"); + } + subscriptionTracker.cancel(); + }; + } else { + return subscriptionTracker::cancel; + } + }); } private void addToManager( Broker node, + List candidates, SubscriptionTracker tracker, OffsetSpecification offsetSpecification, boolean isInitialSubscription) { @@ -188,17 +189,16 @@ private void addToManager( } } if (pickedManager == null) { - String name = keyForClientSubscription(node); + String name = keyForNode(node); LOGGER.debug("Creating subscription manager on {}", name); - pickedManager = new ClientSubscriptionsManager(node, clientParameters); + pickedManager = new ClientSubscriptionsManager(node, candidates, clientParameters); LOGGER.debug("Created subscription manager on {}, id {}", name, pickedManager.id); } try { pickedManager.add(tracker, offsetSpecification, isInitialSubscription); LOGGER.debug( - "Assigned tracker {} (stream '{}') to manager {} (node {}), subscription ID {}", - tracker.id, - tracker.stream, + "Assigned tracker {} to manager {} (node {}), subscription ID {}", + tracker.label(), pickedManager.id, pickedManager.name, tracker.subscriptionIdInClient); @@ -230,7 +230,7 @@ int managerCount() { } // package protected for testing - List findBrokersForStream(String stream, boolean forceReplica) { + List findCandidateNodes(String stream, boolean forceReplica) { LOGGER.debug( "Candidate lookup to consumer from '{}', forcing replica? {}", stream, forceReplica); Map metadata = @@ -253,12 +253,13 @@ List findBrokersForStream(String stream, boolean forceReplica) { } } - List replicas = streamMetadata.getReplicas(); - if ((replicas == null || replicas.isEmpty()) && streamMetadata.getLeader() == null) { + Broker leader = streamMetadata.getLeader(); + List replicas = streamMetadata.getReplicas(); + if ((replicas == null || replicas.isEmpty()) && leader == null) { throw new IllegalStateException("No node available to consume from stream " + stream); } - List brokers; + List brokers; if (replicas == null || replicas.isEmpty()) { if (forceReplica) { throw new IllegalStateException( @@ -267,13 +268,18 @@ List findBrokersForStream(String stream, boolean forceReplica) { + "consuming from leader has been deactivated for this consumer", stream)); } else { - brokers = Collections.singletonList(streamMetadata.getLeader()); - LOGGER.debug( - "Only leader node {} for consuming from {}", streamMetadata.getLeader(), stream); + brokers = Collections.singletonList(new BrokerWrapper(leader, true)); + LOGGER.debug("Only leader node {} for consuming from {}", leader, stream); } } else { LOGGER.debug("Replicas for consuming from {}: {}", stream, replicas); - brokers = new ArrayList<>(replicas); + brokers = + replicas.stream() + .map(b -> new BrokerWrapper(b, false)) + .collect(Collectors.toCollection(ArrayList::new)); + if (!forceReplica && leader != null) { + brokers.add(new BrokerWrapper(leader, true)); + } } LOGGER.debug("Candidates to consume from {}: {}", stream, brokers); @@ -281,7 +287,7 @@ List findBrokersForStream(String stream, boolean forceReplica) { return brokers; } - private Callable> findBrokersForStream(String stream) { + private Callable> findCandidateNodes(String stream) { AtomicInteger attemptNumber = new AtomicInteger(); return () -> { boolean mustUseReplica; @@ -293,20 +299,10 @@ private Callable> findBrokersForStream(String stream) { } LOGGER.debug( "Looking for broker(s) for stream {}, forcing replica {}", stream, mustUseReplica); - return findBrokersForStream(stream, mustUseReplica); + return findCandidateNodes(stream, mustUseReplica); }; } - private Client.Broker pickBroker(List brokers) { - if (brokers.isEmpty()) { - return null; - } else if (brokers.size() == 1) { - return brokers.get(0); - } else { - return brokers.get(random.nextInt(brokers.size())); - } - } - public void close() { Iterator iterator = this.managers.iterator(); while (iterator.hasNext()) { @@ -355,6 +351,7 @@ public String toString() { t -> { StringBuilder trackerBuilder = new StringBuilder("{"); trackerBuilder.append(jsonField("stream", t.stream)).append(","); + trackerBuilder.append(jsonField("id", t.id)).append(","); trackerBuilder.append( jsonField("subscription_id", t.subscriptionIdInClient)); return trackerBuilder.append("}").toString(); @@ -365,7 +362,7 @@ public String toString() { }) .collect(Collectors.joining(","))); builder.append("]"); - if (debug) { + if (DEBUG) { builder.append(","); builder.append("\"subscription_count\" : ").append(this.trackers.size()).append(","); builder.append("\"subscriptions\" : ["); @@ -416,9 +413,10 @@ private static class SubscriptionTracker { private volatile boolean hasReceivedSomething = false; private volatile byte subscriptionIdInClient; private volatile ClientSubscriptionsManager manager; - private volatile AtomicReference state = + private final AtomicReference state = new AtomicReference<>(SubscriptionState.OPENING); private final ConsumerFlowStrategy flowStrategy; + private final Lock subscriptionTrackerLock = new ReentrantLock(); private SubscriptionTracker( long id, @@ -451,36 +449,49 @@ private SubscriptionTracker( } } - synchronized void cancel() { - // the flow of messages in the user message handler should stop, we can call the tracking - // closing callback - // with automatic offset tracking, it will store the last dispatched offset - LOGGER.debug("Calling tracking consumer closing callback (may be no-op)"); - this.trackingClosingCallback.run(); - if (this.manager != null) { - LOGGER.debug("Removing consumer from manager " + this.consumer); - this.manager.remove(this); - } else { - LOGGER.debug("No manager to remove consumer from"); - } - this.state(SubscriptionState.CLOSED); + void cancel() { + lock( + this.subscriptionTrackerLock, + () -> { + // the flow of messages in the user message handler should stop, we can call the + // tracking + // closing callback + // with automatic offset tracking, it will store the last dispatched offset + LOGGER.debug("Calling tracking consumer closing callback (may be no-op)"); + this.trackingClosingCallback.run(); + if (this.manager != null) { + LOGGER.debug("Removing tracker {} from manager", this.label()); + this.manager.remove(this); + } else { + LOGGER.debug("No manager to remove consumer from"); + } + this.state(SubscriptionState.CLOSED); + }); } - synchronized void assign(byte subscriptionIdInClient, ClientSubscriptionsManager manager) { - this.subscriptionIdInClient = subscriptionIdInClient; - this.manager = manager; - if (this.manager == null) { - if (consumer != null) { - this.consumer.setSubscriptionClient(null); - } - } else { - this.consumer.setSubscriptionClient(this.manager.client); - } + void assign(byte subscriptionIdInClient, ClientSubscriptionsManager manager) { + lock( + this.subscriptionTrackerLock, + () -> { + this.subscriptionIdInClient = subscriptionIdInClient; + this.manager = manager; + if (this.manager == null) { + if (consumer != null) { + this.consumer.setSubscriptionClient(null); + } + } else { + this.consumer.setSubscriptionClient(this.manager.client); + } + }); } - synchronized void detachFromManager() { - this.manager = null; - this.consumer.setSubscriptionClient(null); + void detachFromManager() { + lock( + this.subscriptionTrackerLock, + () -> { + this.manager = null; + this.consumer.setSubscriptionClient(null); + }); } void state(SubscriptionState state) { @@ -494,6 +505,12 @@ boolean compareAndSet(SubscriptionState expected, SubscriptionState newValue) { SubscriptionState state() { return this.state.get(); } + + String label() { + return String.format( + "[id=%d, stream=%s, name=%s, consumer=%d]", + this.id, this.stream, this.offsetTrackingReference, this.consumer.id()); + } } private enum SubscriptionState { @@ -570,6 +587,7 @@ private class ClientSubscriptionsManager implements Comparable: (actual or advertised) private final String name; // the 2 data structures track the subscriptions, they must remain consistent private final Map> streamToStreamSubscriptions = @@ -580,13 +598,16 @@ private class ClientSubscriptionsManager implements Comparable candidates, + Client.ClientParameters clientParameters) { this.id = managerIdSequence.getAndIncrement(); - this.node = node; - this.name = keyForClientSubscription(node); - LOGGER.debug("creating subscription manager on {}", name); this.trackerCount = 0; + AtomicReference nameReference = new AtomicReference<>(); + AtomicBoolean clientInitializedInManager = new AtomicBoolean(false); ChunkListener chunkListener = (client, subscriptionId, offset, messageCount, dataSize) -> { @@ -638,7 +659,7 @@ private ClientSubscriptionsManager(Broker node, Client.ClientParameters clientPa "Could not find stream subscription {} in manager {}, node {} for message listener", subscriptionId, this.id, - this.name); + nameReference.get()); } }; MessageIgnoredListener messageIgnoredListener = @@ -662,7 +683,7 @@ private ClientSubscriptionsManager(Broker node, Client.ClientParameters clientPa "Could not find stream subscription {} in manager {}, node {} for message ignored listener", subscriptionId, this.id, - this.name); + nameReference.get()); } }; ShutdownListener shutdownListener = @@ -674,7 +695,7 @@ private ClientSubscriptionsManager(Broker node, Client.ClientParameters clientPa if (shutdownContext.isShutdownUnexpected()) { LOGGER.debug( "Unexpected shutdown notification on subscription connection {}, scheduling consumers re-assignment", - name); + nameReference.get()); LOGGER.debug( "Subscription connection has {} consumer(s) over {} stream(s) to recover", this.subscriptionTrackers.stream().filter(Objects::nonNull).count(), @@ -717,7 +738,7 @@ private ClientSubscriptionsManager(Broker node, Client.ClientParameters clientPa } }, "Consumers re-assignment after disconnection from %s", - name)); + nameReference.get())); } }; MetadataListener metadataListener = @@ -726,7 +747,9 @@ private ClientSubscriptionsManager(Broker node, Client.ClientParameters clientPa "Received metadata notification for '{}', stream is likely to have become unavailable", stream); Set affectedSubscriptions; - synchronized (this) { + + this.subscriptionManagerLock.lock(); + try { Set subscriptions = streamToStreamSubscriptions.remove(stream); if (subscriptions != null && !subscriptions.isEmpty()) { List newSubscriptions = createSubscriptionTrackerList(); @@ -735,8 +758,9 @@ private ClientSubscriptionsManager(Broker node, Client.ClientParameters clientPa } for (SubscriptionTracker subscription : subscriptions) { LOGGER.debug( - "Subscription {} was at offset {} (received something? {})", + "Subscription {} ({}) was at offset {} (received something? {})", subscription.subscriptionIdInClient, + subscription.label(), subscription.offset, subscription.hasReceivedSomething); newSubscriptions.set(subscription.subscriptionIdInClient & 0xFF, null); @@ -745,6 +769,8 @@ private ClientSubscriptionsManager(Broker node, Client.ClientParameters clientPa this.setSubscriptionTrackers(newSubscriptions); } affectedSubscriptions = subscriptions; + } finally { + this.subscriptionManagerLock.unlock(); } if (affectedSubscriptions != null && !affectedSubscriptions.isEmpty()) { @@ -791,18 +817,23 @@ private ClientSubscriptionsManager(Broker node, Client.ClientParameters clientPa }; String connectionName = connectionNamingStrategy.apply(ClientConnectionType.CONSUMER); ClientFactoryContext clientFactoryContext = - ClientFactoryContext.fromParameters( - clientParameters - .clientProperty("connection_name", connectionName) - .chunkListener(chunkListener) - .creditNotification(creditNotification) - .messageListener(messageListener) - .messageIgnoredListener(messageIgnoredListener) - .shutdownListener(shutdownListener) - .metadataListener(metadataListener) - .consumerUpdateListener(consumerUpdateListener)) - .key(name); + new ClientFactoryContext( + clientParameters + .clientProperty("connection_name", connectionName) + .chunkListener(chunkListener) + .creditNotification(creditNotification) + .messageListener(messageListener) + .messageIgnoredListener(messageIgnoredListener) + .shutdownListener(shutdownListener) + .metadataListener(metadataListener) + .consumerUpdateListener(consumerUpdateListener), + keyForNode(targetNode), + candidates.stream().map(BrokerWrapper::broker).collect(toList())); this.client = clientFactory.client(clientFactoryContext); + this.node = brokerFromClient(this.client); + this.name = keyForNode(this.node); + nameReference.set(this.name); + LOGGER.debug("creating subscription manager on {}", name); LOGGER.debug("Created consumer connection '{}'", connectionName); clientInitializedInManager.set(true); } @@ -827,7 +858,7 @@ private void assignConsumersToStream( } }; - AsyncRetry.asyncRetry(findBrokersForStream(stream)) + AsyncRetry.asyncRetry(findCandidateNodes(stream)) .description("Candidate lookup to consume from '%s'", stream) .scheduler(environment.scheduledExecutorService()) .retry(ex -> !(ex instanceof StreamDoesNotExistException)) @@ -835,7 +866,7 @@ private void assignConsumersToStream( .build() .thenAccept( candidateNodes -> { - List candidates = candidateNodes; + List candidates = candidateNodes; if (candidates == null) { LOGGER.debug("No candidate nodes to consume from '{}'", stream); consumersClosingCallback.run(); @@ -869,35 +900,35 @@ private List createSubscriptionTrackerList() { return newSubscriptions; } - private void maybeRecoverSubscription(List candidates, SubscriptionTracker tracker) { + private void maybeRecoverSubscription( + List candidates, SubscriptionTracker tracker) { if (tracker.compareAndSet(SubscriptionState.ACTIVE, SubscriptionState.RECOVERING)) { try { recoverSubscription(candidates, tracker); } catch (Exception e) { LOGGER.warn( - "Error while recovering consumer {} from stream '{}'. Reason: {}", - tracker.consumer.id(), - tracker.stream, + "Error while recovering consumer tracker {}. Reason: {}", + tracker.label(), Utils.exceptionMessage(e)); } } else { LOGGER.debug( - "Not recovering consumer {} from stream {}, state is {}, expected is {}", - tracker.consumer.id(), - tracker.stream, + "Not recovering consumer tracker {}, state is {}, expected is {}", + tracker.label(), tracker.state(), SubscriptionState.ACTIVE); } } - private void recoverSubscription(List candidates, SubscriptionTracker tracker) { + private void recoverSubscription(List candidates, SubscriptionTracker tracker) { boolean reassignmentCompleted = false; while (!reassignmentCompleted) { try { if (tracker.consumer.isOpen()) { - Broker broker = pickBroker(candidates); + Broker broker = pickBroker(brokerPicker, candidates); LOGGER.debug("Using {} to resume consuming from {}", broker, tracker.stream); - synchronized (tracker.consumer) { + tracker.consumer.lock(); + try { if (tracker.consumer.isOpen()) { OffsetSpecification offsetSpecification; if (tracker.hasReceivedSomething) { @@ -905,8 +936,10 @@ private void recoverSubscription(List candidates, SubscriptionTracker tr } else { offsetSpecification = tracker.initialOffsetSpecification; } - addToManager(broker, tracker, offsetSpecification, false); + addToManager(broker, candidates, tracker, offsetSpecification, false); } + } finally { + tracker.consumer.unlock(); } } else { LOGGER.debug( @@ -926,13 +959,32 @@ private void recoverSubscription(List candidates, SubscriptionTracker tr // maybe not a good candidate, let's refresh and retry for this one candidates = Utils.callAndMaybeRetry( - findBrokersForStream(tracker.stream), + findCandidateNodes(tracker.stream), ex -> !(ex instanceof StreamDoesNotExistException), recoveryBackOffDelayPolicy(), "Candidate lookup to consume from '%s' (subscription recovery)", tracker.stream); + } catch (StreamException e) { + LOGGER.warn( + "Stream error while re-assigning subscription from stream {} (name {})", + tracker.stream, + tracker.offsetTrackingReference, + e); + if (e.getCode() == RESPONSE_CODE_SUBSCRIPTION_ID_ALREADY_EXISTS) { + LOGGER.debug("Subscription ID already existing, retrying"); + } else { + LOGGER.debug( + "Not re-assigning consumer '{}' because of '{}'", tracker.label(), e.getMessage()); + reassignmentCompleted = true; + } } catch (Exception e) { - LOGGER.warn("Error while re-assigning subscription from stream {}", tracker.stream, e); + LOGGER.warn( + "Error while re-assigning subscription from stream {} (name {})", + tracker.stream, + tracker.offsetTrackingReference, + e); + LOGGER.debug( + "Not re-assigning consumer '{}' because of '{}'", tracker.label(), e.getMessage()); reassignmentCompleted = true; } } @@ -944,125 +996,146 @@ private void checkNotClosed() { } } - synchronized void add( + void add( SubscriptionTracker subscriptionTracker, OffsetSpecification offsetSpecification, boolean isInitialSubscription) { - if (this.isFull()) { - LOGGER.debug( - "Cannot add subscription tracker for stream '{}', manager is full", - subscriptionTracker.stream); - throw new IllegalStateException("Cannot add subscription tracker, the manager is full"); - } - if (this.isClosed()) { - LOGGER.debug( - "Cannot add subscription tracker for stream '{}', manager is closed", - subscriptionTracker.stream); - throw new IllegalStateException("Cannot add subscription tracker, the manager is closed"); - } + this.subscriptionManagerLock.lock(); + try { + if (this.isFull()) { + LOGGER.debug( + "Cannot add subscription tracker for stream '{}', manager is full", + subscriptionTracker.stream); + throw new IllegalStateException("Cannot add subscription tracker, the manager is full"); + } + if (this.isClosed()) { + LOGGER.debug( + "Cannot add subscription tracker for stream '{}', manager is closed", + subscriptionTracker.stream); + throw new IllegalStateException("Cannot add subscription tracker, the manager is closed"); + } - checkNotClosed(); + checkNotClosed(); - byte subscriptionId = (byte) pickSlot(this.subscriptionTrackers, this.consumerIndexSequence); + byte subscriptionId = + (byte) pickSlot(this.subscriptionTrackers, this.consumerIndexSequence); - List previousSubscriptions = this.subscriptionTrackers; + List previousSubscriptions = this.subscriptionTrackers; - LOGGER.debug( - "Subscribing to {}, requested offset specification is {}, offset tracking reference is {}, properties are {}", - subscriptionTracker.stream, - offsetSpecification == null ? DEFAULT_OFFSET_SPECIFICATION : offsetSpecification, - subscriptionTracker.offsetTrackingReference, - subscriptionTracker.subscriptionProperties); - try { - // updating data structures before subscribing - // (to make sure they are up-to-date in case message would arrive super fast) - subscriptionTracker.assign(subscriptionId, this); - streamToStreamSubscriptions - .computeIfAbsent(subscriptionTracker.stream, s -> ConcurrentHashMap.newKeySet()) - .add(subscriptionTracker); - this.setSubscriptionTrackers( - update(previousSubscriptions, subscriptionId, subscriptionTracker)); - - String offsetTrackingReference = subscriptionTracker.offsetTrackingReference; - if (offsetTrackingReference != null) { - checkNotClosed(); - QueryOffsetResponse queryOffsetResponse = - Utils.callAndMaybeRetry( - () -> client.queryOffset(offsetTrackingReference, subscriptionTracker.stream), - RETRY_ON_TIMEOUT, - "Offset query for consumer %s on stream '%s' (reference %s)", - subscriptionTracker.consumer.id(), + LOGGER.debug( + "Subscribing to {}, requested offset specification is {}, offset tracking reference is {}, properties are {}, " + + "subscription ID is {}", + subscriptionTracker.stream, + offsetSpecification == null ? DEFAULT_OFFSET_SPECIFICATION : offsetSpecification, + subscriptionTracker.offsetTrackingReference, + subscriptionTracker.subscriptionProperties, + subscriptionId); + try { + // updating data structures before subscribing + // (to make sure they are up-to-date in case message would arrive super fast) + subscriptionTracker.assign(subscriptionId, this); + streamToStreamSubscriptions + .computeIfAbsent(subscriptionTracker.stream, s -> ConcurrentHashMap.newKeySet()) + .add(subscriptionTracker); + this.setSubscriptionTrackers( + update(previousSubscriptions, subscriptionId, subscriptionTracker)); + + String offsetTrackingReference = subscriptionTracker.offsetTrackingReference; + if (offsetTrackingReference != null) { + checkNotClosed(); + QueryOffsetResponse queryOffsetResponse = + Utils.callAndMaybeRetry( + () -> client.queryOffset(offsetTrackingReference, subscriptionTracker.stream), + RETRY_ON_TIMEOUT, + "Offset query for consumer %s on stream '%s' (reference %s)", + subscriptionTracker.consumer.id(), + subscriptionTracker.stream, + offsetTrackingReference); + if (queryOffsetResponse.isOk() && queryOffsetResponse.getOffset() != 0) { + if (offsetSpecification != null && isInitialSubscription) { + // subscription call (not recovery), so telling the user their offset specification + // is + // ignored + LOGGER.info( + "Requested offset specification {} not used in favor of stored offset found for reference {}", + offsetSpecification, + offsetTrackingReference); + } + LOGGER.debug( + "Using offset {} to start consuming from {} with consumer {} " + + "(instead of {})", + queryOffsetResponse.getOffset(), subscriptionTracker.stream, - offsetTrackingReference); - if (queryOffsetResponse.isOk() && queryOffsetResponse.getOffset() != 0) { - if (offsetSpecification != null && isInitialSubscription) { - // subscription call (not recovery), so telling the user their offset specification - // is - // ignored - LOGGER.info( - "Requested offset specification {} not used in favor of stored offset found for reference {}", - offsetSpecification, - offsetTrackingReference); + offsetTrackingReference, + offsetSpecification); + offsetSpecification = OffsetSpecification.offset(queryOffsetResponse.getOffset() + 1); } - LOGGER.debug( - "Using offset {} to start consuming from {} with consumer {} " + "(instead of {})", - queryOffsetResponse.getOffset(), - subscriptionTracker.stream, - offsetTrackingReference, - offsetSpecification); - offsetSpecification = OffsetSpecification.offset(queryOffsetResponse.getOffset() + 1); } - } - offsetSpecification = - offsetSpecification == null ? DEFAULT_OFFSET_SPECIFICATION : offsetSpecification; + offsetSpecification = + offsetSpecification == null ? DEFAULT_OFFSET_SPECIFICATION : offsetSpecification; - // TODO consider using/emulating ConsumerUpdateListener, to have only one API, not 2 - // even when the consumer is not a SAC. - SubscriptionContext subscriptionContext = - new DefaultSubscriptionContext(offsetSpecification); - subscriptionTracker.subscriptionListener.preSubscribe(subscriptionContext); - LOGGER.info( - "Computed offset specification {}, offset specification used after subscription listener {}", - offsetSpecification, - subscriptionContext.offsetSpecification()); + // TODO consider using/emulating ConsumerUpdateListener, to have only one API, not 2 + // even when the consumer is not a SAC. + SubscriptionContext subscriptionContext = + new DefaultSubscriptionContext(offsetSpecification, subscriptionTracker.stream); + subscriptionTracker.subscriptionListener.preSubscribe(subscriptionContext); + LOGGER.info( + "Computed offset specification {}, offset specification used after subscription listener {}", + offsetSpecification, + subscriptionContext.offsetSpecification()); - checkNotClosed(); - byte subId = subscriptionId; - Client.Response subscribeResponse = - Utils.callAndMaybeRetry( - () -> - client.subscribe( - subId, - subscriptionTracker.stream, - subscriptionContext.offsetSpecification(), - subscriptionTracker.flowStrategy.initialCredits(), - subscriptionTracker.subscriptionProperties), - RETRY_ON_TIMEOUT, - "Subscribe request for consumer %s on stream '%s'", - subscriptionTracker.consumer.id(), - subscriptionTracker.stream); - if (!subscribeResponse.isOk()) { - String message = - "Subscription to stream " - + subscriptionTracker.stream - + " failed with code " - + formatConstant(subscribeResponse.getResponseCode()); - LOGGER.debug(message); - throw convertCodeToException( - subscribeResponse.getResponseCode(), subscriptionTracker.stream, () -> message); + checkNotClosed(); + Client.Response subscribeResponse = + Utils.callAndMaybeRetry( + () -> + client.subscribe( + subscriptionId, + subscriptionTracker.stream, + subscriptionContext.offsetSpecification(), + subscriptionTracker.flowStrategy.initialCredits(), + subscriptionTracker.subscriptionProperties), + RETRY_ON_TIMEOUT, + "Subscribe request for consumer %s on stream '%s'", + subscriptionTracker.consumer.id(), + subscriptionTracker.stream); + if (!subscribeResponse.isOk()) { + String message = + "Subscription to stream " + + subscriptionTracker.stream + + " failed with code " + + formatConstant(subscribeResponse.getResponseCode()); + LOGGER.debug(message); + if (subscribeResponse.getResponseCode() + == RESPONSE_CODE_SUBSCRIPTION_ID_ALREADY_EXISTS) { + if (LOGGER.isDebugEnabled()) { + SubscriptionTracker initialTracker = previousSubscriptions.get(subscriptionId); + LOGGER.debug("Subscription ID already exists"); + LOGGER.debug( + "Initial tracker with sub ID {}: consumer {}, stream {}, name {}", + subscriptionId, + initialTracker.consumer.id(), + initialTracker.stream, + initialTracker.offsetTrackingReference); + } + } + throw convertCodeToException( + subscribeResponse.getResponseCode(), subscriptionTracker.stream, () -> message); + } + } catch (RuntimeException e) { + subscriptionTracker.assign((byte) -1, null); + this.setSubscriptionTrackers(previousSubscriptions); + streamToStreamSubscriptions + .computeIfAbsent(subscriptionTracker.stream, s -> ConcurrentHashMap.newKeySet()) + .remove(subscriptionTracker); + maybeCleanStreamToStreamSubscriptions(subscriptionTracker.stream); + throw e; } - } catch (RuntimeException e) { - subscriptionTracker.assign((byte) -1, null); - this.setSubscriptionTrackers(previousSubscriptions); - streamToStreamSubscriptions - .computeIfAbsent(subscriptionTracker.stream, s -> ConcurrentHashMap.newKeySet()) - .remove(subscriptionTracker); - maybeCleanStreamToStreamSubscriptions(subscriptionTracker.stream); - throw e; + subscriptionTracker.state(SubscriptionState.ACTIVE); + LOGGER.debug("Subscribed to '{}'", subscriptionTracker.stream); + } finally { + this.subscriptionManagerLock.unlock(); } - subscriptionTracker.state(SubscriptionState.ACTIVE); - LOGGER.debug("Subscribed to '{}'", subscriptionTracker.stream); } private void maybeCleanStreamToStreamSubscriptions(String stream) { @@ -1077,49 +1150,54 @@ private void maybeCleanStreamToStreamSubscriptions(String stream) { }); } - synchronized void remove(SubscriptionTracker subscriptionTracker) { - byte subscriptionIdInClient = subscriptionTracker.subscriptionIdInClient; - try { - Client.Response unsubscribeResponse = - Utils.callAndMaybeRetry( - () -> { - if (client.isOpen()) { - return client.unsubscribe(subscriptionIdInClient); + void remove(SubscriptionTracker subscriptionTracker) { + Utils.lock( + this.subscriptionManagerLock, + () -> { + byte subscriptionIdInClient = subscriptionTracker.subscriptionIdInClient; + try { + Client.Response unsubscribeResponse = + Utils.callAndMaybeRetry( + () -> { + if (client.isOpen()) { + return client.unsubscribe(subscriptionIdInClient); + } else { + return Client.responseOk(); + } + }, + RETRY_ON_TIMEOUT, + "Unsubscribe request for consumer %d on stream '%s'", + subscriptionTracker.consumer.id(), + subscriptionTracker.stream); + if (!unsubscribeResponse.isOk()) { + LOGGER.warn( + "Unexpected response code when unsubscribing from {}: {} (subscription ID {})", + subscriptionTracker.stream, + formatConstant(unsubscribeResponse.getResponseCode()), + subscriptionIdInClient); + } + } catch (TimeoutStreamException e) { + LOGGER.debug( + "Reached timeout when trying to unsubscribe consumer {} from stream '{}'", + subscriptionTracker.consumer.id(), + subscriptionTracker.stream); + } + + this.setSubscriptionTrackers( + update(this.subscriptionTrackers, subscriptionIdInClient, null)); + streamToStreamSubscriptions.compute( + subscriptionTracker.stream, + (stream, subscriptionsForThisStream) -> { + if (subscriptionsForThisStream == null || subscriptionsForThisStream.isEmpty()) { + // should not happen + return null; } else { - return Client.responseOk(); + subscriptionsForThisStream.remove(subscriptionTracker); + return subscriptionsForThisStream.isEmpty() ? null : subscriptionsForThisStream; } - }, - RETRY_ON_TIMEOUT, - "Unsubscribe request for consumer %d on stream '%s'", - subscriptionTracker.consumer.id(), - subscriptionTracker.stream); - if (!unsubscribeResponse.isOk()) { - LOGGER.warn( - "Unexpected response code when unsubscribing from {}: {} (subscription ID {})", - subscriptionTracker.stream, - formatConstant(unsubscribeResponse.getResponseCode()), - subscriptionIdInClient); - } - } catch (TimeoutStreamException e) { - LOGGER.debug( - "Reached timeout when trying to unsubscribe consumer {} from stream '{}'", - subscriptionTracker.consumer.id(), - subscriptionTracker.stream); - } - - this.setSubscriptionTrackers(update(this.subscriptionTrackers, subscriptionIdInClient, null)); - streamToStreamSubscriptions.compute( - subscriptionTracker.stream, - (stream, subscriptionsForThisStream) -> { - if (subscriptionsForThisStream == null || subscriptionsForThisStream.isEmpty()) { - // should not happen - return null; - } else { - subscriptionsForThisStream.remove(subscriptionTracker); - return subscriptionsForThisStream.isEmpty() ? null : subscriptionsForThisStream; - } + }); + closeIfEmpty(); }); - closeIfEmpty(); } private List update( @@ -1152,42 +1230,51 @@ boolean isClosed() { return this.closed.get(); } - synchronized void closeIfEmpty() { - if (this.isEmpty()) { - this.close(); - } + void closeIfEmpty() { + Utils.lock( + this.subscriptionManagerLock, + () -> { + if (this.isEmpty()) { + this.close(); + } + }); } - synchronized void close() { - if (this.closed.compareAndSet(false, true)) { - managers.remove(this); - LOGGER.debug("Closing consumer subscription manager on {}, id {}", this.name, this.id); - if (this.client != null && this.client.isOpen()) { - for (int i = 0; i < this.subscriptionTrackers.size(); i++) { - SubscriptionTracker tracker = this.subscriptionTrackers.get(i); - if (tracker != null) { - try { - if (this.client != null && this.client.isOpen() && tracker.consumer.isOpen()) { - this.client.unsubscribe(tracker.subscriptionIdInClient); + void close() { + Utils.lock( + this.subscriptionManagerLock, + () -> { + if (this.closed.compareAndSet(false, true)) { + managers.remove(this); + LOGGER.debug( + "Closing consumer subscription manager on {}, id {}", this.name, this.id); + if (this.client != null && this.client.isOpen()) { + for (int i = 0; i < this.subscriptionTrackers.size(); i++) { + SubscriptionTracker tracker = this.subscriptionTrackers.get(i); + if (tracker != null) { + try { + if (this.client.isOpen() && tracker.consumer.isOpen()) { + this.client.unsubscribe(tracker.subscriptionIdInClient); + } + } catch (Exception e) { + // OK, moving on + LOGGER.debug( + "Error while unsubscribing from {}, registration {}", + tracker.stream, + tracker.subscriptionIdInClient); + } + this.subscriptionTrackers.set(i, null); + } } - } catch (Exception e) { - // OK, moving on - LOGGER.debug( - "Error while unsubscribing from {}, registration {}", - tracker.stream, - tracker.subscriptionIdInClient); - } - this.subscriptionTrackers.set(i, null); - } - } - streamToStreamSubscriptions.clear(); + streamToStreamSubscriptions.clear(); - if (this.client != null && this.client.isOpen()) { - this.client.close(); - } - } - } + if (this.client.isOpen()) { + this.client.close(); + } + } + } + }); } @Override @@ -1216,9 +1303,12 @@ public int hashCode() { private static final class DefaultSubscriptionContext implements SubscriptionContext { private volatile OffsetSpecification offsetSpecification; + private final String name; - private DefaultSubscriptionContext(OffsetSpecification computedOffsetSpecification) { + private DefaultSubscriptionContext( + OffsetSpecification computedOffsetSpecification, String name) { this.offsetSpecification = computedOffsetSpecification; + this.name = name; } @Override @@ -1231,6 +1321,11 @@ public void offsetSpecification(OffsetSpecification offsetSpecification) { this.offsetSpecification = offsetSpecification; } + @Override + public String stream() { + return this.name; + } + @Override public String toString() { return "SubscriptionContext{" + "offsetSpecification=" + offsetSpecification + '}'; @@ -1286,4 +1381,20 @@ static int pickSlot(List list, AtomicInteger sequence) { } return index; } + + private static List keepReplicasIfPossible(Collection brokers) { + if (brokers.size() > 1) { + return brokers.stream() + .filter(w -> !w.isLeader()) + .map(BrokerWrapper::broker) + .collect(toList()); + } else { + return brokers.stream().map(BrokerWrapper::broker).collect(toList()); + } + } + + static Broker pickBroker( + Function, Broker> picker, Collection candidates) { + return picker.apply(keepReplicasIfPossible(candidates)); + } } diff --git a/src/main/java/com/rabbitmq/stream/impl/DefaultExecutorServiceFactory.java b/src/main/java/com/rabbitmq/stream/impl/DefaultExecutorServiceFactory.java index 0c2757d575..63ccb6fea7 100644 --- a/src/main/java/com/rabbitmq/stream/impl/DefaultExecutorServiceFactory.java +++ b/src/main/java/com/rabbitmq/stream/impl/DefaultExecutorServiceFactory.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2023-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -14,7 +14,8 @@ // info@rabbitmq.com. package com.rabbitmq.stream.impl; -import com.rabbitmq.stream.impl.Utils.NamedThreadFactory; +import static com.rabbitmq.stream.impl.ThreadUtils.threadFactory; + import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -24,6 +25,8 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Supplier; import java.util.stream.IntStream; import org.slf4j.Logger; @@ -41,12 +44,13 @@ class DefaultExecutorServiceFactory implements ExecutorServiceFactory { private final int minSize; private final int clientPerExecutor; private final Supplier executorFactory; + private final Lock lock = new ReentrantLock(); DefaultExecutorServiceFactory(int minSize, int clientPerExecutor, String prefix) { this.minSize = minSize; this.clientPerExecutor = clientPerExecutor; - this.threadFactory = new NamedThreadFactory(prefix); - this.executorFactory = () -> newExecutor(); + this.threadFactory = threadFactory(prefix); + this.executorFactory = this::newExecutor; List l = new ArrayList<>(this.minSize); IntStream.range(0, this.minSize).forEach(ignored -> l.add(this.executorFactory.get())); executors = new CopyOnWriteArrayList<>(l); @@ -110,29 +114,39 @@ private Executor newExecutor() { } @Override - public synchronized ExecutorService get() { - if (closed.get()) { - throw new IllegalStateException("Executor service factory is closed"); - } else { - maybeResize(this.executors, this.minSize, this.clientPerExecutor, this.executorFactory); - LOGGER.debug("Looking least used executor in {}", this.executors); - Executor executor = this.executors.stream().min(EXECUTOR_COMPARATOR).get(); - LOGGER.debug("Least used executor is {}", executor); - executor.incrementUsage(); - return executor.executorService; + public ExecutorService get() { + this.lock.lock(); + try { + if (closed.get()) { + throw new IllegalStateException("Executor service factory is closed"); + } else { + maybeResize(this.executors, this.minSize, this.clientPerExecutor, this.executorFactory); + LOGGER.debug("Looking least used executor in {}", this.executors); + Executor executor = this.executors.stream().min(EXECUTOR_COMPARATOR).get(); + LOGGER.debug("Least used executor is {}", executor); + executor.incrementUsage(); + return executor.executorService; + } + } finally { + this.lock.unlock(); } } @Override - public synchronized void clientClosed(ExecutorService executorService) { - if (!closed.get()) { - Executor executor = find(executorService); - if (executor == null) { - LOGGER.info("Could not find executor service wrapper"); - } else { - executor.decrementUsage(); - maybeResize(this.executors, this.minSize, this.clientPerExecutor, this.executorFactory); + public void clientClosed(ExecutorService executorService) { + this.lock.lock(); + try { + if (!closed.get()) { + Executor executor = find(executorService); + if (executor == null) { + LOGGER.info("Could not find executor service wrapper"); + } else { + executor.decrementUsage(); + maybeResize(this.executors, this.minSize, this.clientPerExecutor, this.executorFactory); + } } + } finally { + this.lock.unlock(); } } @@ -147,17 +161,26 @@ private Executor find(ExecutorService executorService) { @Override public synchronized void close() { - if (closed.compareAndSet(false, true)) { - this.executors.forEach(executor -> executor.executorService.shutdownNow()); + this.lock.lock(); + try { + if (closed.compareAndSet(false, true)) { + this.executors.forEach(executor -> executor.executorService.shutdownNow()); + } + } finally { + this.lock.unlock(); } } static class Executor { + private static final AtomicInteger ID_SEQUENCE = new AtomicInteger(); + private final ExecutorService executorService; private AtomicInteger usage = new AtomicInteger(0); + private final int id; Executor(ExecutorService executorService) { + this.id = ID_SEQUENCE.getAndIncrement(); this.executorService = executorService; } @@ -191,7 +214,7 @@ private void close() { @Override public String toString() { - return "Executor{" + "usage=" + usage.get() + '}'; + return "Executor{" + "id=" + id + ", usage=" + usage.get() + '}'; } } } diff --git a/src/main/java/com/rabbitmq/stream/impl/DelegatingExecutorService.java b/src/main/java/com/rabbitmq/stream/impl/DelegatingExecutorService.java new file mode 100644 index 0000000000..d55796955b --- /dev/null +++ b/src/main/java/com/rabbitmq/stream/impl/DelegatingExecutorService.java @@ -0,0 +1,105 @@ +// Copyright (c) 2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// +// This software, the RabbitMQ Stream Java client library, is dual-licensed under the +// Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). +// For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL, +// please see LICENSE-APACHE2. +// +// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, +// either express or implied. See the LICENSE file for specific language governing +// rights and limitations of this software. +// +// If you have any questions regarding licensing, please contact us at +// info@rabbitmq.com. +package com.rabbitmq.stream.impl; + +import java.util.Collection; +import java.util.List; +import java.util.concurrent.*; + +final class DelegatingExecutorService implements ExecutorService { + + private final ExecutorService delegate; + private final int id; + + DelegatingExecutorService(int id, ExecutorService delegate) { + this.id = id; + this.delegate = delegate; + } + + @Override + public void shutdown() { + this.delegate.shutdown(); + } + + @Override + public List shutdownNow() { + return this.delegate.shutdownNow(); + } + + @Override + public boolean isShutdown() { + return this.delegate.isShutdown(); + } + + @Override + public boolean isTerminated() { + return this.delegate.isTerminated(); + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + return this.delegate.awaitTermination(timeout, unit); + } + + @Override + public Future submit(Callable task) { + return this.delegate.submit(task); + } + + @Override + public Future submit(Runnable task, T result) { + return this.delegate.submit(task, result); + } + + @Override + public Future submit(Runnable task) { + return this.delegate.submit(task); + } + + @Override + public List> invokeAll(Collection> tasks) + throws InterruptedException { + return this.delegate.invokeAll(tasks); + } + + @Override + public List> invokeAll( + Collection> tasks, long timeout, TimeUnit unit) + throws InterruptedException { + return this.delegate.invokeAll(tasks, timeout, unit); + } + + @Override + public T invokeAny(Collection> tasks) + throws InterruptedException, ExecutionException { + return this.delegate.invokeAny(tasks); + } + + @Override + public T invokeAny(Collection> tasks, long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { + return this.delegate.invokeAny(tasks, timeout, unit); + } + + @Override + public void execute(Runnable command) { + this.delegate.execute(command); + } + + @Override + public String toString() { + return "DelegatingExecutorService{" + "id=" + id + '}'; + } +} diff --git a/src/main/java/com/rabbitmq/stream/impl/DynamicBatch.java b/src/main/java/com/rabbitmq/stream/impl/DynamicBatch.java new file mode 100644 index 0000000000..7e2d1a7369 --- /dev/null +++ b/src/main/java/com/rabbitmq/stream/impl/DynamicBatch.java @@ -0,0 +1,120 @@ +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// +// This software, the RabbitMQ Stream Java client library, is dual-licensed under the +// Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). +// For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL, +// please see LICENSE-APACHE2. +// +// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, +// either express or implied. See the LICENSE file for specific language governing +// rights and limitations of this software. +// +// If you have any questions regarding licensing, please contact us at +// info@rabbitmq.com. +package com.rabbitmq.stream.impl; + +import static java.lang.Math.max; +import static java.lang.Math.min; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +final class DynamicBatch implements AutoCloseable { + + private static final Logger LOGGER = LoggerFactory.getLogger(DynamicBatch.class); + private static final int MIN_BATCH_SIZE = 32; + private static final int MAX_BATCH_SIZE = 8192; + + private final BlockingQueue requests = new LinkedBlockingQueue<>(); + private final BatchConsumer consumer; + private final int configuredBatchSize; + private final Thread thread; + + DynamicBatch(BatchConsumer consumer, int batchSize) { + this.consumer = consumer; + this.configuredBatchSize = min(max(batchSize, MIN_BATCH_SIZE), MAX_BATCH_SIZE); + this.thread = ConcurrencyUtils.defaultThreadFactory().newThread(this::loop); + this.thread.start(); + } + + void add(T item) { + try { + requests.put(item); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + private void loop() { + State state = new State<>(); + state.batchSize = this.configuredBatchSize; + state.items = new ArrayList<>(state.batchSize); + Thread currentThread = Thread.currentThread(); + T item; + while (!currentThread.isInterrupted()) { + try { + item = this.requests.poll(100, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + currentThread.interrupt(); + return; + } + if (item != null) { + state.items.add(item); + if (state.items.size() >= state.batchSize) { + this.maybeCompleteBatch(state, true); + } else { + item = this.requests.poll(); + if (item == null) { + this.maybeCompleteBatch(state, false); + } else { + state.items.add(item); + if (state.items.size() >= state.batchSize) { + this.maybeCompleteBatch(state, true); + } + } + } + } else { + this.maybeCompleteBatch(state, false); + } + } + } + + private static final class State { + + int batchSize; + List items; + } + + private void maybeCompleteBatch(State state, boolean increaseIfCompleted) { + try { + boolean completed = this.consumer.process(state.items); + if (completed) { + if (increaseIfCompleted) { + state.batchSize = min(state.batchSize * 2, MAX_BATCH_SIZE); + } else { + state.batchSize = max(state.batchSize / 2, MIN_BATCH_SIZE); + } + state.items = new ArrayList<>(state.batchSize); + } + } catch (Exception e) { + LOGGER.warn("Error during dynamic batch completion: {}", e.getMessage()); + } + } + + @Override + public void close() { + this.thread.interrupt(); + } + + @FunctionalInterface + interface BatchConsumer { + + boolean process(List items); + } +} diff --git a/src/main/java/com/rabbitmq/stream/impl/DynamicBatchMessageAccumulator.java b/src/main/java/com/rabbitmq/stream/impl/DynamicBatchMessageAccumulator.java new file mode 100644 index 0000000000..ee8c397e13 --- /dev/null +++ b/src/main/java/com/rabbitmq/stream/impl/DynamicBatchMessageAccumulator.java @@ -0,0 +1,158 @@ +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// +// This software, the RabbitMQ Stream Java client library, is dual-licensed under the +// Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). +// For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL, +// please see LICENSE-APACHE2. +// +// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, +// either express or implied. See the LICENSE file for specific language governing +// rights and limitations of this software. +// +// If you have any questions regarding licensing, please contact us at +// info@rabbitmq.com. +package com.rabbitmq.stream.impl; + +import com.rabbitmq.stream.Codec; +import com.rabbitmq.stream.ConfirmationHandler; +import com.rabbitmq.stream.Message; +import com.rabbitmq.stream.ObservationCollector; +import com.rabbitmq.stream.compression.Compression; +import com.rabbitmq.stream.compression.CompressionCodec; +import com.rabbitmq.stream.impl.ProducerUtils.AccumulatedEntity; +import io.netty.buffer.ByteBufAllocator; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.function.ToLongFunction; + +final class DynamicBatchMessageAccumulator implements MessageAccumulator { + + private final DynamicBatch dynamicBatch; + private final ObservationCollector observationCollector; + private final StreamProducer producer; + private final ProducerUtils.MessageAccumulatorHelper helper; + + @SuppressWarnings("unchecked") + DynamicBatchMessageAccumulator( + int subEntrySize, + int batchSize, + Codec codec, + int maxFrameSize, + ToLongFunction publishSequenceFunction, + Function filterValueExtractor, + Clock clock, + String stream, + CompressionCodec compressionCodec, + ByteBufAllocator byteBufAllocator, + ObservationCollector observationCollector, + StreamProducer producer) { + this.helper = + new ProducerUtils.MessageAccumulatorHelper( + codec, + maxFrameSize, + publishSequenceFunction, + filterValueExtractor, + clock, + stream, + observationCollector); + this.producer = producer; + this.observationCollector = (ObservationCollector) observationCollector; + boolean shouldObserve = !this.observationCollector.isNoop(); + if (subEntrySize <= 1) { + this.dynamicBatch = + new DynamicBatch<>( + items -> { + boolean result = this.publish(items); + if (result && shouldObserve) { + items.forEach( + i -> { + AccumulatedEntity entity = (AccumulatedEntity) i; + this.observationCollector.published( + entity.observationContext(), entity.confirmationCallback().message()); + }); + } + return result; + }, + batchSize); + } else { + byte compressionCode = + compressionCodec == null ? Compression.NONE.code() : compressionCodec.code(); + this.dynamicBatch = + new DynamicBatch<>( + items -> { + List subBatches = new ArrayList<>(); + int count = 0; + ProducerUtils.Batch batch = + this.helper.batch( + byteBufAllocator, compressionCode, compressionCodec, subEntrySize); + AccumulatedEntity lastMessageInBatch = null; + for (Object msg : items) { + AccumulatedEntity message = (AccumulatedEntity) msg; + lastMessageInBatch = message; + batch.add( + (Codec.EncodedMessage) message.encodedEntity(), + message.confirmationCallback()); + count++; + if (count == subEntrySize) { + batch.time = lastMessageInBatch.time(); + batch.publishingId = lastMessageInBatch.publishingId(); + batch.encodedMessageBatch.close(); + subBatches.add(batch); + lastMessageInBatch = null; + batch = + this.helper.batch( + byteBufAllocator, compressionCode, compressionCodec, subEntrySize); + count = 0; + } + } + + if (!batch.isEmpty() && count < subEntrySize) { + batch.time = lastMessageInBatch.time(); + batch.publishingId = lastMessageInBatch.publishingId(); + batch.encodedMessageBatch.close(); + subBatches.add(batch); + } + boolean result = this.publish(subBatches); + if (result && shouldObserve) { + for (Object msg : items) { + AccumulatedEntity message = (AccumulatedEntity) msg; + this.observationCollector.published( + message.observationContext(), message.confirmationCallback().message()); + } + } + return result; + }, + batchSize * subEntrySize); + } + } + + @Override + public void add(Message message, ConfirmationHandler confirmationHandler) { + this.dynamicBatch.add(helper.entity(message, confirmationHandler)); + } + + @Override + public int size() { + // TODO compute dynamic batch message accumulator pending message count + return 0; + } + + @Override + public void flush(boolean force) {} + + private boolean publish(List entities) { + if (this.producer.canSend()) { + this.producer.publishInternal(entities); + return true; + } else { + return false; + } + } + + @Override + public void close() { + this.dynamicBatch.close(); + } +} diff --git a/src/main/java/com/rabbitmq/stream/impl/ExecutorServiceFactory.java b/src/main/java/com/rabbitmq/stream/impl/ExecutorServiceFactory.java index e1971189df..a8b1bf8c53 100644 --- a/src/main/java/com/rabbitmq/stream/impl/ExecutorServiceFactory.java +++ b/src/main/java/com/rabbitmq/stream/impl/ExecutorServiceFactory.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/impl/HashRoutingStrategy.java b/src/main/java/com/rabbitmq/stream/impl/HashRoutingStrategy.java index 3335cf1c88..b696c714b0 100644 --- a/src/main/java/com/rabbitmq/stream/impl/HashRoutingStrategy.java +++ b/src/main/java/com/rabbitmq/stream/impl/HashRoutingStrategy.java @@ -1,5 +1,5 @@ -// Copyright (c) 2021-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2021-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/impl/HashUtils.java b/src/main/java/com/rabbitmq/stream/impl/HashUtils.java index b89a714756..fde6d946b2 100644 --- a/src/main/java/com/rabbitmq/stream/impl/HashUtils.java +++ b/src/main/java/com/rabbitmq/stream/impl/HashUtils.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -14,9 +14,11 @@ // info@rabbitmq.com. package com.rabbitmq.stream.impl; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.nio.charset.StandardCharsets; import java.util.function.ToIntFunction; +@SuppressFBWarnings({"SF_SWITCH_FALLTHROUGH", "SF_SWITCH_NO_DEFAULT"}) final class HashUtils { static final ToIntFunction MURMUR3 = new Murmur3(); @@ -25,6 +27,7 @@ private HashUtils() {} // from // https://github.com/apache/commons-codec/blob/rel/commons-codec-1.15/src/main/java/org/apache/commons/codec/digest/MurmurHash3.java + // hash32x86 method static class Murmur3 implements ToIntFunction { private static final int DEFAULT_SEED = 104729; @@ -37,10 +40,10 @@ static class Murmur3 implements ToIntFunction { private static final int N_32 = 0xe6546b64; private static int getLittleEndianInt(final byte[] data, final int index) { - return ((data[index] & 0xff)) - | ((data[index + 1] & 0xff) << 8) - | ((data[index + 2] & 0xff) << 16) - | ((data[index + 3] & 0xff) << 24); + return data[index] & 0xff + | (data[index + 1] & 0xff) << 8 + | (data[index + 2] & 0xff) << 16 + | (data[index + 3] & 0xff) << 24; } private static int mix32(int k, int hash) { @@ -94,7 +97,7 @@ public int applyAsInt(String value) { case 2: k1 ^= (data[index + 1] & 0xff) << 8; case 1: - k1 ^= (data[index] & 0xff); + k1 ^= data[index] & 0xff; // mix functions k1 *= C1_32; diff --git a/src/main/java/com/rabbitmq/stream/impl/JdkChunkChecksum.java b/src/main/java/com/rabbitmq/stream/impl/JdkChunkChecksum.java index da64d4f0f3..5b4b208560 100644 --- a/src/main/java/com/rabbitmq/stream/impl/JdkChunkChecksum.java +++ b/src/main/java/com/rabbitmq/stream/impl/JdkChunkChecksum.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -16,54 +16,22 @@ import com.rabbitmq.stream.ChunkChecksum; import com.rabbitmq.stream.ChunkChecksumValidationException; -import com.rabbitmq.stream.StreamException; import io.netty.buffer.ByteBuf; -import io.netty.util.ByteProcessor; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.nio.ByteBuffer; import java.util.function.Supplier; import java.util.zip.CRC32; import java.util.zip.Checksum; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; class JdkChunkChecksum implements ChunkChecksum { - static final ChunkChecksum CRC32_SINGLETON; - private static final Logger LOGGER = LoggerFactory.getLogger(JdkChunkChecksum.class); private static final Supplier CRC32_SUPPLIER = CRC32::new; - - static { - if (isChecksumUpdateByteBufferAvailable()) { - LOGGER.debug("Checksum#update(ByteBuffer) method available, using it for direct buffers"); - CRC32_SINGLETON = new ByteBufferDirectByteBufChecksum(CRC32_SUPPLIER); - } else { - LOGGER.debug( - "Checksum#update(ByteBuffer) method not available, using byte-by-byte CRC calculation for direct buffers"); - CRC32_SINGLETON = new JdkChunkChecksum(CRC32_SUPPLIER); - } - } + static final ChunkChecksum CRC32_SINGLETON = new JdkChunkChecksum(CRC32_SUPPLIER); private final Supplier checksumSupplier; - JdkChunkChecksum() { - this(CRC32_SUPPLIER); - } - JdkChunkChecksum(Supplier checksumSupplier) { this.checksumSupplier = checksumSupplier; } - private static boolean isChecksumUpdateByteBufferAvailable() { - try { - Checksum.class.getDeclaredMethod("update", ByteBuffer.class); - return true; - } catch (Exception e) { - return false; - } - } - @Override public void checksum(ByteBuf byteBuf, long dataLength, long expected) { Checksum checksum = checksumSupplier.get(); @@ -71,64 +39,10 @@ public void checksum(ByteBuf byteBuf, long dataLength, long expected) { checksum.update( byteBuf.array(), byteBuf.arrayOffset() + byteBuf.readerIndex(), byteBuf.readableBytes()); } else { - byteBuf.forEachByte( - byteBuf.readerIndex(), byteBuf.readableBytes(), new UpdateProcessor(checksum)); + checksum.update(byteBuf.nioBuffer(byteBuf.readerIndex(), byteBuf.readableBytes())); } if (expected != checksum.getValue()) { throw new ChunkChecksumValidationException(expected, checksum.getValue()); } } - - private static class ByteBufferDirectByteBufChecksum implements ChunkChecksum { - - private final Supplier checksumSupplier; - private final Method updateMethod; - - private ByteBufferDirectByteBufChecksum(Supplier checksumSupplier) { - this.checksumSupplier = checksumSupplier; - try { - this.updateMethod = Checksum.class.getDeclaredMethod("update", ByteBuffer.class); - } catch (NoSuchMethodException e) { - throw new StreamException("Error while looking up Checksum#update(ByteBuffer) method", e); - } - } - - @Override - public void checksum(ByteBuf byteBuf, long dataLength, long expected) { - Checksum checksum = checksumSupplier.get(); - if (byteBuf.hasArray()) { - checksum.update( - byteBuf.array(), - byteBuf.arrayOffset() + byteBuf.readerIndex(), - byteBuf.readableBytes()); - } else { - try { - this.updateMethod.invoke( - checksum, byteBuf.nioBuffer(byteBuf.readerIndex(), byteBuf.readableBytes())); - } catch (IllegalAccessException e) { - throw new StreamException("Error while calculating CRC", e); - } catch (InvocationTargetException e) { - throw new StreamException("Error while calculating CRC", e); - } - } - if (expected != checksum.getValue()) { - throw new ChunkChecksumValidationException(expected, checksum.getValue()); - } - } - } - - private static class UpdateProcessor implements ByteProcessor { - - private final Checksum checksum; - - private UpdateProcessor(Checksum checksum) { - this.checksum = checksum; - } - - @Override - public boolean process(byte value) { - checksum.update(value); - return true; - } - } } diff --git a/src/main/java/com/rabbitmq/stream/impl/MessageAccumulator.java b/src/main/java/com/rabbitmq/stream/impl/MessageAccumulator.java index 18298c7557..673d931527 100644 --- a/src/main/java/com/rabbitmq/stream/impl/MessageAccumulator.java +++ b/src/main/java/com/rabbitmq/stream/impl/MessageAccumulator.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -17,28 +17,14 @@ import com.rabbitmq.stream.ConfirmationHandler; import com.rabbitmq.stream.Message; -interface MessageAccumulator { +interface MessageAccumulator extends AutoCloseable { - boolean add(Message message, ConfirmationHandler confirmationHandler); - - AccumulatedEntity get(); - - boolean isEmpty(); + void add(Message message, ConfirmationHandler confirmationHandler); int size(); - interface AccumulatedEntity { - - long time(); - - long publishingId(); - - String filterValue(); - - Object encodedEntity(); - - StreamProducer.ConfirmationCallback confirmationCallback(); + void flush(boolean force); - Object observationContext(); - } + @Override + void close(); } diff --git a/src/main/java/com/rabbitmq/stream/impl/MessageBatch.java b/src/main/java/com/rabbitmq/stream/impl/MessageBatch.java index 34e5e23bb6..7ea2a3b737 100644 --- a/src/main/java/com/rabbitmq/stream/impl/MessageBatch.java +++ b/src/main/java/com/rabbitmq/stream/impl/MessageBatch.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -16,6 +16,7 @@ import com.rabbitmq.stream.Message; import com.rabbitmq.stream.compression.Compression; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.ArrayList; import java.util.List; @@ -24,18 +25,11 @@ public final class MessageBatch { final Compression compression; final List messages; - public MessageBatch() { - this(Compression.NONE, new ArrayList<>()); - } - public MessageBatch(Compression compression) { this(compression, new ArrayList<>()); } - public MessageBatch(List messages) { - this(Compression.NONE, messages); - } - + @SuppressFBWarnings("EI_EXPOSE_REP2") public MessageBatch(Compression compression, List messages) { this.compression = compression; this.messages = messages; @@ -45,20 +39,4 @@ public MessageBatch add(Message message) { this.messages.add(message); return this; } - - public List getMessages() { - return messages; - } - - /* - 0 = no compression - 1 = gzip - 2 = snappy - 3 = lz4 - 4 = zstd - 5 = reserved - 6 = reserved - 7 = user - */ - } diff --git a/src/main/java/com/rabbitmq/stream/impl/OffsetTrackingCoordinator.java b/src/main/java/com/rabbitmq/stream/impl/OffsetTrackingCoordinator.java index 179dc71353..08b1219e9e 100644 --- a/src/main/java/com/rabbitmq/stream/impl/OffsetTrackingCoordinator.java +++ b/src/main/java/com/rabbitmq/stream/impl/OffsetTrackingCoordinator.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/impl/ParameterizedTypeReference.java b/src/main/java/com/rabbitmq/stream/impl/ParameterizedTypeReference.java index 2a07ef6fa9..abe2fab11b 100644 --- a/src/main/java/com/rabbitmq/stream/impl/ParameterizedTypeReference.java +++ b/src/main/java/com/rabbitmq/stream/impl/ParameterizedTypeReference.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -14,6 +14,7 @@ // info@rabbitmq.com. package com.rabbitmq.stream.impl; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; @@ -26,10 +27,11 @@ * * @param */ -public abstract class ParameterizedTypeReference { +abstract class ParameterizedTypeReference { private final Type type; + @SuppressFBWarnings("CT_CONSTRUCTOR_THROW") protected ParameterizedTypeReference() { Class parameterizedTypeReferenceSubclass = findParameterizedTypeReferenceSubclass(getClass()); diff --git a/src/main/java/com/rabbitmq/stream/impl/ProducerUtils.java b/src/main/java/com/rabbitmq/stream/impl/ProducerUtils.java new file mode 100644 index 0000000000..5ae8faa7dd --- /dev/null +++ b/src/main/java/com/rabbitmq/stream/impl/ProducerUtils.java @@ -0,0 +1,322 @@ +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// +// This software, the RabbitMQ Stream Java client library, is dual-licensed under the +// Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). +// For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL, +// please see LICENSE-APACHE2. +// +// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, +// either express or implied. See the LICENSE file for specific language governing +// rights and limitations of this software. +// +// If you have any questions regarding licensing, please contact us at +// info@rabbitmq.com. +package com.rabbitmq.stream.impl; + +import com.rabbitmq.stream.*; +import com.rabbitmq.stream.compression.CompressionCodec; +import io.netty.buffer.ByteBufAllocator; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.function.ToLongFunction; + +final class ProducerUtils { + + private ProducerUtils() {} + + static MessageAccumulator createMessageAccumulator( + boolean dynamicBatch, + int subEntrySize, + int batchSize, + CompressionCodec compressionCodec, + Codec codec, + ByteBufAllocator byteBufAllocator, + int maxFrameSize, + ToLongFunction publishSequenceFunction, + Function filterValueExtractor, + Clock clock, + String stream, + ObservationCollector observationCollector, + StreamProducer producer) { + if (dynamicBatch) { + return new DynamicBatchMessageAccumulator( + subEntrySize, + batchSize, + codec, + maxFrameSize, + publishSequenceFunction, + filterValueExtractor, + clock, + stream, + compressionCodec, + byteBufAllocator, + observationCollector, + producer); + } else { + if (subEntrySize <= 1) { + return new SimpleMessageAccumulator( + batchSize, + codec, + maxFrameSize, + publishSequenceFunction, + filterValueExtractor, + clock, + stream, + observationCollector, + producer); + } else { + return new SubEntryMessageAccumulator( + subEntrySize, + batchSize, + compressionCodec, + codec, + byteBufAllocator, + maxFrameSize, + publishSequenceFunction, + clock, + stream, + observationCollector, + producer); + } + } + } + + interface ConfirmationCallback { + + int handle(boolean confirmed, short code); + + Message message(); + } + + interface AccumulatedEntity { + + long time(); + + long publishingId(); + + String filterValue(); + + Object encodedEntity(); + + ConfirmationCallback confirmationCallback(); + + Object observationContext(); + } + + static final class SimpleConfirmationCallback implements ConfirmationCallback { + + private final Message message; + private final ConfirmationHandler confirmationHandler; + + SimpleConfirmationCallback(Message message, ConfirmationHandler confirmationHandler) { + this.message = message; + this.confirmationHandler = confirmationHandler; + } + + @Override + public int handle(boolean confirmed, short code) { + confirmationHandler.handle(new ConfirmationStatus(message, confirmed, code)); + return 1; + } + + @Override + public Message message() { + return this.message; + } + } + + static final class SimpleAccumulatedEntity implements AccumulatedEntity { + + private final long time; + private final long publishingId; + private final String filterValue; + private final Codec.EncodedMessage encodedMessage; + private final ConfirmationCallback confirmationCallback; + private final Object observationContext; + + SimpleAccumulatedEntity( + long time, + long publishingId, + String filterValue, + Codec.EncodedMessage encodedMessage, + ConfirmationCallback confirmationCallback, + Object observationContext) { + this.time = time; + this.publishingId = publishingId; + this.encodedMessage = encodedMessage; + this.filterValue = filterValue; + this.confirmationCallback = confirmationCallback; + this.observationContext = observationContext; + } + + @Override + public long publishingId() { + return publishingId; + } + + @Override + public String filterValue() { + return filterValue; + } + + @Override + public Object encodedEntity() { + return encodedMessage; + } + + @Override + public long time() { + return time; + } + + @Override + public ConfirmationCallback confirmationCallback() { + return confirmationCallback; + } + + @Override + public Object observationContext() { + return this.observationContext; + } + } + + static final class CompositeConfirmationCallback implements ConfirmationCallback { + + private final List callbacks; + + CompositeConfirmationCallback(List callbacks) { + this.callbacks = callbacks; + } + + private void add(ConfirmationCallback confirmationCallback) { + this.callbacks.add(confirmationCallback); + } + + @Override + public int handle(boolean confirmed, short code) { + for (ConfirmationCallback callback : callbacks) { + callback.handle(confirmed, code); + } + return callbacks.size(); + } + + @Override + public Message message() { + throw new UnsupportedOperationException( + "composite confirmation callback does not contain just one message"); + } + } + + static final class Batch implements AccumulatedEntity { + + final Client.EncodedMessageBatch encodedMessageBatch; + private final CompositeConfirmationCallback confirmationCallback; + volatile long publishingId; + volatile long time; + + Batch( + Client.EncodedMessageBatch encodedMessageBatch, + CompositeConfirmationCallback confirmationCallback) { + this.encodedMessageBatch = encodedMessageBatch; + this.confirmationCallback = confirmationCallback; + } + + void add(Codec.EncodedMessage encodedMessage, ConfirmationCallback confirmationCallback) { + this.encodedMessageBatch.add(encodedMessage); + this.confirmationCallback.add(confirmationCallback); + } + + boolean isEmpty() { + return this.confirmationCallback.callbacks.isEmpty(); + } + + @Override + public long publishingId() { + return publishingId; + } + + @Override + public String filterValue() { + return null; + } + + @Override + public Object encodedEntity() { + return encodedMessageBatch; + } + + @Override + public long time() { + return time; + } + + @Override + public ConfirmationCallback confirmationCallback() { + return confirmationCallback; + } + + @Override + public Object observationContext() { + throw new UnsupportedOperationException( + "batch entity does not contain only one observation context"); + } + } + + static final class MessageAccumulatorHelper { + + private static final Function NULL_FILTER_VALUE_EXTRACTOR = m -> null; + + private final ObservationCollector observationCollector; + private final ToLongFunction publishSequenceFunction; + private final String stream; + private final Codec codec; + private final int maxFrameSize; + private final Clock clock; + private final Function filterValueExtractor; + + @SuppressWarnings("unchecked") + MessageAccumulatorHelper( + Codec codec, + int maxFrameSize, + ToLongFunction publishSequenceFunction, + Function filterValueExtractor, + Clock clock, + String stream, + ObservationCollector observationCollector) { + this.publishSequenceFunction = publishSequenceFunction; + this.codec = codec; + this.clock = clock; + this.maxFrameSize = maxFrameSize; + this.filterValueExtractor = + filterValueExtractor == null ? NULL_FILTER_VALUE_EXTRACTOR : filterValueExtractor; + this.observationCollector = (ObservationCollector) observationCollector; + this.stream = stream; + } + + AccumulatedEntity entity(Message message, ConfirmationHandler confirmationHandler) { + Object observationContext = this.observationCollector.prePublish(this.stream, message); + Codec.EncodedMessage encodedMessage = this.codec.encode(message); + Client.checkMessageFitsInFrame(this.maxFrameSize, encodedMessage); + long publishingId = this.publishSequenceFunction.applyAsLong(message); + return new ProducerUtils.SimpleAccumulatedEntity( + this.clock.time(), + publishingId, + this.filterValueExtractor.apply(message), + this.codec.encode(message), + new ProducerUtils.SimpleConfirmationCallback(message, confirmationHandler), + observationContext); + } + + Batch batch( + ByteBufAllocator bba, + byte compressionCode, + CompressionCodec compressionCodec, + int subEntrySize) { + return new ProducerUtils.Batch( + Client.EncodedMessageBatch.create(bba, compressionCode, compressionCodec, subEntrySize), + new ProducerUtils.CompositeConfirmationCallback(new ArrayList<>(subEntrySize))); + } + } +} diff --git a/src/main/java/com/rabbitmq/stream/impl/ProducersCoordinator.java b/src/main/java/com/rabbitmq/stream/impl/ProducersCoordinator.java index 143ef57204..8ac1017f8a 100644 --- a/src/main/java/com/rabbitmq/stream/impl/ProducersCoordinator.java +++ b/src/main/java/com/rabbitmq/stream/impl/ProducersCoordinator.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -14,12 +14,9 @@ // info@rabbitmq.com. package com.rabbitmq.stream.impl; -import static com.rabbitmq.stream.impl.Utils.callAndMaybeRetry; -import static com.rabbitmq.stream.impl.Utils.formatConstant; -import static com.rabbitmq.stream.impl.Utils.jsonField; -import static com.rabbitmq.stream.impl.Utils.namedFunction; -import static com.rabbitmq.stream.impl.Utils.namedRunnable; -import static com.rabbitmq.stream.impl.Utils.quote; +import static com.rabbitmq.stream.impl.Tuples.pair; +import static com.rabbitmq.stream.impl.Utils.*; +import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toSet; import com.rabbitmq.stream.BackOffDelayPolicy; @@ -34,16 +31,11 @@ import com.rabbitmq.stream.impl.Client.PublishErrorListener; import com.rabbitmq.stream.impl.Client.Response; import com.rabbitmq.stream.impl.Client.ShutdownListener; +import com.rabbitmq.stream.impl.Tuples.Pair; import com.rabbitmq.stream.impl.Utils.ClientConnectionType; import com.rabbitmq.stream.impl.Utils.ClientFactory; import com.rabbitmq.stream.impl.Utils.ClientFactoryContext; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.NavigableSet; -import java.util.Objects; -import java.util.Set; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentSkipListSet; @@ -52,16 +44,19 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -class ProducersCoordinator { +final class ProducersCoordinator implements AutoCloseable { static final int MAX_PRODUCERS_PER_CLIENT = 256; static final int MAX_TRACKING_CONSUMERS_PER_CLIENT = 50; + private static final boolean DEBUG = false; private static final Logger LOGGER = LoggerFactory.getLogger(ProducersCoordinator.class); private final StreamEnvironment environment; private final ClientFactory clientFactory; @@ -70,51 +65,58 @@ class ProducersCoordinator { private final AtomicLong managerIdSequence = new AtomicLong(0); private final NavigableSet managers = new ConcurrentSkipListSet<>(); private final AtomicLong trackerIdSequence = new AtomicLong(0); - private final boolean debug = false; private final List producerTrackers = new CopyOnWriteArrayList<>(); private final ExecutorServiceFactory executorServiceFactory = new DefaultExecutorServiceFactory( - Runtime.getRuntime().availableProcessors(), 10, "rabbitmq-stream-producer-connection-"); + AVAILABLE_PROCESSORS, 10, "rabbitmq-stream-producer-connection-"); + private final Lock coordinatorLock = new ReentrantLock(); + private final boolean forceLeader; ProducersCoordinator( StreamEnvironment environment, int maxProducersByClient, int maxTrackingConsumersByClient, Function connectionNamingStrategy, - ClientFactory clientFactory) { + ClientFactory clientFactory, + boolean forceLeader) { this.environment = environment; this.clientFactory = clientFactory; this.maxProducersByClient = maxProducersByClient; this.maxTrackingConsumersByClient = maxTrackingConsumersByClient; this.connectionNamingStrategy = connectionNamingStrategy; - } - - private static String keyForNode(Client.Broker broker) { - return broker.getHost() + ":" + broker.getPort(); + this.forceLeader = forceLeader; } Runnable registerProducer(StreamProducer producer, String reference, String stream) { - ProducerTracker tracker = - new ProducerTracker(trackerIdSequence.getAndIncrement(), reference, stream, producer); - if (debug) { - this.producerTrackers.add(tracker); - } - return registerAgentTracker(tracker, stream); + return lock( + this.coordinatorLock, + () -> { + ProducerTracker tracker = + new ProducerTracker(trackerIdSequence.getAndIncrement(), reference, stream, producer); + if (DEBUG) { + this.producerTrackers.add(tracker); + } + return registerAgentTracker(tracker, stream); + }); } Runnable registerTrackingConsumer(StreamConsumer consumer) { - return registerAgentTracker( - new TrackingConsumerTracker( - trackerIdSequence.getAndIncrement(), consumer.stream(), consumer), - consumer.stream()); + return lock( + this.coordinatorLock, + () -> + registerAgentTracker( + new TrackingConsumerTracker( + trackerIdSequence.getAndIncrement(), consumer.stream(), consumer), + consumer.stream())); } private Runnable registerAgentTracker(AgentTracker tracker, String stream) { - Client.Broker broker = getBrokerForProducer(stream); + List candidates = findCandidateNodes(stream, this.forceLeader); + Broker broker = pickBroker(candidates); - addToManager(broker, tracker); + addToManager(broker, candidates, tracker); - if (debug) { + if (DEBUG) { return () -> { if (tracker instanceof ProducerTracker) { try { @@ -130,7 +132,7 @@ private Runnable registerAgentTracker(AgentTracker tracker, String stream) { } } - private void addToManager(Broker node, AgentTracker tracker) { + private void addToManager(Broker node, List candidates, AgentTracker tracker) { ClientParameters clientParameters = environment .clientParametersCopy() @@ -157,8 +159,9 @@ private void addToManager(Broker node, AgentTracker tracker) { } if (pickedManager == null) { String name = keyForNode(node); - LOGGER.debug("Creating producer manager on {}", name); - pickedManager = new ClientProducersManager(node, this.clientFactory, clientParameters); + LOGGER.debug("Trying to create producer manager on {}", name); + pickedManager = + new ClientProducersManager(node, candidates, this.clientFactory, clientParameters); LOGGER.debug("Created producer manager on {}, id {}", name, pickedManager.id); } try { @@ -197,11 +200,12 @@ private void addToManager(Broker node, AgentTracker tracker) { } } - private Client.Broker getBrokerForProducer(String stream) { + // package protected for testing + List findCandidateNodes(String stream, boolean forceLeader) { Map metadata = this.environment.locatorOperation( namedFunction(c -> c.metadata(stream), "Candidate lookup to publish to '%s'", stream)); - if (metadata.size() == 0 || metadata.get(stream) == null) { + if (metadata.isEmpty() || metadata.get(stream) == null) { throw new StreamDoesNotExistException(stream); } @@ -215,17 +219,41 @@ private Client.Broker getBrokerForProducer(String stream) { } } + List candidates = new ArrayList<>(); Client.Broker leader = streamMetadata.getLeader(); if (leader == null) { - throw new IllegalStateException("Not leader available for stream " + stream); + if (forceLeader) { + throw new IllegalStateException("Not leader available for stream " + stream); + } + } else { + candidates.add(new BrokerWrapper(leader, true)); } - LOGGER.debug( - "Using client on {}:{} to publish to {}", leader.getHost(), leader.getPort(), stream); - return leader; + if (!forceLeader && streamMetadata.hasReplicas()) { + candidates.addAll( + streamMetadata.getReplicas().stream() + .map(b -> new BrokerWrapper(b, false)) + .collect(toList())); + } + + if (candidates.isEmpty()) { + throw new IllegalStateException("No stream member available to publish for stream " + stream); + } else { + LOGGER.debug("Candidates to publish to {}: {}", stream, candidates); + } + + return List.copyOf(candidates); + } + + static Broker pickBroker(List candidates) { + return candidates.stream() + .filter(BrokerWrapper::isLeader) + .findFirst() + .map(BrokerWrapper::broker) + .orElseThrow(() -> new IllegalStateException("Not leader available")); } - void close() { + public void close() { Iterator iterator = this.managers.iterator(); while (iterator.hasNext()) { ClientProducersManager manager = iterator.next(); @@ -270,7 +298,7 @@ public String toString() { "tracking_consumer_count", this.managers.stream().mapToInt(m -> m.trackingConsumerTrackers.size()).sum())) .append(","); - if (debug) { + if (DEBUG) { builder.append(jsonField("producer_tracker_count", this.producerTrackers.size())).append(","); } builder.append(quote("clients")).append(" : ["); @@ -316,7 +344,7 @@ public String toString() { }) .collect(Collectors.joining(","))); builder.append("]"); - if (debug) { + if (DEBUG) { builder.append(","); builder.append("\"producer_trackers\" : ["); builder.append( @@ -382,6 +410,7 @@ private static class ProducerTracker implements AgentTracker { private volatile byte publisherId; private volatile ClientProducersManager clientProducersManager; private final AtomicBoolean recovering = new AtomicBoolean(false); + private final Lock trackerLock = new ReentrantLock(); private ProducerTracker( long uniqueId, String reference, String stream, StreamProducer producer) { @@ -393,10 +422,12 @@ private ProducerTracker( @Override public void assign(byte producerId, Client client, ClientProducersManager manager) { - synchronized (ProducerTracker.this) { - this.publisherId = producerId; - this.clientProducersManager = manager; - } + lock( + this.trackerLock, + () -> { + this.publisherId = producerId; + this.clientProducersManager = manager; + }); this.producer.setPublisherId(producerId); this.producer.setClient(client); } @@ -423,9 +454,7 @@ public String stream() { @Override public void unavailable() { - synchronized (ProducerTracker.this) { - this.clientProducersManager = null; - } + lock(this.trackerLock, () -> this.clientProducersManager = null); this.producer.unavailable(); } @@ -436,10 +465,14 @@ public void running() { } @Override - public synchronized void cancel() { - if (this.clientProducersManager != null) { - this.clientProducersManager.unregister(this); - } + public void cancel() { + lock( + this.trackerLock, + () -> { + if (this.clientProducersManager != null) { + this.clientProducersManager.unregister(this); + } + }); } @Override @@ -475,6 +508,7 @@ private static class TrackingConsumerTracker implements AgentTracker { private final StreamConsumer consumer; private volatile ClientProducersManager clientProducersManager; private final AtomicBoolean recovering = new AtomicBoolean(false); + private final Lock trackerLock = new ReentrantLock(); private TrackingConsumerTracker(long uniqueId, String stream, StreamConsumer consumer) { this.uniqueId = uniqueId; @@ -484,9 +518,7 @@ private TrackingConsumerTracker(long uniqueId, String stream, StreamConsumer con @Override public void assign(byte producerId, Client client, ClientProducersManager manager) { - synchronized (TrackingConsumerTracker.this) { - this.clientProducersManager = manager; - } + lock(this.trackerLock, () -> this.clientProducersManager = manager); this.consumer.setTrackingClient(client); } @@ -512,9 +544,7 @@ public String stream() { @Override public void unavailable() { - synchronized (TrackingConsumerTracker.this) { - this.clientProducersManager = null; - } + lock(this.trackerLock, () -> this.clientProducersManager = null); this.consumer.unavailable(); } @@ -525,10 +555,8 @@ public void running() { } @Override - public synchronized void cancel() { - if (this.clientProducersManager != null) { - this.clientProducersManager.unregister(this); - } + public void cancel() { + lock(this.trackerLock, () -> this.clientProducersManager.unregister(this)); } @Override @@ -571,12 +599,15 @@ private class ClientProducersManager implements Comparable> streamToTrackers = new ConcurrentHashMap<>(); private final Client client; private final AtomicBoolean closed = new AtomicBoolean(false); + private final Lock managerLock = new ReentrantLock(); private ClientProducersManager( - Broker node, ClientFactory cf, Client.ClientParameters clientParameters) { + Broker targetNode, + List candidates, + ClientFactory cf, + Client.ClientParameters clientParameters) { this.id = managerIdSequence.getAndIncrement(); - this.name = keyForNode(node); - this.node = node; + AtomicReference nameReference = new AtomicReference<>(); AtomicReference ref = new AtomicReference<>(); AtomicBoolean clientInitializedInManager = new AtomicBoolean(false); PublishConfirmListener publishConfirmListener = @@ -631,7 +662,7 @@ private ClientProducersManager( }); }, "Producer recovery after disconnection from %s", - name)); + nameReference.get())); } }; MetadataListener metadataListener = @@ -640,7 +671,8 @@ private ClientProducersManager( "Received metadata notification for '{}', stream is likely to have become unavailable", stream); Set affectedTrackers; - synchronized (ClientProducersManager.this) { + this.managerLock.lock(); + try { affectedTrackers = streamToTrackers.remove(stream); LOGGER.debug( "Affected publishers and consumer trackers after metadata update: {}", @@ -656,6 +688,8 @@ private ClientProducersManager( } }); } + } finally { + this.managerLock.unlock(); } if (affectedTrackers != null && !affectedTrackers.isEmpty()) { environment @@ -680,15 +714,19 @@ private ClientProducersManager( }; String connectionName = connectionNamingStrategy.apply(ClientConnectionType.PRODUCER); ClientFactoryContext connectionFactoryContext = - ClientFactoryContext.fromParameters( - clientParameters - .publishConfirmListener(publishConfirmListener) - .publishErrorListener(publishErrorListener) - .shutdownListener(shutdownListener) - .metadataListener(metadataListener) - .clientProperty("connection_name", connectionName)) - .key(name); + new ClientFactoryContext( + clientParameters + .publishConfirmListener(publishConfirmListener) + .publishErrorListener(publishErrorListener) + .shutdownListener(shutdownListener) + .metadataListener(metadataListener) + .clientProperty("connection_name", connectionName), + keyForNode(targetNode), + candidates.stream().map(BrokerWrapper::broker).collect(toList())); this.client = cf.client(connectionFactoryContext); + this.node = Utils.brokerFromClient(this.client); + this.name = keyForNode(this.node); + nameReference.set(this.name); LOGGER.debug("Created producer connection '{}'", connectionName); clientInitializedInManager.set(true); ref.set(this.client); @@ -696,18 +734,27 @@ private ClientProducersManager( private void assignProducersToNewManagers( Collection trackers, String stream, BackOffDelayPolicy delayPolicy) { - AsyncRetry.asyncRetry(() -> getBrokerForProducer(stream)) + AsyncRetry.asyncRetry( + () -> { + List candidates = findCandidateNodes(stream, forceLeader); + return pair(pickBroker(candidates), candidates); + }) .description("Candidate lookup to publish to " + stream) .scheduler(environment.scheduledExecutorService()) .retry(ex -> !(ex instanceof StreamDoesNotExistException)) .delayPolicy(delayPolicy) .build() .thenAccept( - broker -> { + brokerAndCandidates -> { + Broker broker = brokerAndCandidates.v1(); + List candidates = brokerAndCandidates.v2(); String key = keyForNode(broker); LOGGER.debug( - "Assigning {} producer(s) and consumer tracker(s) to {}", trackers.size(), key); - trackers.forEach(tracker -> maybeRecoverAgent(broker, tracker)); + "Assigning {} producer(s) and consumer tracker(s) to {} (stream '{}')", + trackers.size(), + key, + stream); + trackers.forEach(tracker -> maybeRecoverAgent(broker, candidates, tracker)); }) .exceptionally( ex -> { @@ -732,10 +779,11 @@ private void assignProducersToNewManagers( }); } - private void maybeRecoverAgent(Broker broker, AgentTracker tracker) { + private void maybeRecoverAgent( + Broker broker, List candidates, AgentTracker tracker) { if (tracker.markRecoveryInProgress()) { try { - recoverAgent(broker, tracker); + recoverAgent(broker, candidates, tracker); } catch (Exception e) { LOGGER.warn( "Error while recovering {} tracker {} (stream '{}'). Reason: {}", @@ -752,14 +800,14 @@ private void maybeRecoverAgent(Broker broker, AgentTracker tracker) { } } - private void recoverAgent(Broker node, AgentTracker tracker) { + private void recoverAgent(Broker node, List candidates, AgentTracker tracker) { boolean reassignmentCompleted = false; while (!reassignmentCompleted) { try { if (tracker.isOpen()) { LOGGER.debug( "Using {} to resume {} to {}", node.label(), tracker.type(), tracker.stream()); - addToManager(node, tracker); + addToManager(node, candidates, tracker); tracker.running(); } else { LOGGER.debug( @@ -778,14 +826,19 @@ private void recoverAgent(Broker node, AgentTracker tracker) { tracker.identifiable() ? tracker.id() : "N/A", tracker.stream()); // maybe not a good candidate, let's refresh and retry for this one - node = - Utils.callAndMaybeRetry( - () -> getBrokerForProducer(tracker.stream()), + Pair> brokerAndCandidates = + callAndMaybeRetry( + () -> { + List cs = findCandidateNodes(tracker.stream(), forceLeader); + return pair(pickBroker(cs), cs); + }, ex -> !(ex instanceof StreamDoesNotExistException), environment.recoveryBackOffDelayPolicy(), "Candidate lookup for %s on stream '%s'", tracker.type(), tracker.stream()); + node = brokerAndCandidates.v1(); + candidates = brokerAndCandidates.v2(); } catch (Exception e) { LOGGER.warn( "Error while re-assigning {} (stream '{}')", tracker.type(), tracker.stream(), e); @@ -794,79 +847,97 @@ private void recoverAgent(Broker node, AgentTracker tracker) { } } - private synchronized void register(AgentTracker tracker) { - if (this.isFullFor(tracker)) { - throw new IllegalStateException("Cannot add subscription tracker, the manager is full"); - } - if (this.isClosed()) { - throw new IllegalStateException("Cannot add subscription tracker, the manager is closed"); - } - checkNotClosed(); - if (tracker.identifiable()) { - ProducerTracker producerTracker = (ProducerTracker) tracker; - int index = pickSlot(this.producers, producerTracker, this.producerIndexSequence); - this.checkNotClosed(); - Response response = - callAndMaybeRetry( - () -> - this.client.declarePublisher( - (byte) index, tracker.reference(), tracker.stream()), - RETRY_ON_TIMEOUT, - "Declare publisher request for publisher %d on stream '%s'", - producerTracker.uniqueId(), - producerTracker.stream()); - if (response.isOk()) { - tracker.assign((byte) index, this.client, this); - } else { - String message = - "Error while declaring publisher: " - + formatConstant(response.getResponseCode()) - + ". Could not assign producer to client."; - LOGGER.info(message); - throw new StreamException(message, response.getResponseCode()); - } - producers.put(tracker.id(), producerTracker); - } else { - tracker.assign((byte) 0, this.client, this); - trackingConsumerTrackers.add(tracker); - } - streamToTrackers - .computeIfAbsent(tracker.stream(), s -> ConcurrentHashMap.newKeySet()) - .add(tracker); + private void register(AgentTracker tracker) { + lock( + this.managerLock, + () -> { + if (this.isFullFor(tracker)) { + throw new IllegalStateException( + "Cannot add subscription tracker, the manager is full"); + } + if (this.isClosed()) { + throw new IllegalStateException( + "Cannot add subscription tracker, the manager is closed"); + } + checkNotClosed(); + if (tracker.identifiable()) { + ProducerTracker producerTracker = (ProducerTracker) tracker; + int index = pickSlot(this.producers, producerTracker, this.producerIndexSequence); + this.checkNotClosed(); + Response response = + callAndMaybeRetry( + () -> + this.client.declarePublisher( + (byte) index, tracker.reference(), tracker.stream()), + RETRY_ON_TIMEOUT, + "Declare publisher request for publisher %d on stream '%s'", + producerTracker.uniqueId(), + producerTracker.stream()); + if (response.isOk()) { + tracker.assign((byte) index, this.client, this); + } else { + String message = + "Error while declaring publisher: " + + formatConstant(response.getResponseCode()) + + ". Could not assign producer to client."; + LOGGER.info(message); + throw new StreamException(message, response.getResponseCode()); + } + producers.put(tracker.id(), producerTracker); + } else { + tracker.assign((byte) 0, this.client, this); + trackingConsumerTrackers.add(tracker); + } + streamToTrackers + .computeIfAbsent(tracker.stream(), s -> ConcurrentHashMap.newKeySet()) + .add(tracker); + }); } - private synchronized void unregister(AgentTracker tracker) { - LOGGER.debug( - "Unregistering {} {} from manager on {}", tracker.type(), tracker.uniqueId(), this.name); - if (tracker.identifiable()) { - producers.remove(tracker.id()); - } else { - trackingConsumerTrackers.remove(tracker); - } - streamToTrackers.compute( - tracker.stream(), - (s, trackersForThisStream) -> { - if (s == null || trackersForThisStream == null) { - // should not happen - return null; + private void unregister(AgentTracker tracker) { + lock( + this.managerLock, + () -> { + LOGGER.debug( + "Unregistering {} {} from manager on {}", + tracker.type(), + tracker.uniqueId(), + this.name); + if (tracker.identifiable()) { + producers.remove(tracker.id()); } else { - trackersForThisStream.remove(tracker); - return trackersForThisStream.isEmpty() ? null : trackersForThisStream; + trackingConsumerTrackers.remove(tracker); } + streamToTrackers.compute( + tracker.stream(), + (s, trackersForThisStream) -> { + if (s == null || trackersForThisStream == null) { + // should not happen + return null; + } else { + trackersForThisStream.remove(tracker); + return trackersForThisStream.isEmpty() ? null : trackersForThisStream; + } + }); + closeIfEmpty(); }); - closeIfEmpty(); } - synchronized boolean isFullFor(AgentTracker tracker) { - if (tracker.identifiable()) { - return producers.size() == maxProducersByClient; - } else { - return trackingConsumerTrackers.size() == maxTrackingConsumersByClient; - } + boolean isFullFor(AgentTracker tracker) { + return lock( + this.managerLock, + () -> { + if (tracker.identifiable()) { + return producers.size() == maxProducersByClient; + } else { + return trackingConsumerTrackers.size() == maxTrackingConsumersByClient; + } + }); } - synchronized boolean isEmpty() { - return producers.isEmpty() && trackingConsumerTrackers.isEmpty(); + boolean isEmpty() { + return lock( + this.managerLock, () -> producers.isEmpty() && trackingConsumerTrackers.isEmpty()); } private void checkNotClosed() { @@ -884,13 +955,15 @@ boolean isClosed() { private void closeIfEmpty() { if (!closed.get()) { - synchronized (this) { - if (this.isEmpty()) { - this.close(); - } else { - LOGGER.debug("Not closing producer manager {} because it is not empty", this.id); - } - } + lock( + this.managerLock, + () -> { + if (this.isEmpty()) { + this.close(); + } else { + LOGGER.debug("Not closing producer manager {} because it is not empty", this.id); + } + }); } } diff --git a/src/main/java/com/rabbitmq/stream/impl/RoutingKeyRoutingStrategy.java b/src/main/java/com/rabbitmq/stream/impl/RoutingKeyRoutingStrategy.java index 46d33d2bba..7c51d7ecbd 100644 --- a/src/main/java/com/rabbitmq/stream/impl/RoutingKeyRoutingStrategy.java +++ b/src/main/java/com/rabbitmq/stream/impl/RoutingKeyRoutingStrategy.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/impl/ScheduledExecutorServiceWrapper.java b/src/main/java/com/rabbitmq/stream/impl/ScheduledExecutorServiceWrapper.java index 47308b7d57..76eaded5e8 100644 --- a/src/main/java/com/rabbitmq/stream/impl/ScheduledExecutorServiceWrapper.java +++ b/src/main/java/com/rabbitmq/stream/impl/ScheduledExecutorServiceWrapper.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2023-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -14,7 +14,8 @@ // info@rabbitmq.com. package com.rabbitmq.stream.impl; -import com.rabbitmq.stream.impl.Utils.NamedThreadFactory; +import static com.rabbitmq.stream.impl.ThreadUtils.threadFactory; + import java.time.Duration; import java.util.ArrayList; import java.util.Collection; @@ -39,7 +40,7 @@ class ScheduledExecutorServiceWrapper implements ScheduledExecutorService { private final Set tasks = ConcurrentHashMap.newKeySet(); private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor( - new NamedThreadFactory("rabbitmq-stream-scheduled-executor-service-wrapper-")); + threadFactory("rabbitmq-stream-scheduled-executor-service-wrapper-")); ScheduledExecutorServiceWrapper(ScheduledExecutorService delegate) { this.delegate = delegate; @@ -122,10 +123,12 @@ public ScheduledFuture scheduleWithFixedDelay( @Override public void shutdown() { this.delegate.shutdown(); + this.scheduler.shutdown(); } @Override public List shutdownNow() { + this.delegate.shutdownNow(); return this.delegate.shutdownNow(); } diff --git a/src/main/java/com/rabbitmq/stream/impl/ServerFrameHandler.java b/src/main/java/com/rabbitmq/stream/impl/ServerFrameHandler.java index bd0dc3c84e..d63da3ab30 100644 --- a/src/main/java/com/rabbitmq/stream/impl/ServerFrameHandler.java +++ b/src/main/java/com/rabbitmq/stream/impl/ServerFrameHandler.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -251,6 +251,13 @@ public void handle(Client client, int frameSize, ChannelHandlerContext ctx, Byte } abstract int doHandle(Client client, ChannelHandlerContext ctx, ByteBuf message); + + protected void logMissingOutstandingRequest(int correlationId) { + LOGGER.warn( + "Could not find outstanding request with correlation ID {} ({})", + correlationId, + this.getClass().getSimpleName()); + } } private static class ConfirmFrameHandler extends BaseFrameHandler { @@ -325,9 +332,14 @@ static int handleMessage( if (ignore && Long.compareUnsigned(offset, offsetLimit) < 0) { messageIgnored.set(true); } else { - Message message = codec.decode(data); - messageListener.handle( - subscriptionId, offset, chunkTimestamp, committedChunkId, chunkContext, message); + try { + Message message = codec.decode(data); + messageListener.handle( + subscriptionId, offset, chunkTimestamp, committedChunkId, chunkContext, message); + } catch (RuntimeException e) { + LOGGER.warn("Error while decoding message at offset {}", offset, e); + throw e; + } } return read; } @@ -444,7 +456,6 @@ static int handleDeliver( } metricsCollector.chunk(numEntries); - long messagesRead = 0; MutableBoolean messageIgnored = new MutableBoolean(false); while (numRecords != 0) { @@ -475,7 +486,7 @@ static int handleDeliver( subscriptionId, offset, chunkTimestamp, committedOffset, chunkContext); messageIgnored.set(false); } else { - messagesRead++; + metricsCollector.consume(1); } numRecords--; offset++; // works even for unsigned long @@ -544,7 +555,7 @@ static int handleDeliver( subscriptionId, offset, chunkTimestamp, committedOffset, chunkContext); messageIgnored.set(false); } else { - messagesRead++; + metricsCollector.consume(1); } numRecordsInBatch--; offset++; // works even for unsigned long @@ -557,7 +568,6 @@ static int handleDeliver( } } } - metricsCollector.consume(messagesRead); return read; } @@ -670,7 +680,7 @@ int doHandle(Client client, ChannelHandlerContext ctx, ByteBuf message) { OutstandingRequest outstandingRequest = remove(client.outstandingRequests, correlationId, QueryPublisherSequenceResponse.class); if (outstandingRequest == null) { - LOGGER.warn("Could not find outstanding request with correlation ID {}", correlationId); + logMissingOutstandingRequest(correlationId); } else { QueryPublisherSequenceResponse response = new QueryPublisherSequenceResponse(responseCode, sequence); @@ -695,7 +705,7 @@ int doHandle(Client client, ChannelHandlerContext ctx, ByteBuf message) { OutstandingRequest outstandingRequest = remove(client.outstandingRequests, correlationId, QueryOffsetResponse.class); if (outstandingRequest == null) { - LOGGER.warn("Could not find outstanding request with correlation ID {}", correlationId); + logMissingOutstandingRequest(correlationId); } else { QueryOffsetResponse response = new QueryOffsetResponse(responseCode, offset); outstandingRequest.response().set(response); @@ -744,7 +754,13 @@ private static class PeerPropertiesFrameHandler extends BaseFrameHandler { @Override int doHandle(Client client, ChannelHandlerContext ctx, ByteBuf message) { + LOGGER.debug( + "Handling peer properties response for connection {}", client.clientConnectionName()); int correlationId = message.readInt(); + LOGGER.debug( + "Handling peer properties response for connection {}, correlation ID is {}", + client.clientConnectionName(), + correlationId); int read = 4; short responseCode = message.readShort(); @@ -771,12 +787,9 @@ int doHandle(Client client, ChannelHandlerContext ctx, ByteBuf message) { } OutstandingRequest> outstandingRequest = - remove( - client.outstandingRequests, - correlationId, - new ParameterizedTypeReference>() {}); + remove(client.outstandingRequests, correlationId, new ParameterizedTypeReference<>() {}); if (outstandingRequest == null) { - LOGGER.warn("Could not find outstanding request with correlation ID {}", correlationId); + logMissingOutstandingRequest(correlationId); } else { outstandingRequest.response().set(Collections.unmodifiableMap(serverProperties)); outstandingRequest.countDown(); @@ -814,7 +827,7 @@ int doHandle(Client client, ChannelHandlerContext ctx, ByteBuf message) { OutstandingRequest outstandingRequest = remove(client.outstandingRequests, correlationId, OpenResponse.class); if (outstandingRequest == null) { - LOGGER.warn("Could not find outstanding request with correlation ID {}", correlationId); + logMissingOutstandingRequest(correlationId); } else { outstandingRequest.response().set(new OpenResponse(responseCode, connectionProperties)); outstandingRequest.countDown(); @@ -903,7 +916,7 @@ int doHandle(Client client, ChannelHandlerContext ctx, ByteBuf message) { OutstandingRequest outstandingRequest = remove(client.outstandingRequests, correlationId, SaslAuthenticateResponse.class); if (outstandingRequest == null) { - LOGGER.warn("Could not find outstanding request with correlation ID {}", correlationId); + logMissingOutstandingRequest(correlationId); } else { outstandingRequest.response().set(response); outstandingRequest.countDown(); @@ -946,7 +959,7 @@ int doHandle(Client client, ChannelHandlerContext ctx, ByteBuf message) { correlationId, new ParameterizedTypeReference>() {}); if (outstandingRequest == null) { - LOGGER.warn("Could not find outstanding request with correlation ID {}", correlationId); + logMissingOutstandingRequest(correlationId); } else { outstandingRequest.response().set(mechanisms); outstandingRequest.countDown(); @@ -1008,7 +1021,7 @@ int doHandle(Client client, ChannelHandlerContext ctx, ByteBuf message) { correlationId, new ParameterizedTypeReference>() {}); if (outstandingRequest == null) { - LOGGER.warn("Could not find outstanding request with correlation ID {}", correlationId); + logMissingOutstandingRequest(correlationId); } else { outstandingRequest.response().set(results); outstandingRequest.countDown(); @@ -1029,7 +1042,7 @@ int doHandle(Client client, ChannelHandlerContext ctx, ByteBuf message) { OutstandingRequest outstandingRequest = remove(client.outstandingRequests, correlationId, Response.class); if (outstandingRequest == null) { - LOGGER.warn("Could not find outstanding request with correlation ID {}", correlationId); + logMissingOutstandingRequest(correlationId); } else { Response response = new Response(responseCode); outstandingRequest.response().set(response); @@ -1066,12 +1079,9 @@ int doHandle(Client client, ChannelHandlerContext ctx, ByteBuf message) { } OutstandingRequest> outstandingRequest = - remove( - client.outstandingRequests, - correlationId, - new ParameterizedTypeReference>() {}); + remove(client.outstandingRequests, correlationId, new ParameterizedTypeReference<>() {}); if (outstandingRequest == null) { - LOGGER.warn("Could not find outstanding request with correlation ID {}", correlationId); + logMissingOutstandingRequest(correlationId); } else { outstandingRequest.response().set(streams); outstandingRequest.countDown(); @@ -1113,7 +1123,7 @@ int doHandle(Client client, ChannelHandlerContext ctx, ByteBuf message) { correlationId, new ParameterizedTypeReference>() {}); if (outstandingRequest == null) { - LOGGER.warn("Could not find outstanding request with correlation ID {}", correlationId); + logMissingOutstandingRequest(correlationId); } else { outstandingRequest.response().set(streams); outstandingRequest.countDown(); @@ -1158,7 +1168,7 @@ int doHandle(Client client, ChannelHandlerContext ctx, ByteBuf message) { correlationId, new ParameterizedTypeReference>() {}); if (outstandingRequest == null) { - LOGGER.warn("Could not find outstanding request with correlation ID {}", correlationId); + logMissingOutstandingRequest(correlationId); } else { outstandingRequest.response().set(commandVersions); outstandingRequest.countDown(); @@ -1192,7 +1202,7 @@ int doHandle(Client client, ChannelHandlerContext ctx, ByteBuf message) { OutstandingRequest outstandingRequest = remove(client.outstandingRequests, correlationId, StreamStatsResponse.class); if (outstandingRequest == null) { - LOGGER.warn("Could not find outstanding request with correlation ID {}", correlationId); + logMissingOutstandingRequest(correlationId); } else { outstandingRequest.response().set(new StreamStatsResponse(responseCode, info)); outstandingRequest.countDown(); diff --git a/src/main/java/com/rabbitmq/stream/impl/SimpleMessageAccumulator.java b/src/main/java/com/rabbitmq/stream/impl/SimpleMessageAccumulator.java index dc18c8687d..5370b79887 100644 --- a/src/main/java/com/rabbitmq/stream/impl/SimpleMessageAccumulator.java +++ b/src/main/java/com/rabbitmq/stream/impl/SimpleMessageAccumulator.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -15,6 +15,9 @@ package com.rabbitmq.stream.impl; import com.rabbitmq.stream.*; +import com.rabbitmq.stream.impl.ProducerUtils.AccumulatedEntity; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; @@ -23,17 +26,11 @@ class SimpleMessageAccumulator implements MessageAccumulator { - private static final Function NULL_FILTER_VALUE_EXTRACTOR = m -> null; - protected final BlockingQueue messages; - protected final Clock clock; private final int capacity; - protected final Codec codec; - private final int maxFrameSize; - private final ToLongFunction publishSequenceFunction; - private final Function filterValueExtractor; - final String stream; final ObservationCollector observationCollector; + private final StreamProducer producer; + final ProducerUtils.MessageAccumulatorHelper helper; @SuppressWarnings("unchecked") SimpleMessageAccumulator( @@ -44,47 +41,39 @@ class SimpleMessageAccumulator implements MessageAccumulator { Function filterValueExtractor, Clock clock, String stream, - ObservationCollector observationCollector) { + ObservationCollector observationCollector, + StreamProducer producer) { + this.helper = + new ProducerUtils.MessageAccumulatorHelper( + codec, + maxFrameSize, + publishSequenceFunction, + filterValueExtractor, + clock, + stream, + observationCollector); this.capacity = capacity; - this.messages = new LinkedBlockingQueue<>(capacity); - this.codec = codec; - this.maxFrameSize = maxFrameSize; - this.publishSequenceFunction = publishSequenceFunction; - this.filterValueExtractor = - filterValueExtractor == null ? NULL_FILTER_VALUE_EXTRACTOR : filterValueExtractor; - this.clock = clock; - this.stream = stream; + this.messages = new LinkedBlockingQueue<>(this.capacity); this.observationCollector = (ObservationCollector) observationCollector; + this.producer = producer; } - public boolean add(Message message, ConfirmationHandler confirmationHandler) { - Object observationContext = this.observationCollector.prePublish(this.stream, message); - Codec.EncodedMessage encodedMessage = this.codec.encode(message); - Client.checkMessageFitsInFrame(this.maxFrameSize, encodedMessage); - long publishingId = this.publishSequenceFunction.applyAsLong(message); + public void add(Message message, ConfirmationHandler confirmationHandler) { + AccumulatedEntity entity = this.helper.entity(message, confirmationHandler); try { - boolean offered = - messages.offer( - new SimpleAccumulatedEntity( - clock.time(), - publishingId, - this.filterValueExtractor.apply(message), - encodedMessage, - new SimpleConfirmationCallback(message, confirmationHandler), - observationContext), - 60, - TimeUnit.SECONDS); + boolean offered = messages.offer(entity, 60, TimeUnit.SECONDS); if (!offered) { throw new StreamException("Could not accumulate outbound message"); } } catch (InterruptedException e) { throw new StreamException("Error while accumulating outbound message", e); } - return this.messages.size() == this.capacity; + if (this.messages.size() == this.capacity) { + publishBatch(true); + } } - @Override - public AccumulatedEntity get() { + AccumulatedEntity get() { AccumulatedEntity entity = this.messages.poll(); if (entity != null) { this.observationCollector.published( @@ -93,91 +82,38 @@ public AccumulatedEntity get() { return entity; } - @Override - public boolean isEmpty() { - return messages.isEmpty(); - } - @Override public int size() { return messages.size(); } - private static final class SimpleAccumulatedEntity implements AccumulatedEntity { - - private final long time; - private final long publishingId; - private final String filterValue; - private final Codec.EncodedMessage encodedMessage; - private final StreamProducer.ConfirmationCallback confirmationCallback; - private final Object observationContext; - - private SimpleAccumulatedEntity( - long time, - long publishingId, - String filterValue, - Codec.EncodedMessage encodedMessage, - StreamProducer.ConfirmationCallback confirmationCallback, - Object observationContext) { - this.time = time; - this.publishingId = publishingId; - this.encodedMessage = encodedMessage; - this.filterValue = filterValue; - this.confirmationCallback = confirmationCallback; - this.observationContext = observationContext; - } - - @Override - public long publishingId() { - return publishingId; - } - - @Override - public String filterValue() { - return filterValue; - } - - @Override - public Object encodedEntity() { - return encodedMessage; - } - - @Override - public long time() { - return time; - } - - @Override - public StreamProducer.ConfirmationCallback confirmationCallback() { - return confirmationCallback; - } - - @Override - public Object observationContext() { - return this.observationContext; - } + @Override + public void flush(boolean force) { + boolean stateCheck = !force; + publishBatch(stateCheck); } - private static final class SimpleConfirmationCallback - implements StreamProducer.ConfirmationCallback { - - private final Message message; - private final ConfirmationHandler confirmationHandler; - - private SimpleConfirmationCallback(Message message, ConfirmationHandler confirmationHandler) { - this.message = message; - this.confirmationHandler = confirmationHandler; - } - - @Override - public int handle(boolean confirmed, short code) { - confirmationHandler.handle(new ConfirmationStatus(message, confirmed, code)); - return 1; - } - - @Override - public Message message() { - return this.message; + private void publishBatch(boolean stateCheck) { + this.producer.lock(); + try { + if ((!stateCheck || this.producer.canSend()) && !this.messages.isEmpty()) { + List entities = new ArrayList<>(this.capacity); + int batchCount = 0; + while (batchCount != this.capacity) { + AccumulatedEntity entity = this.get(); + if (entity == null) { + break; + } + entities.add(entity); + batchCount++; + } + producer.publishInternal(entities); + } + } finally { + this.producer.unlock(); } } + + @Override + public void close() {} } diff --git a/src/main/java/com/rabbitmq/stream/impl/StreamConsumer.java b/src/main/java/com/rabbitmq/stream/impl/StreamConsumer.java index 4e11f3cfdf..591b6db2e2 100644 --- a/src/main/java/com/rabbitmq/stream/impl/StreamConsumer.java +++ b/src/main/java/com/rabbitmq/stream/impl/StreamConsumer.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -17,24 +17,26 @@ import static com.rabbitmq.stream.BackOffDelayPolicy.fixedWithInitialDelay; import static com.rabbitmq.stream.impl.AsyncRetry.asyncRetry; import static com.rabbitmq.stream.impl.Utils.offsetBefore; +import static java.lang.String.format; import static java.time.Duration.ofMillis; import com.rabbitmq.stream.*; import com.rabbitmq.stream.MessageHandler.Context; import com.rabbitmq.stream.impl.Client.QueryOffsetResponse; import com.rabbitmq.stream.impl.StreamConsumerBuilder.TrackingConfiguration; +import com.rabbitmq.stream.impl.StreamEnvironment.LocatorNotAvailableException; import com.rabbitmq.stream.impl.StreamEnvironment.TrackingConsumerRegistration; import com.rabbitmq.stream.impl.Utils.CompositeConsumerUpdateListener; -import java.util.Collections; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.time.Duration; import java.util.Map; import java.util.Objects; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.LongConsumer; import java.util.function.LongSupplier; import java.util.function.Supplier; @@ -63,7 +65,9 @@ class StreamConsumer implements Consumer { private volatile boolean sacActive; private final boolean sac; private final OffsetSpecification initialOffsetSpecification; + private final Lock lock = new ReentrantLock(); + @SuppressFBWarnings("CT_CONSTRUCTOR_THROW") StreamConsumer( String stream, OffsetSpecification offsetSpecification, @@ -157,18 +161,7 @@ class StreamConsumer implements Consumer { if (context.isActive()) { LOGGER.debug("Looking up offset (stream {})", this.stream); StreamConsumer consumer = (StreamConsumer) context.consumer(); - try { - long offset = getStoredOffsetSafely(consumer, this.environment); - LOGGER.debug( - "Stored offset is {}, returning the value + 1 to the server", offset); - result = OffsetSpecification.offset(offset + 1); - } catch (NoOffsetException e) { - LOGGER.debug( - "No stored offset, using initial offset specification: {}", - this.initialOffsetSpecification); - result = initialOffsetSpecification; - } - return result; + return getStoredOffset(consumer, environment, initialOffsetSpecification); } else { if (receivedSomething.get()) { LOGGER.debug( @@ -188,7 +181,8 @@ class StreamConsumer implements Consumer { return result; }; // just a trick for testing - if (consumerUpdateListener instanceof CompositeConsumerUpdateListener) { + // we know the update listener is either null or a composite one + if (consumerUpdateListener != null) { ((CompositeConsumerUpdateListener) consumerUpdateListener).add(defaultListener); this.consumerUpdateListener = consumerUpdateListener; } else { @@ -204,17 +198,7 @@ class StreamConsumer implements Consumer { if (context.isActive()) { LOGGER.debug("Going from passive to active, looking up offset"); StreamConsumer consumer = (StreamConsumer) context.consumer(); - try { - long offset = getStoredOffsetSafely(consumer, this.environment); - LOGGER.debug( - "Stored offset is {}, returning the value + 1 to the server", offset); - result = OffsetSpecification.offset(offset + 1); - } catch (NoOffsetException e) { - LOGGER.debug( - "No stored offset, using initial offset specification: {}", - this.initialOffsetSpecification); - result = initialOffsetSpecification; - } + result = getStoredOffset(consumer, environment, initialOffsetSpecification); } return result; }; @@ -252,7 +236,7 @@ class StreamConsumer implements Consumer { subscriptionListener, trackingClosingCallback, closedAwareMessageHandler, - Collections.unmodifiableMap(subscriptionProperties), + Map.copyOf(subscriptionProperties), flowStrategy); this.status = Status.RUNNING; @@ -269,38 +253,74 @@ class StreamConsumer implements Consumer { } } + static OffsetSpecification getStoredOffset( + StreamConsumer consumer, StreamEnvironment environment, OffsetSpecification fallback) { + OffsetSpecification result = null; + while (result == null) { + try { + long offset = getStoredOffsetSafely(consumer, environment); + LOGGER.debug("Stored offset is {}, returning the value + 1 to the server", offset); + result = OffsetSpecification.offset(offset + 1); + } catch (NoOffsetException e) { + LOGGER.debug("No stored offset, using initial offset specification: {}", fallback); + result = fallback; + } catch (TimeoutStreamException e) { + LOGGER.debug("Timeout when looking up stored offset, retrying"); + } + } + return result; + } + static long getStoredOffsetSafely(StreamConsumer consumer, StreamEnvironment environment) { long offset; try { offset = consumer.storedOffset(); } catch (IllegalStateException e) { - LOGGER.debug("Leader connection not available to retrieve offset, retrying"); - // no connection to leader to retrieve the offset, retrying + LOGGER.debug( + "Leader connection for '{}' not available to retrieve offset, retrying", consumer.stream); + // no connection to leader to retrieve the offset, trying with environment connections + String description = + format("Stored offset retrieval for '%s' on stream '%s'", consumer.name, consumer.stream); CompletableFuture storedOffetRetrievalFuture = - asyncRetry(() -> consumer.storedOffset(() -> consumer.trackingClient())) - .description( - "Stored offset retrieval for '%s' on stream '%s'", consumer.name, consumer.stream) + asyncRetry(() -> consumer.storedOffset(() -> environment.locator().client())) + .description(description) .scheduler(environment.scheduledExecutorService()) - .retry(ex -> ex instanceof IllegalStateException) + .retry( + ex -> + ex instanceof IllegalStateException + || ex instanceof LocatorNotAvailableException) .delayPolicy( fixedWithInitialDelay( - environment.recoveryBackOffDelayPolicy().delay(0), + Duration.ZERO, environment.recoveryBackOffDelayPolicy().delay(1), environment.recoveryBackOffDelayPolicy().delay(0).multipliedBy(3))) .build(); try { - offset = storedOffetRetrievalFuture.get(); + offset = + storedOffetRetrievalFuture.get( + environment.rpcTimeout().toMillis(), TimeUnit.MILLISECONDS); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); throw new StreamException( - String.format( + format( "Could not get stored offset for '%s' on stream '%s'", consumer.name, consumer.stream), ex); } catch (ExecutionException ex) { - throw new StreamException( - String.format( - "Could not get stored offset for '%s' on stream '%s'", + if (ex.getCause() instanceof StreamException) { + throw (StreamException) ex.getCause(); + } else { + throw new StreamException( + format( + "Could not get stored offset for '%s' on stream '%s'", + consumer.name, consumer.stream), + ex); + } + } catch (TimeoutException ex) { + storedOffetRetrievalFuture.cancel(true); + throw new TimeoutStreamException( + format( + "Could not get stored offset for '%s' on stream '%s' (timeout)", consumer.name, consumer.stream), ex); } @@ -308,10 +328,6 @@ static long getStoredOffsetSafely(StreamConsumer consumer, StreamEnvironment env return offset; } - Client trackingClient() { - return this.trackingClient; - } - void waitForOffsetToBeStored(long expectedStoredOffset) { CompletableFuture storedTask = asyncRetry( @@ -464,8 +480,9 @@ boolean sacActive() { return this.sacActive; } - private boolean canTrack() { - return (this.status == Status.INITIALIZING || this.status == Status.RUNNING) + boolean canTrack() { + return ((this.status == Status.INITIALIZING || this.status == Status.RUNNING) + || (this.trackingClient == null && this.status == Status.NOT_AVAILABLE)) && this.name != null; } @@ -478,6 +495,7 @@ public void close() { } void closeFromEnvironment() { + this.maybeNotifyActiveToInactiveSac(); LOGGER.debug("Calling consumer {} closing callback (stream {})", this.id, this.stream); this.closingCallback.run(); closed.set(true); @@ -487,6 +505,7 @@ void closeFromEnvironment() { void closeAfterStreamDeletion() { if (closed.compareAndSet(false, true)) { + this.maybeNotifyActiveToInactiveSac(); this.environment.removeConsumer(this); this.status = Status.CLOSED; } @@ -503,14 +522,38 @@ void setTrackingClient(Client client) { void setSubscriptionClient(Client client) { this.subscriptionClient = client; if (client == null && this.isSac()) { + maybeNotifyActiveToInactiveSac(); // we lost the connection this.sacActive = false; } } + private void maybeNotifyActiveToInactiveSac() { + if (this.isSac() && this.sacActive) { + LOGGER.debug( + "Single active consumer {} from stream {} with name {} is unavailable, calling consumer update listener", + this.id, + this.stream, + this.name); + this.consumerUpdate(false); + } + } + synchronized void unavailable() { - this.status = Status.NOT_AVAILABLE; - this.trackingClient = null; + Utils.lock( + this.lock, + () -> { + this.status = Status.NOT_AVAILABLE; + this.trackingClient = null; + }); + } + + void lock() { + this.lock.lock(); + } + + void unlock() { + this.lock.unlock(); } void running() { @@ -526,7 +569,7 @@ long storedOffset(Supplier clientSupplier) { response = clientSupplier.get().queryOffset(this.name, this.stream); } catch (Exception e) { throw new IllegalStateException( - String.format( + format( "Not possible to query offset for consumer %s on stream %s for now: %s", this.name, this.stream, e.getMessage()), e); @@ -535,23 +578,22 @@ long storedOffset(Supplier clientSupplier) { return response.getOffset(); } else if (response.getResponseCode() == Constants.RESPONSE_CODE_NO_OFFSET) { throw new NoOffsetException( - String.format( + format( "No offset stored for consumer %s on stream %s (%s)", this.name, this.stream, Utils.formatConstant(response.getResponseCode()))); } else { throw new StreamException( - String.format( + format( "QueryOffset for consumer %s on stream %s returned an error (%s)", this.name, this.stream, Utils.formatConstant(response.getResponseCode())), response.getResponseCode()); } - } else if (this.name == null) { throw new UnsupportedOperationException( "Not possible to query stored offset for a consumer without a name"); } else { throw new IllegalStateException( - String.format( + format( "Not possible to query offset for consumer %s on stream %s for now, consumer status is %s", this.name, this.stream, this.status.name())); } @@ -620,4 +662,13 @@ private void checkNotClosed() { long id() { return this.id; } + + String subscriptionConnectionName() { + Client client = this.subscriptionClient; + if (client == null) { + return ""; + } else { + return client.clientConnectionName(); + } + } } diff --git a/src/main/java/com/rabbitmq/stream/impl/StreamConsumerBuilder.java b/src/main/java/com/rabbitmq/stream/impl/StreamConsumerBuilder.java index 3a077ae918..ec00136800 100644 --- a/src/main/java/com/rabbitmq/stream/impl/StreamConsumerBuilder.java +++ b/src/main/java/com/rabbitmq/stream/impl/StreamConsumerBuilder.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -18,6 +18,7 @@ import static com.rabbitmq.stream.impl.Utils.SUBSCRIPTION_PROPERTY_MATCH_UNFILTERED; import com.rabbitmq.stream.*; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.time.Duration; @@ -30,7 +31,7 @@ class StreamConsumerBuilder implements ConsumerBuilder { - private static final int NAME_MAX_SIZE = 256; // server-side limitation + private static final int NAME_MAX_SIZE = Client.MAX_REFERENCE_SIZE; // server-side limitation private static final TrackingConfiguration DISABLED_TRACKING_CONFIGURATION = new TrackingConfiguration(false, false, -1, Duration.ZERO, Duration.ZERO); private final StreamEnvironment environment; @@ -82,9 +83,9 @@ MessageHandler messageHandler() { @Override public ConsumerBuilder name(String name) { - if (name == null || name.length() > NAME_MAX_SIZE) { + if (name == null || name.length() >= NAME_MAX_SIZE) { throw new IllegalArgumentException( - "The consumer name must be non-null and under 256 characters"); + "The consumer name must be non-null and less than 256 characters"); } this.name = name; return this; @@ -112,6 +113,7 @@ public ConsumerBuilder subscriptionListener(SubscriptionListener subscriptionLis } @Override + @SuppressFBWarnings("AT_STALE_THREAD_WRITE_OF_PRIMITIVE") public ManualTrackingStrategy manualTrackingStrategy() { this.manualTrackingStrategy = new DefaultManualTrackingStrategy(this); this.autoTrackingStrategy = null; @@ -120,6 +122,7 @@ public ManualTrackingStrategy manualTrackingStrategy() { } @Override + @SuppressFBWarnings("AT_STALE_THREAD_WRITE_OF_PRIMITIVE") public AutoTrackingStrategy autoTrackingStrategy() { this.autoTrackingStrategy = new DefaultAutoTrackingStrategy(this); this.manualTrackingStrategy = null; @@ -128,6 +131,7 @@ public AutoTrackingStrategy autoTrackingStrategy() { } @Override + @SuppressFBWarnings("AT_STALE_THREAD_WRITE_OF_PRIMITIVE") public ConsumerBuilder noTrackingStrategy() { this.noTrackingStrategy = true; this.autoTrackingStrategy = null; @@ -140,6 +144,7 @@ public FlowConfiguration flow() { return this.flowConfiguration; } + @SuppressFBWarnings("AT_STALE_THREAD_WRITE_OF_PRIMITIVE") StreamConsumerBuilder lazyInit(boolean lazyInit) { this.lazyInit = lazyInit; return this; @@ -428,7 +433,7 @@ private DefaultFlowConfiguration(ConsumerBuilder consumerBuilder) { this.consumerBuilder = consumerBuilder; } - private ConsumerFlowStrategy strategy = ConsumerFlowStrategy.creditOnChunkArrival(); + private ConsumerFlowStrategy strategy = ConsumerFlowStrategy.creditOnChunkArrival(10); @Override public FlowConfiguration initialCredits(int initialCredits) { diff --git a/src/main/java/com/rabbitmq/stream/impl/StreamEnvironment.java b/src/main/java/com/rabbitmq/stream/impl/StreamEnvironment.java index ac97403f9d..8bf503beab 100644 --- a/src/main/java/com/rabbitmq/stream/impl/StreamEnvironment.java +++ b/src/main/java/com/rabbitmq/stream/impl/StreamEnvironment.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -15,9 +15,13 @@ package com.rabbitmq.stream.impl; import static com.rabbitmq.stream.impl.AsyncRetry.asyncRetry; +import static com.rabbitmq.stream.impl.Client.DEFAULT_RPC_TIMEOUT; +import static com.rabbitmq.stream.impl.ThreadUtils.threadFactory; import static com.rabbitmq.stream.impl.Utils.*; import static java.lang.String.format; +import static java.util.Optional.ofNullable; import static java.util.concurrent.TimeUnit.SECONDS; +import static java.util.stream.Collectors.toList; import com.rabbitmq.stream.*; import com.rabbitmq.stream.MessageHandler.Context; @@ -31,9 +35,9 @@ import com.rabbitmq.stream.impl.Utils.ClientConnectionType; import com.rabbitmq.stream.sasl.CredentialsProvider; import com.rabbitmq.stream.sasl.UsernamePasswordCredentialsProvider; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import io.netty.buffer.ByteBufAllocator; import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import java.io.IOException; @@ -41,10 +45,7 @@ import java.net.URLDecoder; import java.time.Duration; import java.time.LocalDateTime; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -55,6 +56,7 @@ import java.util.function.LongSupplier; import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.IntStream; import javax.net.ssl.SSLException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,6 +67,7 @@ class StreamEnvironment implements Environment { private final EventLoopGroup eventLoopGroup; private final ScheduledExecutorService scheduledExecutorService; + private final ScheduledExecutorService locatorReconnectionScheduledExecutorService; private final boolean privateScheduleExecutorService; private final Client.ClientParameters clientParametersPrototype; private final List
addresses; @@ -83,10 +86,12 @@ class StreamEnvironment implements Environment { private final ByteBufAllocator byteBufAllocator; private final AtomicBoolean locatorsInitialized = new AtomicBoolean(false); private final Runnable locatorInitializationSequence; - private final List locators = new CopyOnWriteArrayList<>(); + private final List locators; private final ExecutorServiceFactory executorServiceFactory; private final ObservationCollector observationCollector; + private final Duration rpcTimeout; + @SuppressFBWarnings("CT_CONSTRUCTOR_THROW") StreamEnvironment( ScheduledExecutorService scheduledExecutorService, Client.ClientParameters clientParametersPrototype, @@ -103,10 +108,16 @@ class StreamEnvironment implements Environment { Function connectionNamingStrategy, Function clientFactory, ObservationCollector observationCollector, - boolean forceReplicaForConsumers) { + boolean forceReplicaForConsumers, + boolean forceLeaderForProducers, + Duration producerNodeRetryDelay, + Duration consumerNodeRetryDelay, + int expectedLocatorCount) { this.recoveryBackOffDelayPolicy = recoveryBackOffDelayPolicy; this.topologyUpdateBackOffDelayPolicy = topologyBackOffDelayPolicy; this.byteBufAllocator = byteBufAllocator; + this.rpcTimeout = + ofNullable(clientParametersPrototype.rpcTimeout()).orElse(DEFAULT_RPC_TIMEOUT); clientParametersPrototype = clientParametersPrototype.byteBufAllocator(byteBufAllocator); clientParametersPrototype = maybeSetUpClientParametersFromUris(uris, clientParametersPrototype); @@ -122,8 +133,6 @@ class StreamEnvironment implements Environment { : tlsConfiguration.sslContext(); clientParametersPrototype.sslContext(sslContext); - clientParametersPrototype.tlsHostnameVerification( - tlsConfiguration.hostnameVerificationEnabled()); } catch (SSLException e) { throw new StreamException("Error while creating Netty SSL context", e); @@ -145,7 +154,7 @@ class StreamEnvironment implements Environment { new Address( uriItem.getHost() == null ? "localhost" : uriItem.getHost(), uriItem.getPort() == -1 ? defaultPort : uriItem.getPort())) - .collect(Collectors.toList()); + .collect(toList()); } AddressResolver addressResolverToUse = addressResolver; @@ -157,7 +166,16 @@ class StreamEnvironment implements Environment { String username = ((UsernamePasswordCredentialsProvider) credentialsProvider).getUsername(); if (DEFAULT_USERNAME.equals(username)) { Address address = new Address("localhost", clientParametersPrototype.port()); - addressResolverToUse = ignored -> address; + Set
passedInAddresses = ConcurrentHashMap.newKeySet(); + addressResolverToUse = + addr -> { + passedInAddresses.add(addr); + if (passedInAddresses.size() > 1) { + LOGGER.warn("Assumed development environment but it seems incorrect."); + passedInAddresses.clear(); + } + return address; + }; LOGGER.info( "Connecting to localhost with {} user, assuming development environment", DEFAULT_USERNAME); @@ -168,13 +186,30 @@ class StreamEnvironment implements Environment { this.addressResolver = addressResolverToUse; - this.addresses.forEach(address -> this.locators.add(new Locator(address))); + int locatorCount; + if (expectedLocatorCount > 0) { + locatorCount = expectedLocatorCount; + } else { + locatorCount = Math.min(this.addresses.size(), 3); + } + LOGGER.debug("Using {} locator connection(s)", locatorCount); + + List lctrs = + IntStream.range(0, locatorCount) + .mapToObj( + i -> { + Address addr = this.addresses.get(i % this.addresses.size()); + return new Locator(i, addr); + }) + .collect(toList()); + this.locators = List.copyOf(lctrs); + this.executorServiceFactory = new DefaultExecutorServiceFactory( this.addresses.size(), 1, "rabbitmq-stream-locator-connection-"); if (clientParametersPrototype.eventLoopGroup == null) { - this.eventLoopGroup = new NioEventLoopGroup(); + this.eventLoopGroup = Utils.eventLoopGroup(); this.clientParametersPrototype = clientParametersPrototype.duplicate().eventLoopGroup(this.eventLoopGroup); } else { @@ -186,10 +221,9 @@ class StreamEnvironment implements Environment { } ScheduledExecutorService executorService; if (scheduledExecutorService == null) { - int threads = Runtime.getRuntime().availableProcessors(); + int threads = AVAILABLE_PROCESSORS; LOGGER.debug("Creating scheduled executor service with {} thread(s)", threads); - ThreadFactory threadFactory = - new Utils.NamedThreadFactory("rabbitmq-stream-environment-scheduler-"); + ThreadFactory threadFactory = threadFactory("rabbitmq-stream-environment-scheduler-"); executorService = Executors.newScheduledThreadPool(threads, threadFactory); this.privateScheduleExecutorService = true; } else { @@ -204,21 +238,28 @@ class StreamEnvironment implements Environment { maxProducersByConnection, maxTrackingConsumersByConnection, connectionNamingStrategy, - Utils.coordinatorClientFactory(this)); + coordinatorClientFactory(this, producerNodeRetryDelay), + forceLeaderForProducers); this.consumersCoordinator = new ConsumersCoordinator( this, maxConsumersByConnection, connectionNamingStrategy, - Utils.coordinatorClientFactory(this), - forceReplicaForConsumers); + coordinatorClientFactory(this, consumerNodeRetryDelay), + forceReplicaForConsumers, + Utils.brokerPicker()); this.offsetTrackingCoordinator = new OffsetTrackingCoordinator(this); + + ThreadFactory threadFactory = threadFactory("rabbitmq-stream-environment-locator-scheduler-"); + this.locatorReconnectionScheduledExecutorService = + Executors.newScheduledThreadPool(this.locators.size(), threadFactory); + ClientParameters clientParametersForInit = locatorParametersCopy(); Runnable locatorInitSequence = () -> { RuntimeException lastException = null; - for (int i = 0; i < addresses.size(); i++) { - Address address = addresses.get(i); + for (int i = 0; i < locators.size(); i++) { + Address address = addresses.get(i % addresses.size()); Locator locator = locator(i); address = addressResolver.resolve(address); String connectionName = connectionNamingStrategy.apply(ClientConnectionType.LOCATOR); @@ -248,7 +289,19 @@ class StreamEnvironment implements Environment { this.locators.forEach( l -> { if (l.isNotSet()) { - scheduleLocatorConnection(l, connectionNamingStrategy, clientFactory); + ShutdownListener shutdownListener = + shutdownListener(l, connectionNamingStrategy, clientFactory); + Client.ClientParameters newLocatorParameters = + this.locatorParametersCopy().shutdownListener(shutdownListener); + scheduleLocatorConnection( + newLocatorParameters, + this.addressResolver, + l, + connectionNamingStrategy, + clientFactory, + this.locatorReconnectionScheduledExecutorService, + this.recoveryBackOffDelayPolicy, + l.label()); } }); } @@ -276,64 +329,46 @@ private ShutdownListener shutdownListener( AtomicReference shutdownListenerReference = new AtomicReference<>(); Client.ShutdownListener shutdownListener = shutdownContext -> { + String label = locator.label(); + LOGGER.debug("Locator {} disconnected", label); if (shutdownContext.isShutdownUnexpected()) { locator.client(null); + BackOffDelayPolicy delayPolicy = recoveryBackOffDelayPolicy; LOGGER.debug( - "Unexpected locator disconnection for locator on '{}', trying to reconnect", - locator.label()); - try { - Client.ClientParameters newLocatorParameters = - this.locatorParametersCopy().shutdownListener(shutdownListenerReference.get()); - asyncRetry( - () -> { - LOGGER.debug("Locator reconnection..."); - Address resolvedAddress = addressResolver.resolve(locator.address()); - String connectionName = - connectionNamingStrategy.apply(ClientConnectionType.LOCATOR); - LOGGER.debug( - "Trying to reconnect locator on {}, with client connection name '{}'", - resolvedAddress, - connectionName); - Client newLocator = - clientFactory.apply( - newLocatorParameters - .host(resolvedAddress.host()) - .port(resolvedAddress.port()) - .clientProperty("connection_name", connectionName)); - LOGGER.debug("Created locator connection '{}'", connectionName); - LOGGER.debug("Locator connected on {}", resolvedAddress); - return newLocator; - }) - .description("Locator recovery") - .scheduler(this.scheduledExecutorService) - .delayPolicy(recoveryBackOffDelayPolicy) - .build() - .thenAccept(locator::client) - .exceptionally( - ex -> { - LOGGER.debug("Locator recovery failed", ex); - return null; - }); - } catch (Exception e) { - LOGGER.debug("Error while scheduling locator reconnection", e); - } + "Unexpected locator disconnection for on '{}', scheduling recovery with {}", + label, + delayPolicy); + Client.ClientParameters newLocatorParameters = + this.locatorParametersCopy().shutdownListener(shutdownListenerReference.get()); + scheduleLocatorConnection( + newLocatorParameters, + this.addressResolver, + locator, + connectionNamingStrategy, + clientFactory, + this.locatorReconnectionScheduledExecutorService, + delayPolicy, + label); } else { - LOGGER.debug("Locator connection '{}' closing normally", locator.label()); + LOGGER.debug("Locator connection '{}' closing normally", label); } }; shutdownListenerReference.set(shutdownListener); return shutdownListener; } - private void scheduleLocatorConnection( + private static void scheduleLocatorConnection( + ClientParameters newLocatorParameters, + AddressResolver addressResolver, Locator locator, Function connectionNamingStrategy, - Function clientFactory) { - ShutdownListener shutdownListener = - shutdownListener(locator, connectionNamingStrategy, clientFactory); + Function clientFactory, + ScheduledExecutorService scheduler, + BackOffDelayPolicy delayPolicy, + String locatorLabel) { + LOGGER.debug( + "Scheduling locator '{}' connection with delay policy {}", locatorLabel, delayPolicy); try { - Client.ClientParameters newLocatorParameters = - this.locatorParametersCopy().shutdownListener(shutdownListener); asyncRetry( () -> { LOGGER.debug("Locator reconnection..."); @@ -354,18 +389,18 @@ private void scheduleLocatorConnection( LOGGER.debug("Locator connected on {}", resolvedAddress); return newLocator; }) - .description("Locator recovery") - .scheduler(this.scheduledExecutorService) - .delayPolicy(recoveryBackOffDelayPolicy) + .description("Locator '%s' connection", locatorLabel) + .scheduler(scheduler) + .delayPolicy(delayPolicy) .build() .thenAccept(locator::client) .exceptionally( ex -> { - LOGGER.debug("Locator recovery failed", ex); + LOGGER.debug("Locator connection failed", ex); return null; }); } catch (Exception e) { - LOGGER.debug("Error while scheduling locator reconnection", e); + LOGGER.debug("Error while scheduling locator '{}' reconnection", locatorLabel, e); } } @@ -449,7 +484,7 @@ public StreamCreator streamCreator() { public void deleteStream(String stream) { checkNotClosed(); this.maybeInitializeLocator(); - Client.Response response = this.locator().delete(stream); + Client.Response response = this.locator().client().delete(stream); if (!response.isOk()) { throw new StreamException( "Error while deleting stream " @@ -465,7 +500,7 @@ public void deleteStream(String stream) { public void deleteSuperStream(String superStream) { checkNotClosed(); this.maybeInitializeLocator(); - Client.Response response = this.locator().deleteSuperStream(superStream); + Client.Response response = this.locator().client().deleteSuperStream(superStream); if (!response.isOk()) { throw new StreamException( "Error while deleting super stream " @@ -572,6 +607,16 @@ public long firstOffset() { public long committedChunkId() { return committedOffsetSupplier.getAsLong(); } + + @Override + public String toString() { + return "StreamStats{" + + "firstOffset=" + + firstOffset() + + ", committedOffset=" + + committedChunkId() + + '}'; + } } @Override @@ -646,6 +691,9 @@ public void close() { if (privateScheduleExecutorService) { this.scheduledExecutorService.shutdownNow(); } + if (this.locatorReconnectionScheduledExecutorService != null) { + this.locatorReconnectionScheduledExecutorService.shutdownNow(); + } try { if (this.eventLoopGroup != null && (!this.eventLoopGroup.isShuttingDown() || !this.eventLoopGroup.isShutdown())) { @@ -667,6 +715,10 @@ ScheduledExecutorService scheduledExecutorService() { return this.scheduledExecutorService; } + Duration rpcTimeout() { + return this.rpcTimeout; + } + void execute(Runnable task, String description, Object... args) { this.scheduledExecutorService().execute(namedRunnable(task, description, args)); } @@ -713,12 +765,22 @@ Runnable registerProducer(StreamProducer producer, String reference, String stre return producersCoordinator.registerProducer(producer, reference, stream); } - Client locator() { + Locator locator() { + if (LOGGER.isDebugEnabled()) { + try { + LOGGER.debug( + "Locators: {}", + this.locators.stream() + .map(l -> l.label() + " is set " + l.isSet()) + .collect(Collectors.joining(", "))); + } catch (Exception e) { + LOGGER.debug("Error while listing locators: {}", e.getMessage()); + } + } return this.locators.stream() .filter(Locator::isSet) .findAny() - .orElseThrow(LocatorNotAvailableException::new) - .client(); + .orElseThrow(LocatorNotAvailableException::new); } T locatorOperation(Function operation) { @@ -727,7 +789,7 @@ T locatorOperation(Function operation) { static T locatorOperation( Function operation, - Supplier clientSupplier, + Supplier locatorSupplier, BackOffDelayPolicy backOffDelayPolicy) { int maxAttempt = 3; int attempt = 0; @@ -738,9 +800,11 @@ static T locatorOperation( long start = System.nanoTime(); while (attempt < maxAttempt) { try { - Client client = clientSupplier.get(); + Locator locator = locatorSupplier.get(); + Client client = locator.client(); LOGGER.debug( - "Using locator on {}:{} to run operation '{}'", + "Using locator {} on {}:{} to run operation '{}'", + locator.id(), client.getHost(), client.getPort(), operation); @@ -855,13 +919,7 @@ TrackingConsumerRegistration registerTrackingConsumer( @Override public String toString() { return "{ \"locators\" : [" - + this.locators.stream() - .map( - l -> { - Client c = l.nullableClient(); - return c == null ? "null" : ("\"" + c.connectionName() + "\""); - }) - .collect(Collectors.joining(",")) + + this.locators.stream().map(l -> quote(l.label())).collect(Collectors.joining(",")) + "], " + Utils.jsonField("producer_client_count", this.producersCoordinator.clientCount()) + "," @@ -916,6 +974,10 @@ static class LocatorNotAvailableException extends StreamException { public LocatorNotAvailableException() { super("Locator not available"); } + + public LocatorNotAvailableException(long id) { + super(String.format("Locator %d not available", id)); + } } private void checkNotClosed() { @@ -924,13 +986,15 @@ private void checkNotClosed() { } } - private static class Locator { + static class Locator { + private final long id; private final Address address; private volatile Optional client; private volatile LocalDateTime lastChanged; - private Locator(Address address) { + Locator(long id, Address address) { + this.id = id; this.address = address; this.client = Optional.empty(); lastChanged = LocalDateTime.now(); @@ -940,7 +1004,7 @@ private Locator(Address address) { Locator client(Client client) { Client previous = this.nullableClient(); - this.client = Optional.ofNullable(client); + this.client = ofNullable(client); LocalDateTime now = LocalDateTime.now(); LOGGER.debug( "Locator wrapper '{}' updated from {} to {}, last changed {}, {} ago", @@ -953,6 +1017,10 @@ Locator client(Client client) { return this; } + private long id() { + return this.id; + } + private boolean isNotSet() { return !this.isSet(); } @@ -961,8 +1029,8 @@ private boolean isSet() { return this.client.isPresent(); } - private Client client() { - return this.client.orElseThrow(LocatorNotAvailableException::new); + Client client() { + return this.client.orElseThrow(() -> new LocatorNotAvailableException(id)); } private Client nullableClient() { @@ -974,7 +1042,18 @@ private Address address() { } private String label() { - return address.host() + ":" + address.port(); + Client c = this.nullableClient(); + if (c == null) { + return String.format("%s:%d (id %d)", address.host(), address.port(), this.id); + } else { + return String.format( + "%s:%d [id %d, advertised %s:%d]", + c.getHost(), + c.getPort(), + this.id(), + c.serverAdvertisedHost(), + c.serverAdvertisedPort()); + } } @Override diff --git a/src/main/java/com/rabbitmq/stream/impl/StreamEnvironmentBuilder.java b/src/main/java/com/rabbitmq/stream/impl/StreamEnvironmentBuilder.java index b31f93ba6d..f3b38cb884 100644 --- a/src/main/java/com/rabbitmq/stream/impl/StreamEnvironmentBuilder.java +++ b/src/main/java/com/rabbitmq/stream/impl/StreamEnvironmentBuilder.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -65,8 +65,12 @@ public class StreamEnvironmentBuilder implements EnvironmentBuilder { private CompressionCodecFactory compressionCodecFactory; private boolean lazyInit = false; private boolean forceReplicaForConsumers = false; + private boolean forceLeaderForProducers = true; private Function clientFactory = Client::new; private ObservationCollector observationCollector = ObservationCollector.NO_OP; + private Duration producerNodeRetryDelay = Duration.ofMillis(500); + private Duration consumerNodeRetryDelay = Duration.ofMillis(1000); + private int locatorConnectionCount = -1; public StreamEnvironmentBuilder() {} @@ -274,6 +278,12 @@ public EnvironmentBuilder forceReplicaForConsumers(boolean forceReplica) { return this; } + @Override + public EnvironmentBuilder forceLeaderForProducers(boolean forceLeader) { + this.forceLeaderForProducers = forceLeader; + return this; + } + @Override public TlsConfiguration tls() { this.tls.enable(); @@ -296,6 +306,22 @@ public EnvironmentBuilder observationCollector(ObservationCollector observati return this; } + StreamEnvironmentBuilder producerNodeRetryDelay(Duration producerNodeRetryDelay) { + this.producerNodeRetryDelay = producerNodeRetryDelay; + return this; + } + + StreamEnvironmentBuilder consumerNodeRetryDelay(Duration consumerNodeRetryDelay) { + this.consumerNodeRetryDelay = consumerNodeRetryDelay; + return this; + } + + @Override + public StreamEnvironmentBuilder locatorConnectionCount(int locatorCount) { + this.locatorConnectionCount = locatorCount; + return this; + } + @Override public Environment build() { if (this.compressionCodecFactory == null) { @@ -327,7 +353,11 @@ public Environment build() { connectionNamingStrategy, this.clientFactory, this.observationCollector, - this.forceReplicaForConsumers); + this.forceReplicaForConsumers, + this.forceLeaderForProducers, + this.producerNodeRetryDelay, + this.consumerNodeRetryDelay, + this.locatorConnectionCount); } static final class DefaultTlsConfiguration implements TlsConfiguration { @@ -342,18 +372,6 @@ private DefaultTlsConfiguration(EnvironmentBuilder environmentBuilder) { this.environmentBuilder = environmentBuilder; } - @Override - public TlsConfiguration hostnameVerification() { - this.hostnameVerification = true; - return this; - } - - @Override - public TlsConfiguration hostnameVerification(boolean hostnameVerification) { - this.hostnameVerification = hostnameVerification; - return this; - } - @Override public TlsConfiguration sslContext(SslContext sslContext) { this.sslContext = sslContext; @@ -365,11 +383,12 @@ public TlsConfiguration trustEverything() { LOGGER.warn( "SECURITY ALERT: this feature trusts every server certificate, effectively disabling peer verification. " + "This is convenient for local development but offers no protection against man-in-the-middle attacks. " - + "Please see https://www.rabbitmq.com/ssl.html to learn more about peer certificate verification."); + + "Please see https://www.rabbitmq.com/docs/ssl to learn more about peer certificate verification."); try { this.sslContext( SslContextBuilder.forClient() .trustManager(Utils.TRUST_EVERYTHING_TRUST_MANAGER) + .endpointIdentificationAlgorithm(null) .build()); } catch (SSLException e) { throw new StreamException("Error while creating Netty SSL context", e); diff --git a/src/main/java/com/rabbitmq/stream/impl/StreamProducer.java b/src/main/java/com/rabbitmq/stream/impl/StreamProducer.java index 5a72ac4c2b..27552512c8 100644 --- a/src/main/java/com/rabbitmq/stream/impl/StreamProducer.java +++ b/src/main/java/com/rabbitmq/stream/impl/StreamProducer.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -18,17 +18,12 @@ import static com.rabbitmq.stream.impl.Utils.formatConstant; import static com.rabbitmq.stream.impl.Utils.namedRunnable; -import com.rabbitmq.stream.Codec; -import com.rabbitmq.stream.ConfirmationHandler; -import com.rabbitmq.stream.ConfirmationStatus; -import com.rabbitmq.stream.Constants; -import com.rabbitmq.stream.Message; -import com.rabbitmq.stream.MessageBuilder; -import com.rabbitmq.stream.Producer; -import com.rabbitmq.stream.StreamException; +import com.rabbitmq.stream.*; import com.rabbitmq.stream.compression.Compression; +import com.rabbitmq.stream.compression.CompressionCodec; import com.rabbitmq.stream.impl.Client.Response; -import com.rabbitmq.stream.impl.MessageAccumulator.AccumulatedEntity; +import com.rabbitmq.stream.impl.ProducerUtils.AccumulatedEntity; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import io.netty.buffer.ByteBuf; import java.nio.charset.StandardCharsets; import java.time.Duration; @@ -48,6 +43,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; import java.util.function.ToLongFunction; import org.slf4j.Logger; @@ -77,22 +74,27 @@ class StreamProducer implements Producer { entity -> ((AccumulatedEntity) entity).publishingId(); private final long enqueueTimeoutMs; private final boolean blockOnMaxUnconfirmed; + private final boolean retryOnRecovery; private volatile Client client; private volatile byte publisherId; private volatile Status status; private volatile ScheduledFuture confirmTimeoutFuture; private final short publishVersion; + private final Lock lock = new ReentrantLock(); + @SuppressFBWarnings("CT_CONSTRUCTOR_THROW") StreamProducer( String name, String stream, int subEntrySize, int batchSize, + boolean dynamicBatch, Compression compression, Duration batchPublishingDelay, int maxUnconfirmedMessages, Duration confirmTimeout, Duration enqueueTimeout, + boolean retryOnRecovery, Function filterValueExtractor, StreamEnvironment environment) { if (filterValueExtractor != null && !environment.filteringSupported()) { @@ -105,6 +107,7 @@ class StreamProducer implements Producer { this.name = name; this.stream = stream; this.enqueueTimeoutMs = enqueueTimeout.toMillis(); + this.retryOnRecovery = retryOnRecovery; this.blockOnMaxUnconfirmed = enqueueTimeout.isZero(); this.closingCallback = environment.registerProducer(this, name, this.stream); final Client.OutboundEntityWriteCallback delegateWriteCallback; @@ -117,37 +120,14 @@ class StreamProducer implements Producer { return publishingSequence.getAndIncrement(); } }; + if (subEntrySize <= 1) { - this.accumulator = - new SimpleMessageAccumulator( - batchSize, - environment.codec(), - client.maxFrameSize(), - accumulatorPublishSequenceFunction, - filterValueExtractor, - this.environment.clock(), - stream, - this.environment.observationCollector()); if (filterValueExtractor == null) { delegateWriteCallback = Client.OUTBOUND_MESSAGE_WRITE_CALLBACK; } else { delegateWriteCallback = OUTBOUND_MSG_FILTER_VALUE_WRITE_CALLBACK; } } else { - this.accumulator = - new SubEntryMessageAccumulator( - subEntrySize, - batchSize, - compression == Compression.NONE - ? null - : environment.compressionCodecFactory().get(compression), - environment.codec(), - this.environment.byteBufAllocator(), - client.maxFrameSize(), - accumulatorPublishSequenceFunction, - this.environment.clock(), - stream, - environment.observationCollector()); delegateWriteCallback = Client.OUTBOUND_MESSAGE_BATCH_WRITE_CALLBACK; } @@ -161,8 +141,7 @@ class StreamProducer implements Producer { new Client.OutboundEntityWriteCallback() { @Override public int write(ByteBuf bb, Object entity, long publishingId) { - MessageAccumulator.AccumulatedEntity accumulatedEntity = - (MessageAccumulator.AccumulatedEntity) entity; + AccumulatedEntity accumulatedEntity = (AccumulatedEntity) entity; unconfirmedMessages.put(publishingId, accumulatedEntity); return delegateWriteCallback.write( bb, accumulatedEntity.encodedEntity(), publishingId); @@ -171,7 +150,7 @@ public int write(ByteBuf bb, Object entity, long publishingId) { @Override public int fragmentLength(Object entity) { return delegateWriteCallback.fragmentLength( - ((MessageAccumulator.AccumulatedEntity) entity).encodedEntity()); + ((AccumulatedEntity) entity).encodedEntity()); } }; } else { @@ -180,8 +159,7 @@ public int fragmentLength(Object entity) { new Client.OutboundEntityWriteCallback() { @Override public int write(ByteBuf bb, Object entity, long publishingId) { - MessageAccumulator.AccumulatedEntity accumulatedEntity = - (MessageAccumulator.AccumulatedEntity) entity; + AccumulatedEntity accumulatedEntity = (AccumulatedEntity) entity; unconfirmedMessages.put(publishingId, accumulatedEntity); return delegateWriteCallback.write(bb, accumulatedEntity, publishingId); } @@ -193,14 +171,36 @@ public int fragmentLength(Object entity) { }; } - if (!batchPublishingDelay.isNegative() && !batchPublishingDelay.isZero()) { + CompressionCodec compressionCodec = null; + if (compression != null) { + compressionCodec = environment.compressionCodecFactory().get(compression); + } + this.accumulator = + ProducerUtils.createMessageAccumulator( + dynamicBatch, + subEntrySize, + batchSize, + compressionCodec, + environment.codec(), + environment.byteBufAllocator(), + client.maxFrameSize(), + accumulatorPublishSequenceFunction, + filterValueExtractor, + environment.clock(), + stream, + environment.observationCollector(), + this); + + boolean backgroundBatchPublishingTaskRequired = + !dynamicBatch && batchPublishingDelay.toMillis() > 0; + LOGGER.debug( + "Background batch publishing task required? {}", backgroundBatchPublishingTaskRequired); + if (backgroundBatchPublishingTaskRequired) { AtomicReference taskReference = new AtomicReference<>(); Runnable task = () -> { if (canSend()) { - synchronized (StreamProducer.this) { - publishBatch(true); - } + this.accumulator.flush(false); } if (status != Status.CLOSED) { environment @@ -284,7 +284,7 @@ private Runnable confirmTimeoutTask(Duration confirmTimeout) { error(unconfirmedEntry.getKey(), Constants.CODE_PUBLISH_CONFIRM_TIMEOUT); count++; } else { - // everything else is after, so we can stop + // everything else is after, we can stop break; } } @@ -313,8 +313,10 @@ private long computeFirstValueOfPublishingSequence() { } } + // visible for testing void confirm(long publishingId) { AccumulatedEntity accumulatedEntity = this.unconfirmedMessages.remove(publishingId); + if (accumulatedEntity != null) { int confirmedCount = accumulatedEntity.confirmationCallback().handle(true, Constants.RESPONSE_CODE_OK); @@ -324,6 +326,11 @@ void confirm(long publishingId) { } } + // for testing + int unconfirmedCount() { + return this.unconfirmedMessages.size(); + } + void error(long publishingId, short errorCode) { AccumulatedEntity accumulatedEntity = unconfirmedMessages.remove(publishingId); if (accumulatedEntity != null) { @@ -392,11 +399,7 @@ public void send(Message message, ConfirmationHandler confirmationHandler) { private void doSend(Message message, ConfirmationHandler confirmationHandler) { if (canSend()) { - if (accumulator.add(message, confirmationHandler)) { - synchronized (this) { - publishBatch(true); - } - } + this.accumulator.add(message, confirmationHandler); } else { failPublishing(message, confirmationHandler); } @@ -414,7 +417,7 @@ private void failPublishing(Message message, ConfirmationHandler confirmationHan } } - private boolean canSend() { + boolean canSend() { return this.status == Status.RUNNING; } @@ -440,6 +443,7 @@ public void close() { } void closeFromEnvironment() { + this.accumulator.close(); this.closingCallback.run(); cancelConfirmTimeoutTask(); this.closed.set(true); @@ -471,25 +475,13 @@ private void cancelConfirmTimeoutTask() { } } - private void publishBatch(boolean stateCheck) { - if ((!stateCheck || canSend()) && !accumulator.isEmpty()) { - List messages = new ArrayList<>(this.batchSize); - int batchCount = 0; - while (batchCount != this.batchSize) { - Object accMessage = accumulator.get(); - if (accMessage == null) { - break; - } - messages.add(accMessage); - batchCount++; - } - client.publishInternal( - this.publishVersion, - this.publisherId, - messages, - this.writeCallback, - this.publishSequenceFunction); - } + void publishInternal(List messages) { + client.publishInternal( + this.publishVersion, + this.publisherId, + messages, + this.writeCallback, + this.publishSequenceFunction); } boolean isOpen() { @@ -501,56 +493,80 @@ void unavailable() { } void running() { - synchronized (this) { - LOGGER.debug( - "Re-publishing {} unconfirmed message(s) and {} accumulated message(s)", - this.unconfirmedMessages.size(), - this.accumulator.size()); - if (!this.unconfirmedMessages.isEmpty()) { - Map messagesToResend = new TreeMap<>(this.unconfirmedMessages); - this.unconfirmedMessages.clear(); - Iterator> resendIterator = - messagesToResend.entrySet().iterator(); - while (resendIterator.hasNext()) { - List messages = new ArrayList<>(this.batchSize); - int batchCount = 0; - while (batchCount != this.batchSize) { - Object accMessage = resendIterator.hasNext() ? resendIterator.next().getValue() : null; - if (accMessage == null) { - break; + this.executeInLock( + () -> { + LOGGER.debug( + "Recovering producer with {} unconfirmed message(s) and {} accumulated message(s)", + this.unconfirmedMessages.size(), + this.accumulator.size()); + if (this.retryOnRecovery) { + LOGGER.debug( + "Re-publishing {} unconfirmed message(s)", this.unconfirmedMessages.size()); + if (!this.unconfirmedMessages.isEmpty()) { + Map messagesToResend = + new TreeMap<>(this.unconfirmedMessages); + this.unconfirmedMessages.clear(); + Iterator> resendIterator = + messagesToResend.entrySet().iterator(); + while (resendIterator.hasNext()) { + List messages = new ArrayList<>(this.batchSize); + int batchCount = 0; + while (batchCount != this.batchSize) { + Object accMessage = + resendIterator.hasNext() ? resendIterator.next().getValue() : null; + if (accMessage == null) { + break; + } + messages.add(accMessage); + batchCount++; + } + client.publishInternal( + this.publishVersion, + this.publisherId, + messages, + this.writeCallback, + this.publishSequenceFunction); + } + } + } else { + LOGGER.debug( + "Skipping republishing of {} unconfirmed messages", + this.unconfirmedMessages.size()); + Map messagesToFail = new TreeMap<>(this.unconfirmedMessages); + this.unconfirmedMessages.clear(); + for (AccumulatedEntity accumulatedEntity : messagesToFail.values()) { + try { + int permits = + accumulatedEntity + .confirmationCallback() + .handle(false, CODE_PUBLISH_CONFIRM_TIMEOUT); + this.unconfirmedMessagesSemaphore.release(permits); + } catch (Exception e) { + LOGGER.debug("Error while nack-ing outbound message: {}", e.getMessage()); + this.unconfirmedMessagesSemaphore.release(1); + } } - messages.add(accMessage); - batchCount++; } - client.publishInternal( - this.publishVersion, - this.publisherId, - messages, - this.writeCallback, - this.publishSequenceFunction); - } - } - publishBatch(false); - - int toRelease = maxUnconfirmedMessages - unconfirmedMessagesSemaphore.availablePermits(); - if (toRelease > 0) { - unconfirmedMessagesSemaphore.release(toRelease); - if (!unconfirmedMessagesSemaphore.tryAcquire(this.unconfirmedMessages.size())) { - LOGGER.debug( - "Could not acquire {} permit(s) for message republishing", - this.unconfirmedMessages.size()); - } - } - } + this.accumulator.flush(true); + int toRelease = maxUnconfirmedMessages - unconfirmedMessagesSemaphore.availablePermits(); + if (toRelease > 0) { + unconfirmedMessagesSemaphore.release(toRelease); + if (!unconfirmedMessagesSemaphore.tryAcquire(this.unconfirmedMessages.size())) { + LOGGER.debug( + "Could not acquire {} permit(s) for message republishing", + this.unconfirmedMessages.size()); + } + } + }); this.status = Status.RUNNING; } - synchronized void setClient(Client client) { - this.client = client; + void setClient(Client client) { + this.executeInLock(() -> this.client = client); } - synchronized void setPublisherId(byte publisherId) { - this.publisherId = publisherId; + void setPublisherId(byte publisherId) { + this.executeInLock(() -> this.publisherId = publisherId); } Status status() { @@ -563,13 +579,6 @@ enum Status { CLOSED } - interface ConfirmationCallback { - - int handle(boolean confirmed, short code); - - Message message(); - } - @Override public boolean equals(Object o) { if (this == o) { @@ -643,4 +652,21 @@ public int fragmentLength(Object entity) { } } } + + void lock() { + this.lock.lock(); + } + + void unlock() { + this.lock.unlock(); + } + + private void executeInLock(Runnable action) { + this.lock(); + try { + action.run(); + } finally { + this.unlock(); + } + } } diff --git a/src/main/java/com/rabbitmq/stream/impl/StreamProducerBuilder.java b/src/main/java/com/rabbitmq/stream/impl/StreamProducerBuilder.java index b5f5f30263..43a57bbc5c 100644 --- a/src/main/java/com/rabbitmq/stream/impl/StreamProducerBuilder.java +++ b/src/main/java/com/rabbitmq/stream/impl/StreamProducerBuilder.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -21,12 +21,16 @@ import com.rabbitmq.stream.StreamException; import com.rabbitmq.stream.compression.Compression; import java.lang.reflect.Field; +import java.lang.reflect.Modifier; import java.time.Duration; import java.util.function.Function; import java.util.function.ToIntFunction; class StreamProducerBuilder implements ProducerBuilder { + static final boolean DEFAULT_DYNAMIC_BATCH = true; +// Boolean.parseBoolean(System.getProperty("rabbitmq.stream.producer.dynamic.batch", "true")); + private final StreamEnvironment environment; private String name; @@ -47,10 +51,14 @@ class StreamProducerBuilder implements ProducerBuilder { private Duration enqueueTimeout = Duration.ofSeconds(10); + private boolean retryOnRecovery = true; + private DefaultRoutingConfiguration routingConfiguration; private Function filterValueExtractor; + private boolean dynamicBatch = DEFAULT_DYNAMIC_BATCH; + StreamProducerBuilder(StreamEnvironment environment) { this.environment = environment; } @@ -95,11 +103,18 @@ public ProducerBuilder compression(Compression compression) { return this; } + @Override public StreamProducerBuilder batchPublishingDelay(Duration batchPublishingDelay) { this.batchPublishingDelay = batchPublishingDelay; return this; } + @Override + public ProducerBuilder dynamicBatch(boolean dynamicBatch) { + this.dynamicBatch = dynamicBatch; + return this; + } + @Override public ProducerBuilder maxUnconfirmedMessages(int maxUnconfirmedMessages) { if (maxUnconfirmedMessages <= 0) { @@ -131,6 +146,12 @@ public ProducerBuilder enqueueTimeout(Duration timeout) { return this; } + @Override + public ProducerBuilder retryOnRecovery(boolean retryOnRecovery) { + this.retryOnRecovery = retryOnRecovery; + return this; + } + @Override public ProducerBuilder filterValue(Function filterValueExtractor) { this.filterValueExtractor = filterValueExtractor; @@ -190,11 +211,13 @@ public Producer build() { stream, subEntrySize, batchSize, + dynamicBatch, compression, batchPublishingDelay, maxUnconfirmedMessages, confirmTimeout, enqueueTimeout, + retryOnRecovery, filterValueExtractor, environment); this.environment.addProducer((StreamProducer) producer); @@ -220,11 +243,13 @@ public Producer build() { StreamProducerBuilder duplicate() { StreamProducerBuilder duplicate = new StreamProducerBuilder(this.environment); for (Field field : StreamProducerBuilder.class.getDeclaredFields()) { - field.setAccessible(true); - try { - field.set(duplicate, field.get(this)); - } catch (IllegalAccessException e) { - throw new StreamException("Error while duplicating stream producer builder", e); + if (!Modifier.isStatic(field.getModifiers())) { + field.setAccessible(true); + try { + field.set(duplicate, field.get(this)); + } catch (IllegalAccessException e) { + throw new StreamException("Error while duplicating stream producer builder", e); + } } } return duplicate; diff --git a/src/main/java/com/rabbitmq/stream/impl/StreamStreamCreator.java b/src/main/java/com/rabbitmq/stream/impl/StreamStreamCreator.java index c7986f72d3..c3d216046e 100644 --- a/src/main/java/com/rabbitmq/stream/impl/StreamStreamCreator.java +++ b/src/main/java/com/rabbitmq/stream/impl/StreamStreamCreator.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -32,7 +32,7 @@ class StreamStreamCreator implements StreamCreator { private final StreamEnvironment environment; private final Client.StreamParametersBuilder streamParametersBuilder = - new Client.StreamParametersBuilder().leaderLocator(LeaderLocator.LEAST_LEADERS); + new Client.StreamParametersBuilder().leaderLocator(LeaderLocator.BALANCED); private String name; private DefaultSuperStreamConfiguration superStreamConfiguration; @@ -86,6 +86,18 @@ public StreamCreator filterSize(int size) { return this; } + @Override + public StreamCreator initialMemberCount(int initialMemberCount) { + streamParametersBuilder.initialMemberCount(initialMemberCount); + return this; + } + + @Override + public StreamCreator argument(String key, String value) { + streamParametersBuilder.put(key, value); + return this; + } + @Override public SuperStreamConfiguration superStream() { if (this.superStreamConfiguration == null) { diff --git a/src/main/java/com/rabbitmq/stream/impl/SubEntryMessageAccumulator.java b/src/main/java/com/rabbitmq/stream/impl/SubEntryMessageAccumulator.java index c2c0d8b1e6..6361b95dea 100644 --- a/src/main/java/com/rabbitmq/stream/impl/SubEntryMessageAccumulator.java +++ b/src/main/java/com/rabbitmq/stream/impl/SubEntryMessageAccumulator.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -20,18 +20,15 @@ import com.rabbitmq.stream.ObservationCollector; import com.rabbitmq.stream.compression.Compression; import com.rabbitmq.stream.compression.CompressionCodec; -import com.rabbitmq.stream.impl.Client.EncodedMessageBatch; import io.netty.buffer.ByteBufAllocator; -import java.util.ArrayList; -import java.util.List; import java.util.function.ToLongFunction; -class SubEntryMessageAccumulator extends SimpleMessageAccumulator { +final class SubEntryMessageAccumulator extends SimpleMessageAccumulator { private final int subEntrySize; private final CompressionCodec compressionCodec; private final ByteBufAllocator byteBufAllocator; - private final byte compression; + private final byte compressionCode; public SubEntryMessageAccumulator( int subEntrySize, @@ -43,7 +40,8 @@ public SubEntryMessageAccumulator( ToLongFunction publishSequenceFunction, Clock clock, String stream, - ObservationCollector observationCollector) { + ObservationCollector observationCollector, + StreamProducer producer) { super( subEntrySize * batchSize, codec, @@ -52,30 +50,30 @@ public SubEntryMessageAccumulator( null, clock, stream, - observationCollector); + observationCollector, + producer); this.subEntrySize = subEntrySize; this.compressionCodec = compressionCodec; - this.compression = compressionCodec == null ? Compression.NONE.code() : compressionCodec.code(); + this.compressionCode = + compressionCodec == null ? Compression.NONE.code() : compressionCodec.code(); this.byteBufAllocator = byteBufAllocator; } - private Batch createBatch() { - return new Batch( - EncodedMessageBatch.create( - byteBufAllocator, compression, compressionCodec, this.subEntrySize), - new CompositeConfirmationCallback(new ArrayList<>(this.subEntrySize))); + private ProducerUtils.Batch createBatch() { + return this.helper.batch( + this.byteBufAllocator, this.compressionCode, this.compressionCodec, this.subEntrySize); } @Override - public AccumulatedEntity get() { + protected ProducerUtils.AccumulatedEntity get() { if (this.messages.isEmpty()) { return null; } int count = 0; - Batch batch = createBatch(); - AccumulatedEntity lastMessageInBatch = null; + ProducerUtils.Batch batch = this.createBatch(); + ProducerUtils.AccumulatedEntity lastMessageInBatch = null; while (count != this.subEntrySize) { - AccumulatedEntity message = messages.poll(); + ProducerUtils.AccumulatedEntity message = messages.poll(); if (message == null) { break; } @@ -94,89 +92,4 @@ public AccumulatedEntity get() { return batch; } } - - private static class Batch implements AccumulatedEntity { - - private final EncodedMessageBatch encodedMessageBatch; - private final CompositeConfirmationCallback confirmationCallback; - private volatile long publishingId; - private volatile long time; - - private Batch( - EncodedMessageBatch encodedMessageBatch, - CompositeConfirmationCallback confirmationCallback) { - this.encodedMessageBatch = encodedMessageBatch; - this.confirmationCallback = confirmationCallback; - } - - void add( - Codec.EncodedMessage encodedMessage, - StreamProducer.ConfirmationCallback confirmationCallback) { - this.encodedMessageBatch.add(encodedMessage); - this.confirmationCallback.add(confirmationCallback); - } - - boolean isEmpty() { - return this.confirmationCallback.callbacks.isEmpty(); - } - - @Override - public long publishingId() { - return publishingId; - } - - @Override - public String filterValue() { - return null; - } - - @Override - public Object encodedEntity() { - return encodedMessageBatch; - } - - @Override - public long time() { - return time; - } - - @Override - public StreamProducer.ConfirmationCallback confirmationCallback() { - return confirmationCallback; - } - - @Override - public Object observationContext() { - throw new UnsupportedOperationException( - "batch entity does not contain only one observation context"); - } - } - - private static class CompositeConfirmationCallback - implements StreamProducer.ConfirmationCallback { - - private final List callbacks; - - private CompositeConfirmationCallback(List callbacks) { - this.callbacks = callbacks; - } - - private void add(StreamProducer.ConfirmationCallback confirmationCallback) { - this.callbacks.add(confirmationCallback); - } - - @Override - public int handle(boolean confirmed, short code) { - for (StreamProducer.ConfirmationCallback callback : callbacks) { - callback.handle(confirmed, code); - } - return callbacks.size(); - } - - @Override - public Message message() { - throw new UnsupportedOperationException( - "composite confirmation callback does not contain just one message"); - } - } } diff --git a/src/main/java/com/rabbitmq/stream/impl/SuperStreamConsumer.java b/src/main/java/com/rabbitmq/stream/impl/SuperStreamConsumer.java index 7b0d844548..898a665336 100644 --- a/src/main/java/com/rabbitmq/stream/impl/SuperStreamConsumer.java +++ b/src/main/java/com/rabbitmq/stream/impl/SuperStreamConsumer.java @@ -1,5 +1,5 @@ -// Copyright (c) 2021-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2021-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -187,6 +187,10 @@ public void store(long offset) { "Consumer#store(long) does not work for super streams, use MessageHandler.Context#storeOffset() instead"); } + Consumer consumer(String partition) { + return this.consumers.get(partition); + } + @Override public long storedOffset() { throw new UnsupportedOperationException( diff --git a/src/main/java/com/rabbitmq/stream/impl/SuperStreamProducer.java b/src/main/java/com/rabbitmq/stream/impl/SuperStreamProducer.java index 4f4af0ea62..91cc4722f0 100644 --- a/src/main/java/com/rabbitmq/stream/impl/SuperStreamProducer.java +++ b/src/main/java/com/rabbitmq/stream/impl/SuperStreamProducer.java @@ -1,5 +1,5 @@ -// Copyright (c) 2021-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2021-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -160,7 +160,7 @@ public void close() { } } - private static class DefaultSuperStreamMetadata implements Metadata { + private static final class DefaultSuperStreamMetadata implements Metadata { private final String superStream; private final StreamEnvironment environment; diff --git a/src/main/java/com/rabbitmq/stream/impl/ThreadUtils.java b/src/main/java/com/rabbitmq/stream/impl/ThreadUtils.java new file mode 100644 index 0000000000..4d73c19123 --- /dev/null +++ b/src/main/java/com/rabbitmq/stream/impl/ThreadUtils.java @@ -0,0 +1,140 @@ +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// +// This software, the RabbitMQ Stream Java client library, is dual-licensed under the +// Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). +// For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL, +// please see LICENSE-APACHE2. +// +// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, +// either express or implied. See the LICENSE file for specific language governing +// rights and limitations of this software. +// +// If you have any questions regarding licensing, please contact us at +// info@rabbitmq.com. +package com.rabbitmq.stream.impl; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; +import java.util.function.Predicate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +final class ThreadUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(ThreadUtils.class); + + private static final ThreadFactory THREAD_FACTORY; + private static final Function EXECUTOR_SERVICE_FACTORY; + private static final Predicate IS_VIRTUAL; + + static { + if (isJava21OrMore()) { + LOGGER.debug("Running Java 21 or more, using virtual threads"); + Class builderClass = + Arrays.stream(Thread.class.getDeclaredClasses()) + .filter(c -> "Builder".equals(c.getSimpleName())) + .findFirst() + .get(); + // Reflection code is the same as: + // Thread.ofVirtual().factory(); + try { + Object builder = Thread.class.getDeclaredMethod("ofVirtual").invoke(null); + THREAD_FACTORY = (ThreadFactory) builderClass.getDeclaredMethod("factory").invoke(builder); + } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + throw new RuntimeException(e); + } + EXECUTOR_SERVICE_FACTORY = + prefix -> { + try { + // Reflection code is the same as the 2 following lines: + // ThreadFactory factory = Thread.ofVirtual().name(prefix, 0).factory(); + // Executors.newThreadPerTaskExecutor(factory); + Object builder = Thread.class.getDeclaredMethod("ofVirtual").invoke(null); + if (prefix != null) { + builder = + builderClass + .getDeclaredMethod("name", String.class, Long.TYPE) + .invoke(builder, prefix, 0L); + } + ThreadFactory factory = + (ThreadFactory) builderClass.getDeclaredMethod("factory").invoke(builder); + return (ExecutorService) + Executors.class + .getDeclaredMethod("newThreadPerTaskExecutor", ThreadFactory.class) + .invoke(null, factory); + } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + throw new RuntimeException(e); + } + }; + IS_VIRTUAL = + thread -> { + Method method = null; + try { + method = Thread.class.getDeclaredMethod("isVirtual"); + return (boolean) method.invoke(thread); + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { + LOGGER.info("Error while checking if a thread is virtual: {}", e.getMessage()); + return false; + } + }; + } else { + THREAD_FACTORY = Executors.defaultThreadFactory(); + EXECUTOR_SERVICE_FACTORY = prefix -> Executors.newCachedThreadPool(threadFactory(prefix)); + IS_VIRTUAL = ignored -> false; + } + } + + private ThreadUtils() {} + + static ThreadFactory threadFactory(String prefix) { + if (prefix == null) { + return Executors.defaultThreadFactory(); + } else { + return new NamedThreadFactory(prefix); + } + } + + static ThreadFactory internalThreadFactory(String prefix) { + return new NamedThreadFactory(THREAD_FACTORY, prefix); + } + + static boolean isVirtual(Thread thread) { + return IS_VIRTUAL.test(thread); + } + + private static boolean isJava21OrMore() { + return Runtime.version().compareTo(Runtime.Version.parse("21")) >= 0; + } + + private static class NamedThreadFactory implements ThreadFactory { + + private final ThreadFactory backingThreadFactory; + + private final String prefix; + + private final AtomicLong count = new AtomicLong(0); + + private NamedThreadFactory(String prefix) { + this(Executors.defaultThreadFactory(), prefix); + } + + private NamedThreadFactory(ThreadFactory backingThreadFactory, String prefix) { + this.backingThreadFactory = backingThreadFactory; + this.prefix = prefix; + } + + @Override + public Thread newThread(Runnable r) { + Thread thread = this.backingThreadFactory.newThread(r); + thread.setName(prefix + count.getAndIncrement()); + return thread; + } + } +} diff --git a/src/main/java/com/rabbitmq/stream/impl/TimeoutStreamException.java b/src/main/java/com/rabbitmq/stream/impl/TimeoutStreamException.java index 092a387704..bbbc9090cb 100644 --- a/src/main/java/com/rabbitmq/stream/impl/TimeoutStreamException.java +++ b/src/main/java/com/rabbitmq/stream/impl/TimeoutStreamException.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/impl/Tuples.java b/src/main/java/com/rabbitmq/stream/impl/Tuples.java new file mode 100644 index 0000000000..937e75c00b --- /dev/null +++ b/src/main/java/com/rabbitmq/stream/impl/Tuples.java @@ -0,0 +1,43 @@ +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// +// This software, the RabbitMQ Stream Java client library, is dual-licensed under the +// Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). +// For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL, +// please see LICENSE-APACHE2. +// +// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, +// either express or implied. See the LICENSE file for specific language governing +// rights and limitations of this software. +// +// If you have any questions regarding licensing, please contact us at +// info@rabbitmq.com. +package com.rabbitmq.stream.impl; + +final class Tuples { + + private Tuples() {} + + static Pair pair(A v1, B v2) { + return new Pair<>(v1, v2); + } + + static class Pair { + + private final A v1; + private final B v2; + + private Pair(A v1, B v2) { + this.v1 = v1; + this.v2 = v2; + } + + A v1() { + return this.v1; + } + + B v2() { + return this.v2; + } + } +} diff --git a/src/main/java/com/rabbitmq/stream/impl/Utils.java b/src/main/java/com/rabbitmq/stream/impl/Utils.java index f600f98809..4ea934e911 100644 --- a/src/main/java/com/rabbitmq/stream/impl/Utils.java +++ b/src/main/java/com/rabbitmq/stream/impl/Utils.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -15,42 +15,37 @@ package com.rabbitmq.stream.impl; import static java.lang.String.format; +import static java.util.Map.copyOf; import com.rabbitmq.stream.*; import com.rabbitmq.stream.impl.Client.ClientParameters; import io.netty.channel.ConnectTimeoutException; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.MultiThreadIoEventLoopGroup; +import io.netty.channel.nio.NioIoHandler; import java.net.UnknownHostException; import java.security.cert.X509Certificate; import java.time.Duration; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; +import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.function.*; import java.util.function.Consumer; -import java.util.function.Function; -import java.util.function.LongConsumer; -import java.util.function.LongSupplier; -import java.util.function.Predicate; -import java.util.function.Supplier; import javax.net.ssl.X509TrustManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; final class Utils { + static final int AVAILABLE_PROCESSORS = Runtime.getRuntime().availableProcessors(); + @SuppressWarnings("rawtypes") private static final Consumer NO_OP_CONSUMER = o -> {}; @@ -79,7 +74,7 @@ final class Utils { LOGGER.info("Error while trying to access field Constants." + field.getName()); } }); - CONSTANT_LABELS = Collections.unmodifiableMap(labels); + CONSTANT_LABELS = copyOf(labels); } static final AddressResolver DEFAULT_ADDRESS_RESOLVER = address -> address; @@ -143,35 +138,43 @@ static short encodeResponseCode(Short code) { return (short) (code | 0B1000_0000_0000_0000); } - static ClientFactory coordinatorClientFactory(StreamEnvironment environment) { + static ClientFactory coordinatorClientFactory( + StreamEnvironment environment, Duration retryInterval) { + String messageFormat = + "%s. %s. " + + "This may be due to the usage of a load balancer that makes topology discovery fail. " + + "Use a custom AddressResolver or the --load-balancer flag if using StreamPerfTest. " + + "See https://rabbitmq.github.io/rabbitmq-stream-java-client/stable/htmlsingle/#understanding-connection-logic " + + "and https://blog.rabbitmq.com/posts/2021/07/connecting-to-streams/#with-a-load-balancer."; return context -> { ClientParameters parametersCopy = context.parameters().duplicate(); Address address = new Address(parametersCopy.host(), parametersCopy.port()); address = environment.addressResolver().resolve(address); parametersCopy.host(address.host()).port(address.port()); - if (context.key() == null) { + if (context.targetKey() == null) { throw new IllegalArgumentException("A key is necessary to create the client connection"); } try { - return Utils.connectToAdvertisedNodeClientFactory( - context.key(), context1 -> new Client(context1.parameters())) - .client(Utils.ClientFactoryContext.fromParameters(parametersCopy).key(context.key())); + ClientFactory delegate = context1 -> new Client(context1.parameters()); + ClientFactoryContext clientFactoryContext = + new ClientFactoryContext(parametersCopy, context.targetKey(), context.candidates()); + return Utils.connectToAdvertisedNodeClientFactory(delegate, retryInterval) + .client(clientFactoryContext); + } catch (TimeoutStreamException e) { + if (e.getCause() == null) { + throw new TimeoutStreamException(format(messageFormat, e.getMessage(), "No root cause")); + } else { + throw new TimeoutStreamException( + format(messageFormat, e.getMessage(), e.getCause().getMessage()), e.getCause()); + } } catch (StreamException e) { if (e.getCause() != null && (e.getCause() instanceof UnknownHostException || e.getCause() instanceof ConnectTimeoutException)) { - String message = - e.getMessage() - + ". " - + e.getCause().getMessage() - + ". " - + "This may be due to the usage of a load balancer that makes topology discovery fail. " - + "Use a custom AddressResolver or the --load-balancer flag if using StreamPerfTest. " - + "See https://rabbitmq.github.io/rabbitmq-stream-java-client/stable/htmlsingle/#understanding-connection-logic " - + "and https://blog.rabbitmq.com/posts/2021/07/connecting-to-streams/#with-a-load-balancer."; - throw new StreamException(message, e.getCause()); + throw new StreamException( + format(messageFormat, e.getMessage(), e.getCause().getMessage()), e.getCause()); } else { throw e; } @@ -180,34 +183,53 @@ static ClientFactory coordinatorClientFactory(StreamEnvironment environment) { } static ClientFactory connectToAdvertisedNodeClientFactory( - String expectedAdvertisedHostPort, ClientFactory clientFactory) { - return connectToAdvertisedNodeClientFactory( - expectedAdvertisedHostPort, clientFactory, ExactNodeRetryClientFactory.RETRY_INTERVAL); - } - - static ClientFactory connectToAdvertisedNodeClientFactory( - String expectedAdvertisedHostPort, ClientFactory clientFactory, Duration retryInterval) { - return new ExactNodeRetryClientFactory( + ClientFactory clientFactory, Duration retryInterval) { + return new ConditionalClientFactory( clientFactory, - client -> { + (ctx, client) -> { String currentKey = client.serverAdvertisedHost() + ":" + client.serverAdvertisedPort(); - boolean success = expectedAdvertisedHostPort.equals(currentKey); + boolean success = ctx.targetKey().equals(currentKey); + if (!success && !ctx.candidates().isEmpty()) { + success = ctx.candidates().stream().anyMatch(b -> currentKey.equals(keyForNode(b))); + } LOGGER.debug( - "Expected client {}, got {}: {}", - expectedAdvertisedHostPort, + "Expected client {}, got {}, viable candidates {}: {}", + ctx.targetKey(), currentKey, + ctx.candidates(), success ? "success" : "failure"); return success; }, retryInterval); } + static String keyForNode(Client.Broker broker) { + return broker.getHost() + ":" + broker.getPort(); + } + + static Client.Broker brokerFromClient(Client client) { + return new Client.Broker(client.serverAdvertisedHost(), client.serverAdvertisedPort()); + } + + static Function, Client.Broker> brokerPicker() { + Random random = new Random(); + return brokers -> { + if (brokers.isEmpty()) { + return null; + } else if (brokers.size() == 1) { + return brokers.get(0); + } else { + return brokers.get(random.nextInt(brokers.size())); + } + }; + } + static Runnable namedRunnable(Runnable task, String format, Object... args) { - return new NamedRunnable(String.format(format, args), task); + return new NamedRunnable(format(format, args), task); } static Function namedFunction(Function task, String format, Object... args) { - return new NamedFunction<>(String.format(format, args), task); + return new NamedFunction<>(format(format, args), task); } static T callAndMaybeRetry( @@ -254,7 +276,7 @@ static T callAndMaybeRetry( try { Thread.sleep(delay.toMillis()); } catch (InterruptedException ex) { - Thread.interrupted(); + Thread.currentThread().interrupt(); lastException = ex; keepTrying = false; } @@ -293,16 +315,16 @@ interface ClientFactory { Client client(ClientFactoryContext context); } - static class ExactNodeRetryClientFactory implements ClientFactory { - - private static final Duration RETRY_INTERVAL = Duration.ofSeconds(1); + static class ConditionalClientFactory implements ClientFactory { private final ClientFactory delegate; - private final Predicate condition; + private final BiPredicate condition; private final Duration retryInterval; - ExactNodeRetryClientFactory( - ClientFactory delegate, Predicate condition, Duration retryInterval) { + ConditionalClientFactory( + ClientFactory delegate, + BiPredicate condition, + Duration retryInterval) { this.delegate = delegate; this.condition = condition; this.retryInterval = retryInterval; @@ -312,7 +334,7 @@ static class ExactNodeRetryClientFactory implements ClientFactory { public Client client(ClientFactoryContext context) { while (true) { Client client = this.delegate.client(context); - if (condition.test(client)) { + if (condition.test(context, client)) { return client; } else { try { @@ -324,7 +346,7 @@ public Client client(ClientFactoryContext context) { try { Thread.sleep(this.retryInterval.toMillis()); } catch (InterruptedException e) { - Thread.interrupted(); + Thread.currentThread().interrupt(); return null; } } @@ -333,29 +355,27 @@ public Client client(ClientFactoryContext context) { static class ClientFactoryContext { - private ClientParameters parameters; - private String key; + private final ClientParameters parameters; + private final String targetKey; + private final List candidates; - static ClientFactoryContext fromParameters(ClientParameters parameters) { - return new ClientFactoryContext().parameters(parameters); + ClientFactoryContext( + ClientParameters parameters, String targetKey, List candidates) { + this.parameters = parameters; + this.targetKey = targetKey; + this.candidates = candidates == null ? Collections.emptyList() : List.copyOf(candidates); } ClientParameters parameters() { return parameters; } - ClientFactoryContext parameters(ClientParameters parameters) { - this.parameters = parameters; - return this; - } - - String key() { - return key; + String targetKey() { + return targetKey; } - ClientFactoryContext key(String key) { - this.key = key; - return this; + List candidates() { + return candidates; } } @@ -391,6 +411,10 @@ static Function defaultConnectionNamingStrategy(St prefixes.get(clientConnectionType) + sequences.get(clientConnectionType).getAndIncrement(); } + static EventLoopGroup eventLoopGroup() { + return new MultiThreadIoEventLoopGroup(NioIoHandler.newFactory()); + } + /* class to help testing SAC on super streams */ @@ -537,31 +561,6 @@ static String jsonField(String name, String value) { return quote(name) + " : " + quote(value); } - static class NamedThreadFactory implements ThreadFactory { - - private final ThreadFactory backingThreaFactory; - - private final String prefix; - - private final AtomicLong count = new AtomicLong(0); - - public NamedThreadFactory(String prefix) { - this(Executors.defaultThreadFactory(), prefix); - } - - public NamedThreadFactory(ThreadFactory backingThreadFactory, String prefix) { - this.backingThreaFactory = backingThreadFactory; - this.prefix = prefix; - } - - @Override - public Thread newThread(Runnable r) { - Thread thread = this.backingThreaFactory.newThread(r); - thread.setName(prefix + count.getAndIncrement()); - return thread; - } - } - static final ExecutorServiceFactory NO_OP_EXECUTOR_SERVICE_FACTORY = new NoOpExecutorServiceFactory(); @@ -662,4 +661,58 @@ boolean get() { return this.value; } } + + static void lock(Lock lock, Runnable action) { + lock( + lock, + () -> { + action.run(); + return null; + }); + } + + static T lock(Lock lock, Supplier action) { + lock.lock(); + try { + return action.get(); + } finally { + lock.unlock(); + } + } + + static class BrokerWrapper { + + private final Client.Broker broker; + private final boolean leader; + + BrokerWrapper(Client.Broker broker, boolean leader) { + this.broker = broker; + this.leader = leader; + } + + Client.Broker broker() { + return broker; + } + + boolean isLeader() { + return this.leader; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + BrokerWrapper that = (BrokerWrapper) o; + return leader == that.leader && Objects.equals(broker, that.broker); + } + + @Override + public int hashCode() { + return Objects.hash(broker, leader); + } + + @Override + public String toString() { + return "BrokerWrapper{" + "broker=" + broker + ", leader=" + leader + '}'; + } + } } diff --git a/src/main/java/com/rabbitmq/stream/metrics/DropwizardMetricsCollector.java b/src/main/java/com/rabbitmq/stream/metrics/DropwizardMetricsCollector.java index 859dfddbe4..8e12a2ba3a 100644 --- a/src/main/java/com/rabbitmq/stream/metrics/DropwizardMetricsCollector.java +++ b/src/main/java/com/rabbitmq/stream/metrics/DropwizardMetricsCollector.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/metrics/MetricsCollector.java b/src/main/java/com/rabbitmq/stream/metrics/MetricsCollector.java index 87e9474c33..e9b7d016f2 100644 --- a/src/main/java/com/rabbitmq/stream/metrics/MetricsCollector.java +++ b/src/main/java/com/rabbitmq/stream/metrics/MetricsCollector.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/metrics/MicrometerMetricsCollector.java b/src/main/java/com/rabbitmq/stream/metrics/MicrometerMetricsCollector.java index c3fb86abea..e5a2d73151 100644 --- a/src/main/java/com/rabbitmq/stream/metrics/MicrometerMetricsCollector.java +++ b/src/main/java/com/rabbitmq/stream/metrics/MicrometerMetricsCollector.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/metrics/NoOpMetricsCollector.java b/src/main/java/com/rabbitmq/stream/metrics/NoOpMetricsCollector.java index fdd6c34a4b..9ee511eeaa 100644 --- a/src/main/java/com/rabbitmq/stream/metrics/NoOpMetricsCollector.java +++ b/src/main/java/com/rabbitmq/stream/metrics/NoOpMetricsCollector.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/observation/micrometer/DefaultProcessObservationConvention.java b/src/main/java/com/rabbitmq/stream/observation/micrometer/DefaultProcessObservationConvention.java index db20684965..077cb933db 100644 --- a/src/main/java/com/rabbitmq/stream/observation/micrometer/DefaultProcessObservationConvention.java +++ b/src/main/java/com/rabbitmq/stream/observation/micrometer/DefaultProcessObservationConvention.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/observation/micrometer/DefaultPublishObservationConvention.java b/src/main/java/com/rabbitmq/stream/observation/micrometer/DefaultPublishObservationConvention.java index d01bfc7138..fc068b1357 100644 --- a/src/main/java/com/rabbitmq/stream/observation/micrometer/DefaultPublishObservationConvention.java +++ b/src/main/java/com/rabbitmq/stream/observation/micrometer/DefaultPublishObservationConvention.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/observation/micrometer/MicrometerObservationCollector.java b/src/main/java/com/rabbitmq/stream/observation/micrometer/MicrometerObservationCollector.java index 75367f1ca2..829c05fe03 100644 --- a/src/main/java/com/rabbitmq/stream/observation/micrometer/MicrometerObservationCollector.java +++ b/src/main/java/com/rabbitmq/stream/observation/micrometer/MicrometerObservationCollector.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/observation/micrometer/MicrometerObservationCollectorBuilder.java b/src/main/java/com/rabbitmq/stream/observation/micrometer/MicrometerObservationCollectorBuilder.java index bfcdd9b447..e7101898d2 100644 --- a/src/main/java/com/rabbitmq/stream/observation/micrometer/MicrometerObservationCollectorBuilder.java +++ b/src/main/java/com/rabbitmq/stream/observation/micrometer/MicrometerObservationCollectorBuilder.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -15,6 +15,7 @@ package com.rabbitmq.stream.observation.micrometer; import com.rabbitmq.stream.ObservationCollector; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import io.micrometer.observation.Observation; import io.micrometer.observation.ObservationConvention; import io.micrometer.observation.ObservationRegistry; @@ -44,6 +45,7 @@ public class MicrometerObservationCollectorBuilder { * @param registry the registry * @return this builder instance */ + @SuppressFBWarnings("EI_EXPOSE_REP2") public MicrometerObservationCollectorBuilder registry(ObservationRegistry registry) { this.registry = registry; return this; diff --git a/src/main/java/com/rabbitmq/stream/observation/micrometer/ProcessContext.java b/src/main/java/com/rabbitmq/stream/observation/micrometer/ProcessContext.java index bdb155d76f..737bb63d72 100644 --- a/src/main/java/com/rabbitmq/stream/observation/micrometer/ProcessContext.java +++ b/src/main/java/com/rabbitmq/stream/observation/micrometer/ProcessContext.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/observation/micrometer/ProcessObservationConvention.java b/src/main/java/com/rabbitmq/stream/observation/micrometer/ProcessObservationConvention.java index b5ca7474f7..fd7c3c110c 100644 --- a/src/main/java/com/rabbitmq/stream/observation/micrometer/ProcessObservationConvention.java +++ b/src/main/java/com/rabbitmq/stream/observation/micrometer/ProcessObservationConvention.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/observation/micrometer/PublishContext.java b/src/main/java/com/rabbitmq/stream/observation/micrometer/PublishContext.java index 535e0cc791..be9be0d8d6 100644 --- a/src/main/java/com/rabbitmq/stream/observation/micrometer/PublishContext.java +++ b/src/main/java/com/rabbitmq/stream/observation/micrometer/PublishContext.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/observation/micrometer/PublishObservationConvention.java b/src/main/java/com/rabbitmq/stream/observation/micrometer/PublishObservationConvention.java index e098b566df..c535be20aa 100644 --- a/src/main/java/com/rabbitmq/stream/observation/micrometer/PublishObservationConvention.java +++ b/src/main/java/com/rabbitmq/stream/observation/micrometer/PublishObservationConvention.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/observation/micrometer/StreamObservationDocumentation.java b/src/main/java/com/rabbitmq/stream/observation/micrometer/StreamObservationDocumentation.java index cb4e4f4d43..c8624b1d8b 100644 --- a/src/main/java/com/rabbitmq/stream/observation/micrometer/StreamObservationDocumentation.java +++ b/src/main/java/com/rabbitmq/stream/observation/micrometer/StreamObservationDocumentation.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/sasl/AnonymousSaslMechanism.java b/src/main/java/com/rabbitmq/stream/sasl/AnonymousSaslMechanism.java new file mode 100644 index 0000000000..14b056c2ff --- /dev/null +++ b/src/main/java/com/rabbitmq/stream/sasl/AnonymousSaslMechanism.java @@ -0,0 +1,33 @@ +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// +// This software, the RabbitMQ Stream Java client library, is dual-licensed under the +// Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). +// For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL, +// please see LICENSE-APACHE2. +// +// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, +// either express or implied. See the LICENSE file for specific language governing +// rights and limitations of this software. +// +// If you have any questions regarding licensing, please contact us at +// info@rabbitmq.com. +package com.rabbitmq.stream.sasl; + +import java.nio.charset.StandardCharsets; + +/** The ANONYMOUS {@link SaslMechanism}. */ +public final class AnonymousSaslMechanism implements SaslMechanism { + + public static final SaslMechanism INSTANCE = new AnonymousSaslMechanism(); + + @Override + public String getName() { + return "ANONYMOUS"; + } + + @Override + public byte[] handleChallenge(byte[] challenge, CredentialsProvider credentialsProvider) { + return "".getBytes(StandardCharsets.UTF_8); + } +} diff --git a/src/main/java/com/rabbitmq/stream/sasl/CredentialsProvider.java b/src/main/java/com/rabbitmq/stream/sasl/CredentialsProvider.java index 1e98c5a5f7..0f7f2b3fd2 100644 --- a/src/main/java/com/rabbitmq/stream/sasl/CredentialsProvider.java +++ b/src/main/java/com/rabbitmq/stream/sasl/CredentialsProvider.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/sasl/DefaultSaslConfiguration.java b/src/main/java/com/rabbitmq/stream/sasl/DefaultSaslConfiguration.java index 70affae74f..5cef5c45f6 100644 --- a/src/main/java/com/rabbitmq/stream/sasl/DefaultSaslConfiguration.java +++ b/src/main/java/com/rabbitmq/stream/sasl/DefaultSaslConfiguration.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -16,27 +16,30 @@ import static java.lang.String.format; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** {@link SaslConfiguration} that supports our built-in mechanisms. */ -public class DefaultSaslConfiguration implements SaslConfiguration { +public final class DefaultSaslConfiguration implements SaslConfiguration { public static final SaslConfiguration PLAIN = new DefaultSaslConfiguration(PlainSaslMechanism.INSTANCE.getName()); public static final SaslConfiguration EXTERNAL = new DefaultSaslConfiguration(ExternalSaslMechanism.INSTANCE.getName()); + public static final SaslConfiguration ANONYMOUS = + new DefaultSaslConfiguration(AnonymousSaslMechanism.INSTANCE.getName()); private final Map mechanisms = Collections.unmodifiableMap( - new HashMap() { - { - put(PlainSaslMechanism.INSTANCE.getName(), PlainSaslMechanism.INSTANCE); - put(ExternalSaslMechanism.INSTANCE.getName(), ExternalSaslMechanism.INSTANCE); - } - }); + Stream.of( + PlainSaslMechanism.INSTANCE, + ExternalSaslMechanism.INSTANCE, + AnonymousSaslMechanism.INSTANCE) + .collect( + Collectors.toMap( + SaslMechanism::getName, m -> m, (k1, k2) -> k1, LinkedHashMap::new))); + private final String mechanism; public DefaultSaslConfiguration() { diff --git a/src/main/java/com/rabbitmq/stream/sasl/DefaultUsernamePasswordCredentialsProvider.java b/src/main/java/com/rabbitmq/stream/sasl/DefaultUsernamePasswordCredentialsProvider.java index 5fbfecebc8..a7b4b43462 100644 --- a/src/main/java/com/rabbitmq/stream/sasl/DefaultUsernamePasswordCredentialsProvider.java +++ b/src/main/java/com/rabbitmq/stream/sasl/DefaultUsernamePasswordCredentialsProvider.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/sasl/ExternalSaslMechanism.java b/src/main/java/com/rabbitmq/stream/sasl/ExternalSaslMechanism.java index 67c7e64a0f..4c31eee11c 100644 --- a/src/main/java/com/rabbitmq/stream/sasl/ExternalSaslMechanism.java +++ b/src/main/java/com/rabbitmq/stream/sasl/ExternalSaslMechanism.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/sasl/JdkSaslConfiguration.java b/src/main/java/com/rabbitmq/stream/sasl/JdkSaslConfiguration.java index 1c732e94e0..ca2a251e94 100644 --- a/src/main/java/com/rabbitmq/stream/sasl/JdkSaslConfiguration.java +++ b/src/main/java/com/rabbitmq/stream/sasl/JdkSaslConfiguration.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -72,7 +72,7 @@ public SaslMechanism getSaslMechanism(List serverMechanisms) { return null; } - private class JdkSaslMechanism implements SaslMechanism { + private static class JdkSaslMechanism implements SaslMechanism { private final SaslClient client; public JdkSaslMechanism(SaslClient client) { @@ -94,13 +94,12 @@ public byte[] handleChallenge(byte[] challenge, CredentialsProvider credentialsP } } - private class UsernamePasswordCallbackHandler implements CallbackHandler { + private static final class UsernamePasswordCallbackHandler implements CallbackHandler { private final UsernamePasswordCredentialsProvider credentialsProvider; public UsernamePasswordCallbackHandler(CredentialsProvider credentialsProvider) { - if (credentialsProvider == null - || !(credentialsProvider instanceof UsernamePasswordCredentialsProvider)) { + if (!(credentialsProvider instanceof UsernamePasswordCredentialsProvider)) { throw new IllegalArgumentException( "Only username/password credentials provider is supported, not " + CredentialsProvider.class.getSimpleName()); diff --git a/src/main/java/com/rabbitmq/stream/sasl/PlainSaslMechanism.java b/src/main/java/com/rabbitmq/stream/sasl/PlainSaslMechanism.java index 36ad4083b0..ad80e6ef61 100644 --- a/src/main/java/com/rabbitmq/stream/sasl/PlainSaslMechanism.java +++ b/src/main/java/com/rabbitmq/stream/sasl/PlainSaslMechanism.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/sasl/SaslConfiguration.java b/src/main/java/com/rabbitmq/stream/sasl/SaslConfiguration.java index 92c2d08e5f..b5fa3aaf78 100644 --- a/src/main/java/com/rabbitmq/stream/sasl/SaslConfiguration.java +++ b/src/main/java/com/rabbitmq/stream/sasl/SaslConfiguration.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/sasl/SaslMechanism.java b/src/main/java/com/rabbitmq/stream/sasl/SaslMechanism.java index 2f9adc8cb5..58adfaab60 100644 --- a/src/main/java/com/rabbitmq/stream/sasl/SaslMechanism.java +++ b/src/main/java/com/rabbitmq/stream/sasl/SaslMechanism.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/sasl/StreamSaslException.java b/src/main/java/com/rabbitmq/stream/sasl/StreamSaslException.java index 12f8a035d6..f2a3169c11 100644 --- a/src/main/java/com/rabbitmq/stream/sasl/StreamSaslException.java +++ b/src/main/java/com/rabbitmq/stream/sasl/StreamSaslException.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/main/java/com/rabbitmq/stream/sasl/UsernamePasswordCredentialsProvider.java b/src/main/java/com/rabbitmq/stream/sasl/UsernamePasswordCredentialsProvider.java index ca849c9a27..2c46921397 100644 --- a/src/main/java/com/rabbitmq/stream/sasl/UsernamePasswordCredentialsProvider.java +++ b/src/main/java/com/rabbitmq/stream/sasl/UsernamePasswordCredentialsProvider.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/SanityCheck.java b/src/test/java/SanityCheck.java index 61065fc0b5..4e550c8bd2 100755 --- a/src/test/java/SanityCheck.java +++ b/src/test/java/SanityCheck.java @@ -1,6 +1,6 @@ ///usr/bin/env jbang "$0" "$@" ; exit $? //REPOS mavencentral,ossrh-staging=https://oss.sonatype.org/content/groups/staging/,rabbitmq-packagecloud-milestones=https://packagecloud.io/rabbitmq/maven-milestones/maven2 -//DEPS com.rabbitmq:stream-client:${version} +//DEPS com.rabbitmq:stream-client:${env.RABBITMQ_LIBRARY_VERSION} //DEPS org.slf4j:slf4j-simple:1.7.36 import com.rabbitmq.stream.Environment; diff --git a/src/test/java/com/rabbitmq/stream/BackOffDelayPolicyTest.java b/src/test/java/com/rabbitmq/stream/BackOffDelayPolicyTest.java index 21f157d1b1..a5983a299d 100644 --- a/src/test/java/com/rabbitmq/stream/BackOffDelayPolicyTest.java +++ b/src/test/java/com/rabbitmq/stream/BackOffDelayPolicyTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/ByteCapacityTest.java b/src/test/java/com/rabbitmq/stream/ByteCapacityTest.java index 6fd87de86d..f359c434ec 100644 --- a/src/test/java/com/rabbitmq/stream/ByteCapacityTest.java +++ b/src/test/java/com/rabbitmq/stream/ByteCapacityTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/Host.java b/src/test/java/com/rabbitmq/stream/Cli.java similarity index 50% rename from src/test/java/com/rabbitmq/stream/Host.java rename to src/test/java/com/rabbitmq/stream/Cli.java index 10a263a01c..7097725af4 100644 --- a/src/test/java/com/rabbitmq/stream/Host.java +++ b/src/test/java/com/rabbitmq/stream/Cli.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -26,61 +26,57 @@ import java.util.*; import java.util.concurrent.Callable; import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class Host { - - private static final Logger LOGGER = LoggerFactory.getLogger(Host.class); +public class Cli { private static final String DOCKER_PREFIX = "DOCKER:"; private static final Gson GSON = new Gson(); - public static String capture(InputStream is) throws IOException { - BufferedReader br = new BufferedReader(new InputStreamReader(is)); - String line; - StringBuilder buff = new StringBuilder(); - while ((line = br.readLine()) != null) { - buff.append(line).append("\n"); - } - return buff.toString(); - } + private static final Map DOCKER_NODES_TO_CONTAINERS = + Map.of( + "rabbit@node0", "rabbitmq0", + "rabbit@node1", "rabbitmq1", + "rabbit@node2", "rabbitmq2"); - private static Process executeCommand(String command) throws IOException { + private static ProcessState executeCommand(String command) { return executeCommand(command, false); } - private static Process executeCommand(String command, boolean ignoreError) throws IOException { + private static ProcessState executeCommand(String command, boolean ignoreError) { Process pr = executeCommandProcess(command); + InputStreamPumpState inputState = new InputStreamPumpState(pr.getInputStream()); + InputStreamPumpState errorState = new InputStreamPumpState(pr.getErrorStream()); - int ev = waitForExitValue(pr); + int ev = waitForExitValue(pr, inputState, errorState); + inputState.pump(); + errorState.pump(); if (ev != 0 && !ignoreError) { - String stdout = capture(pr.getInputStream()); - String stderr = capture(pr.getErrorStream()); - throw new IOException( + throw new RuntimeException( "unexpected command exit value: " + ev + "\ncommand: " + command + "\n" + "\nstdout:\n" - + stdout + + inputState.buffer.toString() + "\nstderr:\n" - + stderr + + errorState.buffer.toString() + "\n"); } - return pr; + return new ProcessState(inputState); } - public static String hostname() throws IOException { - Process process = executeCommand("hostname"); - return capture(process.getInputStream()).trim(); + public static String hostname() { + return executeCommand("hostname").output(); } - private static int waitForExitValue(Process pr) { + private static int waitForExitValue( + Process pr, InputStreamPumpState inputState, InputStreamPumpState errorState) { while (true) { try { + inputState.pump(); + errorState.pump(); pr.waitFor(); break; } catch (InterruptedException ignored) { @@ -89,7 +85,7 @@ private static int waitForExitValue(Process pr) { return pr.exitValue(); } - private static Process executeCommandProcess(String command) throws IOException { + private static Process executeCommandProcess(String command) { String[] finalCommand; if (System.getProperty("os.name").toLowerCase().contains("windows")) { finalCommand = new String[4]; @@ -103,66 +99,59 @@ private static Process executeCommandProcess(String command) throws IOException finalCommand[1] = "-c"; finalCommand[2] = command; } - return Runtime.getRuntime().exec(finalCommand); + try { + return Runtime.getRuntime().exec(finalCommand); + } catch (IOException e) { + throw new RuntimeException(e); + } } - public static Process rabbitmqctl(String command) throws IOException { + public static ProcessState rabbitmqctl(String command) { return executeCommand(rabbitmqctlCommand() + " " + command); } - public static Process rabbitmqctlIgnoreError(String command) throws IOException { + static ProcessState rabbitmqStreams(String command) { + return executeCommand(rabbitmqStreamsCommand() + " " + command); + } + + public static ProcessState rabbitmqctlIgnoreError(String command) { return executeCommand(rabbitmqctlCommand() + " " + command, true); } - public static Process killConnection(String connectionName) { - try { - List cs = listConnections(); - if (cs.stream().filter(c -> connectionName.equals(c.clientProvidedName())).count() != 1) { - throw new IllegalArgumentException( - format( - "Could not find 1 connection '%s' in stream connections: %s", - connectionName, - cs.stream() - .map(ConnectionInfo::clientProvidedName) - .collect(Collectors.joining(", ")))); - } - return rabbitmqctl("eval 'rabbit_stream:kill_connection(\"" + connectionName + "\").'"); - } catch (IOException e) { - throw new RuntimeException(e); + public static ProcessState killConnection(String connectionName) { + List cs = listConnections(); + if (cs.stream().filter(c -> connectionName.equals(c.clientProvidedName())).count() != 1) { + throw new IllegalArgumentException( + format( + "Could not find 1 connection '%s' in stream connections: %s", + connectionName, + cs.stream() + .map(ConnectionInfo::clientProvidedName) + .collect(Collectors.joining(", ")))); } + return rabbitmqctl("eval 'rabbit_stream:kill_connection(\"" + connectionName + "\").'"); } public static List listConnections() { - try { - Process process = - rabbitmqctl("list_stream_connections -q --formatter table conn_name,client_properties"); - List connectionInfoList = Collections.emptyList(); - if (process.exitValue() != 0) { - LOGGER.warn( - "Error while trying to list stream connections. Standard output: {}, error output: {}", - capture(process.getInputStream()), - capture(process.getErrorStream())); - return connectionInfoList; - } - String content = capture(process.getInputStream()); - String[] lines = content.split(System.getProperty("line.separator")); - if (lines.length > 1) { - connectionInfoList = new ArrayList<>(lines.length - 1); - for (int i = 1; i < lines.length; i++) { - String line = lines[i]; - String[] fields = line.split("\t"); - String connectionName = fields[0]; - Map clientProperties = Collections.emptyMap(); - if (fields.length > 1 && fields[1].length() > 1) { - clientProperties = buildClientProperties(fields); - } - connectionInfoList.add(new ConnectionInfo(connectionName, clientProperties)); + ProcessState process = + rabbitmqctl("list_stream_connections -q --formatter table conn_name,client_properties"); + List connectionInfoList = Collections.emptyList(); + String content = process.output(); + String[] lines = content.split(System.lineSeparator()); + if (lines.length > 1) { + connectionInfoList = new ArrayList<>(lines.length - 1); + for (int i = 1; i < lines.length; i++) { + String line = lines[i]; + String[] fields = line.split("\t"); + String connectionName = fields[0]; + Map clientProperties = Collections.emptyMap(); + if (fields.length > 1 && fields[1].length() > 1) { + clientProperties = buildClientProperties(fields); } + connectionInfoList.add(new ConnectionInfo(connectionName, clientProperties)); } - return connectionInfoList; - } catch (IOException e) { - throw new RuntimeException(e); } + return connectionInfoList; } private static Map buildClientProperties(String[] fields) { @@ -189,28 +178,34 @@ static List toConnectionInfoList(String json) { return GSON.fromJson(json, new TypeToken>() {}.getType()); } - public static Process killStreamLeaderProcess(String stream) throws IOException { - return rabbitmqctl( + public static void restartStream(String stream) { + rabbitmqStreams(" restart_stream " + stream); + } + + public static String streamStatus(String stream) { + return rabbitmqStreams(" stream_status --formatter table " + stream).output(); + } + + public static void killStreamLeaderProcess(String stream) { + rabbitmqctl( "eval 'case rabbit_stream_manager:lookup_leader(<<\"/\">>, <<\"" + stream + "\">>) of {ok, Pid} -> exit(Pid, kill); Pid -> exit(Pid, kill) end.'"); } - public static void addUser(String username, String password) throws IOException { + public static void addUser(String username, String password) { rabbitmqctl(format("add_user %s %s", username, password)); } - public static void setPermissions(String username, List permissions) throws IOException { + public static void setPermissions(String username, List permissions) { setPermissions(username, "/", permissions); } - public static void setPermissions(String username, String vhost, String permission) - throws IOException { + public static void setPermissions(String username, String vhost, String permission) { setPermissions(username, vhost, asList(permission, permission, permission)); } - public static void setPermissions(String username, String vhost, List permissions) - throws IOException { + public static void setPermissions(String username, String vhost, List permissions) { if (permissions.size() != 3) { throw new IllegalArgumentException(); } @@ -220,30 +215,30 @@ public static void setPermissions(String username, String vhost, List pe vhost, username, permissions.get(0), permissions.get(1), permissions.get(2))); } - public static void changePassword(String username, String newPassword) throws IOException { + public static void changePassword(String username, String newPassword) { rabbitmqctl(format("change_password %s %s", username, newPassword)); } - public static void deleteUser(String username) throws IOException { + public static void deleteUser(String username) { rabbitmqctl(format("delete_user %s", username)); } - public static void addVhost(String vhost) throws IOException { + public static void addVhost(String vhost) { rabbitmqctl("add_vhost " + vhost); } - public static void deleteVhost(String vhost) throws Exception { + public static void deleteVhost(String vhost) { rabbitmqctl("delete_vhost " + vhost); } - public static void setEnv(String parameter, String value) throws IOException { + public static void setEnv(String parameter, String value) { rabbitmqctl(format("eval 'application:set_env(rabbitmq_stream, %s, %s).'", parameter, value)); } public static String rabbitmqctlCommand() { String rabbitmqCtl = System.getProperty("rabbitmqctl.bin"); if (rabbitmqCtl == null) { - throw new IllegalStateException("Please define the rabbitmqctl.bin system property"); + rabbitmqCtl = DOCKER_PREFIX + "rabbitmq"; } if (rabbitmqCtl.startsWith(DOCKER_PREFIX)) { String containerId = rabbitmqCtl.split(":")[1]; @@ -253,6 +248,15 @@ public static String rabbitmqctlCommand() { } } + private static String rabbitmqStreamsCommand() { + String rabbitmqctl = rabbitmqctlCommand(); + int lastIndex = rabbitmqctl.lastIndexOf("rabbitmqctl"); + if (lastIndex == -1) { + throw new IllegalArgumentException("Not a valid rabbitqmctl command: " + rabbitmqctl); + } + return rabbitmqctl.substring(0, lastIndex) + "rabbitmq-streams"; + } + public static AutoCloseable diskAlarm() throws Exception { return new CallableAutoCloseable( () -> { @@ -297,7 +301,7 @@ private static void setResourceAlarm(String source) throws IOException { rabbitmqctl("eval 'rabbit_alarm:set_alarm({{resource_limit, " + source + ", node()}, []}).'"); } - private static void clearResourceAlarm(String source) throws IOException { + private static void clearResourceAlarm(String source) { rabbitmqctl("eval 'rabbit_alarm:clear_alarm({resource_limit, " + source + ", node()}).'"); } @@ -309,6 +313,68 @@ public static boolean isOnDocker() { return rabbitmqCtl.startsWith(DOCKER_PREFIX); } + public static List nodes() { + List clusterNodes = new ArrayList<>(); + clusterNodes.add(rabbitmqctl("eval 'node().'").output().trim()); + List nodes = + Arrays.stream( + rabbitmqctl("eval 'nodes().'") + .output() + .replace("[", "") + .replace("]", "") + .split(",")) + .map(String::trim) + .collect(Collectors.toList()); + clusterNodes.addAll(nodes); + return List.copyOf(clusterNodes); + } + + public static void restartNode(String node) { + String container = nodeToDockerContainer(node); + String dockerCommand = "docker exec " + container + " "; + String rabbitmqUpgradeCommand = dockerCommand + "rabbitmq-upgrade "; + executeCommand(rabbitmqUpgradeCommand + "await_online_quorum_plus_one -t 300"); + executeCommand(rabbitmqUpgradeCommand + "drain"); + executeCommand("docker stop " + container); + executeCommand("docker start " + container); + String otherContainer = + DOCKER_NODES_TO_CONTAINERS.values().stream() + .filter(c -> !c.endsWith(container)) + .findAny() + .get(); + executeCommand( + "docker exec " + + otherContainer + + " rabbitmqctl await_online_nodes " + + DOCKER_NODES_TO_CONTAINERS.size()); + executeCommand(dockerCommand + "rabbitmqctl status"); + } + + public static void rebalance() { + rabbitmqQueues("rebalance all"); + } + + static ProcessState rabbitmqQueues(String command) { + return executeCommand(rabbitmqQueuesCommand() + " " + command); + } + + private static String rabbitmqQueuesCommand() { + String rabbitmqctl = rabbitmqctlCommand(); + int lastIndex = rabbitmqctl.lastIndexOf("rabbitmqctl"); + if (lastIndex == -1) { + throw new IllegalArgumentException("Not a valid rabbitqmctl command: " + rabbitmqctl); + } + return rabbitmqctl.substring(0, lastIndex) + "rabbitmq-queues"; + } + + private static String nodeToDockerContainer(String node) { + String containerId = DOCKER_NODES_TO_CONTAINERS.get(node); + if (containerId == null) { + throw new IllegalArgumentException("No container for node " + node); + } + return containerId; + } + private static final class CallableAutoCloseable implements AutoCloseable { private final Callable end; @@ -353,4 +419,40 @@ public String toString() { + '}'; } } + + public static class ProcessState { + + private final InputStreamPumpState inputState; + + ProcessState(InputStreamPumpState inputState) { + this.inputState = inputState; + } + + public String output() { + return inputState.buffer.toString(); + } + } + + private static class InputStreamPumpState { + + private final BufferedReader reader; + private final StringBuilder buffer; + + private InputStreamPumpState(InputStream in) { + this.reader = new BufferedReader(new InputStreamReader(in)); + this.buffer = new StringBuilder(); + } + + void pump() { + String line; + while (true) { + try { + if ((line = reader.readLine()) == null) break; + } catch (IOException e) { + throw new RuntimeException(e); + } + buffer.append(line).append("\n"); + } + } + } } diff --git a/src/test/java/com/rabbitmq/stream/HostTest.java b/src/test/java/com/rabbitmq/stream/CliTest.java similarity index 85% rename from src/test/java/com/rabbitmq/stream/HostTest.java rename to src/test/java/com/rabbitmq/stream/CliTest.java index 2ed89e0c9a..7c3c4d1167 100644 --- a/src/test/java/com/rabbitmq/stream/HostTest.java +++ b/src/test/java/com/rabbitmq/stream/CliTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2023-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -16,18 +16,18 @@ import static org.assertj.core.api.Assertions.assertThat; -import com.rabbitmq.stream.Host.ConnectionInfo; +import com.rabbitmq.stream.Cli.ConnectionInfo; import java.util.List; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -public class HostTest { +public class CliTest { @Test @Disabled void deserializeConnectionInfo() { List connections = - Host.toConnectionInfoList(LIST_STREAM_CONNECTIONS_JSON_OUTPUT); + Cli.toConnectionInfoList(LIST_STREAM_CONNECTIONS_JSON_OUTPUT); assertThat(connections).hasSize(3); ConnectionInfo c = connections.get(0); assertThat(c.name()).isEqualTo("127.0.0.1:49214 -> 127.0.1.1:5552"); @@ -36,8 +36,8 @@ void deserializeConnectionInfo() { private static final String LIST_STREAM_CONNECTIONS_JSON_OUTPUT = "[\n" - + "{\"conn_name\":\"127.0.0.1:49214 -> 127.0.1.1:5552\",\"client_properties\":[[\"connection_name\",\"longstr\",\"rabbitmq-stream-consumer-0\"],[\"copyright\",\"longstr\",\"Copyright (c) 2020-2023 Broadcom Inc. and/or its subsidiaries.\"],[\"information\",\"longstr\",\"Licensed under the MPL 2.0. See https://www.rabbitmq.com/\"],[\"platform\",\"longstr\",\"Java\"],[\"product\",\"longstr\",\"RabbitMQ Stream\"],[\"version\",\"longstr\",\"0.5.0-SNAPSHOT\"]]}\n" - + ",{\"conn_name\":\"127.0.0.1:49212 -> 127.0.1.1:5552\",\"client_properties\":[[\"connection_name\",\"longstr\",\"rabbitmq-stream-producer-0\"],[\"copyright\",\"longstr\",\"Copyright (c) 2020-2023 Broadcom Inc. and/or its subsidiaries.\"],[\"information\",\"longstr\",\"Licensed under the MPL 2.0. See https://www.rabbitmq.com/\"],[\"platform\",\"longstr\",\"Java\"],[\"product\",\"longstr\",\"RabbitMQ Stream\"],[\"version\",\"longstr\",\"0.5.0-SNAPSHOT\"]]}\n" - + ",{\"conn_name\":\"127.0.0.1:58118 -> 127.0.0.1:5552\",\"client_properties\":[[\"connection_name\",\"longstr\",\"rabbitmq-stream-locator-0\"],[\"copyright\",\"longstr\",\"Copyright (c) 2020-2023 Broadcom Inc. and/or its subsidiaries.\"],[\"information\",\"longstr\",\"Licensed under the MPL 2.0. See https://www.rabbitmq.com/\"],[\"platform\",\"longstr\",\"Java\"],[\"product\",\"longstr\",\"RabbitMQ Stream\"],[\"version\",\"longstr\",\"0.5.0-SNAPSHOT\"]]}\n" + + "{\"conn_name\":\"127.0.0.1:49214 -> 127.0.1.1:5552\",\"client_properties\":[[\"connection_name\",\"longstr\",\"rabbitmq-stream-consumer-0\"],[\"copyright\",\"longstr\",\"Copyright (c) 2020-2025 Broadcom Inc. and/or its subsidiaries.\"],[\"information\",\"longstr\",\"Licensed under the MPL 2.0. See https://www.rabbitmq.com/\"],[\"platform\",\"longstr\",\"Java\"],[\"product\",\"longstr\",\"RabbitMQ Stream\"],[\"version\",\"longstr\",\"0.5.0-SNAPSHOT\"]]}\n" + + ",{\"conn_name\":\"127.0.0.1:49212 -> 127.0.1.1:5552\",\"client_properties\":[[\"connection_name\",\"longstr\",\"rabbitmq-stream-producer-0\"],[\"copyright\",\"longstr\",\"Copyright (c) 2020-2025 Broadcom Inc. and/or its subsidiaries.\"],[\"information\",\"longstr\",\"Licensed under the MPL 2.0. See https://www.rabbitmq.com/\"],[\"platform\",\"longstr\",\"Java\"],[\"product\",\"longstr\",\"RabbitMQ Stream\"],[\"version\",\"longstr\",\"0.5.0-SNAPSHOT\"]]}\n" + + ",{\"conn_name\":\"127.0.0.1:58118 -> 127.0.0.1:5552\",\"client_properties\":[[\"connection_name\",\"longstr\",\"rabbitmq-stream-locator-0\"],[\"copyright\",\"longstr\",\"Copyright (c) 2020-2025 Broadcom Inc. and/or its subsidiaries.\"],[\"information\",\"longstr\",\"Licensed under the MPL 2.0. See https://www.rabbitmq.com/\"],[\"platform\",\"longstr\",\"Java\"],[\"product\",\"longstr\",\"RabbitMQ Stream\"],[\"version\",\"longstr\",\"0.5.0-SNAPSHOT\"]]}\n" + "]"; } diff --git a/src/test/java/com/rabbitmq/stream/DefaultEnvironmentTest.java b/src/test/java/com/rabbitmq/stream/DefaultEnvironmentTest.java index de78b978be..0441f5e6a9 100644 --- a/src/test/java/com/rabbitmq/stream/DefaultEnvironmentTest.java +++ b/src/test/java/com/rabbitmq/stream/DefaultEnvironmentTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -19,7 +19,8 @@ import com.rabbitmq.stream.impl.Client; import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.MultiThreadIoEventLoopGroup; +import io.netty.channel.nio.NioIoHandler; import java.util.UUID; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -31,7 +32,7 @@ public class DefaultEnvironmentTest { @BeforeAll static void initAll() { - eventLoopGroup = new NioEventLoopGroup(); + eventLoopGroup = new MultiThreadIoEventLoopGroup(NioIoHandler.newFactory()); } @AfterAll diff --git a/src/test/java/com/rabbitmq/stream/benchmark/ChecksumAlgorithmBenchmark.java b/src/test/java/com/rabbitmq/stream/benchmark/ChecksumAlgorithmBenchmark.java index b3a03acd58..ea208bf431 100644 --- a/src/test/java/com/rabbitmq/stream/benchmark/ChecksumAlgorithmBenchmark.java +++ b/src/test/java/com/rabbitmq/stream/benchmark/ChecksumAlgorithmBenchmark.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/benchmark/CompressDecompressBenchmark.java b/src/test/java/com/rabbitmq/stream/benchmark/CompressDecompressBenchmark.java index 417a0c5090..6136e8c3ed 100644 --- a/src/test/java/com/rabbitmq/stream/benchmark/CompressDecompressBenchmark.java +++ b/src/test/java/com/rabbitmq/stream/benchmark/CompressDecompressBenchmark.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/benchmark/EncodeDecodeForPerformanceToolBenchmark.java b/src/test/java/com/rabbitmq/stream/benchmark/EncodeDecodeForPerformanceToolBenchmark.java index 86e73459e2..1fcd74e56a 100644 --- a/src/test/java/com/rabbitmq/stream/benchmark/EncodeDecodeForPerformanceToolBenchmark.java +++ b/src/test/java/com/rabbitmq/stream/benchmark/EncodeDecodeForPerformanceToolBenchmark.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/benchmark/EncodingDecodingBenchmark.java b/src/test/java/com/rabbitmq/stream/benchmark/EncodingDecodingBenchmark.java index b2d64070a8..b7673e4a3d 100644 --- a/src/test/java/com/rabbitmq/stream/benchmark/EncodingDecodingBenchmark.java +++ b/src/test/java/com/rabbitmq/stream/benchmark/EncodingDecodingBenchmark.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/benchmark/FilteringBenchmark.java b/src/test/java/com/rabbitmq/stream/benchmark/FilteringBenchmark.java index 5e53182051..dddbd9bf67 100644 --- a/src/test/java/com/rabbitmq/stream/benchmark/FilteringBenchmark.java +++ b/src/test/java/com/rabbitmq/stream/benchmark/FilteringBenchmark.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/benchmark/HashAlgorithmBenchmark.java b/src/test/java/com/rabbitmq/stream/benchmark/HashAlgorithmBenchmark.java index f08e5abb5f..4b2e528b68 100644 --- a/src/test/java/com/rabbitmq/stream/benchmark/HashAlgorithmBenchmark.java +++ b/src/test/java/com/rabbitmq/stream/benchmark/HashAlgorithmBenchmark.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/benchmark/ModuloBenchmark.java b/src/test/java/com/rabbitmq/stream/benchmark/ModuloBenchmark.java index df7344bfce..342b9147e8 100644 --- a/src/test/java/com/rabbitmq/stream/benchmark/ModuloBenchmark.java +++ b/src/test/java/com/rabbitmq/stream/benchmark/ModuloBenchmark.java @@ -1,5 +1,5 @@ -// Copyright (c) 2021-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2021-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/codec/CodecsTest.java b/src/test/java/com/rabbitmq/stream/codec/CodecsTest.java index 9dee3dc74f..b80acf1b2f 100644 --- a/src/test/java/com/rabbitmq/stream/codec/CodecsTest.java +++ b/src/test/java/com/rabbitmq/stream/codec/CodecsTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -33,18 +33,18 @@ import java.math.BigInteger; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.UUID; +import java.util.*; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; import java.util.function.UnaryOperator; import java.util.stream.Stream; +import org.apache.qpid.proton.amqp.Symbol; import org.apache.qpid.proton.amqp.messaging.AmqpValue; +import org.apache.qpid.proton.amqp.messaging.MessageAnnotations; import org.assertj.core.api.InstanceOfAssertFactories; import org.assertj.core.api.ThrowableAssert; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; @@ -248,6 +248,22 @@ void codecs(CodecCouple codecCouple) { .entry("annotations.string", string) .entrySymbol("annotations.symbol", symbol) .entry("annotations.null", (String) null) + .entry( + "list", + List.of("1", "2", 3, List.of("1"), Map.of("k1", "v1"), new String[] {"1"})) + .entry( + "map", + Map.of( + "k1", + "v1", + "k2", + List.of("v2"), + "k3", + Map.of("k1", "v1"), + "k4", + new String[] {"1"})) + .entryArray("arrayString", new String[] {"1", "2", "3"}) + .entryArray("arrayInt", new Integer[] {200, 201, 202}) .messageBuilder() .build(); outboundMessage.annotate("extra.annotation", "extra annotation value"); @@ -474,6 +490,35 @@ void codecs(CodecCouple codecCouple) { .isNotNull() .isInstanceOf(String.class) .isEqualTo("extra annotation value"); + + List list = (List) inboundMessage.getMessageAnnotations().get("list"); + assertThat(list.get(0)).isEqualTo("1"); + assertThat(list.get(1)).isEqualTo("2"); + assertThat(list.get(2)).isEqualTo(3); + assertThat(list.get(3)).isEqualTo(List.of("1")); + assertThat(list.get(4)).isEqualTo(Map.of("k1", "v1")); + assertThat(list.get(5)).isEqualTo(new String[] {"1"}); + + Map map = (Map) inboundMessage.getMessageAnnotations().get("map"); + assertThat(map.get("k1")).isEqualTo("v1"); + assertThat(map.get("k2")).isEqualTo(List.of("v2")); + assertThat(map.get("k3")).isEqualTo(Map.of("k1", "v1")); + assertThat(map.get("k4")).isEqualTo(new String[] {"1"}); + + Object[] arrayString = + (Object[]) inboundMessage.getMessageAnnotations().get("arrayString"); + assertThat(arrayString).containsExactly("1", "2", "3"); + int[] arrayInt; + // QPid codec returns int[] and SwiftMQ codec returns Integer[] + if (inboundMessage.getMessageAnnotations().get("arrayInt") instanceof Integer[]) { + arrayInt = + Arrays.stream((Integer[]) inboundMessage.getMessageAnnotations().get("arrayInt")) + .mapToInt(Integer::intValue) + .toArray(); + } else { + arrayInt = (int[]) inboundMessage.getMessageAnnotations().get("arrayInt"); + } + assertThat(arrayInt).containsExactly(200, 201, 202); }); } @@ -624,6 +669,40 @@ void copy(CodecCouple codecCouple) { .containsEntry("copy", "copy value"); } + @Test + void qpidDoesNotSupportPrimitiveArrayEncodingInMap() { + org.apache.qpid.proton.message.Message message = + org.apache.qpid.proton.message.Message.Factory.create(); + Map map = new LinkedHashMap<>(); + map.put(Symbol.valueOf("foo"), new int[] {1, 2, 3}); + message.setMessageAnnotations(new MessageAnnotations(map)); + assertThatThrownBy(() -> qpidEncodeDecode(message)).isInstanceOf(ClassCastException.class); + } + + @Test + void qpidEncodeIntegerArrayDecodeIntArrayInMap() { + org.apache.qpid.proton.message.Message message = + org.apache.qpid.proton.message.Message.Factory.create(); + Map map = new LinkedHashMap<>(); + map.put(Symbol.valueOf("foo"), new Integer[] {1, 2, 3}); + message.setMessageAnnotations(new MessageAnnotations(map)); + message = qpidEncodeDecode(message); + map = message.getMessageAnnotations().getValue(); + assertThat(map.get(Symbol.valueOf("foo"))).isInstanceOf(int[].class); + } + + private static org.apache.qpid.proton.message.Message qpidEncodeDecode( + org.apache.qpid.proton.message.Message in) { + QpidProtonCodec.ByteArrayWritableBuffer writableBuffer = + new QpidProtonCodec.ByteArrayWritableBuffer(8192); + in.encode(writableBuffer); + + org.apache.qpid.proton.message.Message out = + org.apache.qpid.proton.message.Message.Factory.create(); + out.decode(writableBuffer.getArray(), 0, writableBuffer.getArrayLength()); + return out; + } + MessageTestConfiguration test( Function messageOperation, Consumer messageExpectation) { diff --git a/src/test/java/com/rabbitmq/stream/docs/ConsumerUsage.java b/src/test/java/com/rabbitmq/stream/docs/ConsumerUsage.java index 1d377a4d15..1c96831164 100644 --- a/src/test/java/com/rabbitmq/stream/docs/ConsumerUsage.java +++ b/src/test/java/com/rabbitmq/stream/docs/ConsumerUsage.java @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/docs/EnvironmentUsage.java b/src/test/java/com/rabbitmq/stream/docs/EnvironmentUsage.java index 72a32daa10..bd7a9c6bc2 100644 --- a/src/test/java/com/rabbitmq/stream/docs/EnvironmentUsage.java +++ b/src/test/java/com/rabbitmq/stream/docs/EnvironmentUsage.java @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -19,7 +19,9 @@ import com.rabbitmq.stream.observation.micrometer.MicrometerObservationCollectorBuilder; import io.micrometer.observation.ObservationRegistry; import io.netty.channel.EventLoopGroup; +import io.netty.channel.MultiThreadIoEventLoopGroup; import io.netty.channel.epoll.EpollEventLoopGroup; +import io.netty.channel.epoll.EpollIoHandler; import io.netty.channel.epoll.EpollSocketChannel; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; @@ -97,6 +99,7 @@ void addressResolver() throws Exception { .host(entryPoint.host()) // <2> .port(entryPoint.port()) // <2> .addressResolver(address -> entryPoint) // <3> + .locatorConnectionCount(3) // <4> .build(); // end::address-resolver[] } @@ -139,7 +142,9 @@ void deleteStream() { void nativeEpoll() { // tag::native-epoll[] - EventLoopGroup epollEventLoopGroup = new EpollEventLoopGroup(); // <1> + EventLoopGroup epollEventLoopGroup = new MultiThreadIoEventLoopGroup( // <1> + EpollIoHandler.newFactory() // <1> + ); // <1> Environment environment = Environment.builder() .netty() // <2> .eventLoopGroup(epollEventLoopGroup) // <3> diff --git a/src/test/java/com/rabbitmq/stream/docs/FilteringUsage.java b/src/test/java/com/rabbitmq/stream/docs/FilteringUsage.java index cb882ae1bc..6340b1b935 100644 --- a/src/test/java/com/rabbitmq/stream/docs/FilteringUsage.java +++ b/src/test/java/com/rabbitmq/stream/docs/FilteringUsage.java @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/docs/ProducerUsage.java b/src/test/java/com/rabbitmq/stream/docs/ProducerUsage.java index 5821d2c769..2a11e8ae2e 100644 --- a/src/test/java/com/rabbitmq/stream/docs/ProducerUsage.java +++ b/src/test/java/com/rabbitmq/stream/docs/ProducerUsage.java @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/docs/SampleApplication.java b/src/test/java/com/rabbitmq/stream/docs/SampleApplication.java index 69c811462b..5daaa2e506 100644 --- a/src/test/java/com/rabbitmq/stream/docs/SampleApplication.java +++ b/src/test/java/com/rabbitmq/stream/docs/SampleApplication.java @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/docs/SuperStreamUsage.java b/src/test/java/com/rabbitmq/stream/docs/SuperStreamUsage.java index 583937129f..109a0305d2 100644 --- a/src/test/java/com/rabbitmq/stream/docs/SuperStreamUsage.java +++ b/src/test/java/com/rabbitmq/stream/docs/SuperStreamUsage.java @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/AlarmsTest.java b/src/test/java/com/rabbitmq/stream/impl/AlarmsTest.java index 0a30e807be..78141e8130 100644 --- a/src/test/java/com/rabbitmq/stream/impl/AlarmsTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/AlarmsTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2021-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2021-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -14,8 +14,8 @@ // info@rabbitmq.com. package com.rabbitmq.stream.impl; -import static com.rabbitmq.stream.Host.diskAlarm; -import static com.rabbitmq.stream.Host.memoryAlarm; +import static com.rabbitmq.stream.Cli.diskAlarm; +import static com.rabbitmq.stream.Cli.memoryAlarm; import static com.rabbitmq.stream.impl.TestUtils.ExceptionConditions.responseCode; import static com.rabbitmq.stream.impl.TestUtils.latchAssert; import static java.util.concurrent.TimeUnit.SECONDS; @@ -32,7 +32,6 @@ import com.rabbitmq.stream.Producer; import com.rabbitmq.stream.StreamException; import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; import java.time.Duration; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; @@ -56,7 +55,7 @@ public class AlarmsTest { @BeforeAll static void initAll() { - eventLoopGroup = new NioEventLoopGroup(); + eventLoopGroup = Utils.eventLoopGroup(); } @AfterAll diff --git a/src/test/java/com/rabbitmq/stream/impl/Amqp10InteroperabilityTest.java b/src/test/java/com/rabbitmq/stream/impl/Amqp10InteroperabilityTest.java index e0a84f9fb3..e4753f4017 100644 --- a/src/test/java/com/rabbitmq/stream/impl/Amqp10InteroperabilityTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/Amqp10InteroperabilityTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -32,7 +32,6 @@ import com.swiftmq.amqp.v100.client.Producer; import com.swiftmq.amqp.v100.client.QoS; import com.swiftmq.amqp.v100.client.Session; -import com.swiftmq.amqp.v100.generated.messaging.delivery_state.*; import com.swiftmq.amqp.v100.generated.messaging.message_format.AmqpSequence; import com.swiftmq.amqp.v100.generated.messaging.message_format.AmqpValue; import com.swiftmq.amqp.v100.generated.messaging.message_format.ApplicationProperties; diff --git a/src/test/java/com/rabbitmq/stream/impl/AmqpInteroperabilityTest.java b/src/test/java/com/rabbitmq/stream/impl/AmqpInteroperabilityTest.java index 620030d4c3..95a4df4f91 100644 --- a/src/test/java/com/rabbitmq/stream/impl/AmqpInteroperabilityTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/AmqpInteroperabilityTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -31,8 +31,6 @@ import com.rabbitmq.stream.codec.SwiftMqCodec; import com.rabbitmq.stream.impl.Client.ClientParameters; import com.rabbitmq.stream.impl.Client.Response; -import com.rabbitmq.stream.impl.TestUtils.BrokerVersion; -import com.rabbitmq.stream.impl.TestUtils.BrokerVersionAtLeast; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.*; @@ -578,9 +576,11 @@ void publishToStreamConsumeFromStreamQueue(Codec codec, TestInfo info) { mb -> mb.applicationProperties() .entry("binary", "hello".getBytes(UTF8)), - d -> - assertThat(d.getProperties().getHeaders()) - .containsEntry("binary", "hello".getBytes(UTF8)))); + d -> { + LongString expected = LongStringHelper.asLongString("hello"); + assertThat(d.getProperties().getHeaders()) + .containsEntry("binary", expected); + })); client.declarePublisher(b(1), null, s); IntStream.range(0, messageCount) diff --git a/src/test/java/com/rabbitmq/stream/impl/Assertions.java b/src/test/java/com/rabbitmq/stream/impl/Assertions.java new file mode 100644 index 0000000000..8a3b0e966f --- /dev/null +++ b/src/test/java/com/rabbitmq/stream/impl/Assertions.java @@ -0,0 +1,64 @@ +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// +// This software, the RabbitMQ Stream Java client library, is dual-licensed under the +// Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). +// For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL, +// please see LICENSE-APACHE2. +// +// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, +// either express or implied. See the LICENSE file for specific language governing +// rights and limitations of this software. +// +// If you have any questions regarding licensing, please contact us at +// info@rabbitmq.com. +package com.rabbitmq.stream.impl; + +import static org.assertj.core.api.Assertions.fail; + +import java.time.Duration; +import org.assertj.core.api.AbstractObjectAssert; + +final class Assertions { + + private Assertions() {} + + static SyncAssert assertThat(TestUtils.Sync sync) { + return new SyncAssert(sync); + } + + static class SyncAssert extends AbstractObjectAssert { + + private SyncAssert(TestUtils.Sync sync) { + super(sync, SyncAssert.class); + } + + SyncAssert completes() { + return this.completes(TestUtils.DEFAULT_CONDITION_TIMEOUT); + } + + SyncAssert completes(Duration timeout) { + boolean completed = actual.await(timeout); + if (!completed) { + fail( + "Sync timed out after %d ms (current count is %d)", + timeout.toMillis(), actual.currentCount()); + } + return this; + } + + SyncAssert hasCompleted() { + if (!this.actual.hasCompleted()) { + fail("Sync should have completed but has not"); + } + return this; + } + + SyncAssert hasNotCompleted() { + if (this.actual.hasCompleted()) { + fail("Sync should have not completed"); + } + return this; + } + } +} diff --git a/src/test/java/com/rabbitmq/stream/impl/AsyncRetryTest.java b/src/test/java/com/rabbitmq/stream/impl/AsyncRetryTest.java index 75bcde1d05..7c18ded3b3 100644 --- a/src/test/java/com/rabbitmq/stream/impl/AsyncRetryTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/AsyncRetryTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -14,61 +14,64 @@ // info@rabbitmq.com. package com.rabbitmq.stream.impl; +import static com.rabbitmq.stream.BackOffDelayPolicy.fixedWithInitialDelay; +import static com.rabbitmq.stream.impl.Assertions.assertThat; +import static com.rabbitmq.stream.impl.TestUtils.sync; +import static java.time.Duration.*; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.*; -import com.rabbitmq.stream.BackOffDelayPolicy; -import java.time.Duration; +import com.rabbitmq.stream.impl.TestUtils.Sync; +import java.util.List; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mock; import org.mockito.MockitoAnnotations; public class AsyncRetryTest { - ScheduledExecutorService scheduler; @Mock Callable task; AutoCloseable mocks; @BeforeEach void init() { mocks = MockitoAnnotations.openMocks(this); - this.scheduler = Executors.newSingleThreadScheduledExecutor(); } @AfterEach void tearDown() throws Exception { - this.scheduler.shutdownNow(); mocks.close(); } - @Test - void callbackCalledIfCompletedImmediately() throws Exception { + @ParameterizedTest + @MethodSource("schedulers") + void callbackCalledIfCompletedImmediately(ScheduledExecutorService scheduler) throws Exception { when(task.call()).thenReturn(42); CompletableFuture completableFuture = AsyncRetry.asyncRetry(task) - .delayPolicy( - BackOffDelayPolicy.fixedWithInitialDelay(Duration.ZERO, Duration.ofMillis(10))) + .delayPolicy(fixedWithInitialDelay(ZERO, ofMillis(10))) .scheduler(scheduler) .build(); AtomicInteger result = new AtomicInteger(0); - completableFuture.thenAccept(value -> result.set(value)); + completableFuture.thenAccept(result::set); assertThat(result.get()).isEqualTo(42); verify(task, times(1)).call(); } - @Test - void shouldRetryWhenExecutionFails() throws Exception { + @ParameterizedTest + @MethodSource("schedulers") + void shouldRetryWhenExecutionFails(ScheduledExecutorService scheduler) throws Exception { when(task.call()) .thenThrow(new RuntimeException()) .thenThrow(new RuntimeException()) .thenReturn(42); CompletableFuture completableFuture = - AsyncRetry.asyncRetry(task).scheduler(scheduler).delay(Duration.ofMillis(50)).build(); + AsyncRetry.asyncRetry(task).scheduler(scheduler).delay(ofMillis(50)).build(); CountDownLatch latch = new CountDownLatch(1); AtomicInteger result = new AtomicInteger(0); completableFuture.thenAccept( @@ -81,15 +84,15 @@ void shouldRetryWhenExecutionFails() throws Exception { verify(task, times(3)).call(); } - @Test - void shouldTimeoutWhenExecutionFailsForTooLong() throws Exception { + @ParameterizedTest + @MethodSource("schedulers") + void shouldTimeoutWhenExecutionFailsForTooLong(ScheduledExecutorService scheduler) + throws Exception { when(task.call()).thenThrow(new RuntimeException()); CompletableFuture completableFuture = AsyncRetry.asyncRetry(task) - .scheduler(this.scheduler) - .delayPolicy( - BackOffDelayPolicy.fixedWithInitialDelay( - Duration.ofMillis(50), Duration.ofMillis(50), Duration.ofMillis(500))) + .scheduler(scheduler) + .delayPolicy(fixedWithInitialDelay(ofMillis(50), ofMillis(50), ofMillis(500))) .build(); CountDownLatch latch = new CountDownLatch(1); AtomicBoolean acceptCalled = new AtomicBoolean(false); @@ -111,8 +114,9 @@ void shouldTimeoutWhenExecutionFailsForTooLong() throws Exception { verify(task, atLeast(5)).call(); } - @Test - void shouldRetryWhenPredicateAllowsIt() throws Exception { + @ParameterizedTest + @MethodSource("schedulers") + void shouldRetryWhenPredicateAllowsIt(ScheduledExecutorService scheduler) throws Exception { when(task.call()) .thenThrow(new IllegalStateException()) .thenThrow(new IllegalStateException()) @@ -121,7 +125,7 @@ void shouldRetryWhenPredicateAllowsIt() throws Exception { AsyncRetry.asyncRetry(task) .scheduler(scheduler) .retry(e -> e instanceof IllegalStateException) - .delay(Duration.ofMillis(50)) + .delay(ofMillis(50)) .build(); CountDownLatch latch = new CountDownLatch(1); AtomicInteger result = new AtomicInteger(0); @@ -135,8 +139,10 @@ void shouldRetryWhenPredicateAllowsIt() throws Exception { verify(task, times(3)).call(); } - @Test - void shouldFailWhenPredicateDoesNotAllowRetry() throws Exception { + @ParameterizedTest + @MethodSource("schedulers") + void shouldFailWhenPredicateDoesNotAllowRetry(ScheduledExecutorService scheduler) + throws Exception { when(task.call()) .thenThrow(new IllegalStateException()) .thenThrow(new IllegalStateException()) @@ -145,7 +151,7 @@ void shouldFailWhenPredicateDoesNotAllowRetry() throws Exception { AsyncRetry.asyncRetry(task) .scheduler(scheduler) .retry(e -> !(e instanceof IllegalArgumentException)) - .delay(Duration.ofMillis(50)) + .delay(ofMillis(50)) .build(); CountDownLatch latch = new CountDownLatch(1); AtomicBoolean acceptCalled = new AtomicBoolean(false); @@ -166,4 +172,33 @@ void shouldFailWhenPredicateDoesNotAllowRetry() throws Exception { assertThat(exceptionallyCalled.get()).isTrue(); verify(task, times(3)).call(); } + + @ParameterizedTest + @MethodSource("schedulers") + void completeExceptionally(ScheduledExecutorService scheduler) throws Exception { + when(task.call()).thenThrow(new UnsupportedOperationException()); + CompletableFuture completableFuture = + AsyncRetry.asyncRetry(task) + .delayPolicy(fixedWithInitialDelay(ZERO, ofMillis(10))) + .retry(ex -> ex instanceof IllegalStateException) + .scheduler(scheduler) + .build(); + Sync sync = sync(); + completableFuture.handleAsync( + (v, ex) -> { + if (ex != null) { + sync.down(); + } + return null; + }, + scheduler); + assertThat(sync).completes(); + } + + static List schedulers() { + return List.of( + Executors.newSingleThreadScheduledExecutor(), + Executors.newScheduledThreadPool( + 0, ThreadUtils.internalThreadFactory("async-retry-test-"))); + } } diff --git a/src/test/java/com/rabbitmq/stream/impl/AuthenticationTest.java b/src/test/java/com/rabbitmq/stream/impl/AuthenticationTest.java index 83c1e091bf..ceb5fb805b 100644 --- a/src/test/java/com/rabbitmq/stream/impl/AuthenticationTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/AuthenticationTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -14,7 +14,7 @@ // info@rabbitmq.com. package com.rabbitmq.stream.impl; -import static com.rabbitmq.stream.Host.*; +import static com.rabbitmq.stream.Cli.*; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -23,6 +23,7 @@ import com.rabbitmq.stream.StreamException; import com.rabbitmq.stream.sasl.*; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.UUID; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -87,6 +88,7 @@ void authenticateShouldFailWhenSendingGarbageToSaslChallenge() { try { cf.get( new Client.ClientParameters() + .rpcTimeout(Duration.ofSeconds(1)) .saslConfiguration( mechanisms -> new SaslMechanism() { @@ -102,7 +104,8 @@ public byte[] handleChallenge( } })); } catch (StreamException e) { - assertThat(e.getMessage()).contains(String.valueOf(Constants.RESPONSE_CODE_SASL_ERROR)); + // there can be a timeout because the connection gets closed before returning the error + assertThat(e).hasMessageContaining(String.valueOf(Constants.RESPONSE_CODE_SASL_ERROR)); } } @@ -144,6 +147,14 @@ void updateSecret() throws Exception { } } + @Test + @TestUtils.BrokerVersionAtLeast(TestUtils.BrokerVersion.RABBITMQ_4_0_0) + void anonymousAuthenticationShouldWork() { + try (Client ignored = + cf.get( + new Client.ClientParameters().saslConfiguration(DefaultSaslConfiguration.ANONYMOUS))) {} + } + private static CredentialsProvider credentialsProvider(String username, String password) { return new DefaultUsernamePasswordCredentialsProvider(username, password); } diff --git a/src/test/java/com/rabbitmq/stream/impl/AuthorisationTest.java b/src/test/java/com/rabbitmq/stream/impl/AuthorisationTest.java index 47d595da6b..df07c03b76 100644 --- a/src/test/java/com/rabbitmq/stream/impl/AuthorisationTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/AuthorisationTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -14,23 +14,32 @@ // info@rabbitmq.com. package com.rabbitmq.stream.impl; -import static com.rabbitmq.stream.Host.*; -import static com.rabbitmq.stream.impl.TestUtils.b; -import static com.rabbitmq.stream.impl.TestUtils.waitAtMost; +import static com.rabbitmq.stream.Cli.*; +import static com.rabbitmq.stream.Constants.*; +import static com.rabbitmq.stream.OffsetSpecification.first; +import static com.rabbitmq.stream.impl.TestUtils.*; +import static com.rabbitmq.stream.impl.TestUtils.ResponseConditions.ok; +import static java.util.Arrays.asList; import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; import com.rabbitmq.stream.Constants; -import com.rabbitmq.stream.OffsetSpecification; +import com.rabbitmq.stream.Message; +import com.rabbitmq.stream.sasl.CredentialsProvider; +import com.rabbitmq.stream.sasl.DefaultUsernamePasswordCredentialsProvider; import java.nio.charset.StandardCharsets; import java.time.Duration; -import java.util.Collections; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; import java.util.stream.IntStream; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.extension.ExtendWith; @ExtendWith(TestUtils.StreamTestInfrastructureExtension.class) @@ -74,11 +83,11 @@ void createStreamWithAuthorisedNameShouldSucceed() { String stream = "stream-authorized" + i; Client.Response response = client.create(stream); assertThat(response.isOk()).isTrue(); - assertThat(response.getResponseCode()).isEqualTo(Constants.RESPONSE_CODE_OK); + assertThat(response.getResponseCode()).isEqualTo(RESPONSE_CODE_OK); response = deletionClient.delete(stream); assertThat(response.isOk()).isTrue(); - assertThat(response.getResponseCode()).isEqualTo(Constants.RESPONSE_CODE_OK); + assertThat(response.getResponseCode()).isEqualTo(RESPONSE_CODE_OK); }); } @@ -105,11 +114,11 @@ void deleteStreamWithAuthorisedNameShouldSucceed() { String stream = "stream-authorized" + i; Client.Response response = creationClient.create(stream); assertThat(response.isOk()).isTrue(); - assertThat(response.getResponseCode()).isEqualTo(Constants.RESPONSE_CODE_OK); + assertThat(response.getResponseCode()).isEqualTo(RESPONSE_CODE_OK); response = client.delete(stream); assertThat(response.isOk()).isTrue(); - assertThat(response.getResponseCode()).isEqualTo(Constants.RESPONSE_CODE_OK); + assertThat(response.getResponseCode()).isEqualTo(RESPONSE_CODE_OK); }); } @@ -123,7 +132,7 @@ void deleteStreamWithUnauthorisedNameShouldFail() { String stream = "not-authorized" + i; Client.Response response = creationClient.create(stream); assertThat(response.isOk()).isTrue(); - assertThat(response.getResponseCode()).isEqualTo(Constants.RESPONSE_CODE_OK); + assertThat(response.getResponseCode()).isEqualTo(RESPONSE_CODE_OK); response = client.delete(stream); assertThat(response.isOk()).isFalse(); @@ -132,7 +141,7 @@ void deleteStreamWithUnauthorisedNameShouldFail() { response = creationClient.delete(stream); assertThat(response.isOk()).isTrue(); - assertThat(response.getResponseCode()).isEqualTo(Constants.RESPONSE_CODE_OK); + assertThat(response.getResponseCode()).isEqualTo(RESPONSE_CODE_OK); }); } @@ -146,15 +155,15 @@ void subscribeToAuthorisedStreamShouldSucceed() { String stream = "stream-authorized" + i; Client.Response response = configurationClient.create(stream); assertThat(response.isOk()).isTrue(); - assertThat(response.getResponseCode()).isEqualTo(Constants.RESPONSE_CODE_OK); + assertThat(response.getResponseCode()).isEqualTo(RESPONSE_CODE_OK); - response = client.subscribe(b(1), stream, OffsetSpecification.first(), 10); + response = client.subscribe(b(1), stream, first(), 10); assertThat(response.isOk()).isTrue(); - assertThat(response.getResponseCode()).isEqualTo(Constants.RESPONSE_CODE_OK); + assertThat(response.getResponseCode()).isEqualTo(RESPONSE_CODE_OK); response = configurationClient.delete(stream); assertThat(response.isOk()).isTrue(); - assertThat(response.getResponseCode()).isEqualTo(Constants.RESPONSE_CODE_OK); + assertThat(response.getResponseCode()).isEqualTo(RESPONSE_CODE_OK); }); } @@ -168,16 +177,16 @@ void subscribeToUnauthorisedStreamShouldFail() { String stream = "not-authorized" + i; Client.Response response = configurationClient.create(stream); assertThat(response.isOk()).isTrue(); - assertThat(response.getResponseCode()).isEqualTo(Constants.RESPONSE_CODE_OK); + assertThat(response.getResponseCode()).isEqualTo(RESPONSE_CODE_OK); - response = client.subscribe(b(1), stream, OffsetSpecification.first(), 10); + response = client.subscribe(b(1), stream, first(), 10); assertThat(response.isOk()).isFalse(); assertThat(response.getResponseCode()) .isEqualTo(Constants.RESPONSE_CODE_ACCESS_REFUSED); response = configurationClient.delete(stream); assertThat(response.isOk()).isTrue(); - assertThat(response.getResponseCode()).isEqualTo(Constants.RESPONSE_CODE_OK); + assertThat(response.getResponseCode()).isEqualTo(RESPONSE_CODE_OK); }); } @@ -191,7 +200,7 @@ void publishToAuthorisedStreamShouldSucceed() { String stream = "stream-authorized" + i; Client.Response response = configurationClient.create(stream); assertThat(response.isOk()).isTrue(); - assertThat(response.getResponseCode()).isEqualTo(Constants.RESPONSE_CODE_OK); + assertThat(response.getResponseCode()).isEqualTo(RESPONSE_CODE_OK); int messageCount = 1000; CountDownLatch publishConfirmLatch = new CountDownLatch(messageCount); @@ -223,7 +232,7 @@ void publishToAuthorisedStreamShouldSucceed() { response = configurationClient.delete(stream); assertThat(response.isOk()).isTrue(); - assertThat(response.getResponseCode()).isEqualTo(Constants.RESPONSE_CODE_OK); + assertThat(response.getResponseCode()).isEqualTo(RESPONSE_CODE_OK); }); } @@ -237,7 +246,7 @@ void publishToUnauthorisedStreamShouldFail() { String stream = "not-authorized" + i; Client.Response response = configurationClient.create(stream); assertThat(response.isOk()).isTrue(); - assertThat(response.getResponseCode()).isEqualTo(Constants.RESPONSE_CODE_OK); + assertThat(response.getResponseCode()).isEqualTo(RESPONSE_CODE_OK); int messageCount = 1000; CountDownLatch publishErrorLatch = new CountDownLatch(messageCount); @@ -270,7 +279,7 @@ void publishToUnauthorisedStreamShouldFail() { response = configurationClient.delete(stream); assertThat(response.isOk()).isTrue(); - assertThat(response.getResponseCode()).isEqualTo(Constants.RESPONSE_CODE_OK); + assertThat(response.getResponseCode()).isEqualTo(RESPONSE_CODE_OK); }); } @@ -304,6 +313,142 @@ void storeQueryOffsetShouldSucceedOnAuthorisedStreamShouldFailOnUnauthorisedStre } } + @Test + @TestUtils.BrokerVersionAtLeast(TestUtils.BrokerVersion.RABBITMQ_3_13_0) + void shouldReceiveMetadataUpdateAfterUpdateSecret(TestInfo info) throws Exception { + try { + String newPassword = "new-password"; + String prefix = "passthrough-"; + String pubSub = TestUtils.streamName(info); + String authorizedPubSub = prefix + TestUtils.streamName(info); + String pub = TestUtils.streamName(info); + String authorizedPub = prefix + TestUtils.streamName(info); + String sub = TestUtils.streamName(info); + String authorizedSub = prefix + TestUtils.streamName(info); + setPermissions(USERNAME, VH, ".*"); + Set metadataUpdates = ConcurrentHashMap.newKeySet(); + ConcurrentMap publishConfirms = new ConcurrentHashMap<>(); + ConcurrentMap creditNotifications = new ConcurrentHashMap<>(); + Set receivedMessages = ConcurrentHashMap.newKeySet(); + Client client = + cf.get( + parameters() + .virtualHost(VH) + .username(USERNAME) + .password(USERNAME) + .publishConfirmListener( + (publisherId, publishingId) -> + publishConfirms.put(publisherId, RESPONSE_CODE_OK)) + .publishErrorListener( + (publisherId, publishingId, errorCode) -> + publishConfirms.put(publisherId, errorCode)) + .creditNotification( + (subscriptionId, responseCode) -> + creditNotifications.put(subscriptionId, responseCode)) + .messageListener( + (subscriptionId, + offset, + chunkTimestamp, + committedChunkId, + chunkContext, + message) -> receivedMessages.add(subscriptionId)) + .metadataListener((stream, code) -> metadataUpdates.add(stream))); + assertThat(client.create(pubSub)).is(ok()); + assertThat(client.create(authorizedPubSub)).is(ok()); + assertThat(client.create(pub)).is(ok()); + assertThat(client.create(authorizedPub)).is(ok()); + assertThat(client.create(sub)).is(ok()); + assertThat(client.create(authorizedSub)).is(ok()); + + Map publishers = new HashMap<>(); + publishers.put(pubSub, b(0)); + publishers.put(authorizedPubSub, b(1)); + publishers.put(pub, b(2)); + publishers.put(authorizedPub, b(3)); + publishers.forEach((s, id) -> assertThat(client.declarePublisher(id, null, s)).is(ok())); + Map subscriptions = new HashMap<>(); + subscriptions.put(pubSub, b(0)); + subscriptions.put(authorizedPubSub, b(1)); + subscriptions.put(sub, b(2)); + subscriptions.put(authorizedSub, b(3)); + subscriptions.forEach((s, id) -> assertThat(client.subscribe(id, s, first(), 1)).is(ok())); + + Function toPub = publishers::get; + Function toSub = subscriptions::get; + + // change password and permissions and re-authenticate + changePassword(USERNAME, newPassword); + setPermissions(USERNAME, VH, "^passthrough.*$"); + client.authenticate(credentialsProvider(USERNAME, newPassword)); + + waitAtMost(() -> metadataUpdates.containsAll(asList(pubSub, pub, sub))); + + List message = Collections.singletonList(client.messageBuilder().build()); + + // publishers for unauthorized streams should be gone + asList(toPub.apply(pubSub), toPub.apply(pub)) + .forEach( + wrap( + pubId -> { + assertThat(publishConfirms).doesNotContainKey(pubId); + client.publish(pubId, message); + waitAtMost(() -> publishConfirms.containsKey(pubId)); + assertThat(publishConfirms) + .containsEntry(pubId, RESPONSE_CODE_PUBLISHER_DOES_NOT_EXIST); + })); + + // subscriptions for unauthorized streams should be gone + asList(toSub.apply(pubSub), toSub.apply(sub)) + .forEach( + wrap( + subId -> { + assertThat(creditNotifications).doesNotContainKey(subId); + client.credit(subId, 1); + waitAtMost(() -> creditNotifications.containsKey(subId)); + assertThat(creditNotifications) + .containsEntry(subId, RESPONSE_CODE_SUBSCRIPTION_ID_DOES_NOT_EXIST); + })); + + // subscriptions for authorized streams still work + asList(toSub.apply(authorizedPubSub), toSub.apply(authorizedSub)) + .forEach(subId -> client.credit(subId, 1)); + + assertThat(receivedMessages).isEmpty(); + // publishers for authorized streams should still work + asList(toPub.apply(authorizedPubSub), toPub.apply(authorizedPub)) + .forEach( + wrap( + pubId -> { + client.publish(pubId, message); + waitAtMost(() -> publishConfirms.containsKey(pubId)); + assertThat(publishConfirms).containsEntry(pubId, RESPONSE_CODE_OK); + })); + + waitAtMost(() -> receivedMessages.contains(b(1))); + + // send message to authorized subscription stream + assertThat(client.declarePublisher(b(5), null, authorizedSub)).is(ok()); + client.publish(b(5), message); + waitAtMost(() -> receivedMessages.contains(toSub.apply(authorizedSub))); + + // last checks to make sure nothing unexpected arrived late + assertThat(metadataUpdates).hasSize(3); + assertThat(creditNotifications).containsOnlyKeys(b(0), b(2)); + assertThat(publishConfirms) + .hasSize(4 + 1) + .containsEntry(toPub.apply(pubSub), RESPONSE_CODE_PUBLISHER_DOES_NOT_EXIST) + .containsEntry(toPub.apply(authorizedPubSub), RESPONSE_CODE_OK) + .containsEntry(toPub.apply(pub), RESPONSE_CODE_PUBLISHER_DOES_NOT_EXIST) + .containsEntry(toPub.apply(authorizedPub), RESPONSE_CODE_OK); + assertThat(receivedMessages).hasSize(2); + + client.close(); + } finally { + changePassword(USERNAME, PASSWORD); + setPermissions(USERNAME, VH, "^stream.*$"); + } + } + Client configurationClient() { return cf.get(new Client.ClientParameters().virtualHost(VH)); } @@ -315,4 +460,12 @@ Client client() { Client client(Client.ClientParameters parameters) { return cf.get(parameters.virtualHost(VH).username(USERNAME).password(PASSWORD)); } + + private static Client.ClientParameters parameters() { + return new Client.ClientParameters(); + } + + private static CredentialsProvider credentialsProvider(String username, String password) { + return new DefaultUsernamePasswordCredentialsProvider(username, password); + } } diff --git a/src/test/java/com/rabbitmq/stream/impl/ClientFlowControlTest.java b/src/test/java/com/rabbitmq/stream/impl/ClientFlowControlTest.java index a0758bd5fa..9695884304 100644 --- a/src/test/java/com/rabbitmq/stream/impl/ClientFlowControlTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/ClientFlowControlTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/ClientParametersTest.java b/src/test/java/com/rabbitmq/stream/impl/ClientParametersTest.java index 947cc4ae1d..322446bb5a 100644 --- a/src/test/java/com/rabbitmq/stream/impl/ClientParametersTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/ClientParametersTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/ClientTest.java b/src/test/java/com/rabbitmq/stream/impl/ClientTest.java index d7deeacfe5..34496ac73d 100644 --- a/src/test/java/com/rabbitmq/stream/impl/ClientTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/ClientTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -23,13 +23,8 @@ import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; import com.rabbitmq.client.ConnectionFactory; -import com.rabbitmq.stream.Codec; -import com.rabbitmq.stream.Constants; -import com.rabbitmq.stream.Message; -import com.rabbitmq.stream.MessageBuilder; -import com.rabbitmq.stream.OffsetSpecification; +import com.rabbitmq.stream.*; import com.rabbitmq.stream.Properties; -import com.rabbitmq.stream.StreamException; import com.rabbitmq.stream.codec.QpidProtonCodec; import com.rabbitmq.stream.codec.SimpleCodec; import com.rabbitmq.stream.codec.SwiftMqCodec; diff --git a/src/test/java/com/rabbitmq/stream/impl/CompressionCodecsTest.java b/src/test/java/com/rabbitmq/stream/impl/CompressionCodecsTest.java index ce5d1044ff..d714a7363b 100644 --- a/src/test/java/com/rabbitmq/stream/impl/CompressionCodecsTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/CompressionCodecsTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/ConsumersCoordinatorTest.java b/src/test/java/com/rabbitmq/stream/impl/ConsumersCoordinatorTest.java index 3bd6d24025..4844a6fdf6 100644 --- a/src/test/java/com/rabbitmq/stream/impl/ConsumersCoordinatorTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/ConsumersCoordinatorTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -15,15 +15,18 @@ package com.rabbitmq.stream.impl; import static com.rabbitmq.stream.BackOffDelayPolicy.fixedWithInitialDelay; -import static com.rabbitmq.stream.impl.ConsumersCoordinator.MAX_SUBSCRIPTIONS_PER_CLIENT; -import static com.rabbitmq.stream.impl.ConsumersCoordinator.pickSlot; +import static com.rabbitmq.stream.impl.ConsumersCoordinator.*; import static com.rabbitmq.stream.impl.TestUtils.b; import static com.rabbitmq.stream.impl.TestUtils.latchAssert; import static com.rabbitmq.stream.impl.TestUtils.metadata; import static com.rabbitmq.stream.impl.TestUtils.namedConsumer; import static com.rabbitmq.stream.impl.TestUtils.waitAtMost; +import static com.rabbitmq.stream.impl.Utils.brokerPicker; import static java.lang.String.format; import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; +import static java.util.stream.Collectors.toList; +import static java.util.stream.IntStream.*; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; @@ -41,6 +44,7 @@ import com.rabbitmq.stream.impl.Client.Response; import com.rabbitmq.stream.impl.MonitoringTestUtils.ConsumerCoordinatorInfo; import com.rabbitmq.stream.impl.Utils.ClientFactory; +import io.netty.channel.ConnectTimeoutException; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -51,8 +55,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; -import java.util.stream.Collectors; -import java.util.stream.IntStream; +import java.util.function.Function; import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -144,12 +147,15 @@ public Client.ClientParameters shutdownListener( } }; mocks = MockitoAnnotations.openMocks(this); - when(environment.locator()).thenReturn(locator); + StreamEnvironment.Locator l = new StreamEnvironment.Locator(-1, new Address("localhost", 5555)); + l.client(locator); + when(environment.locator()).thenReturn(l); when(environment.locatorOperation(any())).thenCallRealMethod(); when(environment.clientParametersCopy()).thenReturn(clientParameters); when(environment.addressResolver()).thenReturn(address -> address); when(client.brokerVersion()).thenReturn("3.11.0"); when(client.isOpen()).thenReturn(true); + clientAdvertises(replica().get(0)); coordinator = new ConsumersCoordinator( @@ -157,7 +163,8 @@ public Client.ClientParameters shutdownListener( ConsumersCoordinator.MAX_SUBSCRIPTIONS_PER_CLIENT, type -> "consumer-connection", clientFactory, - false); + false, + brokerPicker()); } @AfterEach @@ -178,8 +185,7 @@ void tearDown() throws Exception { shouldRetryUntilGettingExactNodeWithAdvertisedHostNameClientFactoryAndNotExactNodeOnFirstTime() { ClientFactory cf = context -> - Utils.connectToAdvertisedNodeClientFactory( - context.key(), clientFactory, Duration.ofMillis(1)) + Utils.connectToAdvertisedNodeClientFactory(clientFactory, Duration.ofMillis(1)) .client(context); ConsumersCoordinator c = new ConsumersCoordinator( @@ -187,7 +193,8 @@ void tearDown() throws Exception { ConsumersCoordinator.MAX_SUBSCRIPTIONS_PER_CLIENT, type -> "consumer-connection", cf, - false); + false, + brokerPicker()); when(locator.metadata("stream")).thenReturn(metadata(null, replica())); when(clientFactory.client(any())).thenReturn(client); @@ -220,8 +227,7 @@ void tearDown() throws Exception { void shouldGetExactNodeImmediatelyWithAdvertisedHostNameClientFactoryAndExactNodeOnFirstTime() { ClientFactory cf = context -> - Utils.connectToAdvertisedNodeClientFactory( - context.key(), clientFactory, Duration.ofMillis(1)) + Utils.connectToAdvertisedNodeClientFactory(clientFactory, Duration.ofMillis(1)) .client(context); ConsumersCoordinator c = new ConsumersCoordinator( @@ -229,7 +235,8 @@ void shouldGetExactNodeImmediatelyWithAdvertisedHostNameClientFactoryAndExactNod ConsumersCoordinator.MAX_SUBSCRIPTIONS_PER_CLIENT, type -> "consumer-connection", cf, - false); + false, + brokerPicker()); when(locator.metadata("stream")).thenReturn(metadata(null, replica())); when(clientFactory.client(any())).thenReturn(client); @@ -258,6 +265,48 @@ void shouldGetExactNodeImmediatelyWithAdvertisedHostNameClientFactoryAndExactNod .subscribe(anyByte(), anyString(), any(OffsetSpecification.class), anyInt(), anyMap()); } + @Test + void shouldAcceptCandidateNode() { + ClientFactory cf = + context -> + Utils.connectToAdvertisedNodeClientFactory(clientFactory, Duration.ofMillis(1)) + .client(context); + ConsumersCoordinator c = + new ConsumersCoordinator( + environment, + ConsumersCoordinator.MAX_SUBSCRIPTIONS_PER_CLIENT, + type -> "consumer-connection", + cf, + false, + brokers -> brokers.get(0)); + + when(locator.metadata("stream")).thenReturn(metadata(null, replicas())); + when(clientFactory.client(any())).thenReturn(client); + when(client.subscribe( + subscriptionIdCaptor.capture(), + anyString(), + any(OffsetSpecification.class), + anyInt(), + anyMap())) + .thenReturn(new Client.Response(Constants.RESPONSE_CODE_OK)); + when(client.serverAdvertisedHost()).thenReturn("foo").thenReturn(replicas().get(1).getHost()); + when(client.serverAdvertisedPort()).thenReturn(42).thenReturn(replicas().get(1).getPort()); + + c.subscribe( + consumer, + "stream", + OffsetSpecification.first(), + null, + NO_OP_SUBSCRIPTION_LISTENER, + NO_OP_TRACKING_CLOSING_CALLBACK, + (offset, message) -> {}, + Collections.emptyMap(), + flowStrategy()); + verify(clientFactory, times(2)).client(any()); + verify(client, times(1)) + .subscribe(anyByte(), anyString(), any(OffsetSpecification.class), anyInt(), anyMap()); + } + @Test @SuppressWarnings("unchecked") void shouldSubscribeWithEmptyPropertiesWithUnamedConsumer() { @@ -395,17 +444,58 @@ void subscribeShouldThrowExceptionIfNoNodeAvailableForStream() { } @Test - void findBrokersForStreamShouldReturnLeaderIfNoReplicas() { + void findCandidateNodesShouldReturnLeaderIfNoReplicas() { when(locator.metadata("stream")).thenReturn(metadata(leader(), null)); - assertThat(coordinator.findBrokersForStream("stream", false)).hasSize(1).contains(leader()); + assertThat(coordinator.findCandidateNodes("stream", false)) + .hasSize(1) + .contains(leaderWrapper()); } @Test - void findBrokersForStreamShouldReturnReplicasIfThereAreSome() { + void findCandidateNodesShouldReturnReplicasIfThereAreSome() { when(locator.metadata("stream")).thenReturn(metadata(null, replicas())); - assertThat(coordinator.findBrokersForStream("stream", false)) + assertThat(coordinator.findCandidateNodes("stream", false)) + .hasSize(2) + .hasSameElementsAs(replicaWrappers()); + } + + @Test + void findCandidateNodesShouldReturnLeaderAndReplicasIfForceReplicaIsFalse() { + when(locator.metadata("stream")).thenReturn(metadata(leader(), replicas())); + assertThat(coordinator.findCandidateNodes("stream", false)) + .hasSize(3) + .contains(leaderWrapper()) + .containsAll(replicaWrappers()); + } + + @Test + void findCandidateNodesShouldReturnOnlyReplicasIfForceReplicaIsTrue() { + when(locator.metadata("stream")).thenReturn(metadata(leader(), replicas())); + assertThat(coordinator.findCandidateNodes("stream", true)) .hasSize(2) - .hasSameElementsAs(replicas()); + .containsAll(replicaWrappers()); + } + + @Test + void pickBrokerShouldPreferReplicas() { + Client.Broker leader = leader(); + List replicas = replicas(); + AtomicInteger sequence = new AtomicInteger(); + Function, Client.Broker> picker = + candidates -> candidates.get(sequence.getAndIncrement() % candidates.size()); + // never pick a leader if replicas are available + List leaderAndOneReplica = + Arrays.asList(leaderWrapper(), replicaWrappers().get(0)); + range(0, 10) + .forEach( + ignored -> { + Client.Broker picked = pickBroker(picker, nodeWrappers()); + assertThat(picked).isNotEqualTo(leader).isIn(replicas); + picked = pickBroker(picker, leaderAndOneReplica); + assertThat(picked).isNotEqualTo(leader).isIn(replicas); + }); + // pick the leader if it is the only one + assertThat(pickBroker(picker, singletonList(leaderWrapper()))).isEqualTo(leader); } @Test @@ -494,6 +584,7 @@ void shouldNotUnsubscribeIfClientIsClosed() { @Test void subscribeShouldSubscribeToStreamAndDispatchMessageWithManySubscriptions() { when(locator.metadata("stream")).thenReturn(metadata(leader(), null)); + clientAdvertises(leader()); when(clientFactory.client(any())).thenReturn(client); when(client.subscribe( @@ -574,7 +665,7 @@ void ignoredMessageShouldTriggerMessageProcessing() { new ConsumerFlowStrategy() { @Override public int initialCredits() { - return 1; + return 10; } @Override @@ -1077,6 +1168,7 @@ void metadataUpdate_shouldCloseConsumerIfRetryTimeoutIsReached() throws Exceptio void shouldUseNewClientsForMoreThanMaxSubscriptionsAndCloseClientAfterUnsubscriptions( int maxConsumersByConnection) { when(locator.metadata("stream")).thenReturn(metadata(leader(), null)); + clientAdvertises(leader()); when(clientFactory.client(any())).thenReturn(client); @@ -1098,10 +1190,11 @@ void shouldUseNewClientsForMoreThanMaxSubscriptionsAndCloseClientAfterUnsubscrip maxConsumersByConnection, type -> "consumer-connection", clientFactory, - false); + false, + brokerPicker()); List closingRunnables = - IntStream.range(0, subscriptionCount) + range(0, subscriptionCount) .mapToObj( i -> coordinator.subscribe( @@ -1114,7 +1207,7 @@ void shouldUseNewClientsForMoreThanMaxSubscriptionsAndCloseClientAfterUnsubscrip (offset, message) -> {}, Collections.emptyMap(), flowStrategy())) - .collect(Collectors.toList()); + .collect(toList()); verify(clientFactory, times(2)).client(any()); verify(client, times(subscriptionCount)) @@ -1162,7 +1255,7 @@ void shouldRemoveClientSubscriptionManagerFromPoolAfterConnectionDies() throws E int extraSubscriptionCount = ConsumersCoordinator.MAX_SUBSCRIPTIONS_PER_CLIENT / 5; int subscriptionCount = ConsumersCoordinator.MAX_SUBSCRIPTIONS_PER_CLIENT + extraSubscriptionCount; - IntStream.range(0, subscriptionCount) + range(0, subscriptionCount) .forEach( i -> { coordinator.subscribe( @@ -1227,7 +1320,7 @@ void shouldRemoveClientSubscriptionManagerFromPoolIfEmptyAfterMetadataUpdate() t int extraSubscriptionCount = ConsumersCoordinator.MAX_SUBSCRIPTIONS_PER_CLIENT / 5; int subscriptionCount = ConsumersCoordinator.MAX_SUBSCRIPTIONS_PER_CLIENT + extraSubscriptionCount; - IntStream.range(0, subscriptionCount) + range(0, subscriptionCount) .forEach( i -> { coordinator.subscribe( @@ -1711,6 +1804,58 @@ void shouldRetryAssignmentOnRecoveryCandidateLookupFailure() throws Exception { verify(locator, times(4)).metadata("stream"); } + @Test + @SuppressWarnings("unchecked") + void shouldRetryAssignmentOnRecoveryConnectionTimeout() throws Exception { + scheduledExecutorService = createScheduledExecutorService(2); + when(environment.scheduledExecutorService()).thenReturn(scheduledExecutorService); + Duration retryDelay = Duration.ofMillis(100); + when(environment.recoveryBackOffDelayPolicy()).thenReturn(BackOffDelayPolicy.fixed(retryDelay)); + when(environment.topologyUpdateBackOffDelayPolicy()) + .thenReturn(BackOffDelayPolicy.fixed(retryDelay)); + when(consumer.isOpen()).thenReturn(true); + when(locator.metadata("stream")).thenReturn(metadata("stream", null, replicas())); + + when(clientFactory.client(any())) + .thenReturn(client) + .thenThrow(new TimeoutStreamException("", new ConnectTimeoutException())) + .thenReturn(client); + + AtomicInteger subscriptionCount = new AtomicInteger(0); + when(client.subscribe( + subscriptionIdCaptor.capture(), + anyString(), + any(OffsetSpecification.class), + anyInt(), + anyMap())) + .thenAnswer( + invocation -> { + subscriptionCount.incrementAndGet(); + return responseOk(); + }); + + coordinator.subscribe( + consumer, + "stream", + null, + null, + NO_OP_SUBSCRIPTION_LISTENER, + NO_OP_TRACKING_CLOSING_CALLBACK, + (offset, message) -> {}, + Collections.emptyMap(), + flowStrategy()); + verify(clientFactory, times(1)).client(any()); + verify(client, times(1)) + .subscribe(anyByte(), anyString(), any(OffsetSpecification.class), anyInt(), anyMap()); + + this.shutdownListener.handle( + new Client.ShutdownContext(Client.ShutdownContext.ShutdownReason.UNKNOWN)); + + waitAtMost(() -> subscriptionCount.get() == 1 + 1); + + verify(locator, times(3)).metadata("stream"); + } + @Test void subscribeUnsubscribeInDifferentThreadsShouldNotDeadlock() { when(locator.metadata("stream")).thenReturn(metadata(null, replicas())); @@ -1866,7 +2011,8 @@ void shouldRetryUntilReplicaIsAvailableWhenForceReplicaIsOn() throws Exception { ConsumersCoordinator.MAX_SUBSCRIPTIONS_PER_CLIENT, type -> "consumer-connection", clientFactory, - true); + true, + brokerPicker()); AtomicInteger messageHandlerCalls = new AtomicInteger(); Runnable closingRunnable = @@ -1937,7 +2083,7 @@ void shouldRetryUntilReplicaIsAvailableWhenForceReplicaIsOn() throws Exception { @Test void pickSlotTest() { List list = new ArrayList<>(ConsumersCoordinator.MAX_SUBSCRIPTIONS_PER_CLIENT); - IntStream.range(0, MAX_SUBSCRIPTIONS_PER_CLIENT).forEach(ignored -> list.add(null)); + range(0, MAX_SUBSCRIPTIONS_PER_CLIENT).forEach(ignored -> list.add(null)); AtomicInteger sequence = new AtomicInteger(0); int index = pickSlot(list, sequence); assertThat(index).isZero(); @@ -1975,14 +2121,28 @@ void pickSlotTest() { assertThat(index).isEqualTo(5); } - Client.Broker leader() { + static Client.Broker leader() { return new Client.Broker("leader", -1); } - List replicas() { + static Utils.BrokerWrapper leaderWrapper() { + return new Utils.BrokerWrapper(leader(), true); + } + + static List replicas() { return Arrays.asList(new Client.Broker("replica1", -1), new Client.Broker("replica2", -1)); } + static List nodeWrappers() { + List wrappers = new ArrayList<>(replicaWrappers()); + wrappers.add(leaderWrapper()); + return wrappers; + } + + static List replicaWrappers() { + return replicas().stream().map(b -> new Utils.BrokerWrapper(b, false)).collect(toList()); + } + List replica() { return replicas().subList(0, 1); } @@ -2013,4 +2173,9 @@ private static Response responseOk() { private static ConsumerFlowStrategy flowStrategy() { return ConsumerFlowStrategy.creditOnChunkArrival(10); } + + private void clientAdvertises(Client.Broker broker) { + when(client.serverAdvertisedHost()).thenReturn(broker.getHost()); + when(client.serverAdvertisedPort()).thenReturn(broker.getPort()); + } } diff --git a/src/test/java/com/rabbitmq/stream/impl/DefaultExecutorServiceFactoryTest.java b/src/test/java/com/rabbitmq/stream/impl/DefaultExecutorServiceFactoryTest.java index 853e7fa5ce..ad187b0f28 100644 --- a/src/test/java/com/rabbitmq/stream/impl/DefaultExecutorServiceFactoryTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/DefaultExecutorServiceFactoryTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/DeliveryTest.java b/src/test/java/com/rabbitmq/stream/impl/DeliveryTest.java index 8371940a0e..02844308dd 100644 --- a/src/test/java/com/rabbitmq/stream/impl/DeliveryTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/DeliveryTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/DynamicBatchTest.java b/src/test/java/com/rabbitmq/stream/impl/DynamicBatchTest.java new file mode 100644 index 0000000000..dca9810762 --- /dev/null +++ b/src/test/java/com/rabbitmq/stream/impl/DynamicBatchTest.java @@ -0,0 +1,121 @@ +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// +// This software, the RabbitMQ Stream Java client library, is dual-licensed under the +// Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). +// For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL, +// please see LICENSE-APACHE2. +// +// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, +// either express or implied. See the LICENSE file for specific language governing +// rights and limitations of this software. +// +// If you have any questions regarding licensing, please contact us at +// info@rabbitmq.com. +package com.rabbitmq.stream.impl; + +import static com.rabbitmq.stream.impl.Assertions.assertThat; +import static com.rabbitmq.stream.impl.TestUtils.sync; +import static com.rabbitmq.stream.impl.TestUtils.waitAtMost; + +import com.codahale.metrics.*; +import com.google.common.util.concurrent.RateLimiter; +import com.rabbitmq.stream.impl.TestUtils.Sync; +import java.util.Locale; +import java.util.Random; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.IntStream; +import org.junit.jupiter.api.Test; + +public class DynamicBatchTest { + + private static void simulateActivity(long duration) { + try { + Thread.sleep(duration); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + private static void printHistogram(Histogram histogram) { + Locale locale = Locale.getDefault(); + System.out.printf(locale, " count = %d%n", histogram.getCount()); + Snapshot snapshot = histogram.getSnapshot(); + System.out.printf(locale, " min = %d%n", snapshot.getMin()); + System.out.printf(locale, " max = %d%n", snapshot.getMax()); + System.out.printf(locale, " mean = %2.2f%n", snapshot.getMean()); + System.out.printf(locale, " stddev = %2.2f%n", snapshot.getStdDev()); + System.out.printf(locale, " median = %2.2f%n", snapshot.getMedian()); + System.out.printf(locale, " 75%% <= %2.2f%n", snapshot.get75thPercentile()); + System.out.printf(locale, " 95%% <= %2.2f%n", snapshot.get95thPercentile()); + System.out.printf(locale, " 98%% <= %2.2f%n", snapshot.get98thPercentile()); + System.out.printf(locale, " 99%% <= %2.2f%n", snapshot.get99thPercentile()); + System.out.printf(locale, " 99.9%% <= %2.2f%n", snapshot.get999thPercentile()); + } + + @Test + void itemAreProcessed() { + MetricRegistry metrics = new MetricRegistry(); + Histogram batchSizeMetrics = metrics.histogram("batch-size"); + int itemCount = 3000; + Sync sync = sync(itemCount); + Random random = new Random(); + DynamicBatch.BatchConsumer action = + items -> { + batchSizeMetrics.update(items.size()); + simulateActivity(random.nextInt(10) + 1); + sync.down(items.size()); + return true; + }; + try (DynamicBatch batch = new DynamicBatch<>(action, 100)) { + RateLimiter rateLimiter = RateLimiter.create(10000); + IntStream.range(0, itemCount) + .forEach( + i -> { + rateLimiter.acquire(); + batch.add(String.valueOf(i)); + }); + assertThat(sync).completes(); + // printHistogram(batchSizeMetrics); + } + } + + @Test + void failedProcessingIsReplayed() throws Exception { + int itemCount = 10000; + AtomicInteger collected = new AtomicInteger(0); + AtomicInteger processed = new AtomicInteger(0); + AtomicBoolean canProcess = new AtomicBoolean(true); + DynamicBatch.BatchConsumer action = + items -> { + boolean result; + if (canProcess.get()) { + collected.addAndGet(items.size()); + processed.addAndGet(items.size()); + result = true; + } else { + result = false; + } + return result; + }; + try (DynamicBatch batch = new DynamicBatch<>(action, 100)) { + int firstRoundCount = itemCount / 5; + IntStream.range(0, firstRoundCount) + .forEach( + i -> { + batch.add(String.valueOf(i)); + }); + waitAtMost(() -> processed.get() == firstRoundCount); + canProcess.set(false); + IntStream.range(firstRoundCount, itemCount) + .forEach( + i -> { + batch.add(String.valueOf(i)); + }); + canProcess.set(true); + waitAtMost(() -> processed.get() == itemCount); + waitAtMost(() -> collected.get() == itemCount); + } + } +} diff --git a/src/test/java/com/rabbitmq/stream/impl/FilteringTest.java b/src/test/java/com/rabbitmq/stream/impl/FilteringTest.java index 2b8ebbb638..4118b78b16 100644 --- a/src/test/java/com/rabbitmq/stream/impl/FilteringTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/FilteringTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/FrameTest.java b/src/test/java/com/rabbitmq/stream/impl/FrameTest.java index 1c12a35557..28422d40ad 100644 --- a/src/test/java/com/rabbitmq/stream/impl/FrameTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/FrameTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/HashRoutingStrategyTest.java b/src/test/java/com/rabbitmq/stream/impl/HashRoutingStrategyTest.java index 56ce9691f8..29799137e3 100644 --- a/src/test/java/com/rabbitmq/stream/impl/HashRoutingStrategyTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/HashRoutingStrategyTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2022-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2022-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/HashUtilsTest.java b/src/test/java/com/rabbitmq/stream/impl/HashUtilsTest.java index a1fa9c9b79..5b9c5cb310 100644 --- a/src/test/java/com/rabbitmq/stream/impl/HashUtilsTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/HashUtilsTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/HeartbeatTest.java b/src/test/java/com/rabbitmq/stream/impl/HeartbeatTest.java index 2618b21fb7..c9d525fd15 100644 --- a/src/test/java/com/rabbitmq/stream/impl/HeartbeatTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/HeartbeatTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/JdkChunkChecksumTest.java b/src/test/java/com/rabbitmq/stream/impl/JdkChunkChecksumTest.java index 534ed59446..fcce8cc29d 100644 --- a/src/test/java/com/rabbitmq/stream/impl/JdkChunkChecksumTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/JdkChunkChecksumTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -35,10 +35,10 @@ public class JdkChunkChecksumTest { static Charset UTF8 = StandardCharsets.UTF_8; static Map> CHECKSUMS = - new HashMap>() { + new HashMap<>() { { - put("crc32", () -> new CRC32()); - put("adler32", () -> new Adler32()); + put("crc32", CRC32::new); + put("adler32", Adler32::new); } }; diff --git a/src/test/java/com/rabbitmq/stream/impl/LoadBalancerClusterTest.java b/src/test/java/com/rabbitmq/stream/impl/LoadBalancerClusterTest.java new file mode 100644 index 0000000000..3c2adaf58e --- /dev/null +++ b/src/test/java/com/rabbitmq/stream/impl/LoadBalancerClusterTest.java @@ -0,0 +1,251 @@ +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// +// This software, the RabbitMQ Stream Java client library, is dual-licensed under the +// Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). +// For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL, +// please see LICENSE-APACHE2. +// +// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, +// either express or implied. See the LICENSE file for specific language governing +// rights and limitations of this software. +// +// If you have any questions regarding licensing, please contact us at +// info@rabbitmq.com. +package com.rabbitmq.stream.impl; + +import static com.rabbitmq.stream.impl.Assertions.assertThat; +import static com.rabbitmq.stream.impl.TestUtils.waitAtMost; +import static java.lang.Integer.parseInt; +import static java.util.stream.Collectors.toSet; +import static java.util.stream.IntStream.range; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +import com.rabbitmq.stream.*; +import com.rabbitmq.stream.impl.Client.Broker; +import com.rabbitmq.stream.impl.MonitoringTestUtils.ConsumerCoordinatorInfo; +import com.rabbitmq.stream.impl.MonitoringTestUtils.EnvironmentInfo; +import com.rabbitmq.stream.impl.MonitoringTestUtils.ProducersCoordinatorInfo; +import com.rabbitmq.stream.impl.TestUtils.DisabledIfNotCluster; +import io.netty.channel.EventLoopGroup; +import java.time.Duration; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +@DisabledIfNotCluster +@StreamTestInfrastructure +public class LoadBalancerClusterTest { + + private static final int LB_PORT = 5555; + private static final SubscriptionListener NO_OP_SUBSCRIPTION_LISTENER = subscriptionContext -> {}; + private static final Runnable NO_OP_TRACKING_CLOSING_CALLBACK = () -> {}; + + @Mock StreamEnvironment environment; + @Mock StreamConsumer consumer; + @Mock StreamProducer producer; + AutoCloseable mocks; + TestUtils.ClientFactory cf; + String stream; + EventLoopGroup eventLoopGroup; + Client locator; + static final Address LOAD_BALANCER_ADDRESS = new Address("localhost", LB_PORT); + + @BeforeEach + void init() { + mocks = MockitoAnnotations.openMocks(this); + locator = cf.get(new Client.ClientParameters().port(LB_PORT)); + StreamEnvironment.Locator l = new StreamEnvironment.Locator(-1, new Address("localhost", 5555)); + l.client(locator); + when(environment.locator()).thenReturn(l); + when(environment.clientParametersCopy()) + .thenReturn(new Client.ClientParameters().eventLoopGroup(eventLoopGroup).port(LB_PORT)); + when(environment.addressResolver()).thenReturn(address -> LOAD_BALANCER_ADDRESS); + when(environment.locatorOperation(any())).thenCallRealMethod(); + } + + @AfterEach + void tearDown() throws Exception { + mocks.close(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void pickConsumersAmongCandidates(boolean forceReplica) throws Exception { + int maxSubscriptionsPerClient = 2; + int subscriptionCount = maxSubscriptionsPerClient * 10; + try (ConsumersCoordinator c = + new ConsumersCoordinator( + environment, + maxSubscriptionsPerClient, + type -> "consumer-connection", + Utils.coordinatorClientFactory(this.environment, Duration.ofMillis(10)), + forceReplica, + Utils.brokerPicker())) { + + waitAtMost( + () -> locator.metadata(stream).get(stream).hasReplicas(), + "Stream '%s' should have replicas"); + + range(0, subscriptionCount) + .forEach( + ignored -> { + c.subscribe( + consumer, + stream, + OffsetSpecification.first(), + null, + NO_OP_SUBSCRIPTION_LISTENER, + NO_OP_TRACKING_CLOSING_CALLBACK, + (offset, message) -> {}, + Collections.emptyMap(), + flowStrategy()); + }); + + Client.StreamMetadata metadata = locator.metadata(stream).get(stream); + Set allowedNodes = new HashSet<>(metadata.getReplicas()); + if (!forceReplica) { + allowedNodes.add(metadata.getLeader()); + } + + ConsumerCoordinatorInfo info = MonitoringTestUtils.extract(c); + assertThat(info.consumerCount()).isEqualTo(subscriptionCount); + Set usedNodes = + info.clients().stream() + .map(m -> m.node().split(":")) + .map(np -> new Broker(np[0], parseInt(np[1]))) + .collect(toSet()); + assertThat(usedNodes).hasSameSizeAs(allowedNodes).containsAll(allowedNodes); + } + } + + @Test + void pickProducersAmongCandidatesIfInstructed() { + boolean forceLeader = true; + when(consumer.stream()).thenReturn(stream); + int maxAgentPerClient = 2; + int agentCount = maxAgentPerClient * 10; + try (ProducersCoordinator c = + new ProducersCoordinator( + environment, + maxAgentPerClient, + maxAgentPerClient, + type -> "producer-connection", + Utils.coordinatorClientFactory(this.environment, Duration.ofMillis(10)), + forceLeader)) { + + range(0, agentCount) + .forEach( + ignored -> { + c.registerProducer(producer, null, stream); + c.registerTrackingConsumer(consumer); + }); + + Client.StreamMetadata metadata = locator.metadata(stream).get(stream); + Set allowedNodes = new HashSet<>(Collections.singleton(metadata.getLeader())); + if (!forceLeader) { + allowedNodes.addAll(metadata.getReplicas()); + } + + ProducersCoordinatorInfo info = MonitoringTestUtils.extract(c); + assertThat(info.producerCount()).isEqualTo(agentCount); + assertThat(info.trackingConsumerCount()).isEqualTo(agentCount); + Set usedNodes = + info.nodesConnected().stream() + .map(n -> n.split(":")) + .map(np -> new Broker(np[0], parseInt(np[1]))) + .collect(toSet()); + assertThat(usedNodes).hasSameSizeAs(allowedNodes).containsAll(allowedNodes); + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void producersConsumersShouldSpreadAccordingToDataLocalitySettings(boolean forceLocality) { + int maxPerConnection = 2; + int agentCount = maxPerConnection * 20; + StreamEnvironmentBuilder builder = (StreamEnvironmentBuilder) Environment.builder(); + builder + .producerNodeRetryDelay(Duration.ofMillis(10)) + .consumerNodeRetryDelay(Duration.ofMillis(10)); + try (Environment env = + builder + .port(LB_PORT) + .forceReplicaForConsumers(forceLocality) + .forceReplicaForConsumers(forceLocality) + .addressResolver(addr -> LOAD_BALANCER_ADDRESS) + .maxProducersByConnection(maxPerConnection) + .maxConsumersByConnection(maxPerConnection) + .forceLeaderForProducers(forceLocality) + .netty() + .eventLoopGroup(eventLoopGroup) + .environmentBuilder() + .build()) { + TestUtils.Sync consumeSync = TestUtils.sync(agentCount * agentCount); + Set consumersThatReceived = ConcurrentHashMap.newKeySet(agentCount); + List producers = new ArrayList<>(); + range(0, agentCount) + .forEach( + index -> { + producers.add(env.producerBuilder().stream(stream).build()); + env.consumerBuilder().stream(stream) + .messageHandler( + (ctx, msg) -> { + consumersThatReceived.add(index); + consumeSync.down(); + }) + .offset(OffsetSpecification.first()) + .build(); + }); + producers.forEach(p -> p.send(p.messageBuilder().build(), ctx -> {})); + assertThat(consumeSync).completes(); + assertThat(consumersThatReceived).containsAll(range(0, agentCount).boxed().collect(toSet())); + + EnvironmentInfo info = MonitoringTestUtils.extract(env); + ProducersCoordinatorInfo producerInfo = info.getProducers(); + ConsumerCoordinatorInfo consumerInfo = info.getConsumers(); + + assertThat(producerInfo.producerCount()).isEqualTo(agentCount); + assertThat(consumerInfo.consumerCount()).isEqualTo(agentCount); + + Client.StreamMetadata metadata = locator.metadata(stream).get(stream); + + Function, Set> toBrokers = + nodes -> + nodes.stream() + .map(n -> n.split(":")) + .map(n -> new Broker(n[0], parseInt(n[1]))) + .collect(toSet()); + Set usedNodes = toBrokers.apply(producerInfo.nodesConnected()); + assertThat(usedNodes).contains(metadata.getLeader()); + if (forceLocality) { + assertThat(usedNodes).hasSize(1); + } else { + assertThat(usedNodes).hasSize(metadata.getReplicas().size() + 1); + assertThat(usedNodes).containsAll(metadata.getReplicas()); + } + + usedNodes = toBrokers.apply(consumerInfo.nodesConnected()); + assertThat(usedNodes).containsAll(metadata.getReplicas()); + if (forceLocality) { + assertThat(usedNodes).hasSameSizeAs(metadata.getReplicas()); + } else { + assertThat(usedNodes).hasSize(metadata.getReplicas().size() + 1); + assertThat(usedNodes).contains(metadata.getLeader()); + } + } + } + + private static ConsumerFlowStrategy flowStrategy() { + return ConsumerFlowStrategy.creditOnChunkArrival(10); + } +} diff --git a/src/test/java/com/rabbitmq/stream/impl/MessageCountConsumerFlowStrategyTest.java b/src/test/java/com/rabbitmq/stream/impl/MessageCountConsumerFlowStrategyTest.java index 81f5c81f51..ea959a1903 100644 --- a/src/test/java/com/rabbitmq/stream/impl/MessageCountConsumerFlowStrategyTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/MessageCountConsumerFlowStrategyTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2023-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -55,7 +55,7 @@ void smallChunksAndSmallRatiosShouldCredit() { } ConsumerFlowStrategy build(double ratio) { - return creditOnProcessedMessageCount(1, ratio); + return creditOnProcessedMessageCount(10, ratio); } ConsumerFlowStrategy.Context context(long messageCount) { diff --git a/src/test/java/com/rabbitmq/stream/impl/MetadataTest.java b/src/test/java/com/rabbitmq/stream/impl/MetadataTest.java index d1633caac9..de02f23093 100644 --- a/src/test/java/com/rabbitmq/stream/impl/MetadataTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/MetadataTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -19,12 +19,11 @@ import static java.util.Arrays.asList; import static org.assertj.core.api.Assertions.assertThat; +import com.rabbitmq.stream.Cli; import com.rabbitmq.stream.Constants; -import com.rabbitmq.stream.Host; import com.rabbitmq.stream.impl.Client.Broker; import com.rabbitmq.stream.impl.TestUtils.BrokerVersion; import com.rabbitmq.stream.impl.TestUtils.BrokerVersionAtLeast; -import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.*; @@ -143,7 +142,7 @@ void metadataExistingNonExistingStreams(int existingCount, int nonExistingCount, } static void checkHost(Broker broker) { - if (!Host.isOnDocker()) { + if (!Cli.isOnDocker()) { assertThat(broker.getHost()).isEqualTo(hostname()); } } @@ -152,11 +151,7 @@ static String hostname() { try { return InetAddress.getLocalHost().getHostName(); } catch (UnknownHostException e) { - try { - return Host.hostname(); - } catch (IOException ex) { - throw new RuntimeException(ex); - } + return Cli.hostname(); } } @@ -166,9 +161,9 @@ void shouldFilterOutNodesInMaintenance() throws Exception { Client client = cf.get(); BooleanSupplier hasLeader = () -> client.metadata(stream).get(stream).getLeader() != null; waitAtMost(() -> hasLeader.getAsBoolean()); - Host.rabbitmqctl("eval 'rabbit_maintenance:drain().'"); + Cli.rabbitmqctl("eval 'rabbit_maintenance:drain().'"); waitAtMost(() -> !hasLeader.getAsBoolean()); - Host.rabbitmqctl("eval 'rabbit_maintenance:revive().'"); + Cli.rabbitmqctl("eval 'rabbit_maintenance:revive().'"); waitAtMost(() -> hasLeader.getAsBoolean()); } } diff --git a/src/test/java/com/rabbitmq/stream/impl/MetricsCollectionTest.java b/src/test/java/com/rabbitmq/stream/impl/MetricsCollectionTest.java index 374d0472a2..3bbd39d801 100644 --- a/src/test/java/com/rabbitmq/stream/impl/MetricsCollectionTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/MetricsCollectionTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/MonitoringTestUtils.java b/src/test/java/com/rabbitmq/stream/impl/MonitoringTestUtils.java index 7f45b1fec2..58b6e4bf07 100644 --- a/src/test/java/com/rabbitmq/stream/impl/MonitoringTestUtils.java +++ b/src/test/java/com/rabbitmq/stream/impl/MonitoringTestUtils.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -130,7 +130,11 @@ public ConsumerManager(long id, String node, int consumer_count) { } public int getConsumerCount() { - return consumer_count; + return this.consumer_count; + } + + public String node() { + return this.node; } @Override diff --git a/src/test/java/com/rabbitmq/stream/impl/MqttInteroperabilityTest.java b/src/test/java/com/rabbitmq/stream/impl/MqttInteroperabilityTest.java index 797ca242c2..9ee3e5b30c 100644 --- a/src/test/java/com/rabbitmq/stream/impl/MqttInteroperabilityTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/MqttInteroperabilityTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2021-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2021-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -27,7 +27,6 @@ import com.rabbitmq.stream.OffsetSpecification; import com.rabbitmq.stream.amqp.UnsignedByte; import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; import java.nio.charset.StandardCharsets; import java.util.UUID; import java.util.concurrent.CountDownLatch; @@ -57,7 +56,7 @@ public class MqttInteroperabilityTest { @BeforeAll static void initAll() { - eventLoopGroup = new NioEventLoopGroup(); + eventLoopGroup = Utils.eventLoopGroup(); } @AfterAll diff --git a/src/test/java/com/rabbitmq/stream/impl/NotificationTest.java b/src/test/java/com/rabbitmq/stream/impl/NotificationTest.java index f3073d0490..189aba2b02 100644 --- a/src/test/java/com/rabbitmq/stream/impl/NotificationTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/NotificationTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/OffsetTest.java b/src/test/java/com/rabbitmq/stream/impl/OffsetTest.java index 0057925ce2..34dc697141 100644 --- a/src/test/java/com/rabbitmq/stream/impl/OffsetTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/OffsetTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/OffsetTrackingCoordinatorTest.java b/src/test/java/com/rabbitmq/stream/impl/OffsetTrackingCoordinatorTest.java index 54138ac96c..4e116f63ea 100644 --- a/src/test/java/com/rabbitmq/stream/impl/OffsetTrackingCoordinatorTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/OffsetTrackingCoordinatorTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/OffsetTrackingTest.java b/src/test/java/com/rabbitmq/stream/impl/OffsetTrackingTest.java index e1b73f56f2..8649dd0e15 100644 --- a/src/test/java/com/rabbitmq/stream/impl/OffsetTrackingTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/OffsetTrackingTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/OutboundMappingCallbackTest.java b/src/test/java/com/rabbitmq/stream/impl/OutboundMappingCallbackTest.java index c0865c5b49..d424070a5d 100644 --- a/src/test/java/com/rabbitmq/stream/impl/OutboundMappingCallbackTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/OutboundMappingCallbackTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/ProducersCoordinatorTest.java b/src/test/java/com/rabbitmq/stream/impl/ProducersCoordinatorTest.java index 68ed0cd4dd..35aa7aeedb 100644 --- a/src/test/java/com/rabbitmq/stream/impl/ProducersCoordinatorTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/ProducersCoordinatorTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -19,6 +19,7 @@ import static com.rabbitmq.stream.impl.TestUtils.CountDownLatchConditions.completed; import static com.rabbitmq.stream.impl.TestUtils.answer; import static com.rabbitmq.stream.impl.TestUtils.metadata; +import static java.util.stream.Collectors.toList; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; @@ -32,11 +33,13 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.rabbitmq.stream.Address; import com.rabbitmq.stream.BackOffDelayPolicy; import com.rabbitmq.stream.Constants; import com.rabbitmq.stream.StreamDoesNotExistException; import com.rabbitmq.stream.impl.Client.Response; import com.rabbitmq.stream.impl.Utils.ClientFactory; +import io.netty.channel.ConnectTimeoutException; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -78,6 +81,10 @@ static Client.Broker leader() { return new Client.Broker("leader", 5552); } + static Utils.BrokerWrapper leaderWrapper() { + return new Utils.BrokerWrapper(leader(), true); + } + static Client.Broker leader1() { return new Client.Broker("leader-1", 5552); } @@ -90,6 +97,10 @@ static List replicas() { return Arrays.asList(new Client.Broker("replica1", 5552), new Client.Broker("replica2", 5552)); } + static List replicaWrappers() { + return replicas().stream().map(b -> new Utils.BrokerWrapper(b, false)).collect(toList()); + } + @BeforeEach void init() { Client.ClientParameters clientParameters = @@ -109,20 +120,25 @@ public Client.ClientParameters metadataListener( } }; mocks = MockitoAnnotations.openMocks(this); - when(environment.locator()).thenReturn(locator); + StreamEnvironment.Locator l = new StreamEnvironment.Locator(-1, new Address("localhost", 5555)); + l.client(locator); + when(environment.locator()).thenReturn(l); when(environment.locatorOperation(any())).thenCallRealMethod(); when(environment.clientParametersCopy()).thenReturn(clientParameters); when(environment.addressResolver()).thenReturn(address -> address); when(trackingConsumer.stream()).thenReturn("stream"); when(client.declarePublisher(anyByte(), isNull(), anyString())) .thenReturn(new Response(Constants.RESPONSE_CODE_OK)); + when(client.serverAdvertisedHost()).thenReturn(leader().getHost()); + when(client.serverAdvertisedPort()).thenReturn(leader().getPort()); coordinator = new ProducersCoordinator( environment, ProducersCoordinator.MAX_PRODUCERS_PER_CLIENT, ProducersCoordinator.MAX_TRACKING_CONSUMERS_PER_CLIENT, type -> "producer-connection", - clientFactory); + clientFactory, + true); when(client.isOpen()).thenReturn(true); when(client.deletePublisher(anyByte())).thenReturn(new Response(Constants.RESPONSE_CODE_OK)); } @@ -183,8 +199,7 @@ void registerShouldAllowPublishing() { shouldRetryUntilGettingExactNodeWithAdvertisedHostNameClientFactoryAndNotExactNodeOnFirstTime() { ClientFactory cf = context -> - Utils.connectToAdvertisedNodeClientFactory( - context.key(), clientFactory, Duration.ofMillis(1)) + Utils.connectToAdvertisedNodeClientFactory(clientFactory, Duration.ofMillis(1)) .client(context); ProducersCoordinator c = new ProducersCoordinator( @@ -192,7 +207,8 @@ void registerShouldAllowPublishing() { ProducersCoordinator.MAX_PRODUCERS_PER_CLIENT, ProducersCoordinator.MAX_TRACKING_CONSUMERS_PER_CLIENT, type -> "producer-connection", - cf); + cf, + true); when(locator.metadata("stream")).thenReturn(metadata(leader(), replicas())); when(clientFactory.client(any())).thenReturn(client); @@ -211,8 +227,7 @@ void registerShouldAllowPublishing() { void shouldGetExactNodeImmediatelyWithAdvertisedHostNameClientFactoryAndExactNodeOnFirstTime() { ClientFactory cf = context -> - Utils.connectToAdvertisedNodeClientFactory( - context.key(), clientFactory, Duration.ofMillis(1)) + Utils.connectToAdvertisedNodeClientFactory(clientFactory, Duration.ofMillis(1)) .client(context); ProducersCoordinator c = new ProducersCoordinator( @@ -220,7 +235,8 @@ void shouldGetExactNodeImmediatelyWithAdvertisedHostNameClientFactoryAndExactNod ProducersCoordinator.MAX_PRODUCERS_PER_CLIENT, ProducersCoordinator.MAX_TRACKING_CONSUMERS_PER_CLIENT, type -> "producer-connection", - cf); + cf, + true); when(locator.metadata("stream")).thenReturn(metadata(leader(), replicas())); when(clientFactory.client(any())).thenReturn(client); @@ -301,6 +317,39 @@ void shouldRedistributeProducerAndTrackingConsumerIfConnectionIsLost() throws Ex assertThat(coordinator.clientCount()).isEqualTo(1); } + @Test + void shouldRecoverOnConnectionTimeout() throws Exception { + scheduledExecutorService = createScheduledExecutorService(); + when(environment.scheduledExecutorService()).thenReturn(scheduledExecutorService); + Duration retryDelay = Duration.ofMillis(50); + when(environment.recoveryBackOffDelayPolicy()).thenReturn(BackOffDelayPolicy.fixed(retryDelay)); + when(locator.metadata("stream")).thenReturn(metadata(leader(), replicas())); + + when(clientFactory.client(any())) + .thenReturn(client) + .thenThrow(new TimeoutStreamException("", new ConnectTimeoutException())) + .thenReturn(client); + + when(producer.isOpen()).thenReturn(true); + + StreamProducer producer = mock(StreamProducer.class); + when(producer.isOpen()).thenReturn(true); + + CountDownLatch runningLatch = new CountDownLatch(1); + doAnswer(answer(runningLatch::countDown)).when(this.producer).running(); + + coordinator.registerProducer(this.producer, null, "stream"); + + verify(this.producer, times(1)).setClient(client); + + shutdownListener.handle( + new Client.ShutdownContext(Client.ShutdownContext.ShutdownReason.UNKNOWN)); + + assertThat(runningLatch.await(5, TimeUnit.SECONDS)).isTrue(); + verify(this.producer, times(1)).unavailable(); + verify(this.producer, times(2)).setClient(client); + } + @Test void shouldDisposeProducerAndNotTrackingConsumerIfRecoveryTimesOut() throws Exception { scheduledExecutorService = createScheduledExecutorService(); @@ -357,6 +406,10 @@ void shouldRedistributeProducersAndTrackingConsumersOnMetadataUpdate() throws Ex .thenReturn(metadata(movingStream, null, replicas())) .thenReturn(metadata(movingStream, leader2(), replicas())); + // the created client is on leader1 + when(client.serverAdvertisedHost()).thenReturn(leader1().getHost()); + when(client.serverAdvertisedPort()).thenReturn(leader1().getPort()); + String fixedStream = "fixed-stream"; when(locator.metadata(fixedStream)).thenReturn(metadata(fixedStream, leader1(), replicas())); @@ -406,6 +459,10 @@ void shouldRedistributeProducersAndTrackingConsumersOnMetadataUpdate() throws Ex verify(fixedTrackingConsumer, times(1)).setTrackingClient(client); assertThat(coordinator.clientCount()).isEqualTo(1); + // the created client is on leader2 + when(client.serverAdvertisedHost()).thenReturn(leader2().getHost()); + when(client.serverAdvertisedPort()).thenReturn(leader2().getPort()); + metadataListener.handle(movingStream, Constants.RESPONSE_CODE_STREAM_NOT_AVAILABLE); assertThat(setClientLatch.await(5, TimeUnit.SECONDS)).isTrue(); @@ -521,7 +578,8 @@ void growShrinkResourcesBasedOnProducersAndTrackingConsumersCount(int maxProduce maxProducersByClient, ProducersCoordinator.MAX_TRACKING_CONSUMERS_PER_CLIENT, type -> "producer-connection", - clientFactory); + clientFactory, + true); class ProducerInfo { StreamProducer producer; @@ -679,6 +737,43 @@ void pickSlotTest() { assertThat(pickSlot(map, "257", sequence)).isEqualTo(5); } + @Test + void findCandidateNodesShouldReturnOnlyLeaderWhenForceLeaderIsTrue() { + when(locator.metadata("stream")).thenReturn(metadata(leader(), replicas())); + assertThat(coordinator.findCandidateNodes("stream", true)).containsOnly(leaderWrapper()); + } + + @Test + void findCandidateNodesShouldReturnLeaderAndReplicasWhenForceLeaderIsFalse() { + when(locator.metadata("stream")).thenReturn(metadata(leader(), replicas())); + assertThat(coordinator.findCandidateNodes("stream", false)) + .hasSize(3) + .contains(leaderWrapper()) + .containsAll(replicaWrappers()); + } + + @Test + void findCandidateNodesShouldThrowIfThereIsNoLeaderAndForceLeaderIsTrue() { + when(locator.metadata("stream")).thenReturn(metadata(null, replicas())); + assertThatThrownBy(() -> coordinator.findCandidateNodes("stream", true)) + .isInstanceOf(IllegalStateException.class); + } + + @Test + void findCandidateNodesShouldThrowIfNoMembersAndForceLeaderIsFalse() { + when(locator.metadata("stream")).thenReturn(metadata(null, List.of())); + assertThatThrownBy(() -> coordinator.findCandidateNodes("stream", false)) + .isInstanceOf(IllegalStateException.class); + } + + @Test + void findCandidateNodesShouldReturnOnlyReplicasIfNoLeaderAndForceLeaderIsFalse() { + when(locator.metadata("stream")).thenReturn(metadata(null, replicas())); + assertThat(coordinator.findCandidateNodes("stream", false)) + .hasSize(2) + .containsAll(replicaWrappers()); + } + private static ScheduledExecutorService createScheduledExecutorService() { return new ScheduledExecutorServiceWrapper(Executors.newSingleThreadScheduledExecutor()); } diff --git a/src/test/java/com/rabbitmq/stream/impl/PublisherTest.java b/src/test/java/com/rabbitmq/stream/impl/PublisherTest.java index d6a31d4b99..1c171713da 100644 --- a/src/test/java/com/rabbitmq/stream/impl/PublisherTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/PublisherTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/RecoveryClusterTest.java b/src/test/java/com/rabbitmq/stream/impl/RecoveryClusterTest.java new file mode 100644 index 0000000000..24f50ee0cd --- /dev/null +++ b/src/test/java/com/rabbitmq/stream/impl/RecoveryClusterTest.java @@ -0,0 +1,417 @@ +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// +// This software, the RabbitMQ Stream Java client library, is dual-licensed under the +// Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). +// For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL, +// please see LICENSE-APACHE2. +// +// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, +// either express or implied. See the LICENSE file for specific language governing +// rights and limitations of this software. +// +// If you have any questions regarding licensing, please contact us at +// info@rabbitmq.com. +package com.rabbitmq.stream.impl; + +import static com.rabbitmq.stream.impl.Assertions.assertThat; +import static com.rabbitmq.stream.impl.LoadBalancerClusterTest.LOAD_BALANCER_ADDRESS; +import static com.rabbitmq.stream.impl.TestUtils.newLoggerLevel; +import static com.rabbitmq.stream.impl.TestUtils.sync; +import static com.rabbitmq.stream.impl.ThreadUtils.threadFactory; +import static com.rabbitmq.stream.impl.Tuples.pair; +import static java.util.stream.Collectors.toList; +import static java.util.stream.IntStream.range; +import static org.assertj.core.api.Assertions.assertThat; + +import ch.qos.logback.classic.Level; +import com.google.common.collect.Streams; +import com.google.common.util.concurrent.RateLimiter; +import com.rabbitmq.stream.*; +import com.rabbitmq.stream.impl.TestUtils.DisabledIfNotCluster; +import com.rabbitmq.stream.impl.TestUtils.Sync; +import com.rabbitmq.stream.impl.Tuples.Pair; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@DisabledIfNotCluster +@StreamTestInfrastructure +public class RecoveryClusterTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(RecoveryClusterTest.class); + + private static final Duration ASSERTION_TIMEOUT = Duration.ofSeconds(20); + // give some slack before first recovery attempt, especially on Docker + static final Duration RECOVERY_INITIAL_DELAY = Duration.ofSeconds(10); + static final Duration RECOVERY_DELAY = Duration.ofSeconds(2); + static List nodes; + static final List URIS = + range(5552, 5555).mapToObj(p -> "rabbitmq-stream://localhost:" + p).collect(toList()); + static final BackOffDelayPolicy BACK_OFF_DELAY_POLICY = + BackOffDelayPolicy.fixedWithInitialDelay(RECOVERY_INITIAL_DELAY, RECOVERY_DELAY); + Environment environment; + TestInfo testInfo; + EventLoopGroup eventLoopGroup; + EnvironmentBuilder environmentBuilder; + static List logLevels; + static List> logClasses = + List.of( + // ProducersCoordinator.class, + // ConsumersCoordinator.class, + AsyncRetry.class, StreamEnvironment.class, ScheduledExecutorServiceWrapper.class); + ScheduledExecutorService scheduledExecutorService; + + @BeforeAll + static void initAll() { + nodes = Cli.nodes(); + logLevels = logClasses.stream().map(c -> newLoggerLevel(c, Level.DEBUG)).collect(toList()); + } + + @BeforeEach + void init(TestInfo info) { + int availableProcessors = Utils.AVAILABLE_PROCESSORS; + LOGGER.info("Available processors: {}", availableProcessors); + ThreadFactory threadFactory = threadFactory("rabbitmq-stream-environment-scheduler-"); + scheduledExecutorService = Executors.newScheduledThreadPool(availableProcessors, threadFactory); + // add some debug log messages + scheduledExecutorService = new ScheduledExecutorServiceWrapper(scheduledExecutorService); + environmentBuilder = + Environment.builder() + .recoveryBackOffDelayPolicy(BACK_OFF_DELAY_POLICY) + .topologyUpdateBackOffDelayPolicy(BACK_OFF_DELAY_POLICY) + .scheduledExecutorService(scheduledExecutorService) + .requestedHeartbeat(Duration.ofSeconds(3)) + .netty() + .eventLoopGroup(eventLoopGroup) + .environmentBuilder(); + this.testInfo = info; + } + + @AfterEach + void tearDown() { + if (environment != null) { + environment.close(); + } + if (scheduledExecutorService != null) { + scheduledExecutorService.shutdownNow(); + } + } + + @AfterAll + static void tearDownAll() { + if (logLevels != null) { + Streams.zip(logClasses.stream(), logLevels.stream(), Tuples::pair) + .forEach(t -> newLoggerLevel(t.v1(), t.v2())); + } + } + + @ParameterizedTest + @CsvSource({ + "false,false", + "true,true", + "true,false", + }) + void clusterRestart(boolean useLoadBalancer, boolean forceLeader) throws InterruptedException { + LOGGER.info( + "Cluster restart test, use load balancer {}, force leader {}", + useLoadBalancer, + forceLeader); + int streamCount = Utils.AVAILABLE_PROCESSORS; + int producerCount = streamCount * 2; + int consumerCount = streamCount * 2; + + if (useLoadBalancer) { + environmentBuilder + .host(LOAD_BALANCER_ADDRESS.host()) + .port(LOAD_BALANCER_ADDRESS.port()) + .addressResolver(addr -> LOAD_BALANCER_ADDRESS) + .forceLeaderForProducers(forceLeader) + .locatorConnectionCount(URIS.size()); + Duration nodeRetryDelay = Duration.ofMillis(100); + // to make the test faster + ((StreamEnvironmentBuilder) environmentBuilder).producerNodeRetryDelay(nodeRetryDelay); + ((StreamEnvironmentBuilder) environmentBuilder).consumerNodeRetryDelay(nodeRetryDelay); + } else { + environmentBuilder.uris(URIS); + } + + environment = + environmentBuilder + .netty() + .bootstrapCustomizer( + b -> { + b.option( + ChannelOption.CONNECT_TIMEOUT_MILLIS, + (int) BACK_OFF_DELAY_POLICY.delay(0).toMillis()); + }) + .environmentBuilder() + .maxProducersByConnection(producerCount / 4) + .maxConsumersByConnection(consumerCount / 4) + .build(); + List streams = + range(0, streamCount) + .mapToObj(i -> TestUtils.streamName(testInfo) + "-" + i) + .collect(toList()); + streams.forEach(s -> environment.streamCreator().stream(s).create()); + List producers = Collections.emptyList(); + List consumers = Collections.emptyList(); + try { + producers = + range(0, producerCount) + .mapToObj( + i -> { + String s = streams.get(i % streams.size()); + boolean dynamicBatch = i % 2 == 0; + return new ProducerState(s, dynamicBatch, environment); + }) + .collect(toList()); + consumers = + range(0, consumerCount) + .mapToObj( + i -> { + String s = streams.get(i % streams.size()); + return new ConsumerState(s, environment); + }) + .collect(toList()); + + producers.forEach(ProducerState::start); + + List syncs = producers.stream().map(p -> p.waitForNewMessages(100)).collect(toList()); + syncs.forEach(s -> assertThat(s).completes()); + + syncs = consumers.stream().map(c -> c.waitForNewMessages(100)).collect(toList()); + syncs.forEach(s -> assertThat(s).completes()); + + nodes.forEach( + n -> { + LOGGER.info("Restarting node {}...", n); + Cli.restartNode(n); + LOGGER.info("Restarted node {}.", n); + }); + LOGGER.info("Rebalancing..."); + Cli.rebalance(); + LOGGER.info("Rebalancing over."); + + Thread.sleep(BACK_OFF_DELAY_POLICY.delay(0).toMillis()); + + List> streamsSyncs = + producers.stream() + .map(p -> pair(p.stream(), p.waitForNewMessages(1000))) + .collect(toList()); + streamsSyncs.forEach( + p -> { + LOGGER.info("Checking publisher to {} still publishes", p.v1()); + assertThat(p.v2()).completes(ASSERTION_TIMEOUT); + LOGGER.info("Publisher to {} still publishes", p.v1()); + }); + + streamsSyncs = + consumers.stream() + .map(c -> pair(c.stream(), c.waitForNewMessages(1000))) + .collect(toList()); + streamsSyncs.forEach( + p -> { + LOGGER.info("Checking consumer from {} still consumes", p.v1()); + assertThat(p.v2()).completes(ASSERTION_TIMEOUT); + LOGGER.info("Consumer from {} still consumes", p.v1()); + }); + + Map committedChunkIdPerStream = new LinkedHashMap<>(streamCount); + streams.forEach( + s -> + committedChunkIdPerStream.put(s, environment.queryStreamStats(s).committedChunkId())); + + syncs = producers.stream().map(p -> p.waitForNewMessages(1000)).collect(toList()); + syncs.forEach(s -> assertThat(s).completes(ASSERTION_TIMEOUT)); + + streams.forEach( + s -> { + assertThat(environment.queryStreamStats(s).committedChunkId()) + .as("Committed chunk ID did not increase") + .isGreaterThan(committedChunkIdPerStream.get(s)); + }); + + } finally { + LOGGER.info("Environment information:"); + System.out.println(TestUtils.jsonPrettyPrint(environment.toString())); + + LOGGER.info("Producer information:"); + producers.forEach( + p -> { + LOGGER.info("Producer to '{}' (last exception: '{}')", p.stream(), p.lastException); + }); + + LOGGER.info("Closing producers"); + producers.forEach( + p -> { + try { + p.close(); + } catch (Exception e) { + LOGGER.info("Error while closing producer to '{}': {}", p.stream(), e.getMessage()); + } + }); + + LOGGER.info("Stream status..."); + streams.forEach(s -> System.out.println(Cli.streamStatus(s))); + + consumers.forEach( + c -> { + try { + c.close(); + } catch (Exception e) { + LOGGER.info("Error while closing from '{}': {}", c.stream(), e.getMessage()); + } + }); + + LOGGER.info("Deleting streams after test"); + try { + streams.forEach(s -> environment.deleteStream(s)); + } catch (Exception e) { + LOGGER.info("Error while deleting streams: {}", e.getMessage()); + } + } + } + + private static class ProducerState implements AutoCloseable { + + private static final byte[] BODY = "hello".getBytes(StandardCharsets.UTF_8); + + private final String stream; + private final Producer producer; + final RateLimiter limiter = RateLimiter.create(1000); + Thread task; + final AtomicBoolean stopped = new AtomicBoolean(false); + final AtomicInteger acceptedCount = new AtomicInteger(); + final AtomicReference postConfirmed = new AtomicReference<>(() -> {}); + final AtomicReference lastException = new AtomicReference<>(); + final AtomicReference lastExceptionInstant = new AtomicReference<>(); + + private ProducerState(String stream, boolean dynamicBatch, Environment environment) { + this.stream = stream; + this.producer = + environment.producerBuilder().stream(stream).dynamicBatch(dynamicBatch).build(); + } + + void start() { + ConfirmationHandler confirmationHandler = + confirmationStatus -> { + if (confirmationStatus.isConfirmed()) { + acceptedCount.incrementAndGet(); + postConfirmed.get().run(); + } + }; + task = + Executors.defaultThreadFactory() + .newThread( + () -> { + while (!stopped.get() && !Thread.currentThread().isInterrupted()) { + try { + this.limiter.acquire(1); + this.producer.send( + producer.messageBuilder().addData(BODY).build(), confirmationHandler); + } catch (Throwable e) { + this.lastException.set(e); + this.lastExceptionInstant.set(Instant.now()); + } + } + }); + task.start(); + } + + Sync waitForNewMessages(int messageCount) { + Sync sync = sync(messageCount); + AtomicInteger count = new AtomicInteger(); + this.postConfirmed.set( + () -> { + if (count.incrementAndGet() == messageCount) { + this.postConfirmed.set(() -> {}); + } + sync.down(); + }); + return sync; + } + + String stream() { + return this.stream; + } + + String lastException() { + if (this.lastException.get() == null) { + return "no exception"; + } else { + return this.lastException.get().getMessage() + + " at " + + DateTimeFormatter.ISO_INSTANT.format(lastExceptionInstant.get()); + } + } + + @Override + public void close() { + stopped.set(true); + task.interrupt(); + producer.close(); + } + } + + private static class ConsumerState implements AutoCloseable { + + private final String stream; + private final Consumer consumer; + final AtomicInteger receivedCount = new AtomicInteger(); + final AtomicReference postHandle = new AtomicReference<>(() -> {}); + + private ConsumerState(String stream, Environment environment) { + this.stream = stream; + this.consumer = + environment.consumerBuilder().stream(stream) + .offset(OffsetSpecification.first()) + .messageHandler( + (ctx, m) -> { + receivedCount.incrementAndGet(); + postHandle.get().run(); + }) + .build(); + } + + Sync waitForNewMessages(int messageCount) { + Sync sync = sync(messageCount); + AtomicInteger count = new AtomicInteger(); + this.postHandle.set( + () -> { + if (count.incrementAndGet() == messageCount) { + this.postHandle.set(() -> {}); + } + sync.down(); + }); + return sync; + } + + String stream() { + return this.stream; + } + + @Override + public void close() { + this.consumer.close(); + } + } +} diff --git a/src/test/java/com/rabbitmq/stream/impl/RetentionClientTest.java b/src/test/java/com/rabbitmq/stream/impl/RetentionClientTest.java index c76c3db892..1d9fde7aff 100644 --- a/src/test/java/com/rabbitmq/stream/impl/RetentionClientTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/RetentionClientTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -23,7 +23,7 @@ import com.rabbitmq.client.Connection; import com.rabbitmq.client.ConnectionFactory; import com.rabbitmq.stream.ByteCapacity; -import com.rabbitmq.stream.Host; +import com.rabbitmq.stream.Cli; import com.rabbitmq.stream.OffsetSpecification; import com.rabbitmq.stream.impl.TestUtils.CallableConsumer; import com.rabbitmq.stream.impl.TestUtils.RunnableWithException; @@ -85,11 +85,11 @@ static RetentionTestConfig[] retention() { + "'{\"max-length-bytes\":%d,\"stream-max-segment-size-bytes\":%d }' " + "--priority 1 --apply-to queues", stream, maxLengthBytes, maxSegmentSizeBytes); - Host.rabbitmqctl(policyCommand); + Cli.rabbitmqctl(policyCommand); }, firstMessageId -> firstMessageId > 0, firstMessageId -> "First message ID should be positive but is " + firstMessageId, - () -> Host.rabbitmqctl("clear_policy stream-retention-test")), + () -> Cli.rabbitmqctl("clear_policy stream-retention-test")), new RetentionTestConfig( "with size helper to specify bytes", context -> { diff --git a/src/test/java/com/rabbitmq/stream/impl/RoutePartitionsTest.java b/src/test/java/com/rabbitmq/stream/impl/RoutePartitionsTest.java index 93980c1b81..e42d23e796 100644 --- a/src/test/java/com/rabbitmq/stream/impl/RoutePartitionsTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/RoutePartitionsTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2021-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2021-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/SacClientTest.java b/src/test/java/com/rabbitmq/stream/impl/SacClientTest.java index 7eb8924f00..6434af4d19 100644 --- a/src/test/java/com/rabbitmq/stream/impl/SacClientTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/SacClientTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2022-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2022-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -27,8 +27,8 @@ import static java.util.stream.Collectors.toList; import static org.assertj.core.api.Assertions.assertThat; +import com.rabbitmq.stream.Cli; import com.rabbitmq.stream.Constants; -import com.rabbitmq.stream.Host; import com.rabbitmq.stream.OffsetSpecification; import com.rabbitmq.stream.impl.Client.ClientParameters; import com.rabbitmq.stream.impl.Client.ConsumerUpdateListener; @@ -430,7 +430,7 @@ void killingConnectionsShouldTriggerConsumerUpdateNotification() throws Exceptio Map consumerStates = new ConcurrentHashMap<>(); List consumerNames = IntStream.range(0, 5).mapToObj(i -> "foo-" + i).collect(toList()); - int connectionCount = Host.listConnections().size(); + int connectionCount = Cli.listConnections().size(); for (String consumerName : consumerNames) { Client c0 = @@ -467,10 +467,10 @@ void killingConnectionsShouldTriggerConsumerUpdateNotification() throws Exceptio } int cCount = connectionCount; - waitAtMost(() -> Host.listConnections().size() == cCount); + waitAtMost(() -> Cli.listConnections().size() == cCount); for (String consumerName : consumerNames) { - Host.killConnection(consumerName + "-connection-0"); + Cli.killConnection(consumerName + "-connection-0"); waitAtMost( () -> consumerStates.containsKey(consumerName + "-connection-1") @@ -605,7 +605,7 @@ void singleActiveConsumerMustHaveName() { void connectionShouldBeClosedIfConsumerUpdateTakesTooLong() throws Exception { Duration timeout = Duration.ofSeconds(1); try { - Host.setEnv("request_timeout", String.valueOf(timeout.getSeconds())); + Cli.setEnv("request_timeout", String.valueOf(timeout.getSeconds())); CountDownLatch shutdownLatch = new CountDownLatch(1); Client client = cf.get( @@ -629,7 +629,7 @@ void connectionShouldBeClosedIfConsumerUpdateTakesTooLong() throws Exception { assertThat(latchAssert(shutdownLatch)).completes(timeout.multipliedBy(5)); } finally { - Host.setEnv("request_timeout", "60000"); + Cli.setEnv("request_timeout", "60000"); } } } diff --git a/src/test/java/com/rabbitmq/stream/impl/SacStreamConsumerTest.java b/src/test/java/com/rabbitmq/stream/impl/SacStreamConsumerTest.java index beab11cd85..e58dc2e89b 100644 --- a/src/test/java/com/rabbitmq/stream/impl/SacStreamConsumerTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/SacStreamConsumerTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2022-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2022-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -14,24 +14,25 @@ // info@rabbitmq.com. package com.rabbitmq.stream.impl; -import static com.rabbitmq.stream.impl.TestUtils.publishAndWaitForConfirms; -import static com.rabbitmq.stream.impl.TestUtils.waitAtMost; +import static com.rabbitmq.stream.impl.Assertions.assertThat; +import static com.rabbitmq.stream.impl.TestUtils.*; import static org.assertj.core.api.Assertions.assertThat; -import com.rabbitmq.stream.Consumer; -import com.rabbitmq.stream.Environment; -import com.rabbitmq.stream.EnvironmentBuilder; -import com.rabbitmq.stream.OffsetSpecification; +import com.rabbitmq.stream.*; import com.rabbitmq.stream.impl.TestUtils.BrokerVersionAtLeast311Condition; import io.netty.channel.EventLoopGroup; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.BiConsumer; +import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; @ExtendWith({ TestUtils.StreamTestInfrastructureExtension.class, @@ -119,24 +120,24 @@ void autoTrackingSecondConsumerShouldTakeOverWhereTheFirstOneLeftOff() throws Ex void manualTrackingSecondConsumerShouldTakeOverWhereTheFirstOneLeftOff() throws Exception { int messageCount = 10000; int storeEvery = 1000; - Map receivedMessages = new ConcurrentHashMap<>(); - receivedMessages.put(0, new AtomicInteger(0)); - receivedMessages.put(1, new AtomicInteger(0)); + AtomicInteger consumer1MessageCount = new AtomicInteger(0); + AtomicInteger consumer2MessageCount = new AtomicInteger(0); AtomicLong lastReceivedOffset = new AtomicLong(0); String consumerName = "foo"; + BiConsumer handler = + (ctx, count) -> { + lastReceivedOffset.set(ctx.offset()); + if (count.incrementAndGet() % storeEvery == 0) { + ctx.storeOffset(); + } + }; + Consumer consumer1 = environment.consumerBuilder().stream(stream) .name(consumerName) .singleActiveConsumer() - .messageHandler( - (context, message) -> { - lastReceivedOffset.set(context.offset()); - int count = receivedMessages.get(0).incrementAndGet(); - if (count % storeEvery == 0) { - context.storeOffset(); - } - }) + .messageHandler((context, message) -> handler.accept(context, consumer1MessageCount)) .offset(OffsetSpecification.first()) .manualTrackingStrategy() .builder() @@ -146,24 +147,17 @@ void manualTrackingSecondConsumerShouldTakeOverWhereTheFirstOneLeftOff() throws environment.consumerBuilder().stream(stream) .name(consumerName) .singleActiveConsumer() - .messageHandler( - (context, message) -> { - lastReceivedOffset.set(context.offset()); - int count = receivedMessages.get(1).incrementAndGet(); - if (count % storeEvery == 0) { - context.storeOffset(); - } - }) + .messageHandler((context, message) -> handler.accept(context, consumer2MessageCount)) .offset(OffsetSpecification.first()) .manualTrackingStrategy() .builder() .build(); publishAndWaitForConfirms(cf, messageCount, stream); - waitAtMost(() -> receivedMessages.getOrDefault(0, new AtomicInteger(0)).get() == messageCount); + waitAtMost(() -> consumer1MessageCount.get() == messageCount); assertThat(lastReceivedOffset).hasPositiveValue(); - assertThat(receivedMessages.get(1)).hasValue(0); + assertThat(consumer2MessageCount).hasValue(0); long firstWaveLimit = lastReceivedOffset.get(); @@ -174,9 +168,9 @@ void manualTrackingSecondConsumerShouldTakeOverWhereTheFirstOneLeftOff() throws publishAndWaitForConfirms(cf, messageCount, stream); - waitAtMost(() -> receivedMessages.getOrDefault(0, new AtomicInteger(1)).get() == messageCount); + waitAtMost(() -> consumer2MessageCount.get() == messageCount); assertThat(lastReceivedOffset).hasValueGreaterThan(firstWaveLimit); - assertThat(receivedMessages.get(0)).hasValue(messageCount); + assertThat(consumer1MessageCount).hasValue(messageCount); consumer2.close(); } @@ -237,4 +231,72 @@ void externalTrackingSecondConsumerShouldTakeOverWhereTheFirstOneLeftOff() throw // nothing stored on the server side assertThat(cf.get().queryOffset(consumerName, stream).getOffset()).isZero(); } + + public static Stream> + activeConsumerShouldGetUpdateNotificationAfterDisruption() { + return Stream.of( + namedConsumer(consumer -> Cli.killConnection(connectionName(consumer)), "kill connection"), + namedConsumer(consumer -> Cli.restartStream(stream(consumer)), "restart stream"), + namedConsumer(Consumer::close, "close consumer")); + } + + @ParameterizedTest + @MethodSource + @TestUtils.DisabledIfRabbitMqCtlNotSet + void activeConsumerShouldGetUpdateNotificationAfterDisruption( + java.util.function.Consumer disruption) { + String consumerName = "foo"; + Sync consumer1Active = sync(); + Sync consumer1Inactive = sync(); + Consumer consumer1 = + environment.consumerBuilder().stream(stream) + .name(consumerName) + .noTrackingStrategy() + .singleActiveConsumer() + .consumerUpdateListener( + context -> { + if (context.isActive()) { + consumer1Active.down(); + } else { + consumer1Inactive.down(); + } + return OffsetSpecification.next(); + }) + .messageHandler((context, message) -> {}) + .build(); + + Sync consumer2Active = sync(); + Sync consumer2Inactive = sync(); + environment.consumerBuilder().stream(stream) + .name(consumerName) + .noTrackingStrategy() + .singleActiveConsumer() + .consumerUpdateListener( + context -> { + if (!context.isActive()) { + consumer2Inactive.down(); + } + return OffsetSpecification.next(); + }) + .messageHandler((context, message) -> {}) + .build(); + + assertThat(consumer1Active).completes(); + assertThat(consumer2Inactive).hasNotCompleted(); + assertThat(consumer1Inactive).hasNotCompleted(); + assertThat(consumer2Active).hasNotCompleted(); + + disruption.accept(consumer1); + + assertThat(consumer2Inactive).hasNotCompleted(); + assertThat(consumer1Inactive).completes(); + } + + private static String connectionName(Consumer consumer) { + return ((StreamConsumer) consumer).subscriptionConnectionName(); + } + + private static String stream(Consumer consumer) { + return ((StreamConsumer) consumer).stream(); + } } diff --git a/src/test/java/com/rabbitmq/stream/impl/SacSuperStreamConsumerTest.java b/src/test/java/com/rabbitmq/stream/impl/SacSuperStreamConsumerTest.java index 7864118563..df538319cd 100644 --- a/src/test/java/com/rabbitmq/stream/impl/SacSuperStreamConsumerTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/SacSuperStreamConsumerTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2021-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2021-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -14,18 +14,12 @@ // info@rabbitmq.com. package com.rabbitmq.stream.impl; -import static com.rabbitmq.stream.impl.TestUtils.declareSuperStreamTopology; -import static com.rabbitmq.stream.impl.TestUtils.deleteSuperStreamTopology; -import static com.rabbitmq.stream.impl.TestUtils.waitAtMost; +import static com.rabbitmq.stream.impl.Assertions.assertThat; +import static com.rabbitmq.stream.impl.TestUtils.*; import static java.util.stream.Collectors.toList; import static org.assertj.core.api.Assertions.assertThat; -import com.rabbitmq.stream.Consumer; -import com.rabbitmq.stream.ConsumerUpdateListener; -import com.rabbitmq.stream.Environment; -import com.rabbitmq.stream.EnvironmentBuilder; -import com.rabbitmq.stream.NoOffsetException; -import com.rabbitmq.stream.OffsetSpecification; +import com.rabbitmq.stream.*; import com.rabbitmq.stream.impl.TestUtils.BrokerVersionAtLeast311Condition; import com.rabbitmq.stream.impl.TestUtils.CallableBooleanSupplier; import com.rabbitmq.stream.impl.TestUtils.SingleActiveConsumer; @@ -39,11 +33,14 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.stream.IntStream; +import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; @ExtendWith({ TestUtils.StreamTestInfrastructureExtension.class, @@ -287,7 +284,8 @@ void sacAutoOffsetTrackingShouldStoreOnRelanbancing() throws Exception { && consumerStates.get("1" + partitions.get(2))); assertThat(consumerStates) - .containsEntry("0" + partitions.get(0), true) // not changed after closing + .containsEntry( + "0" + partitions.get(0), false) // client library notifies the listener on closing .containsEntry("0" + partitions.get(1), false) // not changed after closing .containsEntry("0" + partitions.get(2), false) // not changed after closing .containsEntry("1" + partitions.get(0), true) // now active @@ -314,12 +312,15 @@ void sacAutoOffsetTrackingShouldStoreOnRelanbancing() throws Exception { && consumerStates.get("2" + partitions.get(2))); assertThat(consumerStates) - .containsEntry("0" + partitions.get(0), true) // not changed after closing + .containsEntry( + "0" + partitions.get(0), false) // client library notifies the listener on closing .containsEntry("0" + partitions.get(1), false) // not changed after closing .containsEntry("0" + partitions.get(2), false) // not changed after closing - .containsEntry("1" + partitions.get(0), true) // not changed after closing + .containsEntry( + "1" + partitions.get(0), false) // client library notifies the listener on closing .containsEntry("1" + partitions.get(1), false) // not changed after closing - .containsEntry("1" + partitions.get(2), true) // not changed after closing + .containsEntry( + "1" + partitions.get(2), false) // client library notifies the listener on closing .containsEntry("2" + partitions.get(0), true) // now active .containsEntry("2" + partitions.get(1), true) // now active .containsEntry("2" + partitions.get(2), true); // now active @@ -809,6 +810,85 @@ void consumerGroupsOnSameSuperStreamShouldBeIndependent() { }); } + public static Stream> + activeConsumerShouldGetUpdateNotificationAfterDisruption() { + return Stream.of( + namedBiConsumer((s, c) -> Cli.killConnection(connectionName(s, c)), "kill connection"), + namedBiConsumer((s, c) -> Cli.restartStream(s), "restart stream"), + namedBiConsumer((s, c) -> c.close(), "close consumer")); + } + + @ParameterizedTest + @MethodSource + @TestUtils.DisabledIfRabbitMqCtlNotSet + void activeConsumerShouldGetUpdateNotificationAfterDisruption( + java.util.function.BiConsumer disruption) { + declareSuperStreamTopology(configurationClient, superStream, partitionCount); + String partition = superStream + "-0"; + + String consumerName = "foo"; + Function, ConsumerUpdateListener> + filteringListener = + action -> + (ConsumerUpdateListener) + context -> { + if (partition.equals(context.stream())) { + action.accept(context); + } + return OffsetSpecification.next(); + }; + + Sync consumer1Active = sync(); + Sync consumer1Inactive = sync(); + + Consumer consumer1 = + environment + .consumerBuilder() + .singleActiveConsumer() + .superStream(superStream) + .name(consumerName) + .noTrackingStrategy() + .consumerUpdateListener( + filteringListener.apply( + context -> { + if (context.isActive()) { + consumer1Active.down(); + } else { + consumer1Inactive.down(); + } + })) + .messageHandler((context, message) -> {}) + .build(); + + Sync consumer2Active = sync(); + Sync consumer2Inactive = sync(); + environment + .consumerBuilder() + .singleActiveConsumer() + .superStream(superStream) + .name(consumerName) + .noTrackingStrategy() + .consumerUpdateListener( + filteringListener.apply( + context -> { + if (!context.isActive()) { + consumer2Inactive.down(); + } + })) + .messageHandler((context, message) -> {}) + .build(); + + assertThat(consumer1Active).completes(); + assertThat(consumer2Inactive).hasNotCompleted(); + assertThat(consumer1Inactive).hasNotCompleted(); + assertThat(consumer2Active).hasNotCompleted(); + + disruption.accept(partition, consumer1); + + assertThat(consumer2Inactive).hasNotCompleted(); + assertThat(consumer1Inactive).completes(); + } + private static void waitUntil(CallableBooleanSupplier action) { try { waitAtMost(action); @@ -816,4 +896,9 @@ private static void waitUntil(CallableBooleanSupplier action) { throw new RuntimeException(e); } } + + private static String connectionName(String partition, Consumer consumer) { + return ((StreamConsumer) ((SuperStreamConsumer) consumer).consumer(partition)) + .subscriptionConnectionName(); + } } diff --git a/src/test/java/com/rabbitmq/stream/impl/ServerFrameHandlerTest.java b/src/test/java/com/rabbitmq/stream/impl/ServerFrameHandlerTest.java index 2581d6580c..7ad3a56173 100644 --- a/src/test/java/com/rabbitmq/stream/impl/ServerFrameHandlerTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/ServerFrameHandlerTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/ShutdownListenerTest.java b/src/test/java/com/rabbitmq/stream/impl/ShutdownListenerTest.java index 93bfee3e0a..dfb8df27c1 100644 --- a/src/test/java/com/rabbitmq/stream/impl/ShutdownListenerTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/ShutdownListenerTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -16,8 +16,8 @@ import static org.assertj.core.api.Assertions.assertThat; +import com.rabbitmq.stream.Cli; import com.rabbitmq.stream.Constants; -import com.rabbitmq.stream.Host; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.util.UUID; @@ -87,7 +87,7 @@ void shutdownListenerShouldBeCalledWhenConnectionIsKilled() throws Exception { shutdownLatch.countDown(); })); - Host.killConnection(connectionName); + Cli.killConnection(connectionName); assertThat(shutdownLatch.await(10, TimeUnit.SECONDS)).isTrue(); assertThat(reason.get()).isNotNull().isEqualTo(Client.ShutdownContext.ShutdownReason.UNKNOWN); assertThat(client.isOpen()).isFalse(); diff --git a/src/test/java/com/rabbitmq/stream/impl/SingleActiveConsumerTestSuite.java b/src/test/java/com/rabbitmq/stream/impl/SingleActiveConsumerTestSuite.java index cf46d49dac..8d06a66fae 100644 --- a/src/test/java/com/rabbitmq/stream/impl/SingleActiveConsumerTestSuite.java +++ b/src/test/java/com/rabbitmq/stream/impl/SingleActiveConsumerTestSuite.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/StompInteroperabilityTest.java b/src/test/java/com/rabbitmq/stream/impl/StompInteroperabilityTest.java index 8566d44043..4faf6cd848 100644 --- a/src/test/java/com/rabbitmq/stream/impl/StompInteroperabilityTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/StompInteroperabilityTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2021-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2021-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -22,7 +22,6 @@ import com.rabbitmq.stream.*; import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; @@ -66,7 +65,7 @@ public class StompInteroperabilityTest { @BeforeAll static void initAll() { - eventLoopGroup = new NioEventLoopGroup(); + eventLoopGroup = Utils.eventLoopGroup(); } @AfterAll diff --git a/src/test/java/com/rabbitmq/stream/impl/StreamConsumerTest.java b/src/test/java/com/rabbitmq/stream/impl/StreamConsumerTest.java index 9c676937d7..6871823f61 100644 --- a/src/test/java/com/rabbitmq/stream/impl/StreamConsumerTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/StreamConsumerTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -17,7 +17,6 @@ import static com.rabbitmq.stream.ConsumerFlowStrategy.creditWhenHalfMessagesProcessed; import static com.rabbitmq.stream.impl.TestUtils.*; import static com.rabbitmq.stream.impl.TestUtils.CountDownLatchConditions.completed; -import static java.lang.Runtime.getRuntime; import static java.lang.String.format; import static java.util.Collections.synchronizedList; import static org.assertj.core.api.Assertions.*; @@ -68,20 +67,19 @@ static Stream> consumerShouldKeepConsumingAf return Stream.of( TestUtils.namedTask( o -> { - Host.killStreamLeaderProcess(o.toString()); + Cli.killStreamLeaderProcess(o.toString()); Thread.sleep(TOPOLOGY_DELAY.toMillis()); }, "stream leader process is killed"), TestUtils.namedTask( - o -> Host.killConnection("rabbitmq-stream-consumer-0"), - "consumer connection is killed"), + o -> Cli.killConnection("rabbitmq-stream-consumer-0"), "consumer connection is killed"), TestUtils.namedTask( o -> { try { - Host.rabbitmqctl("stop_app"); + Cli.rabbitmqctl("stop_app"); Thread.sleep(1000L); } finally { - Host.rabbitmqctl("start_app"); + Cli.rabbitmqctl("start_app"); } Thread.sleep(recoveryInitialDelay.toMillis() * 2); }, @@ -90,7 +88,7 @@ static Stream> consumerShouldKeepConsumingAf @BeforeEach void init() { - if (Host.isOnDocker()) { + if (Cli.isOnDocker()) { // with a containerized broker in bridged network mode, the client should not // reconnect too soon, as it would see the port still open but would not get any response. // This then provokes some cascading timeouts in the test. @@ -204,7 +202,7 @@ void consumeWithAsyncConsumerFlowControl() throws Exception { environment.consumerBuilder().stream(stream) .offset(OffsetSpecification.first()) .flow() - .strategy(creditWhenHalfMessagesProcessed()) + .strategy(creditWhenHalfMessagesProcessed(1)) .builder(); List messageContexts = synchronizedList(new ArrayList<>()); @@ -244,15 +242,13 @@ void consumeWithAsyncConsumerFlowControl() throws Exception { void asynchronousProcessingWithFlowControl() { int messageCount = 100_000; publishAndWaitForConfirms(cf, messageCount, stream); - - ExecutorService executorService = - Executors.newFixedThreadPool(getRuntime().availableProcessors()); + ExecutorService executorService = Executors.newFixedThreadPool(Utils.AVAILABLE_PROCESSORS); try { CountDownLatch latch = new CountDownLatch(messageCount); environment.consumerBuilder().stream(stream) .offset(OffsetSpecification.first()) .flow() - .strategy(creditWhenHalfMessagesProcessed()) + .strategy(creditWhenHalfMessagesProcessed(1)) .builder() .messageHandler( (ctx, message) -> @@ -476,8 +472,7 @@ void manualTrackingConsumerShouldRestartWhereItLeftOff() throws Exception { @Test @DisabledIfRabbitMqCtlNotSet - void consumerShouldReUseInitialOffsetSpecificationAfterDisruptionIfNoMessagesReceived() - throws Exception { + void consumerShouldReUseInitialOffsetSpecificationAfterDisruptionIfNoMessagesReceived() { int messageCountFirstWave = 10_000; Producer producer = environment.producerBuilder().stream(stream).build(); @@ -510,7 +505,7 @@ void consumerShouldReUseInitialOffsetSpecificationAfterDisruptionIfNoMessagesRec .build(); // killing the consumer connection to trigger an internal restart - Host.killConnection("rabbitmq-stream-consumer-0"); + Cli.killConnection("rabbitmq-stream-consumer-0"); // no messages should have been received assertThat(consumedCount.get()).isZero(); @@ -813,7 +808,7 @@ void externalOffsetTrackingWithSubscriptionListener() throws Exception { waitAtMost(5, () -> receivedMessages.get() == messageCount); assertThat(offsetTracking.get()).isGreaterThanOrEqualTo(messageCount - 1); - Host.killConnection("rabbitmq-stream-consumer-0"); + Cli.killConnection("rabbitmq-stream-consumer-0"); waitAtMost( recoveryInitialDelay.multipliedBy(2), () -> subscriptionListenerCallCount.get() == 2); @@ -858,7 +853,7 @@ void duplicatesWhenResubscribeAfterDisconnectionWithLongFlushInterval() throws E }; publish.accept(storeEvery * 2 - 100); waitAtMost(5, () -> receivedMessages.get() == publishedMessages.get()); - Host.killConnection("rabbitmq-stream-consumer-0"); + Cli.killConnection("rabbitmq-stream-consumer-0"); publish.accept(storeEvery * 2); waitAtMost( @@ -924,7 +919,7 @@ void useSubscriptionListenerToRestartExactlyWhereDesired() throws Exception { }; publish.accept(storeEvery * 2 - 100); waitAtMost(5, () -> receivedMessages.get() == publishedMessages.get()); - Host.killConnection("rabbitmq-stream-consumer-0"); + Cli.killConnection("rabbitmq-stream-consumer-0"); publish.accept(storeEvery * 2); producer.send( diff --git a/src/test/java/com/rabbitmq/stream/impl/StreamConsumerUnitTest.java b/src/test/java/com/rabbitmq/stream/impl/StreamConsumerUnitTest.java index 3fab5e8849..226a082028 100644 --- a/src/test/java/com/rabbitmq/stream/impl/StreamConsumerUnitTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/StreamConsumerUnitTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -16,13 +16,13 @@ import static com.rabbitmq.stream.impl.StreamConsumer.getStoredOffsetSafely; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; import com.rabbitmq.stream.BackOffDelayPolicy; +import com.rabbitmq.stream.Constants; +import com.rabbitmq.stream.NoOffsetException; import java.time.Duration; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -34,8 +34,12 @@ public class StreamConsumerUnitTest { + private static final Duration LARGE_RPC_TIMEOUT = Duration.ofSeconds(10); + @Mock StreamConsumer consumer; @Mock StreamEnvironment environment; + @Mock Client client; + @Mock StreamEnvironment.Locator locator; AutoCloseable closeable; @@ -68,9 +72,66 @@ void getStoredOffsetSafely_ShouldRetryWhenLeaderConnectionIsNotAvailable() { when(consumer.storedOffset(any())).thenThrow(new IllegalStateException()).thenReturn(42L); Duration retryDelay = Duration.ofMillis(10); when(environment.recoveryBackOffDelayPolicy()).thenReturn(BackOffDelayPolicy.fixed(retryDelay)); + when(environment.rpcTimeout()).thenReturn(LARGE_RPC_TIMEOUT); assertThat(getStoredOffsetSafely(consumer, environment)).isEqualTo(42); verify(consumer, times(1)).storedOffset(); verify(environment, times(1)).scheduledExecutorService(); verify(consumer, times(2)).storedOffset(any()); } + + @Test + void getStoredOffsetSafely_ShouldThrowNoOffsetException() { + when(consumer.storedOffset()).thenThrow(new NoOffsetException("")); + assertThatThrownBy(() -> getStoredOffsetSafely(consumer, environment)) + .isInstanceOf(NoOffsetException.class); + verify(consumer, times(1)).storedOffset(); + verify(environment, never()).scheduledExecutorService(); + verify(consumer, never()).storedOffset(any()); + } + + @Test + void getStoredOffsetSafely_ShouldReThrowNoOffsetExceptionFromFallback() { + when(consumer.storedOffset()).thenThrow(new IllegalStateException()); + when(consumer.storedOffset(any())).thenThrow(new NoOffsetException("no offset")); + Duration retryDelay = Duration.ofMillis(10); + when(environment.recoveryBackOffDelayPolicy()).thenReturn(BackOffDelayPolicy.fixed(retryDelay)); + assertThatThrownBy(() -> getStoredOffsetSafely(consumer, environment)) + .isInstanceOf(NoOffsetException.class); + verify(consumer, times(1)).storedOffset(); + verify(environment, times(1)).scheduledExecutorService(); + verify(consumer, times(1)).storedOffset(any()); + } + + @Test + void getStoredOffsetSafely_ShouldThrowTimeoutExceptionIfFallbackTimesOut() { + when(consumer.storedOffset()).thenThrow(new IllegalStateException()); + when(consumer.storedOffset(any())).thenThrow(new IllegalStateException()); + Duration retryDelay = Duration.ofMillis(50); + Duration rpcTimeout = retryDelay.multipliedBy(2); + when(environment.rpcTimeout()).thenReturn(rpcTimeout); + when(environment.recoveryBackOffDelayPolicy()).thenReturn(BackOffDelayPolicy.fixed(retryDelay)); + assertThatThrownBy(() -> getStoredOffsetSafely(consumer, environment)) + .isInstanceOf(TimeoutStreamException.class); + verify(consumer, times(1)).storedOffset(); + verify(environment, times(1)).scheduledExecutorService(); + verify(consumer, atLeastOnce()).storedOffset(any()); + } + + @Test + void getStoredOffsetSafely_ShouldUseLocatorConnectionWhenLeaderConnectionIsNotAvailable() { + when(consumer.canTrack()).thenReturn(true); + when(consumer.storedOffset()).thenThrow(new IllegalStateException()); + when(consumer.storedOffset(any())).thenCallRealMethod(); + when(environment.locator()).thenReturn(locator); + when(locator.client()).thenReturn(client); + when(client.queryOffset(isNull(), isNull())) + .thenReturn(new Client.QueryOffsetResponse(Constants.RESPONSE_CODE_OK, 42L)); + Duration retryDelay = Duration.ofMillis(10); + when(environment.recoveryBackOffDelayPolicy()).thenReturn(BackOffDelayPolicy.fixed(retryDelay)); + assertThat(getStoredOffsetSafely(consumer, environment)).isEqualTo(42L); + verify(consumer, times(1)).storedOffset(); + verify(environment, times(1)).scheduledExecutorService(); + verify(environment, times(1)).locator(); + verify(consumer, times(1)).storedOffset(any()); + } } diff --git a/src/test/java/com/rabbitmq/stream/impl/StreamCreatorTest.java b/src/test/java/com/rabbitmq/stream/impl/StreamCreatorTest.java index f0445a01c6..60aee2136f 100644 --- a/src/test/java/com/rabbitmq/stream/impl/StreamCreatorTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/StreamCreatorTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/StreamEnvironmentTest.java b/src/test/java/com/rabbitmq/stream/impl/StreamEnvironmentTest.java index 9a7d23e210..f076a78fde 100644 --- a/src/test/java/com/rabbitmq/stream/impl/StreamEnvironmentTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/StreamEnvironmentTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -14,12 +14,9 @@ // info@rabbitmq.com. package com.rabbitmq.stream.impl; +import static com.rabbitmq.stream.impl.TestUtils.*; +import static com.rabbitmq.stream.impl.TestUtils.CountDownLatchConditions.completed; import static com.rabbitmq.stream.impl.TestUtils.ExceptionConditions.responseCode; -import static com.rabbitmq.stream.impl.TestUtils.latchAssert; -import static com.rabbitmq.stream.impl.TestUtils.localhost; -import static com.rabbitmq.stream.impl.TestUtils.localhostTls; -import static com.rabbitmq.stream.impl.TestUtils.streamName; -import static com.rabbitmq.stream.impl.TestUtils.waitAtMost; import static java.util.concurrent.TimeUnit.SECONDS; import static java.util.stream.Collectors.toList; import static java.util.stream.IntStream.range; @@ -29,14 +26,14 @@ import com.rabbitmq.stream.Address; import com.rabbitmq.stream.AuthenticationFailureException; import com.rabbitmq.stream.BackOffDelayPolicy; +import com.rabbitmq.stream.Cli; +import com.rabbitmq.stream.Cli.ConnectionInfo; import com.rabbitmq.stream.ConfirmationHandler; import com.rabbitmq.stream.Constants; import com.rabbitmq.stream.Consumer; import com.rabbitmq.stream.ConsumerBuilder; import com.rabbitmq.stream.Environment; import com.rabbitmq.stream.EnvironmentBuilder; -import com.rabbitmq.stream.Host; -import com.rabbitmq.stream.Host.ConnectionInfo; import com.rabbitmq.stream.Message; import com.rabbitmq.stream.NoOffsetException; import com.rabbitmq.stream.OffsetSpecification; @@ -54,9 +51,9 @@ import com.rabbitmq.stream.impl.TestUtils.DisabledIfTlsNotEnabled; import io.netty.channel.Channel; import io.netty.channel.EventLoopGroup; -import io.netty.channel.epoll.EpollEventLoopGroup; +import io.netty.channel.MultiThreadIoEventLoopGroup; +import io.netty.channel.epoll.EpollIoHandler; import io.netty.channel.epoll.EpollSocketChannel; -import io.netty.channel.nio.NioEventLoopGroup; import io.netty.handler.ssl.SslHandler; import java.net.ConnectException; import java.nio.charset.StandardCharsets; @@ -69,9 +66,7 @@ import java.util.Random; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.CountDownLatch; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -82,11 +77,7 @@ import javax.net.ssl.SNIHostName; import javax.net.ssl.SSLParameters; import org.assertj.core.api.ThrowableAssert.ThrowingCallable; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.*; import org.junit.jupiter.api.condition.EnabledIfSystemProperty; import org.junit.jupiter.api.condition.EnabledOnOs; import org.junit.jupiter.api.condition.OS; @@ -97,22 +88,11 @@ @ExtendWith(TestUtils.StreamTestInfrastructureExtension.class) public class StreamEnvironmentTest { - static EventLoopGroup eventLoopGroup; - EnvironmentBuilder environmentBuilder; String stream; TestUtils.ClientFactory cf; - - @BeforeAll - static void initAll() { - eventLoopGroup = new NioEventLoopGroup(); - } - - @AfterAll - static void afterAll() throws Exception { - eventLoopGroup.shutdownGracefully(1, 10, SECONDS).get(10, SECONDS); - } + EventLoopGroup eventLoopGroup; @BeforeEach void init() { @@ -368,12 +348,12 @@ void environmentPublishersConsumersShouldCloseSuccessfullyWhenBrokerIsDown() thr latchAssert(consumeLatch).completes(); try { - Host.rabbitmqctl("stop_app"); + Cli.rabbitmqctl("stop_app"); producer.close(); consumer.close(); environment.close(); } finally { - Host.rabbitmqctl("start_app"); + Cli.rabbitmqctl("start_app"); } waitAtMost( 30, @@ -394,7 +374,7 @@ void locatorShouldReconnectIfConnectionIsLost(TestInfo info) { String s = streamName(info); environment.streamCreator().stream(s).create(); environment.deleteStream(s); - Host.killConnection("rabbitmq-stream-locator-0"); + Cli.killConnection("rabbitmq-stream-locator-0"); environment.streamCreator().stream(s).create(); try { Producer producer = environment.producerBuilder().stream(s).build(); @@ -427,14 +407,14 @@ void shouldHaveSeveralLocatorsWhenSeveralUrisSpecifiedAndShouldRecoverThemIfClos Supplier> locatorConnectionNamesSupplier = () -> - Host.listConnections().stream() + Cli.listConnections().stream() .map(ConnectionInfo::clientProvidedName) - .filter(name -> name.contains("-locator-")) + .filter(name -> name != null && name.contains("-locator-")) .collect(toList()); List locatorConnectionNames = locatorConnectionNamesSupplier.get(); assertThat(locatorConnectionNames).hasSameSizeAs(uris); - locatorConnectionNames.forEach(connectionName -> Host.killConnection(connectionName)); + locatorConnectionNames.forEach(Cli::killConnection); environment.streamCreator().stream(s).create(); try { @@ -744,7 +724,8 @@ void nettyInitializersAreCalled() { @EnabledIfSystemProperty(named = "os.arch", matches = "amd64") void nativeEpollWorksOnLinux() { int messageCount = 10_000; - EventLoopGroup epollEventLoopGroup = new EpollEventLoopGroup(); + EventLoopGroup epollEventLoopGroup = + new MultiThreadIoEventLoopGroup(EpollIoHandler.newFactory()); try { Set channels = ConcurrentHashMap.newKeySet(); try (Environment env = @@ -778,4 +759,57 @@ void nativeEpollWorksOnLinux() { epollEventLoopGroup.shutdownGracefully(0, 0, SECONDS); } } + + @Test + void enforceEntityPerConnectionLimits() { + int entityCount = 10; + int limit = 3; + ExecutorService executor = Executors.newCachedThreadPool(); + try (Environment env = + environmentBuilder + .maxProducersByConnection(limit) + .maxConsumersByConnection(limit) + .maxTrackingConsumersByConnection(limit) + .build()) { + CountDownLatch latch = new CountDownLatch(entityCount * 2); + IntStream.range(0, entityCount) + .forEach( + i -> { + executor.execute( + () -> { + env.producerBuilder().stream(stream).name(String.valueOf(i)).build(); + latch.countDown(); + }); + }); + IntStream.range(0, entityCount) + .forEach( + i -> { + executor.execute( + () -> { + env.consumerBuilder().stream(stream).messageHandler((ctx, msg) -> {}).build(); + latch.countDown(); + }); + }); + assertThat(latch).is(completed()); + EnvironmentInfo envInfo = MonitoringTestUtils.extract(env); + int expectedConnectionCount = entityCount / limit + 1; + assertThat(envInfo.getProducers().clientCount()).isEqualTo(expectedConnectionCount); + assertThat(envInfo.getConsumers().clients()).hasSize(expectedConnectionCount); + } finally { + executor.shutdownNow(); + } + } + + @Test + void brokerShouldAcceptInitialMemberCountArgument(TestInfo info) { + String s = streamName(info); + Environment env = environmentBuilder.build(); + try { + env.streamCreator().name(s).initialMemberCount(1).create(); + assertThat(env.streamExists(s)).isTrue(); + } finally { + env.deleteStream(s); + env.close(); + } + } } diff --git a/src/test/java/com/rabbitmq/stream/impl/StreamEnvironmentUnitTest.java b/src/test/java/com/rabbitmq/stream/impl/StreamEnvironmentUnitTest.java index 7d80172660..e8d3f25e34 100644 --- a/src/test/java/com/rabbitmq/stream/impl/StreamEnvironmentUnitTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/StreamEnvironmentUnitTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -20,6 +20,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; +import com.rabbitmq.stream.Address; import com.rabbitmq.stream.BackOffDelayPolicy; import com.rabbitmq.stream.ObservationCollector; import com.rabbitmq.stream.StreamException; @@ -98,7 +99,11 @@ Client.ClientParameters duplicate() { type -> "locator-connection", cf, ObservationCollector.NO_OP, - false); + false, + true, + Duration.ofMillis(100), + Duration.ofMillis(100), + -1); } @AfterEach @@ -163,7 +168,11 @@ void shouldTryUrisOnInitializationFailure() throws Exception { type -> "locator-connection", cf, ObservationCollector.NO_OP, - false); + false, + true, + Duration.ofMillis(100), + Duration.ofMillis(100), + -1); verify(cf, times(3)).apply(any(Client.ClientParameters.class)); } @@ -191,7 +200,11 @@ void shouldNotOpenConnectionWhenLazyInitIsEnabled( type -> "locator-connection", cf, ObservationCollector.NO_OP, - false); + false, + true, + Duration.ofMillis(100), + Duration.ofMillis(100), + -1); verify(cf, times(expectedConnectionCreation)).apply(any(Client.ClientParameters.class)); } @@ -285,5 +298,11 @@ void locatorOperationShouldNotRetryAndReThrowUnexpectedException() { assertThat(counter).hasValue(1); } - private static final Supplier CLIENT_SUPPLIER = () -> mock(Client.class); + private static final Supplier CLIENT_SUPPLIER = + () -> { + StreamEnvironment.Locator locator = + new StreamEnvironment.Locator(-1, new Address("localhost", 5555)); + locator.client(mock(Client.class)); + return locator; + }; } diff --git a/src/test/java/com/rabbitmq/stream/impl/StreamProducerTest.java b/src/test/java/com/rabbitmq/stream/impl/StreamProducerTest.java index 775f12eb97..63254247ad 100644 --- a/src/test/java/com/rabbitmq/stream/impl/StreamProducerTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/StreamProducerTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -14,19 +14,17 @@ // info@rabbitmq.com. package com.rabbitmq.stream.impl; -import static com.rabbitmq.stream.impl.TestUtils.latchAssert; -import static com.rabbitmq.stream.impl.TestUtils.localhost; -import static com.rabbitmq.stream.impl.TestUtils.streamName; -import static com.rabbitmq.stream.impl.TestUtils.waitAtMost; +import static com.rabbitmq.stream.impl.Assertions.assertThat; +import static com.rabbitmq.stream.impl.TestUtils.*; import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import ch.qos.logback.classic.Level; import com.rabbitmq.stream.*; import com.rabbitmq.stream.compression.Compression; import com.rabbitmq.stream.impl.MonitoringTestUtils.ProducerInfo; import com.rabbitmq.stream.impl.StreamProducer.Status; +import com.rabbitmq.stream.impl.TestUtils.Sync; import io.netty.channel.ChannelOption; import io.netty.channel.ConnectTimeoutException; import io.netty.channel.EventLoopGroup; @@ -94,7 +92,7 @@ void tearDown() { void send() throws Exception { int batchSize = 10; int messageCount = 10 * batchSize + 1; // don't want a multiple of batch size - CountDownLatch publishLatch = new CountDownLatch(messageCount); + Sync confirmSync = sync(messageCount); Producer producer = environment.producerBuilder().stream(stream).batchSize(batchSize).build(); AtomicLong count = new AtomicLong(0); AtomicLong sequence = new AtomicLong(0); @@ -117,13 +115,12 @@ void send() throws Exception { idsConfirmed.add( confirmationStatus.getMessage().getProperties().getMessageIdAsLong()); count.incrementAndGet(); - publishLatch.countDown(); + confirmSync.down(); }); }); - boolean completed = publishLatch.await(10, TimeUnit.SECONDS); + assertThat(confirmSync).completes(); assertThat(idsSent).hasSameSizeAs(idsConfirmed); idsSent.forEach(idSent -> assertThat(idsConfirmed).contains(idSent)); - assertThat(completed).isTrue(); ProducerInfo info = MonitoringTestUtils.extract(producer); assertThat(info.getId()).isGreaterThanOrEqualTo(0); @@ -283,7 +280,7 @@ void shouldRecoverAfterConnectionIsKilled(int subEntrySize) throws Exception { Thread.sleep(1000L); - Host.killConnection("rabbitmq-stream-producer-0"); + Cli.killConnection("rabbitmq-stream-producer-0"); waitAtMost(() -> ((StreamProducer) producer).status() == Status.NOT_AVAILABLE); canPublish.set(false); @@ -349,63 +346,59 @@ void shouldRecoverAfterConnectionIsKilled(int subEntrySize) throws Exception { @ParameterizedTest @ValueSource(ints = {1, 7}) void producerShouldBeClosedWhenStreamIsDeleted(int subEntrySize, TestInfo info) throws Exception { - Level initialLogLevel = TestUtils.newLoggerLevel(ProducersCoordinator.class, Level.DEBUG); - try { - String s = streamName(info); - environment.streamCreator().stream(s).create(); - - StreamProducer producer = - (StreamProducer) - environment.producerBuilder().subEntrySize(subEntrySize).stream(s).build(); - - AtomicInteger published = new AtomicInteger(0); - AtomicInteger confirmed = new AtomicInteger(0); - AtomicInteger errored = new AtomicInteger(0); - Set errorCodes = ConcurrentHashMap.newKeySet(); - - AtomicBoolean continuePublishing = new AtomicBoolean(true); - Thread publishThread = - new Thread( - () -> { - ConfirmationHandler confirmationHandler = - confirmationStatus -> { - if (confirmationStatus.isConfirmed()) { - confirmed.incrementAndGet(); - } else { - errored.incrementAndGet(); - errorCodes.add(confirmationStatus.getCode()); - } - }; - while (continuePublishing.get()) { - try { - producer.send( - producer - .messageBuilder() - .addData("".getBytes(StandardCharsets.UTF_8)) - .build(), - confirmationHandler); - published.incrementAndGet(); - } catch (StreamException e) { - // OK - } + String s = streamName(info); + environment.streamCreator().stream(s).create(); + + StreamProducer producer = + (StreamProducer) environment.producerBuilder().subEntrySize(subEntrySize).stream(s).build(); + + AtomicInteger published = new AtomicInteger(0); + AtomicInteger confirmed = new AtomicInteger(0); + AtomicInteger errored = new AtomicInteger(0); + Set errorCodes = ConcurrentHashMap.newKeySet(); + + AtomicBoolean continuePublishing = new AtomicBoolean(true); + Thread publishThread = + new Thread( + () -> { + ConfirmationHandler confirmationHandler = + confirmationStatus -> { + if (confirmationStatus.isConfirmed()) { + confirmed.incrementAndGet(); + } else { + errored.incrementAndGet(); + errorCodes.add(confirmationStatus.getCode()); + } + }; + while (continuePublishing.get()) { + try { + producer.send( + producer + .messageBuilder() + .addData("".getBytes(StandardCharsets.UTF_8)) + .build(), + confirmationHandler); + published.incrementAndGet(); + } catch (StreamException e) { + // OK } - }); - publishThread.start(); + } + }); + publishThread.start(); - Thread.sleep(1000L); + waitAtMost(() -> confirmed.get() > 100); + int confirmedNow = confirmed.get(); + waitAtMost(() -> confirmed.get() > confirmedNow + 1000); - assertThat(producer.isOpen()).isTrue(); + assertThat(producer.isOpen()).isTrue(); - environment.deleteStream(s); + environment.deleteStream(s); - waitAtMost(() -> !producer.isOpen()); - continuePublishing.set(false); - waitAtMost( - () -> !errorCodes.isEmpty(), - () -> "The producer should have received negative publish confirms"); - } finally { - TestUtils.newLoggerLevel(ProducersCoordinator.class, initialLogLevel); - } + waitAtMost(() -> !producer.isOpen()); + continuePublishing.set(false); + waitAtMost( + () -> !errorCodes.isEmpty(), + () -> "The producer should have received negative publish confirms"); } @ParameterizedTest @@ -415,15 +408,14 @@ void messagesShouldBeDeDuplicatedWhenUsingNameAndPublishingId(int subEntrySize) int firstWaveLineCount = lineCount / 5; int backwardCount = firstWaveLineCount / 10; SortedSet document = new TreeSet<>(); - IntStream.range(0, lineCount).forEach(i -> document.add(i)); + IntStream.range(0, lineCount).forEach(document::add); Producer producer = environment.producerBuilder().name("producer-1").stream(stream) .subEntrySize(subEntrySize) .build(); - AtomicReference latch = - new AtomicReference<>(new CountDownLatch(firstWaveLineCount)); - ConfirmationHandler confirmationHandler = confirmationStatus -> latch.get().countDown(); + Sync confirmSync = sync(firstWaveLineCount); + ConfirmationHandler confirmationHandler = confirmationStatus -> confirmSync.down(); Consumer publishMessage = i -> producer.send( @@ -433,15 +425,17 @@ void messagesShouldBeDeDuplicatedWhenUsingNameAndPublishingId(int subEntrySize) .addData(String.valueOf(i).getBytes()) .build(), confirmationHandler); + // publish the first wave document.headSet(firstWaveLineCount).forEach(publishMessage); - assertThat(latch.get().await(10, TimeUnit.SECONDS)).isTrue(); + assertThat(confirmSync).completes(); - latch.set(new CountDownLatch(lineCount - firstWaveLineCount + backwardCount)); + confirmSync.reset(lineCount - firstWaveLineCount + backwardCount); + // publish the rest, but with some overlap from the first wave document.tailSet(firstWaveLineCount - backwardCount).forEach(publishMessage); - assertThat(latch.get().await(5, TimeUnit.SECONDS)).isTrue(); + assertThat(confirmSync).completes(); CountDownLatch consumeLatch = new CountDownLatch(lineCount); AtomicInteger consumed = new AtomicInteger(); @@ -453,14 +447,17 @@ void messagesShouldBeDeDuplicatedWhenUsingNameAndPublishingId(int subEntrySize) consumeLatch.countDown(); }) .build(); - assertThat(consumeLatch.await(10, TimeUnit.SECONDS)).isTrue(); - Thread.sleep(1000); - // if we are using sub-entries, we cannot avoid duplicates. - // here, a sub-entry in the second wave, right at the end of the re-submitted - // values will contain those duplicates, because its publishing ID will be - // the one of its last message, so the server will accept the whole sub-entry, - // including the duplicates. - assertThat(consumed.get()).isEqualTo(lineCount + backwardCount % subEntrySize); + assertThat(consumeLatch.await(5, TimeUnit.SECONDS)).isTrue(); + if (subEntrySize == 1) { + assertThat(consumed.get()).isEqualTo(lineCount); + } else { + // if we are using sub-entries, we cannot avoid duplicates. + // here, a sub-entry in the second wave, right at the end of the re-submitted + // values will contain those duplicates, because its publishing ID will be + // the one of its last message, so the server will accept the whole sub-entry, + // including the duplicates. + assertThat(consumed.get()).isBetween(lineCount, lineCount + subEntrySize); + } } @ParameterizedTest @@ -636,11 +633,10 @@ void subEntryBatchesSentCompressedShouldBeConsumedProperly() { } @Test - void methodsShouldThrowExceptionWhenProducerIsClosed() throws InterruptedException { + void methodsShouldThrowExceptionWhenProducerIsClosed() { Producer producer = environment.producerBuilder().stream(stream).build(); producer.close(); - assertThatThrownBy(() -> producer.getLastPublishingId()) - .isInstanceOf(IllegalStateException.class); + assertThatThrownBy(producer::getLastPublishingId).isInstanceOf(IllegalStateException.class); } @Test diff --git a/src/test/java/com/rabbitmq/stream/impl/StreamProducerUnitTest.java b/src/test/java/com/rabbitmq/stream/impl/StreamProducerUnitTest.java index ecc1440ee6..1150a90842 100644 --- a/src/test/java/com/rabbitmq/stream/impl/StreamProducerUnitTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/StreamProducerUnitTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2021-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2021-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -15,6 +15,7 @@ package com.rabbitmq.stream.impl; import static com.rabbitmq.stream.impl.TestUtils.waitAtMost; +import static java.util.stream.IntStream.range; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.*; @@ -27,6 +28,7 @@ import com.rabbitmq.stream.StreamException; import com.rabbitmq.stream.codec.SimpleCodec; import com.rabbitmq.stream.compression.Compression; +import com.rabbitmq.stream.compression.DefaultCompressionCodecFactory; import com.rabbitmq.stream.impl.Client.OutboundEntityWriteCallback; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufAllocator; @@ -42,8 +44,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.ToLongFunction; -import java.util.stream.IntStream; import org.assertj.core.api.ThrowableAssert.ThrowingCallable; +import org.assertj.core.data.Offset; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; @@ -115,6 +117,8 @@ void init() { when(env.clock()).thenReturn(clock); when(env.codec()).thenReturn(new SimpleCodec()); when(env.observationCollector()).thenAnswer(invocation -> ObservationCollector.NO_OP); + DefaultCompressionCodecFactory ccf = new DefaultCompressionCodecFactory(); + when(env.compressionCodecFactory()).thenReturn(ccf); doAnswer( (Answer) invocationOnMock -> { @@ -172,34 +176,43 @@ void confirmTimeoutTaskShouldFailMessagesAfterTimeout( "stream", subEntrySize, 10, + StreamProducerBuilder.DEFAULT_DYNAMIC_BATCH, Compression.NONE, Duration.ofMillis(100), messageCount * 10, confirmTimeout, Duration.ofSeconds(10), + true, null, env); - IntStream.range(0, messageCount) + range(0, messageCount) .forEach( i -> producer.send( producer.messageBuilder().addData("".getBytes()).build(), confirmationHandler)); - IntStream.range(0, confirmedPart).forEach(publishingId -> producer.confirm(publishingId)); - assertThat(confirmedCount.get()).isEqualTo(expectedConfirmed); + waitAtMost(() -> producer.unconfirmedCount() >= messageCount / subEntrySize); + range(0, confirmedPart).forEach(producer::confirm); + if (subEntrySize == 1) { + assertThat(confirmedCount.get()).isEqualTo(expectedConfirmed); + } else { + assertThat(confirmedCount.get()).isCloseTo(confirmedCount.get(), Offset.offset(subEntrySize)); + } assertThat(erroredCount.get()).isZero(); + int confirmedPreviously = confirmedCount.get(); executorService.scheduleAtFixedRate(() -> clock.refresh(), 100, 100, TimeUnit.MILLISECONDS); Thread.sleep(waitTime.toMillis()); - assertThat(confirmedCount.get()).isEqualTo(expectedConfirmed); + assertThat(confirmedCount.get()).isEqualTo(confirmedPreviously); if (confirmTimeout.isZero()) { assertThat(erroredCount.get()).isZero(); assertThat(responseCodes).isEmpty(); } else { waitAtMost( - waitTime.multipliedBy(2), () -> erroredCount.get() == (messageCount - expectedConfirmed)); + waitTime.multipliedBy(2), + () -> erroredCount.get() == (messageCount - confirmedPreviously)); assertThat(responseCodes).hasSize(1).contains(Constants.CODE_PUBLISH_CONFIRM_TIMEOUT); } } @@ -214,11 +227,13 @@ void enqueueTimeoutMessageShouldBeFailedWhenEnqueueTimeoutIsReached(int subEntry "stream", subEntrySize, 10, + StreamProducerBuilder.DEFAULT_DYNAMIC_BATCH, Compression.NONE, Duration.ZERO, 2, Duration.ofMinutes(1), enqueueTimeout, + true, null, env); @@ -253,11 +268,13 @@ void enqueueTimeoutSendingShouldBlockWhenEnqueueTimeoutIsZero(int subEntrySize) "stream", subEntrySize, 10, + StreamProducerBuilder.DEFAULT_DYNAMIC_BATCH, Compression.NONE, Duration.ZERO, 2, Duration.ofMinutes(1), enqueueTimeout, + true, null, env); diff --git a/src/test/java/com/rabbitmq/stream/impl/StreamTestInfrastructure.java b/src/test/java/com/rabbitmq/stream/impl/StreamTestInfrastructure.java new file mode 100644 index 0000000000..1c02c0c278 --- /dev/null +++ b/src/test/java/com/rabbitmq/stream/impl/StreamTestInfrastructure.java @@ -0,0 +1,24 @@ +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. +// +// This software, the RabbitMQ Stream Java client library, is dual-licensed under the +// Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). +// For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL, +// please see LICENSE-APACHE2. +// +// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, +// either express or implied. See the LICENSE file for specific language governing +// rights and limitations of this software. +// +// If you have any questions regarding licensing, please contact us at +// info@rabbitmq.com. +package com.rabbitmq.stream.impl; + +import java.lang.annotation.*; +import org.junit.jupiter.api.extension.ExtendWith; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@ExtendWith(TestUtils.StreamTestInfrastructureExtension.class) +public @interface StreamTestInfrastructure {} diff --git a/src/test/java/com/rabbitmq/stream/impl/SubEntryBatchingTest.java b/src/test/java/com/rabbitmq/stream/impl/SubEntryBatchingTest.java index c7eb7891f1..65a6b5567a 100644 --- a/src/test/java/com/rabbitmq/stream/impl/SubEntryBatchingTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/SubEntryBatchingTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2021-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2021-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/SubscriptionTest.java b/src/test/java/com/rabbitmq/stream/impl/SubscriptionTest.java index d7cc8df41b..5c3eb4d828 100644 --- a/src/test/java/com/rabbitmq/stream/impl/SubscriptionTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/SubscriptionTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/SuperStreamConsumerTest.java b/src/test/java/com/rabbitmq/stream/impl/SuperStreamConsumerTest.java index fb41100709..a563532c06 100644 --- a/src/test/java/com/rabbitmq/stream/impl/SuperStreamConsumerTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/SuperStreamConsumerTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2021-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2021-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -94,6 +94,37 @@ private static void publishToPartitions( latchAssert(publishLatch).completes(); } + static AutoCloseable publishToPartitions(TestUtils.ClientFactory cf, List partitions) { + Client client = cf.get(); + for (int i = 0; i < partitions.size(); i++) { + assertThat(client.declarePublisher(b(i), null, partitions.get(i)).isOk()).isTrue(); + } + Runnable publish = + () -> { + int count = 0; + while (!Thread.currentThread().isInterrupted()) { + int partitionIndex = count++ % partitions.size(); + String partition = partitions.get(partitionIndex); + client.publish( + b(partitionIndex), + Collections.singletonList( + client + .messageBuilder() + .addData(partition.getBytes(StandardCharsets.UTF_8)) + .build())); + try { + Thread.sleep(10); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + }; + Thread thread = new Thread(publish); + thread.start(); + return thread::interrupt; + } + @Test void consumeAllMessagesFromAllPartitions() { declareSuperStreamTopology(configurationClient, superStream, partitionCount); @@ -299,60 +330,62 @@ void autoOffsetTrackingShouldStoreOffsetZero() { @BrokerVersionAtLeast(RABBITMQ_3_11_11) void rebalancedPartitionShouldGetMessagesWhenItComesBackToOriginalConsumerInstance() throws Exception { + Duration timeout = Duration.ofSeconds(60); declareSuperStreamTopology(configurationClient, superStream, partitionCount); Client client = cf.get(); List partitions = client.partitions(superStream); - int messageCount = 10_000; - publishToPartitions(cf, partitions, messageCount); - String consumerName = "my-app"; - Set receivedPartitions = ConcurrentHashMap.newKeySet(partitionCount); - Runnable processing = - () -> { - try { - Thread.sleep(10); - } catch (InterruptedException e) { - // OK - } - }; - Consumer consumer1 = - environment - .consumerBuilder() - .superStream(superStream) - .singleActiveConsumer() - .offset(OffsetSpecification.first()) - .name(consumerName) - .autoTrackingStrategy() - .messageCountBeforeStorage(messageCount / partitionCount / 50) - .builder() - .messageHandler( - (context, message) -> { - receivedPartitions.add(context.stream()); - processing.run(); - }) - .build(); - waitAtMost(() -> receivedPartitions.size() == partitions.size()); + try (AutoCloseable publish = publishToPartitions(cf, partitions)) { + int messageCountBeforeStorage = 10; + String consumerName = "my-app"; + Set receivedPartitions = ConcurrentHashMap.newKeySet(partitionCount); + Consumer consumer1 = + environment + .consumerBuilder() + .superStream(superStream) + .singleActiveConsumer() + .offset(OffsetSpecification.first()) + .name(consumerName) + .autoTrackingStrategy() + .messageCountBeforeStorage(messageCountBeforeStorage) + .builder() + .messageHandler( + (context, message) -> { + receivedPartitions.add(context.stream()); + }) + .build(); + waitAtMost( + timeout, + () -> receivedPartitions.size() == partitions.size(), + () -> + format( + "Expected to receive messages from all partitions, got %s", receivedPartitions)); - AtomicReference partition = new AtomicReference<>(); - Consumer consumer2 = - environment - .consumerBuilder() - .superStream(superStream) - .singleActiveConsumer() - .offset(OffsetSpecification.first()) - .name(consumerName) - .autoTrackingStrategy() - .messageCountBeforeStorage(messageCount / partitionCount / 50) - .builder() - .messageHandler( - (context, message) -> { - partition.set(context.stream()); - processing.run(); - }) - .build(); - waitAtMost(() -> partition.get() != null); - consumer2.close(); - receivedPartitions.clear(); - waitAtMost(() -> receivedPartitions.size() == partitions.size()); - consumer1.close(); + AtomicReference partition = new AtomicReference<>(); + Consumer consumer2 = + environment + .consumerBuilder() + .superStream(superStream) + .singleActiveConsumer() + .offset(OffsetSpecification.first()) + .name(consumerName) + .autoTrackingStrategy() + .messageCountBeforeStorage(messageCountBeforeStorage) + .builder() + .messageHandler( + (context, message) -> { + partition.set(context.stream()); + }) + .build(); + waitAtMost(timeout, () -> partition.get() != null); + consumer2.close(); + receivedPartitions.clear(); + waitAtMost( + timeout, + () -> receivedPartitions.size() == partitions.size(), + () -> + format( + "Expected to receive messages from all partitions, got %s", receivedPartitions)); + consumer1.close(); + } } } diff --git a/src/test/java/com/rabbitmq/stream/impl/SuperStreamManagementTest.java b/src/test/java/com/rabbitmq/stream/impl/SuperStreamManagementTest.java index 88da50289f..a8464ed04e 100644 --- a/src/test/java/com/rabbitmq/stream/impl/SuperStreamManagementTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/SuperStreamManagementTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2023-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -14,8 +14,8 @@ // info@rabbitmq.com. package com.rabbitmq.stream.impl; +import static com.rabbitmq.stream.Cli.*; import static com.rabbitmq.stream.Constants.*; -import static com.rabbitmq.stream.Host.*; import static com.rabbitmq.stream.impl.TestUtils.ResponseConditions.*; import static com.rabbitmq.stream.impl.TestUtils.streamName; import static com.rabbitmq.stream.impl.TestUtils.waitAtMost; diff --git a/src/test/java/com/rabbitmq/stream/impl/SuperStreamProducerTest.java b/src/test/java/com/rabbitmq/stream/impl/SuperStreamProducerTest.java index a08f6ba50c..e2d43d7aa1 100644 --- a/src/test/java/com/rabbitmq/stream/impl/SuperStreamProducerTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/SuperStreamProducerTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2021-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2021-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/SuperStreamTest.java b/src/test/java/com/rabbitmq/stream/impl/SuperStreamTest.java index 2d5db3f42e..0f3021126e 100644 --- a/src/test/java/com/rabbitmq/stream/impl/SuperStreamTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/SuperStreamTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2021-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2021-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/impl/TestUtils.java b/src/test/java/com/rabbitmq/stream/impl/TestUtils.java index 1b676e53fc..59d88cf6aa 100644 --- a/src/test/java/com/rabbitmq/stream/impl/TestUtils.java +++ b/src/test/java/com/rabbitmq/stream/impl/TestUtils.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -23,13 +23,17 @@ import static org.junit.jupiter.api.Assertions.fail; import ch.qos.logback.classic.Level; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; import com.rabbitmq.client.BuiltinExchangeType; import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; import com.rabbitmq.client.ConnectionFactory; import com.rabbitmq.stream.Address; +import com.rabbitmq.stream.Cli; import com.rabbitmq.stream.Constants; -import com.rabbitmq.stream.Host; import com.rabbitmq.stream.Message; import com.rabbitmq.stream.MessageBuilder; import com.rabbitmq.stream.StreamException; @@ -38,12 +42,8 @@ import com.rabbitmq.stream.impl.Client.Response; import com.rabbitmq.stream.impl.Client.StreamMetadata; import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; import io.vavr.Tuple2; -import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import java.lang.annotation.Documented; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; @@ -83,7 +83,7 @@ public final class TestUtils { private static final Logger LOGGER = LoggerFactory.getLogger(TestUtils.class); - private static final Duration DEFAULT_CONDITION_TIMEOUT = Duration.ofSeconds(10); + static final Duration DEFAULT_CONDITION_TIMEOUT = Duration.ofSeconds(10); private static final ConnectionFactory AMQP_CF = new ConnectionFactory(); @@ -93,6 +93,11 @@ public static Duration waitAtMost(CallableBooleanSupplier condition) throws Exce return waitAtMost(DEFAULT_CONDITION_TIMEOUT, condition, null); } + public static Duration waitAtMost( + CallableBooleanSupplier condition, String format, Object... args) throws Exception { + return waitAtMost(DEFAULT_CONDITION_TIMEOUT, condition, () -> String.format(format, args)); + } + public static Duration waitAtMost(CallableBooleanSupplier condition, Supplier message) throws Exception { return waitAtMost(DEFAULT_CONDITION_TIMEOUT, condition, message); @@ -257,6 +262,20 @@ public String toString() { }; } + static BiConsumer namedBiConsumer(BiConsumer delegate, String description) { + return new BiConsumer() { + @Override + public void accept(T t, U s) { + delegate.accept(t, s); + } + + @Override + public String toString() { + return description; + } + }; + } + static Answer answer(Runnable task) { return invocationOnMock -> { task.run(); @@ -358,28 +377,12 @@ private static String streamName(Class testClass, Method testMethod) { } static boolean tlsAvailable() { - if (Host.rabbitmqctlCommand() == null) { - throw new IllegalStateException( - "rabbitmqctl.bin system property not set, cannot check if TLS is enabled"); - } else { - try { - Process process = Host.rabbitmqctl("status"); - String output = capture(process.getInputStream()); - return output.contains("stream/ssl"); - } catch (Exception e) { - throw new RuntimeException("Error while trying to detect TLS: " + e.getMessage()); - } - } + return Cli.rabbitmqctl("status").output().contains("stream/ssl"); } - private static String capture(InputStream is) throws IOException { - BufferedReader br = new BufferedReader(new InputStreamReader(is)); - String line; - StringBuilder buff = new StringBuilder(); - while ((line = br.readLine()) != null) { - buff.append(line).append("\n"); - } - return buff.toString(); + static boolean isCluster() { + String content = Cli.rabbitmqctl("eval 'nodes().'").output(); + return !content.replace("[", "").replace("]", "").trim().isEmpty(); } static void forEach(Collection in, CallableIndexConsumer consumer) throws Exception { @@ -522,6 +525,12 @@ static boolean atLeastVersion(String expectedVersion, String currentVersion) { @ExtendWith(DisabledIfTlsNotEnabledCondition.class) public @interface DisabledIfTlsNotEnabled {} + @Target({ElementType.TYPE, ElementType.METHOD}) + @Retention(RetentionPolicy.RUNTIME) + @Documented + @ExtendWith(DisabledIfNotClusterCondition.class) + @interface DisabledIfNotCluster {} + @Target({ElementType.TYPE, ElementType.METHOD}) @Retention(RetentionPolicy.RUNTIME) @Documented @@ -617,7 +626,7 @@ static EventLoopGroup eventLoopGroup(ExtensionContext context) { @Override public void beforeAll(ExtensionContext context) { - store(context).put("nettyEventLoopGroup", new NioEventLoopGroup()); + store(context).put("nettyEventLoopGroup", Utils.eventLoopGroup()); } @Override @@ -795,7 +804,7 @@ static class DisabledIfRabbitMqCtlNotSetCondition implements ExecutionCondition @Override public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { - if (Host.rabbitmqctlCommand() == null) { + if (Cli.rabbitmqctlCommand() == null) { return ConditionEvaluationResult.disabled("rabbitmqctl.bin system property not set"); } else { return ConditionEvaluationResult.enabled("rabbitmqctl.bin system property is set"); @@ -815,15 +824,14 @@ abstract static class DisabledIfPluginNotEnabledCondition implements ExecutionCo @Override public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { - if (Host.rabbitmqctlCommand() == null) { + if (Cli.rabbitmqctlCommand() == null) { return ConditionEvaluationResult.disabled( format( "rabbitmqctl.bin system property not set, cannot check if %s plugin is enabled", pluginLabel)); } else { try { - Process process = Host.rabbitmqctl("status"); - String output = capture(process.getInputStream()); + String output = Cli.rabbitmqctl("status").output(); if (condition.test(output)) { return ConditionEvaluationResult.enabled(format("%s plugin enabled", pluginLabel)); } else { @@ -884,6 +892,22 @@ public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext con } } + static class DisabledIfNotClusterCondition implements ExecutionCondition { + + private static final String KEY = "isCluster"; + + @Override + public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) { + ExtensionContext.Store store = context.getRoot().getStore(ExtensionContext.Namespace.GLOBAL); + boolean isCluster = store.getOrComputeIfAbsent(KEY, k -> isCluster(), Boolean.class); + if (isCluster) { + return ConditionEvaluationResult.enabled("Multi-node cluster"); + } else { + return ConditionEvaluationResult.disabled("Not a multi-node cluster"); + } + } + } + private static class BaseBrokerVersionAtLeastCondition implements ExecutionCondition { private final Function versionProvider; @@ -1023,6 +1047,12 @@ static void waitMs(long waitTime) { } } + static String jsonPrettyPrint(String in) { + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + JsonElement element = JsonParser.parseString(in); + return gson.toJson(element); + } + @Target({ElementType.TYPE, ElementType.METHOD}) @Retention(RetentionPolicy.RUNTIME) @Tag("single-active-consumer") @@ -1034,7 +1064,8 @@ public enum BrokerVersion { RABBITMQ_3_11_9("3.11.9"), RABBITMQ_3_11_11("3.11.11"), RABBITMQ_3_11_14("3.11.14"), - RABBITMQ_3_13_0("3.13.0"); + RABBITMQ_3_13_0("3.13.0"), + RABBITMQ_4_0_0("4.0.0"); final String value; @@ -1102,4 +1133,54 @@ static void repeatIfFailure(RunnableWithException test) throws Exception { private static Connection connection() throws IOException, TimeoutException { return AMQP_CF.newConnection(); } + + static Sync sync() { + return sync(1); + } + + static Sync sync(int count) { + return new Sync(count); + } + + static class Sync { + + private final AtomicReference latch = new AtomicReference<>(); + + private Sync(int count) { + this.latch.set(new CountDownLatch(count)); + } + + void down() { + this.latch.get().countDown(); + } + + void down(int count) { + IntStream.range(0, count).forEach(ignored -> this.latch.get().countDown()); + } + + boolean await(Duration timeout) { + try { + return this.latch.get().await(timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new RuntimeException(ie); + } + } + + long currentCount() { + return this.latch.get().getCount(); + } + + void reset(int count) { + this.latch.set(new CountDownLatch(count)); + } + + void reset() { + this.reset(1); + } + + boolean hasCompleted() { + return this.latch.get().getCount() == 0; + } + } } diff --git a/src/test/java/com/rabbitmq/stream/impl/TestUtilsTest.java b/src/test/java/com/rabbitmq/stream/impl/TestUtilsTest.java index b1e889fa25..9938ef90ef 100644 --- a/src/test/java/com/rabbitmq/stream/impl/TestUtilsTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/TestUtilsTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -16,18 +16,41 @@ import static org.assertj.core.api.Assertions.assertThat; +import java.util.concurrent.Executor; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; public class TestUtilsTest { - @ParameterizedTest - @CsvSource({ - "3.9.6,3.9.5,false", - "3.9.6,3.9.0-alpha-stream.232,true", - "3.9.6,3.9.6-alpha.28,true" - }) - void atLeastVersion(String expectedVersion, String currentVersion, boolean expected) { - assertThat(TestUtils.atLeastVersion(expectedVersion, currentVersion)).isEqualTo(expected); - } + @ParameterizedTest + @CsvSource( + { + "3.9.6,3.9.5,false", + "3.9.6,3.9.0-alpha-stream.232,true", + "3.9.6,3.9.6-alpha.28,true", + } + ) + void atLeastVersion( + String expectedVersion, + String currentVersion, + boolean expected + ) { + assertThat( + TestUtils.atLeastVersion(expectedVersion, currentVersion) + ).isEqualTo(expected); + } + + private static class DelegatingExecutor implements Executor { + + private final Executor delegate; + + private DelegatingExecutor(Executor delegate) { + this.delegate = delegate; + } + + @Override + public void execute(Runnable command) { + delegate.execute(command); + } + } } diff --git a/src/test/java/com/rabbitmq/stream/impl/TlsTest.java b/src/test/java/com/rabbitmq/stream/impl/TlsTest.java index e45b89e171..249034555b 100644 --- a/src/test/java/com/rabbitmq/stream/impl/TlsTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/TlsTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2021-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2021-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -36,7 +36,6 @@ import io.netty.handler.ssl.SslHandler; import java.io.File; import java.io.FileInputStream; -import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; import java.nio.charset.Charset; @@ -57,8 +56,6 @@ import javax.net.ssl.SSLException; import javax.net.ssl.SSLHandshakeException; import javax.net.ssl.SSLParameters; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -66,26 +63,6 @@ @ExtendWith(TestUtils.StreamTestInfrastructureExtension.class) public class TlsTest { - static boolean isJava13() { - String javaVersion = System.getProperty("java.version"); - return javaVersion != null && javaVersion.startsWith("13."); - } - - @BeforeEach - public void init() { - if (isJava13()) { - // for Java 13.0.7, see https://github.com/bcgit/bc-java/issues/941 - System.setProperty("keystore.pkcs12.keyProtectionAlgorithm", "PBEWithHmacSHA256AndAES_256"); - } - } - - @AfterEach - public void tearDown() throws Exception { - if (isJava13()) { - System.setProperty("keystore.pkcs12.keyProtectionAlgorithm", ""); - } - } - String stream; TestUtils.ClientFactory cf; @@ -268,10 +245,10 @@ void saslExternalShouldSucceedWithUserForClientCertificate() throws Exception { .build(); String username = clientCertificate.getSubjectX500Principal().getName(); - Host.rabbitmqctlIgnoreError(format("delete_user %s", username)); - Host.rabbitmqctl(format("add_user %s foo", username)); + Cli.rabbitmqctlIgnoreError(format("delete_user %s", username)); + Cli.rabbitmqctl(format("add_user %s foo", username)); try { - Host.rabbitmqctl(format("set_permissions %s '.*' '.*' '.*'", username)); + Cli.rabbitmqctl(format("set_permissions %s '.*' '.*' '.*'", username)); cf.get( new ClientParameters() @@ -279,7 +256,7 @@ void saslExternalShouldSucceedWithUserForClientCertificate() throws Exception { .sslContext(context) .saslConfiguration(DefaultSaslConfiguration.EXTERNAL)); } finally { - Host.rabbitmqctl(format("delete_user %s", username)); + Cli.rabbitmqctl(format("delete_user %s", username)); } } @@ -295,7 +272,7 @@ void saslExternalShouldFailIfNoUserForClientCertificate() throws Exception { .build(); String username = clientCertificate.getSubjectX500Principal().getName(); - Host.rabbitmqctlIgnoreError(format("delete_user %s", username)); + Cli.rabbitmqctlIgnoreError(format("delete_user %s", username)); assertThatThrownBy( () -> cf.get( @@ -318,12 +295,12 @@ void hostnameVerificationShouldFailWhenSettingHostToLoopbackInterface() throws E @Test void shouldConnectWhenSettingHostToLoopbackInterfaceAndDisablingHostnameVerification() throws Exception { - SslContext context = SslContextBuilder.forClient().trustManager(caCertificate()).build(); - cf.get( - new ClientParameters() - .sslContext(context) - .host("127.0.0.1") - .tlsHostnameVerification(false)); + SslContext context = + SslContextBuilder.forClient() + .endpointIdentificationAlgorithm(null) + .trustManager(caCertificate()) + .build(); + cf.get(new ClientParameters().sslContext(context).host("127.0.0.1")); } @Test @@ -385,11 +362,7 @@ private static String hostname() { try { return InetAddress.getLocalHost().getHostName(); } catch (UnknownHostException e) { - try { - return Host.hostname(); - } catch (IOException ex) { - throw new RuntimeException(ex); - } + return Cli.hostname(); } } diff --git a/src/test/java/com/rabbitmq/stream/impl/UtilsTest.java b/src/test/java/com/rabbitmq/stream/impl/UtilsTest.java index dc83319e31..fb2f0bcddf 100644 --- a/src/test/java/com/rabbitmq/stream/impl/UtilsTest.java +++ b/src/test/java/com/rabbitmq/stream/impl/UtilsTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -20,6 +20,7 @@ import static com.rabbitmq.stream.impl.Utils.defaultConnectionNamingStrategy; import static com.rabbitmq.stream.impl.Utils.formatConstant; import static com.rabbitmq.stream.impl.Utils.offsetBefore; +import static java.util.Collections.emptyList; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; @@ -32,10 +33,10 @@ import com.rabbitmq.stream.impl.Utils.ClientConnectionType; import com.rabbitmq.stream.impl.Utils.ClientFactory; import com.rabbitmq.stream.impl.Utils.ClientFactoryContext; -import com.rabbitmq.stream.impl.Utils.ExactNodeRetryClientFactory; +import com.rabbitmq.stream.impl.Utils.ConditionalClientFactory; import java.time.Duration; +import java.util.function.BiPredicate; import java.util.function.Function; -import java.util.function.Predicate; import java.util.stream.IntStream; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -53,14 +54,14 @@ void formatConstantOk() { } @Test - void exactNodeRetryClientFactoryShouldReturnImmediatelyIfConditionOk() { + void conditionalClientFactoryShouldReturnImmediatelyIfConditionOk() { Client client = mock(Client.class); ClientFactory cf = mock(ClientFactory.class); when(cf.client(any())).thenReturn(client); - Predicate condition = c -> true; + BiPredicate condition = (ctx, c) -> true; Client result = - new ExactNodeRetryClientFactory(cf, condition, Duration.ofMillis(1)) - .client(ClientFactoryContext.fromParameters(new ClientParameters())); + new ConditionalClientFactory(cf, condition, Duration.ofMillis(1)) + .client(new ClientFactoryContext(new ClientParameters(), "", emptyList())); assertThat(result).isEqualTo(client); verify(cf, times(1)).client(any()); verify(client, never()).close(); @@ -68,15 +69,15 @@ void exactNodeRetryClientFactoryShouldReturnImmediatelyIfConditionOk() { @Test @SuppressWarnings("unchecked") - void exactNodeRetryClientFactoryShouldRetryUntilConditionOk() { + void conditionalClientFactoryShouldRetryUntilConditionOk() { Client client = mock(Client.class); ClientFactory cf = mock(ClientFactory.class); when(cf.client(any())).thenReturn(client); - Predicate condition = mock(Predicate.class); - when(condition.test(any())).thenReturn(false).thenReturn(false).thenReturn(true); + BiPredicate condition = mock(BiPredicate.class); + when(condition.test(any(), any())).thenReturn(false).thenReturn(false).thenReturn(true); Client result = - new ExactNodeRetryClientFactory(cf, condition, Duration.ofMillis(1)) - .client(ClientFactoryContext.fromParameters(new ClientParameters())); + new ConditionalClientFactory(cf, condition, Duration.ofMillis(1)) + .client(new ClientFactoryContext(new ClientParameters(), "", emptyList())); assertThat(result).isEqualTo(client); verify(cf, times(3)).client(any()); verify(client, times(2)).close(); diff --git a/src/test/java/com/rabbitmq/stream/metrics/MetricsCollectorsTest.java b/src/test/java/com/rabbitmq/stream/metrics/MetricsCollectorsTest.java index 638913de7a..20b5152b07 100644 --- a/src/test/java/com/rabbitmq/stream/metrics/MetricsCollectorsTest.java +++ b/src/test/java/com/rabbitmq/stream/metrics/MetricsCollectorsTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/observation/micrometer/MicrometerObservationCollectorTest.java b/src/test/java/com/rabbitmq/stream/observation/micrometer/MicrometerObservationCollectorTest.java index 026b90e641..704a347f81 100644 --- a/src/test/java/com/rabbitmq/stream/observation/micrometer/MicrometerObservationCollectorTest.java +++ b/src/test/java/com/rabbitmq/stream/observation/micrometer/MicrometerObservationCollectorTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. -// and/or its subsidiaries. +// Copyright (c) 2007-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/java/com/rabbitmq/stream/sasl/DefaultSaslConfigurationTest.java b/src/test/java/com/rabbitmq/stream/sasl/DefaultSaslConfigurationTest.java index 579f955eff..5979406515 100644 --- a/src/test/java/com/rabbitmq/stream/sasl/DefaultSaslConfigurationTest.java +++ b/src/test/java/com/rabbitmq/stream/sasl/DefaultSaslConfigurationTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). @@ -58,6 +58,6 @@ void getSaslMechanismShouldThrowExceptionIfNoMechanismSpecifiedAndNoMatch() { assertThatThrownBy(() -> configuration.getSaslMechanism(asList("FOO", "BAR"))) .isInstanceOf(IllegalStateException.class) .hasMessage( - "Unable to agree on a SASL mechanism. Client: PLAIN, EXTERNAL / server FOO, BAR."); + "Unable to agree on a SASL mechanism. Client: PLAIN, EXTERNAL, ANONYMOUS / server FOO, BAR."); } } diff --git a/src/test/java/com/rabbitmq/stream/sasl/PlainSaslMechanismTest.java b/src/test/java/com/rabbitmq/stream/sasl/PlainSaslMechanismTest.java index 011df695aa..eb3746b836 100644 --- a/src/test/java/com/rabbitmq/stream/sasl/PlainSaslMechanismTest.java +++ b/src/test/java/com/rabbitmq/stream/sasl/PlainSaslMechanismTest.java @@ -1,5 +1,5 @@ -// Copyright (c) 2020-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom -// Inc. and/or its subsidiaries. +// Copyright (c) 2020-2025 Broadcom. All Rights Reserved. +// The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries. // // This software, the RabbitMQ Stream Java client library, is dual-licensed under the // Mozilla Public License 2.0 ("MPL"), and the Apache License version 2 ("ASL"). diff --git a/src/test/resources/logback-test.xml b/src/test/resources/logback-test.xml index 4bec720537..0cf733c381 100644 --- a/src/test/resources/logback-test.xml +++ b/src/test/resources/logback-test.xml @@ -6,6 +6,11 @@ + + + + +