Skip to content

Commit

Permalink
Ledger API Test Tool: support --additional tests [KVL-1100] (#10829)
Browse files Browse the repository at this point in the history 
* Support adding tests as an hidden option

* Simplify existing suites

CHANGELOG_BEGIN
CHANGELOG_END

* Remove stale conformance suites from build.yml

* `--add` -> ``--additional`

* Re-add `--all-tests` as deprecated CLI option to be tested

* Move sandbox-classic pruning test to wall clock again

* Run KVCommandDeduplicationIT for sandbox append-only

* Tidy-up

* Also add participant pruning test to ledger-on-memory/single-participant

* Remove KVCommandDeduplicationIT on ledger-on-memory/append-only

* Run the full suite plus pruning (rather than just pruning) for ledger-on-memory with multiple participants and append-only

* Add KVCommandDeduplicationIT to ledger-on-memory append-only

* Exclude ConfigManagementServiceIT from ledger-on-memory append-only multi-participant

* Tidy-up

* Use KVCommandDeduplicationIT for sandbox-on-x too

* Fix merge

Add max dedup duration arg to all the test suites that include command dedup tests

* Make `--include` and `--additional` mutually exclusive

* Uniform formatting of multi-line strings

* Move exclusions after additions as they are applied last

* Re-disable deduplication test on sandbox with static time

* Re-disable deduplication test on sandbox-on-x
  • Loading branch information
@fabiotudone-da
fabiotudone-da authored Sep 14, 2021
1 parent 97e14de commit be4e064
Show file tree
Hide file tree
Showing 12 changed files with 197 additions and 608 deletions.
6 changes: 1 addition & 5 deletions ci/build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,11 +315,7 @@ jobs:
//triggers/service:test-oracle \
//ledger/participant-integration-api:participant-integration-api-tests-oracle \
//ledger/ledger-on-sql:conformance-test-oracle \
//ledger/ledger-on-sql:conformance-test-append-only-oracle \
//ledger/ledger-on-sql:conformance-test-append-only-pruning-oracle \
//ledger/ledger-on-sql:conformance-test-pruning-oracle \
//ledger/ledger-on-sql:conformance-test-append-only-multi-party-submission-oracle \
//ledger/ledger-on-sql:conformance-test-multi-party-submission-oracle
//ledger/ledger-on-sql:conformance-test-append-only-oracle
env:
DOCKER_LOGIN: $(DOCKER_LOGIN)
DOCKER_PASSWORD: $(DOCKER_PASSWORD)
Expand Down
60 changes: 6 additions & 54 deletions ledger/daml-on-sql/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,37 +109,16 @@ conformance_test(
"--port=6865",
"--eager-package-loading",
],
)

# TODO append-only: cleanup
conformance_test(
name = "conformance-test-append-only",
server = ":daml-on-sql-ephemeral-postgresql",
server_args = [
"--ledgerid=conformance-test",
"--port=6865",
"--eager-package-loading",
"--enable-append-only-schema",
],
)

conformance_test(
name = "conformance-test-multi-party-submissions",
server = ":daml-on-sql-ephemeral-postgresql",
server_args = [
"--ledgerid=conformance-test",
"--port=6865",
"--eager-package-loading",
],
test_tool_args = [
"--verbose",
"--include=MultiPartySubmissionIT",
"--additional=MultiPartySubmissionIT",
"--additional=ContractIdIT:Accept",
],
)

# TODO append-only: cleanup
conformance_test(
name = "conformance-test-multi-party-submissions-append-only",
name = "conformance-test-append-only",
server = ":daml-on-sql-ephemeral-postgresql",
server_args = [
"--ledgerid=conformance-test",
Expand All @@ -149,36 +128,9 @@ conformance_test(
],
test_tool_args = [
"--verbose",
"--include=MultiPartySubmissionIT",
],
)

conformance_test(
name = "conformance-test-contract-ids",
server = ":daml-on-sql-ephemeral-postgresql",
server_args = [
"--ledgerid=conformance-test",
"--port=6865",
],
test_tool_args = [
"--verbose",
"--include=ContractIdIT:Accept",
],
)

conformance_test(
name = "conformance-test-append-only-command-completion-dedup-info",
server = ":daml-on-sql-ephemeral-postgresql",
server_args = [
"--ledgerid=conformance-test",
"--port=6865",
"--enable-append-only-schema",
],
test_tool_args = [
"--verbose",
"--include=" +
"AppendOnlyCompletionDeduplicationInfoITCommandService" +
",AppendOnlyCompletionDeduplicationInfoITCommandSubmissionService",
"--additional=MultiPartySubmissionIT",
"--additional=AppendOnlyCompletionDeduplicationInfoITCommandService",
"--additional=AppendOnlyCompletionDeduplicationInfoITCommandSubmissionService",
],
)

Expand Down
81 changes: 16 additions & 65 deletions ledger/ledger-api-test-tool/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ da_scala_test_suite(
)

conformance_test(
name = "ssl-test",
name = "conformance-test",
extra_data = [
"//ledger/test-common/test-certificates:client.crt",
"//ledger/test-common/test-certificates:client.pem",
Expand All @@ -219,10 +219,12 @@ conformance_test(
"//ledger/test-common/test-certificates:server.pem.enc",
"//ledger/test-common/test-certificates:ca.crt",
],
lf_versions = lf_version_configuration_versions,
ports = [6865],
server = "//ledger/ledger-on-memory:app",
server_args = [
"--contract-id-seeding=testing-weak",
"--participant participant-id=ssl-test,port=6865",
"--participant=participant-id=example,port=6865",
"--crt $$(rlocation $$TEST_WORKSPACE/$(rootpath //ledger/test-common/test-certificates:server.crt))",
"--cacrt $$(rlocation $$TEST_WORKSPACE/$(rootpath //ledger/test-common/test-certificates:ca.crt))",
"--pem $$(rlocation $$TEST_WORKSPACE/$(rootpath //ledger/test-common/test-certificates:server.pem.enc))",
Expand All @@ -231,74 +233,23 @@ conformance_test(
],
test_tool_args = [
"--verbose",
"--include=SemanticTests",
"--crt $$(rlocation $$TEST_WORKSPACE/$(rootpath //ledger/test-common/test-certificates:client.crt))",
"--cacrt $$(rlocation $$TEST_WORKSPACE/$(rootpath //ledger/test-common/test-certificates:ca.crt))",
"--pem $$(rlocation $$TEST_WORKSPACE/$(rootpath //ledger/test-common/test-certificates:client.pem))",
],
)

# Explicitly include retired tests here to make sure existing CI pipelines are not broken
# Retired tests will be eventually removed
conformance_test(
name = "retired-tests",
server = "//ledger/ledger-on-memory:app",
server_args = [
"--contract-id-seeding=testing-weak",
"--participant participant-id=ssl-test,port=6865",
],
test_tool_args = [
"--include=LotsOfPartiesIT",
"--include=TransactionScaleIT",
],
)

# Makes sure that deprecated CLI options can still be used to make sure existing CI pipelines are not broken
# This test should fail if any deprecated CLI option has any effect whatsoever -- they are preserved
# exclusively for backwards-compatibility
# Deprecated CLI options will be eventually removed
conformance_test(
name = "deprecated-cli-options",
server = "//ledger/ledger-on-memory:app",
server_args = [
"--contract-id-seeding=testing-weak",
"--participant participant-id=ssl-test,port=6865",
],
test_tool_args = [
"--include=IdentityIT",
"--all-tests",
"--additional=IdentityIT",
"--additional=IdentityIT:IdNotEmpty",
# Explicitly include retired tests here to make sure existing CI pipelines are not broken.
# Retired tests will be eventually removed.
"--additional=LotsOfPartiesIT",
"--additional=TransactionScaleIT",
"--exclude=CommandDeduplicationIT",
# Makes sure that deprecated CLI options can still be used to make sure existing CI pipelines are not broken.
# This test should fail if any deprecated CLI option has any effect whatsoever -- they are preserved
# exclusively for backwards-compatibility.
# Deprecated CLI options will be eventually removed.
"--load-scale-factor=THIS_OPTION_IS_DEPRECATED_AND_HAS_NO_EFFECT",
"--target-port=THIS_OPTION_IS_DEPRECATED_AND_HAS_NO_EFFECT",
],
)

# Test that both --include ContractKeysIT as well as --include ContractKeysIT:CKFetchOrLookup
# are supported.
conformance_test(
name = "test-name-syntax",
server = "//ledger/ledger-on-memory:app",
server_args = [
"--contract-id-seeding=testing-weak",
"--participant participant-id=test-name,port=6865",
],
test_tool_args = [
"--include=IdentityIT",
"--include=IdentityIT:IdNotEmpty",
],
)

conformance_test(
name = "conformance-test",
lf_versions = lf_version_configuration_versions,
ports = [6865],
server = "//ledger/ledger-on-memory:app",
server_args = [
"--contract-id-seeding=testing-weak",
"--participant=participant-id=example,port=6865",
],
test_tool_args = [
"--verbose",
"--exclude=CommandDeduplicationIT",
"--all-tests",
],
)

Expand Down
84 changes: 60 additions & 24 deletions ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/Cli.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,14 @@

package com.daml.ledger.api.testtool

import java.io.File
import java.nio.file.{Path, Paths}

import com.daml.buildinfo.BuildInfo
import com.daml.ledger.api.testtool.infrastructure.PartyAllocationConfiguration
import com.daml.ledger.api.testtool.tests.Tests
import com.daml.ledger.api.tls.TlsConfiguration
import scopt.{OptionParser, Read}

import java.io.File
import java.nio.file.{Path, Paths}
import scala.collection.compat.immutable.LazyList
import scala.concurrent.duration.{DurationInt, FiniteDuration}
import scala.util.Try
Expand Down Expand Up @@ -72,14 +71,16 @@ object Cli {
private def invalidPerformanceTestName[A](name: String): Either[String, Unit] =
failure(s"$name is not a valid performance test name. Use `--list` to see valid names.")

head("""The Ledger API Test Tool is a command line tool for testing the correctness of
|ledger implementations based on Daml and Ledger API.""".stripMargin)
head(
"""The Ledger API Test Tool is a command line tool for testing the correctness of
|ledger implementations based on Daml and Ledger API.""".stripMargin
)

arg[(String, Int)]("[endpoints...]")(endpointRead)
.action((address, config) => config.copy(participants = config.participants :+ address))
.unbounded()
.optional()
.text("""Addresses of the participants to test, specified as `<host>:<port>`.""")
.text("Addresses of the participants to test, specified as `<host>:<port>`.")

opt[Int]("max-connection-attempts")
.action((maxConnectionAttempts, config) =>
Expand All @@ -103,7 +104,8 @@ object Cli {
opt[String]("crt")
.optional()
.text(
"TLS: The crt file to be used as the cert chain. Required if any other TLS parameters are set. Applied to all endpoints."
"""TLS: The crt file to be used as the cert chain.
|Required if any other TLS parameters are set. Applied to all endpoints.""".stripMargin
)
.action(crtConfig)

Expand All @@ -115,10 +117,12 @@ object Cli {
opt[Double](name = "timeout-scale-factor")
.optional()
.action((v, c) => c.copy(timeoutScaleFactor = v))
.text("""Scale factor for timeouts used in all test suites. Useful to tune timeouts
.text(
"""Scale factor for timeouts used in all test suites. Useful to tune timeouts
|depending on the environment and the Ledger implementation under test.
|Defaults to 1.0. Use numbers higher than 1.0 to make test timeouts more lax,
|use numbers lower than 1.0 to make test timeouts more strict.""".stripMargin)
|use numbers lower than 1.0 to make test timeouts more strict.""".stripMargin
)

opt[String](name = "load-scale-factor")
.optional()
Expand All @@ -138,10 +142,12 @@ object Cli {

opt[Unit]("must-fail")
.action((_, c) => c.copy(mustFail = true))
.text("""Reverse success status logic of the tool. Use this flag if you expect one or
.text(
"""Reverse success status logic of the tool. Use this flag if you expect one or
|more or the scenario tests to fail. If enabled, the tool will succeed when at
|least one test fails, and it will fail when all tests succeed. Defaults to
|false.""".stripMargin)
|false.""".stripMargin
)

opt[Unit]('x', "extract")
.action((_, c) => c.copy(extract = true))
Expand All @@ -154,21 +160,38 @@ object Cli {
.action((ex, c) => c.copy(excluded = c.excluded ++ ex))
.unbounded()
.text(
"""A comma-separated list of exclusion prefixes. Tests whose name start with any of the given prefixes will be skipped. Can be specified multiple times, i.e. `--exclude=a,b` is the same as `--exclude=a --exclude=b`."""
"""A comma-separated list of exclusion prefixes. Tests whose name start with
|any of the given prefixes will be skipped. Can be specified multiple times,
|i.e. `--exclude=a,b` is the same as `--exclude=a --exclude=b`.""".stripMargin
)

opt[Seq[String]]("include")
.action((inc, c) => c.copy(included = c.included ++ inc))
.unbounded()
.text(
"""A comma-separated list of inclusion prefixes. If not specified, all default tests are included. If specified, only tests that match at least one of the given inclusion prefixes (and none of the given exclusion prefixes) will be run. Can be specified multiple times, i.e. `--include=a,b` is the same as `--include=a --include=b`."""
"""A comma-separated list of inclusion prefixes. If not specified,
|all default tests are included. If specified, only tests that match at least one
|of the given inclusion prefixes (and none of the given exclusion prefixes) will be run.
|Can be specified multiple times, i.e. `--include=a,b` is the same as `--include=a --include=b`.
|Mutually exclusive with `--additional`.""".stripMargin
)

opt[Seq[String]]("additional")
.action((additional, c) => c.copy(additional = c.additional ++ additional))
.hidden()
.unbounded()
.text(
"""A comma-separated list of additional prefixes. If specified, also tests that match at least one
|of the given inclusion prefixes (and none of the given exclusion prefixes) will be run.
|Can be specified multiple times, i.e. `--additional=a,b` is the same as `--additional=a --additional=b`.
|Mutually exclusive with `--include`.""".stripMargin
)

opt[Seq[String]]("perf-tests")
.validate(_.find(!Tests.PerformanceTestsKeys(_)).fold(success)(invalidPerformanceTestName))
.action((inc, c) => c.copy(performanceTests = c.performanceTests ++ inc))
.unbounded()
.text("""A comma-separated list of performance tests that should be run.""")
.text("A comma-separated list of performance tests that should be run.")

opt[Path]("perf-tests-report")
.action((inc, c) => c.copy(performanceTestsReport = Some(inc)))
Expand All @@ -184,29 +207,35 @@ object Cli {

opt[Unit]("shuffle-participants")
.action((_, c) => c.copy(shuffleParticipants = true))
.text("""Shuffle the list of participants used in a test.
|By default participants are used in the order they're given.""".stripMargin)
.text(
"""Shuffle the list of participants used in a test.
|By default participants are used in the order they're given.""".stripMargin
)

opt[Unit]("no-wait-for-parties")
.action((_, c) => c.copy(partyAllocation = PartyAllocationConfiguration.ClosedWorld))
.text("""Do not wait for parties to be allocated on all participants.""")
.text("Do not wait for parties to be allocated on all participants.")
.hidden()

opt[Unit]("open-world")
.action((_, c) => c.copy(partyAllocation = PartyAllocationConfiguration.OpenWorld))
.text("""|Do not allocate parties explicitly.
|Instead, expect the ledger to allocate parties dynamically.
|Party names must be their hints.""".stripMargin)
.text(
"""Do not allocate parties explicitly.
|Instead, expect the ledger to allocate parties dynamically.
|Party names must be their hints.""".stripMargin
)

opt[Unit]("list")
.action((_, c) => c.copy(listTestSuites = true))
.text(
"""Lists all available test suites that can be used in the include and exclude options. Test names always start with their suite name, so using the suite name as a prefix matches all tests in a given suite."""
"""Lists all available test suites that can be used in the include and exclude options.
|Test names always start with their suite name, so using the suite name as a prefix
|matches all tests in a given suite.""".stripMargin
)

opt[Unit]("list-all")
.action((_, c) => c.copy(listTests = true))
.text("""Lists all available tests that can be used in the include and exclude options.""")
.text("Lists all available tests that can be used in the include and exclude options.")

opt[Unit]("version")
.optional()
Expand All @@ -222,16 +251,23 @@ object Cli {
.optional()
.action((x, c) => c.copy(ledgerClockGranularity = x))
.text(
"Specify the largest interval that you will see between clock ticks on the ledger under test. The default is \"1s\" (1 second)."
"""Specify the largest interval that you will see between clock ticks
|on the ledger under test. The default is \"1s\" (1 second).""".stripMargin
)

opt[Unit]("skip-dar-upload")
.optional()
.action((_, c) => c.copy(uploadDars = false))
.text("Skip DARs upload into ledger before running tests")

help("help").text("Prints this usage text")
checkConfig(c =>
if (c.included.nonEmpty && c.additional.nonEmpty)
failure("`--include` and `--additional` are mutually exclusive")
else
success
)

help("help").text("Prints this usage text")
}

def parse(args: Array[String]): Option[Config] =
Expand Down
Loading

0 comments on commit be4e064

Please sign in to comment.