Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Let stable packages bypass LF version restrictions. #10377

Merged
merged 16 commits into from
Jul 27, 2021
Merged

Let stable packages bypass LF version restrictions. #10377

Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
f1e6cb4
Stable packages bypass version restrictions.
sofiafaro-da Jul 22, 2021
6dd741f
Add missing bypass
sofiafaro-da Jul 22, 2021
3f095f8
scalafmt
sofiafaro-da Jul 22, 2021
9c7c3a5
Fix tests
sofiafaro-da Jul 22, 2021
40ae433
fix script runner
sofiafaro-da Jul 22, 2021
ee3d4d9
Fix scenario service
sofiafaro-da Jul 23, 2021
fbf4e9e
Dont make stable packages configurable
sofiafaro-da Jul 23, 2021
60ccfcb
unused import
sofiafaro-da Jul 23, 2021
81071c2
revert unnecessary formatting changes
sofiafaro-da Jul 26, 2021
643c1fb
Added a StableOnly engine mode and a conformance test that uses it
sofiafaro-da Jul 26, 2021
f930922
buildifier-fix
sofiafaro-da Jul 26, 2021
fec1b49
Inline LanguageVersions.StableOnly
sofiafaro-da Jul 26, 2021
1e5d575
scalafmt
sofiafaro-da Jul 26, 2021
c63576a
Remove EngineMode in favor of representing the version range directly
sofiafaro-da Jul 26, 2021
d4fda65
Add -unsafe to flag name
sofiafaro-da Jul 27, 2021
d6082c5
Apply suggestions from code review
sofiafaro-da Jul 27, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions daml-lf/engine/src/main/scala/com/digitalasset/daml/lf/engine/Engine.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import com.daml.lf.transaction.Node._
import com.daml.lf.value.Value
import java.nio.file.Files

import com.daml.lf.language.{Interface, LanguageVersion}
import com.daml.lf.language.{Interface, LanguageVersion, StablePackages}
import com.daml.lf.validation.Validation
import com.daml.lf.value.Value.ContractId
import com.daml.nameof.NameOf
Expand Down Expand Up @@ -452,7 +452,9 @@ class Engine(val config: EngineConfig = new EngineConfig(LanguageVersion.StableV
for {
_ <- pkgs
.collectFirst {
case (pkgId, pkg) if !config.allowedLanguageVersions.contains(pkg.languageVersion) =>
case (pkgId, pkg)
if !StablePackages.ids.contains(pkgId) && !config.allowedLanguageVersions
.contains(pkg.languageVersion) =>
Error.Package.AllowedLanguageVersion(
pkgId,
pkg.languageVersion,
Expand Down
6 changes: 4 additions & 2 deletions daml-lf/interpreter/src/main/scala/com/digitalasset/daml/lf/speedy/Compiler.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import java.util
import com.daml.lf.data.Ref._
import com.daml.lf.data.{ImmArray, Numeric, Ref, Struct, Time}
import com.daml.lf.language.Ast._
import com.daml.lf.language.{LanguageVersion, LookupError, Interface}
import com.daml.lf.language.{LanguageVersion, LookupError, Interface, StablePackages}
import com.daml.lf.speedy.Anf.flattenToAnf
import com.daml.lf.speedy.Profile.LabelModule
import com.daml.lf.speedy.SBuiltin._
Expand Down Expand Up @@ -346,7 +346,9 @@ private[lf] final class Compiler(
val t0 = Time.Timestamp.now()

interface.lookupPackage(pkgId) match {
case Right(pkg) if !config.allowedLanguageVersions.contains(pkg.languageVersion) =>
case Right(pkg)
if !StablePackages.ids.contains(pkgId) && !config.allowedLanguageVersions
.contains(pkg.languageVersion) =>
throw LanguageVersionError(pkgId, pkg.languageVersion, config.allowedLanguageVersions)
case _ =>
}
Expand Down
120 changes: 120 additions & 0 deletions daml-lf/language/src/main/scala/com/digitalasset/daml/lf/language/StablePackages.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package com.daml.lf.language

import com.daml.lf.data.Ref.{PackageId, PackageName}

object StablePackages {
// Based on compiler/damlc/tests/src/stable-packages.sh
// TODO: Use this map to generate the mapping used in the test,
miklos-da marked this conversation as resolved.
Show resolved Hide resolved
// or generate both from a single source.
val map = Map[PackageName, PackageId](
sofiafaro-da marked this conversation as resolved.
Show resolved Hide resolved
PackageName.assertFromString(
"daml-prim-DA-Exception-ArithmeticError-cb0552debf219cc909f51cbb5c3b41e9981d39f8f645b1f35e2ef5be2e0b858a"
) -> PackageId.assertFromString(
"cb0552debf219cc909f51cbb5c3b41e9981d39f8f645b1f35e2ef5be2e0b858a"
),
PackageName.assertFromString(
"daml-prim-DA-Exception-AssertionFailed-3f4deaf145a15cdcfa762c058005e2edb9baa75bb7f95a4f8f6f937378e86415"
) -> PackageId.assertFromString(
"3f4deaf145a15cdcfa762c058005e2edb9baa75bb7f95a4f8f6f937378e86415"
),
PackageName.assertFromString(
"daml-prim-DA-Exception-GeneralError-86828b9843465f419db1ef8a8ee741d1eef645df02375ebf509cdc8c3ddd16cb"
) -> PackageId.assertFromString(
"86828b9843465f419db1ef8a8ee741d1eef645df02375ebf509cdc8c3ddd16cb"
),
PackageName.assertFromString(
"daml-prim-DA-Exception-PreconditionFailed-f20de1e4e37b92280264c08bf15eca0be0bc5babd7a7b5e574997f154c00cb78"
) -> PackageId.assertFromString(
"f20de1e4e37b92280264c08bf15eca0be0bc5babd7a7b5e574997f154c00cb78"
),
PackageName.assertFromString(
"daml-prim-DA-Internal-Erased-76bf0fd12bd945762a01f8fc5bbcdfa4d0ff20f8762af490f8f41d6237c6524f"
) -> PackageId.assertFromString(
"76bf0fd12bd945762a01f8fc5bbcdfa4d0ff20f8762af490f8f41d6237c6524f"
),
PackageName.assertFromString(
"daml-prim-DA-Internal-PromotedText-d58cf9939847921b2aab78eaa7b427dc4c649d25e6bee3c749ace4c3f52f5c97"
) -> PackageId.assertFromString(
"d58cf9939847921b2aab78eaa7b427dc4c649d25e6bee3c749ace4c3f52f5c97"
),
PackageName.assertFromString(
"daml-prim-DA-Types-40f452260bef3f29dede136108fc08a88d5a5250310281067087da6f0baddff7"
) -> PackageId.assertFromString(
"40f452260bef3f29dede136108fc08a88d5a5250310281067087da6f0baddff7"
),
PackageName.assertFromString(
"daml-prim-GHC-Prim-e491352788e56ca4603acc411ffe1a49fefd76ed8b163af86cf5ee5f4c38645b"
) -> PackageId.assertFromString(
"e491352788e56ca4603acc411ffe1a49fefd76ed8b163af86cf5ee5f4c38645b"
),
PackageName.assertFromString(
"daml-prim-GHC-Tuple-6839a6d3d430c569b2425e9391717b44ca324b88ba621d597778811b2d05031d"
) -> PackageId.assertFromString(
"6839a6d3d430c569b2425e9391717b44ca324b88ba621d597778811b2d05031d"
),
PackageName.assertFromString(
"daml-prim-GHC-Types-518032f41fd0175461b35ae0c9691e08b4aea55e62915f8360af2cc7a1f2ba6c"
) -> PackageId.assertFromString(
"518032f41fd0175461b35ae0c9691e08b4aea55e62915f8360af2cc7a1f2ba6c"
),
PackageName.assertFromString(
"daml-stdlib-DA-Date-Types-bfcd37bd6b84768e86e432f5f6c33e25d9e7724a9d42e33875ff74f6348e733f"
) -> PackageId.assertFromString(
"bfcd37bd6b84768e86e432f5f6c33e25d9e7724a9d42e33875ff74f6348e733f"
),
PackageName.assertFromString(
"daml-stdlib-DA-Internal-Any-cc348d369011362a5190fe96dd1f0dfbc697fdfd10e382b9e9666f0da05961b7"
) -> PackageId.assertFromString(
"cc348d369011362a5190fe96dd1f0dfbc697fdfd10e382b9e9666f0da05961b7"
),
PackageName.assertFromString(
"daml-stdlib-DA-Internal-Down-057eed1fd48c238491b8ea06b9b5bf85a5d4c9275dd3f6183e0e6b01730cc2ba"
) -> PackageId.assertFromString(
"057eed1fd48c238491b8ea06b9b5bf85a5d4c9275dd3f6183e0e6b01730cc2ba"
),
PackageName.assertFromString(
"daml-stdlib-DA-Internal-Template-d14e08374fc7197d6a0de468c968ae8ba3aadbf9315476fd39071831f5923662"
) -> PackageId.assertFromString(
"d14e08374fc7197d6a0de468c968ae8ba3aadbf9315476fd39071831f5923662"
),
PackageName.assertFromString(
"daml-stdlib-DA-Logic-Types-c1f1f00558799eec139fb4f4c76f95fb52fa1837a5dd29600baa1c8ed1bdccfd"
) -> PackageId.assertFromString(
"c1f1f00558799eec139fb4f4c76f95fb52fa1837a5dd29600baa1c8ed1bdccfd"
),
PackageName.assertFromString(
"daml-stdlib-DA-Monoid-Types-6c2c0667393c5f92f1885163068cd31800d2264eb088eb6fc740e11241b2bf06"
) -> PackageId.assertFromString(
"6c2c0667393c5f92f1885163068cd31800d2264eb088eb6fc740e11241b2bf06"
),
PackageName.assertFromString(
"daml-stdlib-DA-NonEmpty-Types-e22bce619ae24ca3b8e6519281cb5a33b64b3190cc763248b4c3f9ad5087a92c"
) -> PackageId.assertFromString(
"e22bce619ae24ca3b8e6519281cb5a33b64b3190cc763248b4c3f9ad5087a92c"
),
PackageName.assertFromString(
"daml-stdlib-DA-Semigroup-Types-8a7806365bbd98d88b4c13832ebfa305f6abaeaf32cfa2b7dd25c4fa489b79fb"
) -> PackageId.assertFromString(
"8a7806365bbd98d88b4c13832ebfa305f6abaeaf32cfa2b7dd25c4fa489b79fb"
),
PackageName.assertFromString(
"daml-stdlib-DA-Set-Types-97b883cd8a2b7f49f90d5d39c981cf6e110cf1f1c64427a28a6d58ec88c43657"
) -> PackageId.assertFromString(
"97b883cd8a2b7f49f90d5d39c981cf6e110cf1f1c64427a28a6d58ec88c43657"
),
PackageName.assertFromString(
"daml-stdlib-DA-Time-Types-733e38d36a2759688a4b2c4cec69d48e7b55ecc8dedc8067b815926c917a182a"
) -> PackageId.assertFromString(
"733e38d36a2759688a4b2c4cec69d48e7b55ecc8dedc8067b815926c917a182a"
),
PackageName.assertFromString(
"daml-stdlib-DA-Validation-Types-99a2705ed38c1c26cbb8fe7acf36bbf626668e167a33335de932599219e0a235"
) -> PackageId.assertFromString(
"99a2705ed38c1c26cbb8fe7acf36bbf626668e167a33335de932599219e0a235"
),
)
val ids = map.values.toSet
sofiafaro-da marked this conversation as resolved.
Show resolved Hide resolved
}
12 changes: 12 additions & 0 deletions ledger/ledger-on-memory/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,18 @@ conformance_test(
],
)

conformance_test(
name = "conformance-test-min-1.14",
lf_versions = ["1.14"],
ports = [6865],
server = ":app",
server_args = [
"--contract-id-seeding=testing-weak",
"--participant=participant-id=example,port=6865",
"--daml-lf-min-version-1.14",
],
)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

a (non-conformance) test that actually checks that (non-stable) packages older than 1.14 are rejected would be a nice to have.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, there is already some tests for this at the engine level:

daml/daml-lf/engine/src/test/scala/com/digitalasset/daml/lf/engine/EngineTest.scala

Line 2683 in 4b55f1a

val positiveTestCases = Table(

But there's room for a test of the new flag --daml-lf-min-version-1.14


conformance_test(
name = "conformance-test-multi-participant",
ports = [
Expand Down
32 changes: 18 additions & 14 deletions ...ate/kvutils/app/src/main/scala/com/daml/ledger/participant/state/kvutils/app/Config.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,10 @@ import java.util.concurrent.TimeUnit

import com.daml.caching
import com.daml.ledger.api.tls.TlsConfiguration
import com.daml.ledger.participant.state.kvutils.app.Config.EngineMode
import com.daml.ledger.resources.ResourceOwner
import com.daml.lf.data.Ref
import com.daml.lf.language.LanguageVersion
import com.daml.lf.VersionRange
import com.daml.metrics.MetricsReporter
import com.daml.platform.apiserver.SeedService.Seeding
import com.daml.platform.configuration.Readers._
Expand All @@ -40,7 +41,7 @@ final case class Config[Extra](
seeding: Seeding,
metricsReporter: Option[MetricsReporter],
metricsReportingInterval: Duration,
engineMode: EngineMode,
allowedLanguageVersions: VersionRange[LanguageVersion],
enableAppendOnlySchema: Boolean, // TODO append-only: remove after removing support for the current (mutating) schema
enableMutableContractStateCache: Boolean,
enableInMemoryFanOutForLedgerApi: Boolean,
Expand Down Expand Up @@ -73,7 +74,7 @@ object Config {
seeding = Seeding.Strong,
metricsReporter = None,
metricsReportingInterval = Duration.ofSeconds(10),
engineMode = EngineMode.Stable,
allowedLanguageVersions = LanguageVersion.StableVersions,
enableAppendOnlySchema = false,
enableMutableContractStateCache = false,
enableInMemoryFanOutForLedgerApi = false,
Expand Down Expand Up @@ -486,19 +487,31 @@ object Config {

opt[Unit]("early-access")
.optional()
.action((_, c) => c.copy(engineMode = EngineMode.EarlyAccess))
.action((_, c) => c.copy(allowedLanguageVersions = LanguageVersion.EarlyAccessVersions))
miklos-da marked this conversation as resolved.
Show resolved Hide resolved
.text(
"Enable preview version of the next Daml-LF language. Should not be used in production."
)

opt[Unit]("daml-lf-dev-mode-unsafe")
.optional()
.hidden()
.action((_, c) => c.copy(engineMode = EngineMode.Dev))
.action((_, c) => c.copy(allowedLanguageVersions = LanguageVersion.DevVersions))
.text(
"Enable the development version of the Daml-LF language. Highly unstable. Should not be used in production."
)

opt[Unit]("daml-lf-min-version-1.14")
sofiafaro-da marked this conversation as resolved.
Show resolved Hide resolved
.optional()
.hidden()
.action((_, c) =>
c.copy(allowedLanguageVersions =
c.allowedLanguageVersions.copy(min = LanguageVersion.v1_14)
)
)
.text(
"Set minimum LF version for unstable packages to 1.14. Should not be used in production."
)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mziolekda @miklos-da
This PR adds a feature needed by canton. However we need a way to test it through the full stack.
Are you fine we add this flag to be able to use the ledger-on-memory to test the feature ?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, with the above small change. Though, it would be preferred to parse the actual version so that we don't have to modify this command-line parameter for the next Daml-LF version.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Though, it would preferred to parse the actual version so that we don't have to modify this command-line parameter for the next Daml-LF version.

I agree that it would be preferable to parse the actual min version, but note that there's no plan currently for the 1.14 minimum to change in the future, even when the next LF version is released.


// TODO append-only: remove after removing support for the current (mutating) schema
opt[Unit]("index-append-only-schema-unsafe")
.optional()
Expand Down Expand Up @@ -550,13 +563,4 @@ object Config {
extraOptions(parser)
parser
}

sealed abstract class EngineMode extends Product with Serializable

object EngineMode {
final case object Stable extends EngineMode
final case object EarlyAccess extends EngineMode
final case object Dev extends EngineMode
}

}
10 changes: 1 addition & 9 deletions ...ate/kvutils/app/src/main/scala/com/daml/ledger/participant/state/kvutils/app/Runner.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,12 @@ import akka.actor.ActorSystem
import akka.stream.Materializer
import com.codahale.metrics.InstrumentedExecutorService
import com.daml.ledger.api.health.HealthChecks
import com.daml.ledger.participant.state.kvutils.app.Config.EngineMode
import com.daml.ledger.participant.state.v1.WritePackagesService
import com.daml.ledger.participant.state.v1.metrics.{TimedReadService, TimedWriteService}
import com.daml.ledger.resources.{Resource, ResourceContext, ResourceOwner}
import com.daml.lf.archive.DarParser
import com.daml.lf.data.Ref
import com.daml.lf.engine._
import com.daml.lf.language.LanguageVersion
import com.daml.logging.LoggingContext.newLoggingContext
import com.daml.logging.{ContextualizedLogger, LoggingContext}
import com.daml.metrics.JvmMetricSet
Expand Down Expand Up @@ -83,13 +81,7 @@ final class Runner[T <: ReadWriteService, Extra](
)
implicit val materializer: Materializer = Materializer(actorSystem)

val allowedLanguageVersions =
config.engineMode match {
case EngineMode.Stable => LanguageVersion.StableVersions
case EngineMode.EarlyAccess => LanguageVersion.EarlyAccessVersions
case EngineMode.Dev => LanguageVersion.DevVersions
}
val sharedEngine = new Engine(EngineConfig(allowedLanguageVersions))
val sharedEngine = new Engine(EngineConfig(config.allowedLanguageVersions))

newLoggingContext { implicit loggingContext =>
for {
Expand Down