Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Attempt to improve muzzle time by randomly ignoring versions until 10 remain. #1451

Merged
merged 1 commit into from
May 13, 2020
Merged

Attempt to improve muzzle time by randomly ignoring versions until 10 remain. #1451

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ defaults: &defaults
- image: &default_container datadog/dd-trace-java-docker-build:latest

cache_keys: &cache_keys
# Reset the cache approx every release
keys:
- dd-trace-java-{{ checksum "dd-trace-java.gradle" }}-{{ .Branch }}-{{ .Revision }}
- dd-trace-java-{{ checksum "dd-trace-java.gradle" }}-{{ .Branch }}
- dd-trace-java-{{ checksum "dd-trace-java.gradle" }}
# Rev the version when the cache gets too big
- dd-trace-java-v1-{{ .Branch }}-{{ .Revision }}
- dd-trace-java-v1-{{ .Branch }}
# - dd-trace-java-v1-

jobs:
build:
Expand Down Expand Up @@ -225,16 +225,18 @@ jobs:
- checkout

- restore_cache:
# Reset the cache approx every release
keys:
- dd-trace-java-muzzle-{{ checksum "dd-trace-java.gradle" }}
# Rev the version when the cache gets too big
- dd-trace-java-muzzle-v1-{{ .Branch }}-{{ .Revision }}
- dd-trace-java-muzzle-v1-{{ .Branch }}
# - dd-trace-java-muzzle-v1-

- run:
name: Verify Muzzle
command: SKIP_BUILDSCAN="true" GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx4G -Xms64M' -Ddatadog.forkedMaxHeapSize=4G -Ddatadog.forkedMinHeapSize=64M" ./gradlew muzzle --parallel --stacktrace --no-daemon --max-workers=16

- save_cache:
key: dd-trace-java-muzzle-{{ checksum "dd-trace-java.gradle" }}
key: dd-trace-java-muzzle-v1-{{ .Branch }}-{{ .Revision }}
paths: ~/.gradle

workflows:
Expand Down
40 changes: 32 additions & 8 deletions buildSrc/src/main/groovy/MuzzlePlugin.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,16 @@ import java.util.regex.Pattern
* muzzle task plugin which runs muzzle validation against a range of dependencies.
*/
class MuzzlePlugin implements Plugin<Project> {
/**
* Select a random set of versions to test
*/
private static final int RANGE_COUNT_LIMIT = 10
/**
* Remote repositories used to query version ranges and fetch dependencies
*/
private static final List<RemoteRepository> MUZZLE_REPOS
private static final AtomicReference<ClassLoader> TOOLING_LOADER = new AtomicReference<>()

static {
RemoteRepository central = new RemoteRepository.Builder("central", "default", "https://repo1.maven.org/maven2/").build()
RemoteRepository sonatype = new RemoteRepository.Builder("sonatype", "default", "https://oss.sonatype.org/content/repositories/releases/").build()
Expand Down Expand Up @@ -204,20 +209,23 @@ class MuzzlePlugin implements Plugin<Project> {
/**
* Convert a muzzle directive to a list of artifacts
*/
private static List<Artifact> muzzleDirectiveToArtifacts(MuzzleDirective muzzleDirective, RepositorySystem system, RepositorySystemSession session) {
private static Set<Artifact> muzzleDirectiveToArtifacts(MuzzleDirective muzzleDirective, RepositorySystem system, RepositorySystemSession session) {
final Artifact directiveArtifact = new DefaultArtifact(muzzleDirective.group, muzzleDirective.module, "jar", muzzleDirective.versions)

final VersionRangeRequest rangeRequest = new VersionRangeRequest()
rangeRequest.setRepositories(MUZZLE_REPOS)
rangeRequest.setArtifact(directiveArtifact)
final VersionRangeResult rangeResult = system.resolveVersionRange(session, rangeRequest)
final Set<Version> versions = rangeResult.versions.toSet()

limitLargeRanges(rangeResult, versions, muzzleDirective.skipVersions)

// println "Range Request: " + rangeRequest
// println "Range Result: " + rangeResult

final List<Artifact> allVersionArtifacts = filterVersion(rangeResult.versions, muzzleDirective.skipVersions).collect { version ->
final Set<Artifact> allVersionArtifacts = filterVersion(versions, muzzleDirective.skipVersions).collect { version ->
new DefaultArtifact(muzzleDirective.group, muzzleDirective.module, "jar", version.toString())
}
}.toSet()

if (allVersionArtifacts.isEmpty()) {
throw new GradleException("No muzzle artifacts found for $muzzleDirective.group:$muzzleDirective.module $muzzleDirective.versions")
Expand All @@ -229,8 +237,8 @@ class MuzzlePlugin implements Plugin<Project> {
/**
* Create a list of muzzle directives which assert the opposite of the given MuzzleDirective.
*/
private static List<MuzzleDirective> inverseOf(MuzzleDirective muzzleDirective, RepositorySystem system, RepositorySystemSession session) {
List<MuzzleDirective> inverseDirectives = new ArrayList<>()
private static Set<MuzzleDirective> inverseOf(MuzzleDirective muzzleDirective, RepositorySystem system, RepositorySystemSession session) {
Set<MuzzleDirective> inverseDirectives = new HashSet<>()

final Artifact allVerisonsArtifact = new DefaultArtifact(muzzleDirective.group, muzzleDirective.module, "jar", "[,)")
final Artifact directiveArtifact = new DefaultArtifact(muzzleDirective.group, muzzleDirective.module, "jar", muzzleDirective.versions)
Expand All @@ -245,9 +253,10 @@ class MuzzlePlugin implements Plugin<Project> {
rangeRequest.setRepositories(MUZZLE_REPOS)
rangeRequest.setArtifact(directiveArtifact)
final VersionRangeResult rangeResult = system.resolveVersionRange(session, rangeRequest)
final Set<Version> versions = rangeResult.versions.toSet()

filterVersion(allRangeResult.versions, muzzleDirective.skipVersions).collect { version ->
if (!rangeResult.versions.contains(version)) {
filterVersion(allRangeResult.versions.toSet(), muzzleDirective.skipVersions).collect { version ->
if (!versions.contains(version)) {
final MuzzleDirective inverseDirective = new MuzzleDirective()
inverseDirective.group = muzzleDirective.group
inverseDirective.module = muzzleDirective.module
Expand All @@ -260,6 +269,21 @@ class MuzzlePlugin implements Plugin<Project> {
return inverseDirectives
}

private static void limitLargeRanges(VersionRangeResult result, Set<Version> versions, Set<String> skipVersions) {
List<Version> copy = new ArrayList<>(versions)
Collections.shuffle(copy)
copy.removeAll(skipVersions)
while (RANGE_COUNT_LIMIT <= copy.size()) {
Version version = copy.pop()
if (!(version.equals(result.lowestVersion) || version.equals(result.highestVersion))) {
skipVersions.add(version.toString())
}
}
if (skipVersions.size() > 0) {
println "Muzzle skipping " + skipVersions.size() + " versions"
}
}

/**
* Configure a muzzle task to pass or fail a given version.
*
Expand Down Expand Up @@ -360,7 +384,7 @@ class MuzzlePlugin implements Plugin<Project> {
/**
* Filter out snapshot-type builds from versions list.
*/
private static filterVersion(List<Version> list, Set<String> skipVersions) {
private static filterVersion(Set<Version> list, Set<String> skipVersions) {
list.removeIf {
def version = it.toString().toLowerCase()
return version.contains("rc") ||
Expand Down