Skip to content

Commit

Permalink
Update the bom tool to include external transitive dependencies via e…
Browse files Browse the repository at this point in the history 
…xternal BOMs
  • Loading branch information
@pallavit
pallavit committed Jul 7, 2021
1 parent 6a72bac commit a0e66c2
Show file tree
Hide file tree
Showing 4 changed files with 251 additions and 85 deletions.
20 changes: 20 additions & 0 deletions eng/bomgenerator/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,26 @@
<artifactId>maven-core</artifactId>
<version>3.5.4</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.9.6</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>2.9.6</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-xml</artifactId>
<version>2.11.1</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.9.6</version>
</dependency>
</dependencies>
<build>
<plugins>
Expand Down
178 changes: 93 additions & 85 deletions eng/bomgenerator/src/main/java/com/azure/tools/bomgenerator/BomGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,9 @@
import org.apache.maven.model.Model;
import org.apache.maven.model.io.xpp3.MavenXpp3Reader;
import org.apache.maven.model.io.xpp3.MavenXpp3Writer;
import org.jboss.shrinkwrap.resolver.api.maven.Maven;
import org.jboss.shrinkwrap.resolver.api.maven.MavenArtifactInfo;
import org.jboss.shrinkwrap.resolver.api.maven.MavenFormatStage;
import org.jboss.shrinkwrap.resolver.api.maven.MavenResolvedArtifact;
import org.jboss.shrinkwrap.resolver.api.maven.MavenResolverSystemBase;
import org.jboss.shrinkwrap.resolver.api.maven.MavenStrategyStage;
import org.jboss.shrinkwrap.resolver.api.maven.PomEquippedResolveStage;
import org.jboss.shrinkwrap.resolver.api.maven.PomlessResolveStage;
import org.jboss.shrinkwrap.resolver.api.maven.ScopeType;
import org.jboss.shrinkwrap.resolver.api.maven.coordinate.MavenDependency;
import org.codehaus.plexus.util.xml.pull.XmlPullParserException;

import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
Expand All @@ -28,7 +20,6 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.TreeMap;
import java.util.TreeSet;
Expand All @@ -39,25 +30,27 @@
import static com.azure.tools.bomgenerator.Utils.AZURE_PERF_LIBRARY_IDENTIFIER;
import static com.azure.tools.bomgenerator.Utils.AZURE_TEST_LIBRARY_IDENTIFIER;
import static com.azure.tools.bomgenerator.Utils.EXCLUSION_LIST;
import static com.azure.tools.bomgenerator.Utils.EXTERNAL_BOM_DEPENDENCIES;
import static com.azure.tools.bomgenerator.Utils.POM_TYPE;
import static com.azure.tools.bomgenerator.Utils.RESOLVED_EXCLUSION_LIST;
import static com.azure.tools.bomgenerator.Utils.SDK_DEPENDENCY_PATTERN;
import static com.azure.tools.bomgenerator.Utils.SDK_NON_GA_PATTERN;

public class BomGenerator {
private List<BomDependency> scannedDependencies = new ArrayList<>();
private TreeSet<BomDependency> externalDependencies = new TreeSet<>(new BomDependencyComparator());
private List<BomDependency> bomEligibleDependencies = new ArrayList<>();
private TreeSet<BomDependency> bomIneligibleDependencies = new TreeSet<>(new BomDependencyComparator());

private String outputFileName;
private String inputFileName;
private String pomFileName;
private String externalDependenciesFileName;
private String inputGroupId;

TreeMap<BomDependencyNoVersion, HashMap<String, Collection<BomDependency>>> nameToVersionToChildrenDependencyTree = new TreeMap<>(new BomDependencyNonVersionComparator());
TreeMap<BomDependencyNoVersion, HashMap<String, Collection<BomDependency>>> nameToVersionToChildrenDependencyTreeForExternal = new TreeMap<>(new BomDependencyNonVersionComparator());

BomGenerator() {}
BomGenerator() {
}

public void setInputFile(String inputFileName) {
this.inputFileName = inputFileName;
Expand All @@ -71,6 +64,10 @@ public void setPomFile(String pomFileName) {
this.pomFileName = pomFileName;
}

public void setExternalDependenciesFile(String externalDependenciesFileName) {
this.externalDependenciesFileName = externalDependenciesFileName;
}

public void setGroupId(String groupId) {
this.inputGroupId = groupId;
}
Expand All @@ -89,23 +86,23 @@ private void scan() {
Matcher nonGAMatcher = SDK_NON_GA_PATTERN.matcher(version);
if (!nonGAMatcher.matches()) {
BomDependency dependency = new BomDependency(groupId, artifactId, version);
if(inputGroupId.equalsIgnoreCase(groupId)) {
switch (artifactId) {
case "azure-sdk-all":
case "azure-sdk-parent":
case "azure-client-sdk-parent":
break;
default:
if (EXCLUSION_LIST.contains(artifactId)
|| artifactId.contains(AZURE_PERF_LIBRARY_IDENTIFIER)
|| (artifactId.contains(AZURE_TEST_LIBRARY_IDENTIFIER) && !artifactId.equalsIgnoreCase(AZURE_CORE_TEST_LIBRARY))) {
System.out.printf("Skipping dependency %s:%s%n", groupId, artifactId);
continue;
}

scannedDependencies.add(dependency);
break;
}
if (inputGroupId.equalsIgnoreCase(groupId)) {
switch (artifactId) {
case "azure-sdk-all":
case "azure-sdk-parent":
case "azure-client-sdk-parent":
break;
default:
if (EXCLUSION_LIST.contains(artifactId)
|| artifactId.contains(AZURE_PERF_LIBRARY_IDENTIFIER)
|| (artifactId.contains(AZURE_TEST_LIBRARY_IDENTIFIER) && !artifactId.equalsIgnoreCase(AZURE_CORE_TEST_LIBRARY))) {
System.out.printf("Skipping dependency %s:%s%n", groupId, artifactId);
continue;
}

scannedDependencies.add(dependency);
break;
}
}
}
}
Expand All @@ -117,8 +114,7 @@ private void scan() {
}
}

private void filterConflicts() {

private void resolveConflicts() {
// We can do this in a more performant way by going through the iteration once and use more memory
// but choosing this for simplicity.
nameToVersionToChildrenDependencyTree.keySet().stream().forEach(
Expand All @@ -128,22 +124,62 @@ private void filterConflicts() {
// We have multiple versions of this coming up, we will let the latest version win.
List<String> versionList = versionToDependency.keySet().stream().sorted(new DependencyVersionComparator()).collect(Collectors.toList());
String latestVersion = versionList.get(versionList.size() - 1);
System.out.printf("Multiple version of the dependency %s included. Picking the latest version for BOM: %s\r\n", key, latestVersion);
bomEligibleDependencies.add(new BomDependency(key.getGroupId(), key.getArtifactId(), latestVersion));
for (int index = 0; index < versionList.size() - 1; index++) {
String version = versionList.get(index);
bomIneligibleDependencies.addAll(versionToDependency.get(version));

if (key.getArtifactId().startsWith("azure-core")) {
System.out.printf("Multiple version of the dependency %s included\r\n", key);
System.out.printf("\tPicking the latest version for BOM: %s\r\n", latestVersion);

BomDependency dependency = new BomDependency(key.getGroupId(), key.getArtifactId(), latestVersion);
if (!externalDependencies.contains(dependency)) {
bomEligibleDependencies.add(dependency);
}
for (int index = 0; index < versionList.size() - 1; index++) {
String version = versionList.get(index);
bomIneligibleDependencies.addAll(versionToDependency.get(version));
System.out.printf("\tDiscarding version:%s\r\n", version);
System.out.printf("\t\tDiscarding child dependencies %s\r\n", versionToDependency.get(version).toString());
}
} else {
// If this is not a core dependency, we pick the highest version with the largest size.
int largestSize = 0;
String version = null;

for (String orderedVersion : versionList) {
int currentSize = versionToDependency.get(orderedVersion).size();
if (currentSize > largestSize) {
largestSize = currentSize;
version = orderedVersion;
}
}

BomDependency dependency = new BomDependency(key.getGroupId(), key.getArtifactId(), version);
bomEligibleDependencies.add(dependency);
System.out.printf("Multiple version of the dependency %s included\r\n", key);
System.out.printf("\tPicking the highest version with largest dependents: %s\r\n", version);

// Put all the other dependencies into the ineligible list.
for (String orderedVersion : versionList) {
if (orderedVersion != version) {
bomIneligibleDependencies.addAll(versionToDependency.get(orderedVersion));
System.out.printf("\tDiscarding version:%s\r\n", orderedVersion);
System.out.printf("\t\tDiscarding child dependencies %s\r\n", versionToDependency.get(orderedVersion).toString());
}
}
}
}
});
}

private void filterConflicts() {
resolveConflicts();
nameToVersionToChildrenDependencyTree.keySet().stream().forEach(
key -> {
HashMap<String, Collection<BomDependency>> versionToDependency = nameToVersionToChildrenDependencyTree.get(key);

if (versionToDependency.size() == 1) {
BomDependency dependency = new BomDependency(key.getGroupId(), key.getArtifactId(), versionToDependency.keySet().stream().findFirst().get());
if (!bomIneligibleDependencies.contains(dependency)) {
if (!bomIneligibleDependencies.contains(dependency)
&& !externalDependencies.contains(dependency)) {
// No conflict, the library can be added to the list.
bomEligibleDependencies.add(dependency);
}
Expand All @@ -154,6 +190,7 @@ private void filterConflicts() {
public void generate() {
scan();
resolveTree();
resolveExternalDependencies();
filterConflicts();
rewriteBomFile();
writeBom();
Expand Down Expand Up @@ -181,6 +218,7 @@ private void writeBom() {
DependencyManagement management = model.getDependencyManagement();
List<Dependency> externalBomDependencies = management.getDependencies().stream().filter(dependency -> dependency.getType().equals(POM_TYPE)).collect(Collectors.toList());


List<Dependency> dependencies = bomEligibleDependencies.stream().map(bomDependency -> {
Dependency dependency = new Dependency();
dependency.setGroupId(bomDependency.getGroupId());
Expand All @@ -202,55 +240,25 @@ private void writeBom() {
}

public void resolveTree() {
for (MavenDependency gaLibrary : scannedDependencies) {
try {
MavenResolvedArtifact mavenResolvedArtifact = getMavenResolver()
.addDependency(gaLibrary)
.resolve()
.withoutTransitivity()
.asSingleResolvedArtifact();

BomDependency parentDependency = new BomDependency(mavenResolvedArtifact.getCoordinate());
MavenArtifactInfo[] dependencies = mavenResolvedArtifact.getDependencies();

addDependencyToDependencyTree(parentDependency, null, nameToVersionToChildrenDependencyTree);

for (MavenArtifactInfo dependency : dependencies) {
if (dependency.getScope() == ScopeType.TEST) {
continue;
}
if(/*EXTERNAL_BOM_DEPENDENCIES.contains(dependency.getCoordinate().getGroupId())
|| */RESOLVED_EXCLUSION_LIST.contains(dependency.getCoordinate().getArtifactId())) {
// This is coming from the BOM, we may not include this.
continue;
}

BomDependency childDependency = new BomDependency(dependency.getCoordinate());
addDependencyToDependencyTree(childDependency, parentDependency, nameToVersionToChildrenDependencyTree);
}
}
catch(Exception ex) {
System.out.println(ex);
}
}
}

private MavenResolverSystemBase<PomEquippedResolveStage, PomlessResolveStage, MavenStrategyStage, MavenFormatStage> getMavenResolver() {
return Maven.configureResolver().withMavenCentralRepo(true);
Utils.resolveTree(scannedDependencies, nameToVersionToChildrenDependencyTree);
}

private void addDependencyToDependencyTree(BomDependency dependency, BomDependency parentDependency, TreeMap<BomDependencyNoVersion, HashMap<String, Collection<BomDependency>>> dependencyTree) {
if (!dependencyTree.containsKey(dependency)) {
dependencyTree.put(new BomDependencyNoVersion(dependency.getGroupId(), dependency.getArtifactId()), new HashMap<>());
}

HashMap<String, Collection<BomDependency>> versionToParents = dependencyTree.get(dependency);
if(!versionToParents.containsKey(dependency.getVersion())) {
versionToParents.put(dependency.getVersion(), new ArrayList<>());
}
public void resolveExternalDependencies() {
MavenXpp3Reader reader = new MavenXpp3Reader();
try {
Model model = reader.read(new FileReader(this.pomFileName));
DependencyManagement management = model.getDependencyManagement();
List<Dependency> externalBomDependencies = management.getDependencies().stream().filter(dependency -> dependency.getType().equals(POM_TYPE)).collect(Collectors.toList());
for (Dependency externalDependency : externalBomDependencies) {
externalDependencies.addAll(Utils.getPomFileContent(externalDependency));
}

if(parentDependency != null) {
versionToParents.get(dependency.getVersion()).add(parentDependency);
} catch (XmlPullParserException e) {
e.printStackTrace();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException exception) {
exception.printStackTrace();
}
}
}
6 changes: 6 additions & 0 deletions eng/bomgenerator/src/main/java/com/azure/tools/bomgenerator/Main.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import java.util.regex.Matcher;

import static com.azure.tools.bomgenerator.Utils.AZURE_CORE_GROUPID;
import static com.azure.tools.bomgenerator.Utils.COMMANDLINE_EXTERNALDEPENDENCIES;
import static com.azure.tools.bomgenerator.Utils.COMMANDLINE_GROUPID;
import static com.azure.tools.bomgenerator.Utils.COMMANDLINE_INPUTFILE;
import static com.azure.tools.bomgenerator.Utils.COMMANDLINE_OUTPUTFILE;
Expand Down Expand Up @@ -38,6 +39,11 @@ private static void parseCommandLine(String[] args, BomGenerator generator) {
generator.setPomFile(argValue);
break;

case COMMANDLINE_EXTERNALDEPENDENCIES:
generator.setExternalDependenciesFile((argValue));
break;


// case COMMANDLINE_GROUPID:
// switch(argValue) {
// case AZURE_CORE_GROUPID:
Expand Down
Loading

0 comments on commit a0e66c2

Please sign in to comment.