Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enhanced global type search, and prefs revisions #167

Merged
merged 1 commit into from
Oct 7, 2020
Merged

Enhanced global type search, and prefs revisions #167

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions bzl-java-sdk/META-INF/MANIFEST.MF
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ Export-Package: com.salesforce.bazel.sdk.aspect,
com.salesforce.bazel.sdk.console,
com.salesforce.bazel.sdk.index;version="1.2.1.qualifier",
com.salesforce.bazel.sdk.lang.jvm;version="1.2.1.qualifier",
com.salesforce.bazel.sdk.lang.jvm.external;version="1.2.1.qualifier",
com.salesforce.bazel.sdk.logging;version="1.2.1.qualifier",
com.salesforce.bazel.sdk.model;version="1.2.1.qualifier",
com.salesforce.bazel.sdk.project;version="1.2.1.qualifier",
Expand Down
118 changes: 59 additions & 59 deletions bzl-java-sdk/src/main/java/com/salesforce/bazel/sdk/index/BazelJvmIndexClasspath.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,16 @@
import java.util.ArrayList;
import java.util.List;

import com.salesforce.bazel.sdk.index.jar.JavaJarCrawler;
import com.salesforce.bazel.sdk.index.jar.JarIdentiferResolver;
import com.salesforce.bazel.sdk.index.jar.JavaJarCrawler;
import com.salesforce.bazel.sdk.index.model.CodeLocationDescriptor;
import com.salesforce.bazel.sdk.lang.jvm.BazelJvmClasspathResponse;
import com.salesforce.bazel.sdk.lang.jvm.JvmClasspathEntry;
import com.salesforce.bazel.sdk.lang.jvm.external.BazelExternalJarRuleManager;
import com.salesforce.bazel.sdk.lang.jvm.external.BazelExternalJarRuleType;
import com.salesforce.bazel.sdk.model.BazelWorkspace;
import com.salesforce.bazel.sdk.util.WorkProgressMonitor;
import com.salesforce.bazel.sdk.workspace.OperatingEnvironmentDetectionStrategy;

/**
* Creates a classpath containing all downloaded jars (binary and source) and all workspace built jars
Expand All @@ -42,89 +45,82 @@
* that needs to enumerate all available types in a workspace.
*/
public class BazelJvmIndexClasspath {
/**
* Associated workspace.
*/
protected BazelWorkspace bazelWorkspace;
protected File bazelBin;
protected File jarCacheDir;
private List<File> bazelBinInternals = new ArrayList<>();

// collaborators
protected OperatingEnvironmentDetectionStrategy os;
protected BazelExternalJarRuleManager externalJarRuleManager;

/**
* User provided locations to search.
*/
protected List<File> additionalJarLocations;

private BazelJvmClasspathResponse cacheResponse;
protected BazelJvmClasspathResponse cacheResponse;

/**
* Ctor with workspace.
*/
public BazelJvmIndexClasspath(BazelWorkspace bazelWorkspace, File jarCacheDir) {
public BazelJvmIndexClasspath(BazelWorkspace bazelWorkspace, OperatingEnvironmentDetectionStrategy os,
BazelExternalJarRuleManager externalJarRuleManager, List<File> additionalJarLocations) {
this.bazelWorkspace = bazelWorkspace;
this.bazelBin = bazelWorkspace.getBazelBinDirectory();

if (jarCacheDir != null) {
// maven_install for example stores all downloaded jars outside of the bazel-out directory structure
this.jarCacheDir = jarCacheDir;
} else {
// default is to hunt around in the bazel-bin/external directory
this.jarCacheDir = new File(bazelBin, "external");
}

File bazelBinInternal = new File(bazelBin, "projects"); // TODO needs to be configurable/computable, this is just our convention
if (bazelBinInternal.exists()) {
bazelBinInternals.add(bazelBinInternal);
}
this.os = os;
this.externalJarRuleManager = externalJarRuleManager;
this.additionalJarLocations = additionalJarLocations;
}

/**
* Ctor with workspace, and the names of the root directories in the Bazel workspace that are interesting to
* index. This will typically exclude //tools and other non-production code. For example, if your production code
* base is rooted in //projects and //libs you would pass ['projects', 'libs'] for the second param.
*/
public BazelJvmIndexClasspath(BazelWorkspace bazelWorkspace, File jarCacheDir, List<String> internalRootDirectoryNames) {
this(bazelWorkspace, jarCacheDir);

for (String name : internalRootDirectoryNames) {
File internalRoot = new File(bazelBin, name);
if (internalRoot.exists()) {
bazelBinInternals.add(internalRoot);
}
}
}

/**
* Computes the JVM classpath for the associated BazelProject
* Computes the JVM classpath for the associated Bazel workspace
*/
public BazelJvmClasspathResponse getClasspathEntries(WorkProgressMonitor progressMonitor) {
if (cacheResponse != null) {
return cacheResponse; // TODO expire the cache
return cacheResponse;
}

return getClasspathEntriesInternal(this.bazelBinInternals, this.jarCacheDir, progressMonitor);
}

/* visible for testing */
BazelJvmClasspathResponse getClasspathEntriesInternal(List<File> bazelBinInternals, File jarCacheDir,
WorkProgressMonitor progressMonitor) {
JvmCodeIndex index = new JvmCodeIndex();
List<File> locations = new ArrayList<>();

// EXTERNAL (aka maven_install downloaded)
if (jarCacheDir != null) {
JarIdentiferResolver jarResolver = new JarIdentiferResolver("/public/"); // TODO this is only maven_install convention
JavaJarCrawler jarCrawler = new JavaJarCrawler(index, jarResolver);
jarCrawler.index(jarCacheDir, false);
// for each jar downloading rule type in the workspace, add the appropriate local directories of the downloaded jars
List<BazelExternalJarRuleType> ruleTypes = externalJarRuleManager.findInUseExternalJarRuleTypes(this.bazelWorkspace);
for (BazelExternalJarRuleType ruleType : ruleTypes) {
System.out.println("");
List<File> ruleSpecificLocations = ruleType.getDownloadedJarLocations(bazelWorkspace);
locations.addAll(ruleSpecificLocations);
}

// INTERNAL (jars produced by Bazel)
if (bazelBinInternals != null && bazelBinInternals.size() > 0) {

// TODO we should actually add the source files directly instead of indexing the built jars

JarIdentiferResolver jarResolver = new JarIdentiferResolver("/bin/");
JavaJarCrawler jarCrawler = new JavaJarCrawler(index, jarResolver);
for (File bazelBinInternal : bazelBinInternals) {
jarCrawler.index(bazelBinInternal, false);
}
// add internal location (jars built by the bazel workspace
addInternalLocations(locations);

// add the additional directories the user wants to search
if (additionalJarLocations != null) {
locations.addAll(additionalJarLocations);
}

// now do the searching
for (File location : locations) {
getClasspathEntriesInternal(location, index, progressMonitor);
}

cacheResponse = convertIndexIntoResponse(index);
return cacheResponse;
}

public void clearCache() {
cacheResponse = null;
}

/* visible for testing */
void getClasspathEntriesInternal(File location, JvmCodeIndex index, WorkProgressMonitor progressMonitor) {
if (location != null && location.exists()) {
JarIdentiferResolver jarResolver = new JarIdentiferResolver();
JavaJarCrawler jarCrawler = new JavaJarCrawler(index, jarResolver);
jarCrawler.index(location, false);
}
}

protected BazelJvmClasspathResponse convertIndexIntoResponse(JvmCodeIndex index) {
BazelJvmClasspathResponse response = new BazelJvmClasspathResponse();
List<JvmClasspathEntry> entries = new ArrayList<>();
Expand Down Expand Up @@ -167,4 +163,8 @@ protected void addJarEntry(List<JvmClasspathEntry> entries, CodeLocationDescript
}
entries.add(cpEntry);
}

protected void addInternalLocations(List<File> locations) {
// TODO INTERNAL (jars produced by Bazel)
}
}
26 changes: 12 additions & 14 deletions bzl-java-sdk/src/main/java/com/salesforce/bazel/sdk/index/JvmCodeIndexer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,30 +104,28 @@ public JvmCodeIndex buildIndex() {
if (jarResolver != null) {
JavaJarCrawler jarCrawler = new JavaJarCrawler(index, jarResolver);
jarCrawler.index(externalJarRootFile, true);
} else {
logInfo("Could not determine the build system (maven/bazel) from the jar root. Skipping jar scanning...");
}

if (sourceRoot == null) {
logInfo("The provided source code root directory does not exist. This is ok.");
return index;
if (sourceRoot != null) {
File sourceRootFile = new File(sourceRoot);
if (!sourceRootFile.exists()) {
logInfo("The provided source code root directory does not exist. This is ok.");
return index;
}
JavaSourceCrawler sourceCrawler = new JavaSourceCrawler(index, pickJavaSourceArtifactMarker(externalJarRoot));
sourceCrawler.index(sourceRootFile);
}
File sourceRootFile = new File(sourceRoot);
if (!sourceRootFile.exists()) {
else {
logInfo("The provided source code root directory does not exist. This is ok.");
return index;
}
JavaSourceCrawler sourceCrawler = new JavaSourceCrawler(index, pickJavaSourceArtifactMarker(externalJarRoot));
sourceCrawler.index(sourceRootFile);

return index;
}

private static JarIdentiferResolver pickJavaJarResolver(String jarRepoPath) {
if (jarRepoPath.contains(".m2/repository")) {
return new JarIdentiferResolver("repository");
} else if (jarRepoPath.contains("bazel-out")) {
return new JarIdentiferResolver("public");
}
return null;
return new JarIdentiferResolver();
}

private static String pickJavaSourceArtifactMarker(String jarRepoPath) {
Expand Down
51 changes: 26 additions & 25 deletions bzl-java-sdk/src/main/java/com/salesforce/bazel/sdk/index/jar/JarIdentiferResolver.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,27 +26,18 @@
import java.io.File;
import java.util.zip.ZipFile;

/**
* Resolver that decides if the passed File is an interesting jar file, and if so it will deduce
* GAV information from the path.
*/
public class JarIdentiferResolver {
private String startWord;

/**
* Creates a resolver, with the important startWord. The startWord delimits the path part that
* divides the location on disk for the directory structure from the important structure information
* that identifies the artifact. For example, in a Maven repository, the path information below the
* .m2/repository directory contains the groupId and versionId.
*
* @param startWord for Maven repo, "repository"; for a bazel workspace using maven_install, "public"
*/
public JarIdentiferResolver(String startWord) {
this.startWord = startWord;

public JarIdentiferResolver() {
}

public JarIdentifier resolveJarIdentifier(File pathFile, ZipFile zipFile) {
public JarIdentifier resolveJarIdentifier(File gavRoot, File pathFile, ZipFile zipFile) {
String path = pathFile.getAbsolutePath();
int startIndex = path.lastIndexOf(startWord);
if (startIndex == -1) {
return null;
}

// do some quick Bazel exclusions (TODO how best to identify the interesting jars in the Bazel output dirs?)
if (path.contains("-hjar.jar")) {
return null; // perf optimization jar, not intended for non-Bazel consumption
Expand All @@ -64,7 +55,10 @@ public JarIdentifier resolveJarIdentifier(File pathFile, ZipFile zipFile) {
return null; // source jar, this will be pulled in as an attribute of the main jar
}
if (path.contains("-gensrc.jar")) {
return null; // source jar, this will be pulled in as an attribute of the main jar
return null; // generated source jar, this will be pulled in as an attribute of the main jar
}
if (path.contains("_deploy.jar")) {
return null; // uber jar, which contains exploded classes from other jars
}
if (path.contains("Test.jar")) {
return null; // by convention, a jar that contains a test to run in bazel
Expand All @@ -73,10 +67,10 @@ public JarIdentifier resolveJarIdentifier(File pathFile, ZipFile zipFile) {
return null; // by convention, a jar that contains a test to run in bazel
}

// Maven: com/acme/libs/my-blue-impl/0.1.8/my-blue-impl-0.1.8.jar
// Bazel: com/acme/libs/my-blue-impl/0.1.8/my-blue-impl-0.1.8-ijar.jar
// Maven compatible: com/acme/libs/my-blue-impl/0.1.8/my-blue-impl-0.1.8.jar
// Bazel internal: com/acme/libs/my-blue-impl/0.1.8/my-blue-impl-0.1.8-ijar.jar

String gavPart = path.substring(startIndex+startWord.length());
String gavPart = path.substring(gavRoot.getAbsolutePath().length());
String[] gavParts = gavPart.split("/");

// open-context-impl-0.1.8.jar => open-context-impl
Expand Down Expand Up @@ -115,15 +109,22 @@ public JarIdentifier resolveJarIdentifier(File pathFile, ZipFile zipFile) {
public static void main(String[] args) {

// Maven build system
JarIdentiferResolver resolver = new JarIdentiferResolver("/repository/");
JarIdentiferResolver resolver = new JarIdentiferResolver();
File gavRoot = new File("/Users/mbenioff/.m2/repository");
File pathFile = new File("/Users/mbenioff/.m2/repository/com/acme/libs/my-blue-impl/0.1.8/my-blue-impl-0.1.8.jar");
JarIdentifier id = resolver.resolveJarIdentifier(pathFile, null);
JarIdentifier id = resolver.resolveJarIdentifier(gavRoot, pathFile, null);
System.out.println(id.locationIdentifier);
if (!id.locationIdentifier.equals("com.acme.libs:my-blue-impl:0.1.8")) {
System.err.println("FAIL!");
}

// Bazel build system
resolver = new JarIdentiferResolver("/public/");
gavRoot = new File("/tmp/_bazel_benioff/dsf87dsfsl/execroot/__main__/bazel-out/darwin-fastbuild/bin/external/maven/v1/https/benioff%40nexus.acme.com/nexus/content/groups/public");
pathFile = new File("/tmp/_bazel_benioff/dsf87dsfsl/execroot/__main__/bazel-out/darwin-fastbuild/bin/external/maven/v1/https/benioff%40nexus.acme.com/nexus/content/groups/public/com/acme/libs/my-blue-impl/0.1.8/my-blue-impl-0.1.8-ijar.jar");
id = resolver.resolveJarIdentifier(pathFile, null);
id = resolver.resolveJarIdentifier(gavRoot, pathFile, null);
System.out.println(id.locationIdentifier);
if (!id.locationIdentifier.equals("com.acme.libs:my-blue-impl:0.1.8")) {
System.err.println("FAIL!");
}
}
}
31 changes: 25 additions & 6 deletions bzl-java-sdk/src/main/java/com/salesforce/bazel/sdk/index/jar/JavaJarCrawler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
package com.salesforce.bazel.sdk.index.jar;

import java.io.File;
import java.io.FilenameFilter;
import java.util.Enumeration;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
Expand All @@ -47,14 +48,31 @@ public JavaJarCrawler(JvmCodeIndex index, JarIdentiferResolver resolver) {
}

public void index(File basePath, boolean doIndexClasses) {
indexRecur(basePath, doIndexClasses);
indexRecur(null, basePath, doIndexClasses);
}

protected void indexRecur(File path, boolean doIndexClasses) {
protected void indexRecur(File gavRoot, File path, boolean doIndexClasses) {
File[] children = path.listFiles();
if (children == null) {
return;
}

// some file system layouts put gav information in the path, e.g.
// ~/.m2/repository/com/acme/blue/1.0.0/blue.jar
// we want to track the start of the gav info in the path if possible
if (gavRoot == null) {
File[] gavRootIndicators = path.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
// TODO sketchy logic here, we assume at least one downloaded jar comes from a common domain
return name.equals("com") || name.equals("org") || name.equals("net");
}
});
if (gavRootIndicators.length > 0) {
gavRoot = path;
}
}

for (File child : children) {
ZipFile zipFile = null;
try {
Expand All @@ -63,11 +81,11 @@ protected void indexRecur(File path, boolean doIndexClasses) {
// bazel test sandbox, stay out of here as the jars in here are for running tests
return;
}
indexRecur(child, doIndexClasses);
indexRecur(gavRoot, child, doIndexClasses);
} else if (child.canRead()) {
if (child.getName().endsWith(".jar")) {
zipFile = new ZipFile(child);
foundJar(child, zipFile, doIndexClasses);
foundJar(gavRoot, child, zipFile, doIndexClasses);
}
}
}
Expand All @@ -84,9 +102,10 @@ protected void indexRecur(File path, boolean doIndexClasses) {
}
}

protected void foundJar(File jarFile, ZipFile zipFile, boolean doIndexClasses) {
protected void foundJar(File gavRoot, File jarFile, ZipFile zipFile, boolean doIndexClasses) {
// precisely identify the jar file
JarIdentifier jarId = resolver.resolveJarIdentifier(jarFile, zipFile);
log("jar:" , jarFile.getName(), null);
JarIdentifier jarId = resolver.resolveJarIdentifier(gavRoot, jarFile, zipFile);
if (jarId == null) {
// this jar is not part of the typical dependencies (e.g. it is a jar used in the build toolchain); ignore
return;
Expand Down
Loading