From 199aed0e12e7db50e9058e92166c7ea0cd92f42c Mon Sep 17 00:00:00 2001 From: Pranav Gaikwad Date: Fri, 6 Oct 2023 15:26:06 -0400 Subject: [PATCH] :sparkles: Identify interesting Jars to decomp and improve decomp performance (#352) Fixes #317 Fixes #319 Summary of changes: * When decompiling binaries, for every JAR we find, we attempt to look at its metadata to get artifact and group. If we dont find metadata, we look it up on maven central using its sha. For all such JARs that we find accurate information about, we add them to java project's pom as dependencies. For all other JARs, we send them to decompile. * In decompile, we are running concurrently. Also, instead of decompiling each class file individually, we are now decompiling whole JAR using fernflower and then exploding it (this is faster than individual .class file decompile) * Prior to initing Java provider, we are now downloading sources for all dependencies. If we find any that don't have sources, we are passing them to decompile. * When getting dependencies for a binary, we are now using same logic from step 1 to get more fine grained info for a JAR. --------- Signed-off-by: David Zager Signed-off-by: Pranav Gaikwad Co-authored-by: David Zager --- provider/internal/java/dependency.go | 36 ++- provider/internal/java/provider.go | 138 ++++++-- provider/internal/java/provider_test.go | 48 +++ provider/internal/java/service_client.go | 1 - provider/internal/java/util.go | 385 ++++++++++++++++++++--- provider/internal/java/util_test.go | 98 ++++++ 6 files changed, 640 insertions(+), 66 deletions(-) create mode 100644 provider/internal/java/provider_test.go create mode 100644 provider/internal/java/util_test.go diff --git a/provider/internal/java/dependency.go b/provider/internal/java/dependency.go index e3db2306..5d75492f 100644 --- a/provider/internal/java/dependency.go +++ b/provider/internal/java/dependency.go @@ -78,12 +78,12 @@ func (p *javaServiceClient) GetDependencies(ctx context.Context) (map[uri.URI][] return m, nil } -func (p *javaServiceClient) getLocalRepoPath() string { +func getMavenLocalRepoPath(mvnSettingsFile string) string { args := []string{ "help:evaluate", "-Dexpression=settings.localRepository", "-q", "-DforceStdout", } - if p.mvnSettingsFile != "" { - args = append(args, "-s", p.mvnSettingsFile) + if mvnSettingsFile != "" { + args = append(args, "-s", mvnSettingsFile) } cmd := exec.Command("mvn", args...) var outb bytes.Buffer @@ -147,7 +147,7 @@ func (p *javaServiceClient) GetDependencyFallback(ctx context.Context) (map[uri. } func (p *javaServiceClient) GetDependenciesDAG(ctx context.Context) (map[uri.URI][]provider.DepDAGItem, error) { - localRepoPath := p.getLocalRepoPath() + localRepoPath := getMavenLocalRepoPath(p.mvnSettingsFile) path := p.findPom() file := uri.File(path) @@ -209,13 +209,17 @@ func (p *javaServiceClient) GetDependenciesDAG(ctx context.Context) (map[uri.URI func (p *javaServiceClient) discoverDepsFromJars(path string, ll map[uri.URI][]konveyor.DepDAGItem) { // for binaries we only find JARs embedded in archive w := walker{ - deps: ll, + deps: ll, + depToLabels: p.depToLabels, + m2RepoPath: getMavenLocalRepoPath(p.mvnSettingsFile), } filepath.WalkDir(path, w.walkDirForJar) } type walker struct { - deps map[uri.URI][]provider.DepDAGItem + deps map[uri.URI][]provider.DepDAGItem + depToLabels map[string]*depLabelItem + m2RepoPath string } func (w *walker) walkDirForJar(path string, info fs.DirEntry, err error) error { @@ -229,6 +233,20 @@ func (w *walker) walkDirForJar(path string, info fs.DirEntry, err error) error { d := provider.Dep{ Name: info.Name(), } + artifact, _ := toDependency(context.TODO(), path) + if (artifact != javaArtifact{}) { + d.Name = fmt.Sprintf("%s.%s", artifact.GroupId, artifact.ArtifactId) + d.Version = artifact.Version + d.Labels = addDepLabels(w.depToLabels, d.Name) + d.ResolvedIdentifier = artifact.sha1 + // when we can successfully get javaArtifact from a jar + // we added it to the pom and it should be in m2Repo path + if w.m2RepoPath != "" { + d.FileURIPrefix = filepath.Join(w.m2RepoPath, + strings.Replace(artifact.GroupId, ".", "/", -1), artifact.ArtifactId, artifact.Version) + } + } + w.deps[uri.URI(filepath.Join(path, info.Name()))] = []provider.DepDAGItem{ { Dep: d, @@ -271,15 +289,15 @@ func (p *javaServiceClient) parseDepString(dep, localRepoPath string) (provider. d.ResolvedIdentifier = string(b) } - d.Labels = p.addDepLabels(d.Name) + d.Labels = addDepLabels(p.depToLabels, d.Name) d.FileURIPrefix = fmt.Sprintf("file://%v", filepath.Dir(fp)) return d, nil } -func (p *javaServiceClient) addDepLabels(depName string) []string { +func addDepLabels(depToLabels map[string]*depLabelItem, depName string) []string { m := map[string]interface{}{} - for _, d := range p.depToLabels { + for _, d := range depToLabels { if d.r.Match([]byte(depName)) { for label, _ := range d.labels { m[label] = nil diff --git a/provider/internal/java/provider.go b/provider/internal/java/provider.go index 6b9623aa..7fa37e96 100644 --- a/provider/internal/java/provider.go +++ b/provider/internal/java/provider.go @@ -1,11 +1,14 @@ package java import ( + "bufio" "context" "fmt" + "io" "os" "os/exec" "path" + "path/filepath" "strings" "github.com/getkin/kin-openapi/openapi3" @@ -17,6 +20,7 @@ import ( ) const ( + JavaFile = ".java" JavaArchive = ".jar" WebArchive = ".war" EnterpriseArchive = ".ear" @@ -186,26 +190,7 @@ func (p *javaProvider) Init(ctx context.Context, log logr.Logger, config provide } log = log.WithValues("provider", "java") - isBinary := false - var returnErr error - // each service client should have their own context - ctx, cancelFunc := context.WithCancel(ctx) - extension := strings.ToLower(path.Ext(config.Location)) - switch extension { - case JavaArchive, WebArchive, EnterpriseArchive: - depLocation, sourceLocation, err := decompileJava(ctx, log, config.Location) - if err != nil { - cancelFunc() - return nil, err - } - config.Location = sourceLocation - // for binaries, we fallback to looking at .jar files only for deps - config.DependencyPath = depLocation - // for binaries, always run in source-only mode as we don't know how to correctly resolve deps - config.AnalysisMode = provider.SourceOnlyAnalysisMode - isBinary = true - } - + // read provider settings bundlesString, ok := config.ProviderSpecificConfig[BUNDLES_INIT_OPTION].(string) if !ok { bundlesString = "" @@ -224,10 +209,35 @@ func (p *javaProvider) Init(ctx context.Context, log logr.Logger, config provide lspServerPath, ok := config.ProviderSpecificConfig[provider.LspServerPathConfigKey].(string) if !ok || lspServerPath == "" { - cancelFunc() return nil, fmt.Errorf("invalid lspServerPath provided, unable to init java provider") } + isBinary := false + var returnErr error + // each service client should have their own context + ctx, cancelFunc := context.WithCancel(ctx) + extension := strings.ToLower(path.Ext(config.Location)) + switch extension { + case JavaArchive, WebArchive, EnterpriseArchive: + depLocation, sourceLocation, err := decompileJava(ctx, log, config.Location) + if err != nil { + cancelFunc() + return nil, err + } + config.Location = sourceLocation + // for binaries, we fallback to looking at .jar files only for deps + config.DependencyPath = depLocation + isBinary = true + } + + // we attempt to decompile JARs of dependencies that don't have a sources JAR attached + // we need to do this for jdtls to correctly recognize source attachment for dep + err := resolveSourcesJars(ctx, log, config.Location, mavenSettingsFile) + if err != nil { + // TODO (pgaikwad): should we ignore this failure? + log.Error(err, "failed to resolve sources jar for location", "location", config.Location) + } + // handle proxy settings for k, v := range config.Proxy.ToEnvVars() { os.Setenv(k, v) @@ -304,3 +314,89 @@ func (p *javaProvider) GetDependencies(ctx context.Context) (map[uri.URI][]*prov func (p *javaProvider) GetDependenciesDAG(ctx context.Context) (map[uri.URI][]provider.DepDAGItem, error) { return provider.FullDepDAGResponse(ctx, p.clients) } + +// resolveSourcesJars for a given source code location, runs maven to find +// deps that don't have sources attached and decompiles them +func resolveSourcesJars(ctx context.Context, log logr.Logger, location, mavenSettings string) error { + decompileJobs := []decompileJob{} + mvnOutput, err := os.CreateTemp("", "mvn-sources-") + if err != nil { + return err + } + defer mvnOutput.Close() + args := []string{ + "dependency:sources", + "-Djava.net.useSystemProxies=true", + fmt.Sprintf("-DoutputFile=%s", mvnOutput.Name()), + } + if mavenSettings != "" { + args = append(args, "-s", mavenSettings) + } + cmd := exec.CommandContext(ctx, "mvn", args...) + cmd.Dir = location + err = cmd.Run() + if err != nil { + return err + } + artifacts, err := parseUnresolvedSources(mvnOutput) + if err != nil { + return err + } + m2Repo := getMavenLocalRepoPath(mavenSettings) + if m2Repo == "" { + return nil + } + for _, artifact := range artifacts { + groupDirs := filepath.Join(strings.Split(artifact.GroupId, ".")...) + artifactDirs := filepath.Join(strings.Split(artifact.ArtifactId, ".")...) + jarName := fmt.Sprintf("%s-%s.jar", artifact.ArtifactId, artifact.Version) + decompileJobs = append(decompileJobs, decompileJob{ + artifact: artifact, + inputPath: filepath.Join( + m2Repo, groupDirs, artifactDirs, artifact.Version, jarName), + outputPath: filepath.Join( + m2Repo, groupDirs, artifactDirs, artifact.Version, "decompiled", jarName), + }) + } + err = decompile(ctx, log, alwaysDecompileFilter(true), 10, decompileJobs, "") + if err != nil { + return err + } + // move decompiled files to base location of the jar + for _, decompileJob := range decompileJobs { + jarName := strings.TrimSuffix(filepath.Base(decompileJob.inputPath), ".jar") + moveFile(decompileJob.outputPath, + filepath.Join(filepath.Dir(decompileJob.inputPath), + fmt.Sprintf("%s-sources.jar", jarName))) + } + return nil +} + +func parseUnresolvedSources(output io.Reader) ([]javaArtifact, error) { + artifacts := []javaArtifact{} + scanner := bufio.NewScanner(output) + unresolvedSeparatorSeen := false + for scanner.Scan() { + line := scanner.Text() + line = strings.TrimLeft(line, " ") + if strings.HasPrefix(line, "The following files have NOT been resolved:") { + unresolvedSeparatorSeen = true + } else if unresolvedSeparatorSeen { + parts := strings.Split(line, ":") + if len(parts) != 6 { + continue + } + groupId := parts[0] + artifactId := parts[1] + version := parts[4] + artifacts = append(artifacts, + javaArtifact{ + packaging: JavaArchive, + ArtifactId: artifactId, + GroupId: groupId, + Version: version, + }) + } + } + return artifacts, scanner.Err() +} diff --git a/provider/internal/java/provider_test.go b/provider/internal/java/provider_test.go new file mode 100644 index 00000000..d315224c --- /dev/null +++ b/provider/internal/java/provider_test.go @@ -0,0 +1,48 @@ +package java + +import ( + "reflect" + "strings" + "testing" +) + +func Test_parseUnresolvedSources(t *testing.T) { + tests := []struct { + name string + mvnOutput string + wantErr bool + wantList []javaArtifact + }{ + { + name: "valid sources output", + mvnOutput: ` +The following files have been resolved: + org.springframework.boot:spring-boot:jar:sources:2.5.0:compile + +The following files have NOT been resolved: + io.konveyor.demo:config-utils:jar:sources:1.0.0:compile +`, + wantErr: false, + wantList: []javaArtifact{ + { + packaging: JavaArchive, + GroupId: "io.konveyor.demo", + ArtifactId: "config-utils", + Version: "1.0.0", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + outputReader := strings.NewReader(tt.mvnOutput) + gotList, gotErr := parseUnresolvedSources(outputReader) + if (gotErr != nil) != tt.wantErr { + t.Errorf("parseUnresolvedSources() gotErr = %v, wantErr %v", gotErr, tt.wantErr) + } + if !reflect.DeepEqual(gotList, tt.wantList) { + t.Errorf("parseUnresolvedSources() gotList = %v, wantList %v", gotList, tt.wantList) + } + }) + } +} diff --git a/provider/internal/java/service_client.go b/provider/internal/java/service_client.go index 05d08305..29f30670 100644 --- a/provider/internal/java/service_client.go +++ b/provider/internal/java/service_client.go @@ -19,7 +19,6 @@ import ( type javaServiceClient struct { rpc *jsonrpc2.Conn - ctx context.Context cancelFunc context.CancelFunc config provider.InitConfig log logr.Logger diff --git a/provider/internal/java/util.go b/provider/internal/java/util.go index e6fea506..40d2f5af 100644 --- a/provider/internal/java/util.go +++ b/provider/internal/java/util.go @@ -2,15 +2,26 @@ package java import ( "archive/zip" + "bufio" "context" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "fmt" "io" + "math" + "net/http" "os" "os/exec" "path" "path/filepath" "strings" + "sync" + "text/template" "github.com/go-logr/logr" + "github.com/konveyor/analyzer-lsp/engine/labels" + "github.com/konveyor/analyzer-lsp/provider" "github.com/konveyor/analyzer-lsp/tracing" ) @@ -31,6 +42,13 @@ const javaProjectPom = ` +{{range .}} + + {{.GroupId}} + {{.ArtifactId}} + {{.Version}} + +{{end}} @@ -38,6 +56,111 @@ const javaProjectPom = ` ` +type javaArtifact struct { + packaging string + GroupId string + ArtifactId string + Version string + sha1 string +} + +type decompileFilter interface { + shouldDecompile(javaArtifact) bool +} + +type alwaysDecompileFilter bool + +func (a alwaysDecompileFilter) shouldDecompile(j javaArtifact) bool { + return bool(a) +} + +type excludeOpenSourceDecompileFilter map[string]*depLabelItem + +func (o excludeOpenSourceDecompileFilter) shouldDecompile(j javaArtifact) bool { + matchWith := fmt.Sprintf("%s.%s", j.GroupId, j.ArtifactId) + for _, r := range o { + if r.r.MatchString(matchWith) { + if _, ok := r.labels[labels.AsString(provider.DepSourceLabel, javaDepSourceOpenSource)]; ok { + return false + } + } + } + return true +} + +type decompileJob struct { + inputPath string + outputPath string + artifact javaArtifact +} + +// decompile decompiles files submitted via a list of decompileJob concurrently +// if a .class file is encountered, it will be decompiled to output path right away +// if a .jar file is encountered, it will be decompiled as a whole, then exploded to project path +func decompile(ctx context.Context, log logr.Logger, filter decompileFilter, workerCount int, jobs []decompileJob, projectPath string) error { + wg := &sync.WaitGroup{} + jobChan := make(chan decompileJob) + + workerCount = int(math.Min(float64(len(jobs)), float64(workerCount))) + // init workers + for i := 0; i < workerCount; i++ { + logger := log.WithName(fmt.Sprintf("decompileWorker-%d", i)) + wg.Add(1) + go func(log logr.Logger) { + defer log.V(6).Info("shutting down decompile worker") + defer wg.Done() + log.V(6).Info("init decompile worker") + for job := range jobChan { + // apply decompile filter + if !filter.shouldDecompile(job.artifact) { + continue + } + if _, err := os.Stat(job.outputPath); err == nil { + // already decompiled, duplicate... + continue + } + outputPathDir := filepath.Dir(job.outputPath) + if err := os.MkdirAll(outputPathDir, 0755); err != nil { + log.V(3).Error(err, + "failed to create directories for decompiled file", "path", outputPathDir) + continue + } + cmd := exec.CommandContext( + ctx, "java", "-jar", "/bin/fernflower.jar", job.inputPath, outputPathDir) + err := cmd.Run() + if err != nil { + log.V(5).Error(err, "failed to decompile file", "file", job.inputPath, job.outputPath) + } else { + log.V(5).Info("decompiled file", "source", job.inputPath, "dest", job.outputPath) + } + // if we just decompiled a java archive, we need to + // explode it further and copy files to project + if job.artifact.packaging == JavaArchive && projectPath != "" { + _, _, _, err = explode(ctx, log, job.outputPath, projectPath) + if err != nil { + log.V(5).Error(err, "failed to explode decompiled jar", "path", job.inputPath) + } + } + } + }(logger) + } + + seenJobs := map[string]bool{} + for _, job := range jobs { + jobKey := fmt.Sprintf("%s-%s", job.inputPath, job.outputPath) + if _, ok := seenJobs[jobKey]; !ok { + seenJobs[jobKey] = true + jobChan <- job + } + } + + close(jobChan) + + wg.Wait() + + return nil +} + // decompileJava unpacks archive at archivePath, decompiles all .class files in it // creates new java project and puts the java files in the tree of the project // returns path to exploded archive, path to java project, and an error when encountered @@ -47,49 +170,76 @@ func decompileJava(ctx context.Context, log logr.Logger, archivePath string) (ex projectPath = filepath.Join(filepath.Dir(archivePath), "java-project") - err = createJavaProject(ctx, projectPath) + decompFilter := alwaysDecompileFilter(true) + + explodedPath, decompJobs, deps, err := explode(ctx, log, archivePath, projectPath) + if err != nil { + log.Error(err, "failed to decompile archive", "path", archivePath) + return "", "", err + } + + err = createJavaProject(ctx, projectPath, deduplicateJavaArtifacts(deps)) if err != nil { log.Error(err, "failed to create java project", "path", projectPath) return "", "", err } log.V(5).Info("created java project", "path", projectPath) - explodedPath, err = decompile(ctx, log, archivePath, projectPath) + err = decompile(ctx, log, decompFilter, 10, decompJobs, projectPath) if err != nil { - log.Error(err, "failed to decompile archive", "path", archivePath) + log.Error(err, "failed to decompile", "path", archivePath) return "", "", err } + return explodedPath, projectPath, err } -// decompile is a function that extracts the contents of a Java archive (.jar|.war|.ear) and -// decompiles any .class files found using the fernflower decompiler into java project location -// maintaining the tree. swallows decomp and copy errors, returns others -func decompile(ctx context.Context, log logr.Logger, archivePath, projectPath string) (string, error) { +func deduplicateJavaArtifacts(artifacts []javaArtifact) []javaArtifact { + uniq := []javaArtifact{} + seen := map[string]bool{} + for _, a := range artifacts { + key := fmt.Sprintf("%s-%s-%s%s", + a.ArtifactId, a.GroupId, a.Version, a.packaging) + if _, ok := seen[key]; !ok { + seen[key] = true + uniq = append(uniq, a) + } + } + return uniq +} + +// explode explodes the given JAR, WAR or EAR archive, generates javaArtifact struct for given archive +// and identifies all .class found recursively. returns output path, a list of decompileJob for .class files +// it also returns a list of any javaArtifact we could interpret from jars +func explode(ctx context.Context, log logr.Logger, archivePath, projectPath string) (string, []decompileJob, []javaArtifact, error) { + var dependencies []javaArtifact fileInfo, err := os.Stat(archivePath) if err != nil { - return "", err + return "", nil, dependencies, err } + // Create the destDir directory using the same permissions as the Java archive file - // java.jar should become java-jar-decompiled - destDir := filepath.Join(path.Dir(archivePath), strings.Replace(path.Base(archivePath), ".", "-", -1)+"-decompiled") + // java.jar should become java-jar-exploded + destDir := filepath.Join(path.Dir(archivePath), strings.Replace(path.Base(archivePath), ".", "-", -1)+"-exploded") // make sure execute bits are set so that fernflower can decompile err = os.MkdirAll(destDir, fileInfo.Mode()|0111) if err != nil { - return "", err + return "", nil, dependencies, err } archive, err := zip.OpenReader(archivePath) if err != nil { - return "", err + return "", nil, dependencies, err } defer archive.Close() + decompileJobs := []decompileJob{} + for _, f := range archive.File { // Stop processing if our context is cancelled select { case <-ctx.Done(): - return "", ctx.Err() + return "", decompileJobs, dependencies, ctx.Err() default: } @@ -107,24 +257,24 @@ func decompile(ctx context.Context, log logr.Logger, archivePath, projectPath st continue } - if err = os.MkdirAll(filepath.Dir(filePath), f.Mode()); err != nil { - return "", err + if err = os.MkdirAll(filepath.Dir(filePath), f.Mode()|0111); err != nil { + return "", decompileJobs, dependencies, err } - dstFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + dstFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()|0111) if err != nil { - return "", err + return "", decompileJobs, dependencies, err } defer dstFile.Close() archiveFile, err := f.Open() if err != nil { - return "", err + return "", decompileJobs, dependencies, err } defer archiveFile.Close() if _, err := io.Copy(dstFile, archiveFile); err != nil { - return "", err + return "", decompileJobs, dependencies, err } switch { // when it's a .class file, decompile it into java project @@ -135,41 +285,206 @@ func decompile(ctx context.Context, log logr.Logger, archivePath, projectPath st strings.Replace(filePath, destDir, "", -1)) destPath = strings.ReplaceAll(destPath, "WEB-INF/classes", "") destPath = strings.ReplaceAll(destPath, "META-INF/classes", "") - if _, err = os.Stat(destPath); err == nil { - // already decompiled, duplicate... + destPath = strings.TrimSuffix(destPath, ClassFile) + ".java" + decompileJobs = append(decompileJobs, decompileJob{ + inputPath: filePath, + outputPath: destPath, + artifact: javaArtifact{ + packaging: ClassFile, + }, + }) + // when it's a java file, it's already decompiled, move it to project path + case strings.HasSuffix(f.Name, JavaFile): + destPath := filepath.Join( + projectPath, "src", "main", "java", + strings.Replace(filePath, destDir, "", -1)) + destPath = strings.ReplaceAll(destPath, "WEB-INF/classes", "") + destPath = strings.ReplaceAll(destPath, "META-INF/classes", "") + if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { + log.V(8).Error(err, "error creating directory for java file", "path", destPath) continue } - if err = os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { - log.V(8).Error(err, "failed to create directories for decompiled file", "path", filepath.Dir(destPath)) + if err := moveFile(filePath, destPath); err != nil { + log.V(8).Error(err, "error moving decompiled file to project path", + "src", filePath, "dest", destPath) continue } - cmd := exec.CommandContext( - ctx, "java", "-jar", "/bin/fernflower.jar", filePath, path.Dir(destPath)) - err := cmd.Run() - if err != nil { - log.Error(err, "failed to decompile file", "file", filePath) - } else { - log.V(8).Info("decompiled file", "file", filePath) - } // decompile web archives case strings.HasSuffix(f.Name, WebArchive): - if _, err := decompile(ctx, log, filePath, projectPath); err != nil { + // TODO(djzager): Should we add these deps to the pom? + _, nestedJobs, deps, err := explode(ctx, log, filePath, projectPath) + if err != nil { log.Error(err, "failed to decompile file", "file", filePath) } + decompileJobs = append(decompileJobs, nestedJobs...) + dependencies = append(dependencies, deps...) + // attempt to add nested jars as dependency before decompiling + case strings.HasSuffix(f.Name, JavaArchive): + dep, err := toDependency(ctx, filePath) + if err != nil { + log.Error(err, "failed to add dep", "file", filePath) + // when we fail to identify a dep we will fallback to + // decompiling it ourselves and adding as source + outputPath := filepath.Join( + filepath.Dir(filePath), fmt.Sprintf("%s-decompiled", + strings.TrimSuffix(f.Name, JavaArchive)), filepath.Base(f.Name)) + // TODO(djzager): Is it possible for the javaArtifact be empty + // it is not an issue right now, but if we used a decompileFilter + // javaArtifact has to be accurate for filter to work + decompileJobs = append(decompileJobs, decompileJob{ + inputPath: filePath, + outputPath: outputPath, + artifact: javaArtifact{ + packaging: JavaArchive, + GroupId: dep.GroupId, + ArtifactId: dep.ArtifactId, + }, + }) + } + if (dep != javaArtifact{}) { + dependencies = append(dependencies, dep) + } } } - return destDir, nil + return destDir, decompileJobs, dependencies, nil } -func createJavaProject(ctx context.Context, dir string) error { +func createJavaProject(ctx context.Context, dir string, dependencies []javaArtifact) error { + tmpl := template.Must(template.New("javaProjectPom").Parse(javaProjectPom)) + err := os.MkdirAll(filepath.Join(dir, "src", "main", "java"), 0755) if err != nil { return err } - err = os.WriteFile(filepath.Join(dir, "pom.xml"), []byte(javaProjectPom), 0755) + + pom, err := os.OpenFile(filepath.Join(dir, "pom.xml"), os.O_CREATE|os.O_WRONLY, 0755) if err != nil { return err } + + err = tmpl.Execute(pom, dependencies) + if err != nil { + return err + } + return nil +} + +func moveFile(srcPath string, destPath string) error { + inputFile, err := os.Open(srcPath) + if err != nil { + return err + } + outputFile, err := os.Create(destPath) + if err != nil { + inputFile.Close() + return err + } + _, err = io.Copy(outputFile, inputFile) + inputFile.Close() + if err != nil { + return err + } + err = os.Remove(srcPath) + if err != nil { + return err + } + defer outputFile.Close() return nil } + +func toDependency(ctx context.Context, jarFile string) (javaArtifact, error) { + dep := javaArtifact{} + jar, err := zip.OpenReader(jarFile) + if err != nil { + return dep, err + } + defer jar.Close() + + for _, file := range jar.File { + match, err := filepath.Match("META-INF/maven/*/*/pom.properties", file.Name) + if err != nil { + return dep, err + } + + if match { + // Open the file in the ZIP archive + rc, err := file.Open() + if err != nil { + return dep, err + } + defer rc.Close() + + // Read and process the lines in the properties file + scanner := bufio.NewScanner(rc) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "version=") { + dep.Version = strings.TrimSpace(strings.TrimPrefix(line, "version=")) + } else if strings.HasPrefix(line, "artifactId=") { + dep.ArtifactId = strings.TrimSpace(strings.TrimPrefix(line, "artifactId=")) + } else if strings.HasPrefix(line, "groupId=") { + dep.GroupId = strings.TrimSpace(strings.TrimPrefix(line, "groupId=")) + } + } + + return dep, err + } + } + + // we didn't find a pom.properties. Look it up in maven + file, err := os.Open(jarFile) + if err != nil { + return dep, err + } + defer file.Close() + + hash := sha1.New() + _, err = io.Copy(hash, file) + if err != nil { + return dep, err + } + + sha1sum := hex.EncodeToString(hash.Sum(nil)) + + // Make an HTTP request to search.maven.org + searchURL := fmt.Sprintf("http://search.maven.org/solrsearch/select?q=1:%s&rows=20&wt=json", sha1sum) + resp, err := http.Get(searchURL) + if err != nil { + return dep, err + } + defer resp.Body.Close() + + // Read and parse the JSON response + body, err := io.ReadAll(resp.Body) + if err != nil { + return dep, err + } + + var data map[string]interface{} + err = json.Unmarshal(body, &data) + if err != nil { + return dep, err + } + + // Check if a single result is found + response, ok := data["response"].(map[string]interface{}) + if !ok { + return dep, err + } + + numFound, ok := response["numFound"].(float64) + if !ok { + return dep, err + } + + if numFound == 1 { + jarInfo := response["docs"].([]interface{})[0].(map[string]interface{}) + dep.GroupId = jarInfo["g"].(string) + dep.ArtifactId = jarInfo["a"].(string) + dep.Version = jarInfo["v"].(string) + dep.sha1 = sha1sum + return dep, nil + } + return dep, fmt.Errorf("failed to construct artifact from jar") +} diff --git a/provider/internal/java/util_test.go b/provider/internal/java/util_test.go new file mode 100644 index 00000000..30df477a --- /dev/null +++ b/provider/internal/java/util_test.go @@ -0,0 +1,98 @@ +package java + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "testing" +) + +func TestRenderPom(t *testing.T) { + // Create a temporary directory for testing + tmpDir := t.TempDir() + + // Define some sample dependencies + dependencies := []javaArtifact{ + { + GroupId: "com.example", + ArtifactId: "example-artifact", + Version: "1.0.0", + }, + { + GroupId: "org.another", + ArtifactId: "another-artifact", + Version: "2.0.0", + }, + } + + // Call the function with the temporary directory and sample dependencies + err := createJavaProject(nil, tmpDir, dependencies) + if err != nil { + t.Fatalf("createJavaProject returned an error: %v", err) + } + + // Verify that the project directory and pom.xml file were created + projectDir := filepath.Join(tmpDir, "src", "main", "java") + pomFile := filepath.Join(tmpDir, "pom.xml") + + if _, err := os.Stat(projectDir); os.IsNotExist(err) { + t.Errorf("Java source directory not created: %v", err) + } + + if _, err := os.Stat(pomFile); os.IsNotExist(err) { + t.Errorf("pom.xml file not created: %v", err) + } + + // Read the generated pom.xml content + pomContent, err := os.ReadFile(pomFile) + if err != nil { + t.Fatalf("error reading pom.xml file: %v", err) + } + + // Define the expected pom.xml content + expectedPom := ` + + 4.0.0 + + io.konveyor + java-project + 1.0-SNAPSHOT + + java-project + http://www.konveyor.io + + + UTF-8 + + + + + + com.example + example-artifact + 1.0.0 + + + + org.another + another-artifact + 2.0.0 + + + + + + + +` + + // Compare the generated pom.xml content with the expected content + if !bytes.Equal(pomContent, []byte(expectedPom)) { + t.Errorf("Generated pom.xml content does not match the expected content") + fmt.Println(string(pomContent)) + fmt.Println("expected POM") + fmt.Println(expectedPom) + } +}