+Copyright (C) 2015 WhiteSource Ltd.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+[1]: http://www.github.com/whitesource/agents
+[2]: http://www.whitesourcesoftware.com
+[3]: https://whitesource.atlassian.net/wiki/spaces/WD/pages/33718339/File+System+Agent
+[4]: https://github.com/whitesource/fs-agent
+[5]: mailto:support@whitesourcesoftware.com
+[6]: http://www.apache.org/licenses/LICENSE-2.0.html
diff --git a/folder-offline-test/whitesource/update-request.txt b/folder-offline-test/whitesource/update-request.txt
new file mode 100644
index 0000000..03c2624
--- /dev/null
+++ b/folder-offline-test/whitesource/update-request.txt
@@ -0,0 +1 @@
+{"updateType":"OVERRIDE","type":"UPDATE","agent":"fs-agent","agentVersion":"2.7.7","pluginVersion":"18.7.2-SNAPSHOT","orgToken":"token","product":"fsAgentMain","productVersion":"","timeStamp":1532939765748,"projects":[{"coordinates":{"artifactId":"npm","version":""},"dependencies":[{"groupId":"request","artifactId":"request-2.83.0.tgz","version":"2.83.0","sha1":"ca0b65da02ed62935887808e6f510381034e3356","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\request\\package.json","optional":false,"children":[{"groupId":"aws-sign2","artifactId":"aws-sign2-0.7.0.tgz","version":"0.7.0","sha1":"b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\aws-sign2\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\aws-sign2\\package.json","dependencyType":"NPM","checksums":{"SHA1":"b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8"}},{"groupId":"aws4","artifactId":"aws4-1.7.0.tgz","version":"1.7.0","sha1":"d4d0e9b9dbfca77bf08eeb0a8a471550fe39e289","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\aws4\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\aws4\\package.json","dependencyType":"NPM","checksums":{"SHA1":"d4d0e9b9dbfca77bf08eeb0a8a471550fe39e289"}},{"groupId":"caseless","artifactId":"caseless-0.12.0.tgz","version":"0.12.0","sha1":"1b681c21ff84033c826543090689420d187151dc","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\caseless\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\caseless\\package.json","dependencyType":"NPM","checksums":{"SHA1":"1b681c21ff84033c826543090689420d187151dc"}},{"groupId":"combined-stream","artifactId":"combined-stream-1.0.6.tgz","version":"1.0.6","sha1":"723e7df6e801ac5613113a7e445a9b69cb632818","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\combined-stream\\package.json","optional":false,"children":[{"groupId":"delayed-stream","artifactId":"delayed-stream-1.0.0.tgz","version":"1.0.0","sha1":"df3ae199acadfb7d440aaae0b29e2272b24ec619","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\delayed-stream\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\delayed-stream\\package.json","dependencyType":"NPM","checksums":{"SHA1":"df3ae199acadfb7d440aaae0b29e2272b24ec619"}}],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\combined-stream\\package.json","dependencyType":"NPM","checksums":{"SHA1":"723e7df6e801ac5613113a7e445a9b69cb632818"}},{"groupId":"extend","artifactId":"extend-3.0.2.tgz","version":"3.0.2","sha1":"f8b1136b4071fbd8eb140aff858b1019ec2915fa","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\extend\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\extend\\package.json","dependencyType":"NPM","checksums":{"SHA1":"f8b1136b4071fbd8eb140aff858b1019ec2915fa"}},{"groupId":"forever-agent","artifactId":"forever-agent-0.6.1.tgz","version":"0.6.1","sha1":"fbc71f0c41adeb37f96c577ad1ed42d8fdacca91","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\forever-agent\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\forever-agent\\package.json","dependencyType":"NPM","checksums":{"SHA1":"fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"}},{"groupId":"form-data","artifactId":"form-data-2.3.2.tgz","version":"2.3.2","sha1":"4970498be604c20c005d4f5c23aecd21d6b49099","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\form-data\\package.json","optional":false,"children":[{"groupId":"asynckit","artifactId":"asynckit-0.4.0.tgz","version":"0.4.0","sha1":"c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\asynckit\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\asynckit\\package.json","dependencyType":"NPM","checksums":{"SHA1":"c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"}}],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\form-data\\package.json","dependencyType":"NPM","checksums":{"SHA1":"4970498be604c20c005d4f5c23aecd21d6b49099"}},{"groupId":"har-validator","artifactId":"har-validator-5.0.3.tgz","version":"5.0.3","sha1":"ba402c266194f15956ef15e0fcf242993f6a7dfd","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\har-validator\\package.json","optional":false,"children":[{"groupId":"ajv","artifactId":"ajv-5.5.2.tgz","version":"5.5.2","sha1":"73b5eeca3fab653e3d3f9422b341ad42205dc965","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\ajv\\package.json","optional":false,"children":[{"groupId":"co","artifactId":"co-4.6.0.tgz","version":"4.6.0","sha1":"6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\co\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\co\\package.json","dependencyType":"NPM","checksums":{"SHA1":"6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184"}},{"groupId":"fast-deep-equal","artifactId":"fast-deep-equal-1.1.0.tgz","version":"1.1.0","sha1":"c053477817c86b51daa853c81e059b733d023614","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\fast-deep-equal\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\fast-deep-equal\\package.json","dependencyType":"NPM","checksums":{"SHA1":"c053477817c86b51daa853c81e059b733d023614"}},{"groupId":"fast-json-stable-stringify","artifactId":"fast-json-stable-stringify-2.0.0.tgz","version":"2.0.0","sha1":"d5142c0caee6b1189f87d3a76111064f86c8bbf2","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\fast-json-stable-stringify\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\fast-json-stable-stringify\\package.json","dependencyType":"NPM","checksums":{"SHA1":"d5142c0caee6b1189f87d3a76111064f86c8bbf2"}},{"groupId":"json-schema-traverse","artifactId":"json-schema-traverse-0.3.1.tgz","version":"0.3.1","sha1":"349a6d44c53a51de89b40805c5d5e59b417d3340","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\json-schema-traverse\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\json-schema-traverse\\package.json","dependencyType":"NPM","checksums":{"SHA1":"349a6d44c53a51de89b40805c5d5e59b417d3340"}}],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\ajv\\package.json","dependencyType":"NPM","checksums":{"SHA1":"73b5eeca3fab653e3d3f9422b341ad42205dc965"}},{"groupId":"har-schema","artifactId":"har-schema-2.0.0.tgz","version":"2.0.0","sha1":"a94c2224ebcac04782a0d9035521f24735b7ec92","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\har-schema\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\har-schema\\package.json","dependencyType":"NPM","checksums":{"SHA1":"a94c2224ebcac04782a0d9035521f24735b7ec92"}}],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\har-validator\\package.json","dependencyType":"NPM","checksums":{"SHA1":"ba402c266194f15956ef15e0fcf242993f6a7dfd"}},{"groupId":"hawk","artifactId":"hawk-6.0.2.tgz","version":"6.0.2","sha1":"af4d914eb065f9b5ce4d9d11c1cb2126eecc3038","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\hawk\\package.json","optional":false,"children":[{"groupId":"boom","artifactId":"boom-4.3.1.tgz","version":"4.3.1","sha1":"4f8a3005cb4a7e3889f749030fd25b96e01d2e31","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\boom\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\boom\\package.json","dependencyType":"NPM","checksums":{"SHA1":"4f8a3005cb4a7e3889f749030fd25b96e01d2e31"}},{"groupId":"cryptiles","artifactId":"cryptiles-3.1.2.tgz","version":"3.1.2","sha1":"a89fbb220f5ce25ec56e8c4aa8a4fd7b5b0d29fe","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\cryptiles\\package.json","optional":false,"children":[{"groupId":"boom","artifactId":"boom-5.2.0.tgz","version":"5.2.0","sha1":"5dd9da6ee3a5f302077436290cb717d3f4a54e02","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\cryptiles\\node_modules\\boom\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\cryptiles\\node_modules\\boom\\package.json","dependencyType":"NPM","checksums":{"SHA1":"5dd9da6ee3a5f302077436290cb717d3f4a54e02"}}],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\cryptiles\\package.json","dependencyType":"NPM","checksums":{"SHA1":"a89fbb220f5ce25ec56e8c4aa8a4fd7b5b0d29fe"}},{"groupId":"hoek","artifactId":"hoek-4.2.1.tgz","version":"4.2.1","sha1":"9634502aa12c445dd5a7c5734b572bb8738aacbb","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\hoek\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\hoek\\package.json","dependencyType":"NPM","checksums":{"SHA1":"9634502aa12c445dd5a7c5734b572bb8738aacbb"}},{"groupId":"sntp","artifactId":"sntp-2.1.0.tgz","version":"2.1.0","sha1":"2c6cec14fedc2222739caf9b5c3d85d1cc5a2cc8","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\sntp\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\sntp\\package.json","dependencyType":"NPM","checksums":{"SHA1":"2c6cec14fedc2222739caf9b5c3d85d1cc5a2cc8"}}],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\hawk\\package.json","dependencyType":"NPM","checksums":{"SHA1":"af4d914eb065f9b5ce4d9d11c1cb2126eecc3038"}},{"groupId":"http-signature","artifactId":"http-signature-1.2.0.tgz","version":"1.2.0","sha1":"9aecd925114772f3d95b65a60abb8f7c18fbace1","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\http-signature\\package.json","optional":false,"children":[{"groupId":"assert-plus","artifactId":"assert-plus-1.0.0.tgz","version":"1.0.0","sha1":"f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\assert-plus\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\assert-plus\\package.json","dependencyType":"NPM","checksums":{"SHA1":"f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525"}},{"groupId":"jsprim","artifactId":"jsprim-1.4.1.tgz","version":"1.4.1","sha1":"313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\jsprim\\package.json","optional":false,"children":[{"groupId":"extsprintf","artifactId":"extsprintf-1.3.0.tgz","version":"1.3.0","sha1":"96918440e3041a7a414f8c52e3c574eb3c3e1e05","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\extsprintf\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\extsprintf\\package.json","dependencyType":"NPM","checksums":{"SHA1":"96918440e3041a7a414f8c52e3c574eb3c3e1e05"}},{"groupId":"json-schema","artifactId":"json-schema-0.2.3.tgz","version":"0.2.3","sha1":"b480c892e59a2f05954ce727bd3f2a4e882f9e13","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\json-schema\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\json-schema\\package.json","dependencyType":"NPM","checksums":{"SHA1":"b480c892e59a2f05954ce727bd3f2a4e882f9e13"}},{"groupId":"verror","artifactId":"verror-1.10.0.tgz","version":"1.10.0","sha1":"3a105ca17053af55d6e270c1f8288682e18da400","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\verror\\package.json","optional":false,"children":[{"groupId":"core-util-is","artifactId":"core-util-is-1.0.2.tgz","version":"1.0.2","sha1":"b5fd54220aa2bc5ab57aab7140c940754503c1a7","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\core-util-is\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\core-util-is\\package.json","dependencyType":"NPM","checksums":{"SHA1":"b5fd54220aa2bc5ab57aab7140c940754503c1a7"}}],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\verror\\package.json","dependencyType":"NPM","checksums":{"SHA1":"3a105ca17053af55d6e270c1f8288682e18da400"}}],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\jsprim\\package.json","dependencyType":"NPM","checksums":{"SHA1":"313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2"}},{"groupId":"sshpk","artifactId":"sshpk-1.14.2.tgz","version":"1.14.2","sha1":"c6fc61648a3d9c4e764fd3fcdf4ea105e492ba98","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\sshpk\\package.json","optional":false,"children":[{"groupId":"asn1","artifactId":"asn1-0.2.3.tgz","version":"0.2.3","sha1":"dac8787713c9966849fc8180777ebe9c1ddf3b86","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\asn1\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\asn1\\package.json","dependencyType":"NPM","checksums":{"SHA1":"dac8787713c9966849fc8180777ebe9c1ddf3b86"}},{"groupId":"bcrypt-pbkdf","artifactId":"bcrypt-pbkdf-1.0.2.tgz","version":"1.0.2","sha1":"a4301d389b6a43f9b67ff3ca11a3f6637e360e9e","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\bcrypt-pbkdf\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\bcrypt-pbkdf\\package.json","dependencyType":"NPM","checksums":{"SHA1":"a4301d389b6a43f9b67ff3ca11a3f6637e360e9e"}},{"groupId":"dashdash","artifactId":"dashdash-1.14.1.tgz","version":"1.14.1","sha1":"853cfa0f7cbe2fed5de20326b8dd581035f6e2f0","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\dashdash\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\dashdash\\package.json","dependencyType":"NPM","checksums":{"SHA1":"853cfa0f7cbe2fed5de20326b8dd581035f6e2f0"}},{"groupId":"ecc-jsbn","artifactId":"ecc-jsbn-0.1.2.tgz","version":"0.1.2","sha1":"3a83a904e54353287874c564b7549386849a98c9","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\ecc-jsbn\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\ecc-jsbn\\package.json","dependencyType":"NPM","checksums":{"SHA1":"3a83a904e54353287874c564b7549386849a98c9"}},{"groupId":"getpass","artifactId":"getpass-0.1.7.tgz","version":"0.1.7","sha1":"5eff8e3e684d569ae4cb2b1282604e8ba62149fa","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\getpass\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\getpass\\package.json","dependencyType":"NPM","checksums":{"SHA1":"5eff8e3e684d569ae4cb2b1282604e8ba62149fa"}},{"groupId":"jsbn","artifactId":"jsbn-0.1.1.tgz","version":"0.1.1","sha1":"a5e654c2e5a2deb5f201d96cefbca80c0ef2f513","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\jsbn\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\jsbn\\package.json","dependencyType":"NPM","checksums":{"SHA1":"a5e654c2e5a2deb5f201d96cefbca80c0ef2f513"}},{"groupId":"safer-buffer","artifactId":"safer-buffer-2.1.2.tgz","version":"2.1.2","sha1":"44fa161b0187b9549dd84bb91802f9bd8385cd6a","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\safer-buffer\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\safer-buffer\\package.json","dependencyType":"NPM","checksums":{"SHA1":"44fa161b0187b9549dd84bb91802f9bd8385cd6a"}},{"groupId":"tweetnacl","artifactId":"tweetnacl-0.14.5.tgz","version":"0.14.5","sha1":"5ae68177f192d4456269d108afa93ff8743f4f64","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\tweetnacl\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\tweetnacl\\package.json","dependencyType":"NPM","checksums":{"SHA1":"5ae68177f192d4456269d108afa93ff8743f4f64"}}],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\sshpk\\package.json","dependencyType":"NPM","checksums":{"SHA1":"c6fc61648a3d9c4e764fd3fcdf4ea105e492ba98"}}],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\http-signature\\package.json","dependencyType":"NPM","checksums":{"SHA1":"9aecd925114772f3d95b65a60abb8f7c18fbace1"}},{"groupId":"is-typedarray","artifactId":"is-typedarray-1.0.0.tgz","version":"1.0.0","sha1":"e479c80858df0c1b11ddda6940f96011fcda4a9a","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\is-typedarray\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\is-typedarray\\package.json","dependencyType":"NPM","checksums":{"SHA1":"e479c80858df0c1b11ddda6940f96011fcda4a9a"}},{"groupId":"isstream","artifactId":"isstream-0.1.2.tgz","version":"0.1.2","sha1":"47e63f7af55afa6f92e1500e690eb8b8529c099a","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\isstream\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\isstream\\package.json","dependencyType":"NPM","checksums":{"SHA1":"47e63f7af55afa6f92e1500e690eb8b8529c099a"}},{"groupId":"json-stringify-safe","artifactId":"json-stringify-safe-5.0.1.tgz","version":"5.0.1","sha1":"1296a2d58fd45f19a0f6ce01d65701e2c735b6eb","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\json-stringify-safe\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\json-stringify-safe\\package.json","dependencyType":"NPM","checksums":{"SHA1":"1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"}},{"groupId":"mime-types","artifactId":"mime-types-2.1.19.tgz","version":"2.1.19","sha1":"71e464537a7ef81c15f2db9d97e913fc0ff606f0","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\mime-types\\package.json","optional":false,"children":[{"groupId":"mime-db","artifactId":"mime-db-1.35.0.tgz","version":"1.35.0","sha1":"0569d657466491283709663ad379a99b90d9ab47","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\mime-db\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\mime-db\\package.json","dependencyType":"NPM","checksums":{"SHA1":"0569d657466491283709663ad379a99b90d9ab47"}}],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\mime-types\\package.json","dependencyType":"NPM","checksums":{"SHA1":"71e464537a7ef81c15f2db9d97e913fc0ff606f0"}},{"groupId":"oauth-sign","artifactId":"oauth-sign-0.8.2.tgz","version":"0.8.2","sha1":"46a6ab7f0aead8deae9ec0565780b7d4efeb9d43","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\oauth-sign\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\oauth-sign\\package.json","dependencyType":"NPM","checksums":{"SHA1":"46a6ab7f0aead8deae9ec0565780b7d4efeb9d43"}},{"groupId":"performance-now","artifactId":"performance-now-2.1.0.tgz","version":"2.1.0","sha1":"6309f4e0e5fa913ec1c69307ae364b4b377c9e7b","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\performance-now\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\performance-now\\package.json","dependencyType":"NPM","checksums":{"SHA1":"6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"}},{"groupId":"qs","artifactId":"qs-6.5.2.tgz","version":"6.5.2","sha1":"cb3ae806e8740444584ef154ce8ee98d403f3e36","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\qs\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\qs\\package.json","dependencyType":"NPM","checksums":{"SHA1":"cb3ae806e8740444584ef154ce8ee98d403f3e36"}},{"groupId":"safe-buffer","artifactId":"safe-buffer-5.1.2.tgz","version":"5.1.2","sha1":"991ec69d296e0313747d59bdfd2b745c35f8828d","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\safe-buffer\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\safe-buffer\\package.json","dependencyType":"NPM","checksums":{"SHA1":"991ec69d296e0313747d59bdfd2b745c35f8828d"}},{"groupId":"stringstream","artifactId":"stringstream-0.0.6.tgz","version":"0.0.6","sha1":"7880225b0d4ad10e30927d167a1d6f2fd3b33a72","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\stringstream\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\stringstream\\package.json","dependencyType":"NPM","checksums":{"SHA1":"7880225b0d4ad10e30927d167a1d6f2fd3b33a72"}},{"groupId":"tough-cookie","artifactId":"tough-cookie-2.3.4.tgz","version":"2.3.4","sha1":"ec60cee38ac675063ffc97a5c18970578ee83655","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\tough-cookie\\package.json","optional":false,"children":[{"groupId":"punycode","artifactId":"punycode-1.4.1.tgz","version":"1.4.1","sha1":"c0d5a63b2718800ad8e1eb0fa5269c84dd41845e","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\punycode\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\punycode\\package.json","dependencyType":"NPM","checksums":{"SHA1":"c0d5a63b2718800ad8e1eb0fa5269c84dd41845e"}}],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\tough-cookie\\package.json","dependencyType":"NPM","checksums":{"SHA1":"ec60cee38ac675063ffc97a5c18970578ee83655"}},{"groupId":"tunnel-agent","artifactId":"tunnel-agent-0.6.0.tgz","version":"0.6.0","sha1":"27a5dea06b36b04a0a9966774b290868f0fc40fd","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\tunnel-agent\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\tunnel-agent\\package.json","dependencyType":"NPM","checksums":{"SHA1":"27a5dea06b36b04a0a9966774b290868f0fc40fd"}}],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\request\\package.json","dependencyType":"NPM","checksums":{"SHA1":"ca0b65da02ed62935887808e6f510381034e3356"}},{"groupId":"uuid","artifactId":"uuid-3.1.0.tgz","version":"3.1.0","sha1":"3dd3d3e790abc24d7b0d3a034ffababe28ebbc04","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\uuid\\package.json","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample\\node_modules\\uuid\\package.json","dependencyType":"NPM","checksums":{"SHA1":"3dd3d3e790abc24d7b0d3a034ffababe28ebbc04"}},{"artifactId":"sample.js","sha1":"60639b7e53e467dadcbd6bf00a6daaf5738b278e","fullHash":"1660f12ea5e40066471ee989dd3a4b393eb908b4","mostSigBitsHash":"dc25a8c05c41be6d828d0bd404c172ada6b5fd6e","leastSigBitsHash":"7ccfad2d66f476830cccdb173950a64c81006f4d","otherPlatformSha1":"33dbd00f8459b279344daefbf7118f2eea001504","systemPath":"C:\\Gitub-Projects\\fs-agent\\target\\test-classes\\resolver\\npm\\sample.js","optional":false,"children":[],"exclusions":[],"licenses":[],"copyrights":[],"filename":"sample.js","checksums":{"SHA1":"60639b7e53e467dadcbd6bf00a6daaf5738b278e","SHA1_SUPER_HASH":"1660f12ea5e40066471ee989dd3a4b393eb908b4","SHA1_SUPER_HASH_MSB":"dc25a8c05c41be6d828d0bd404c172ada6b5fd6e","SHA1_SUPER_HASH_LSB":"7ccfad2d66f476830cccdb173950a64c81006f4d","SHA1_NO_HEADER":"e489eba188497f066442d3890943d68b5a0ed7f4","SHA1_NO_COMMENTS_SUPER_HASH":"0376304b0fd59234ca5e336dd83d580c7579b299","SHA1_OTHER_PLATFORM":"33dbd00f8459b279344daefbf7118f2eea001504"}}]}],"aggregateModules":false,"preserveModuleStructure":false}
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..29e0c31
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,537 @@
+
+
+ 4.0.0
+
+ org.whitesource
+ whitesource-fs-agent
+ 1.8
+ jar
+
+ White Source File System Agent
+ File System Agent is a simple java command line tool which extracts descriptive information from your open source libraries
+ https://github.com/whitesource/fs-agent
+
+
+ UTF-8
+ 2.9.5
+ 1.7.5
+ false
+
+
+
+ WhiteSource Software
+ http://whitesourcesoftware.com/
+
+ 2014
+
+
+
+ The Apache Software License, Version 2.0
+ http://www.apache.org/licenses/LICENSE-2.0.txt
+
+
+
+
+ scm:git:git@github.com:whitesource/fs-agent.git
+ scm:git:git@github.com:whitesource/fs-agent.git
+ git@github.com:whitesource/fs-agent.git
+
+
+ https://github.com/whitesource/fs-agent/issues
+ GitHub Issues
+
+
+
+
+ anna.rozin
+ Anna Rozin
+ anna.rozin@whitesourcesoftware.com
+
+
+ tom.shapira
+ Tom Shapira
+ tom.shapira@whitesourcesoftware.com
+
+
+
+
+
+ javax.servlet
+ javax.servlet-api
+ 3.1.0
+
+
+
+ net.lingala.zip4j
+ zip4j
+ 1.3.2
+
+
+ org.apache.httpcomponents
+ httpclient
+ 4.3.4
+
+
+ org.zeroturnaround
+ zt-exec
+ 1.9
+
+
+ org.jasypt
+ jasypt
+ 1.9.2
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+ 2.8.7
+
+
+ javax.mail
+ mail
+ 1.5.0-b01
+
+
+ commons-io
+ commons-io
+ 2.5
+
+
+ org.apache.commons
+ commons-lang3
+ 3.5
+
+
+ org.apache.logging.log4j
+ log4j-core
+ 2.10.0
+
+
+ org.junit.jupiter
+ junit-jupiter-api
+ 5.4.0
+ test
+
+
+ org.junit.jupiter
+ junit-jupiter-engine
+ 5.4.0
+ test
+
+
+ org.junit.platform
+ junit-platform-launcher
+ 1.4.0
+ test
+
+
+ io.vertx
+ vertx-core
+ 3.5.4
+
+
+
+ io.vertx
+ vertx-web
+ 3.5.4
+
+
+
+
+ org.apache.maven
+ maven-model-builder
+ 3.5.2
+
+
+ org.whitesource
+ maven-dependency-tree-parser
+ 1.0.5
+
+
+
+
+ org.whitesource
+ wss-agent-api-client
+ ${agent.api.version}
+
+
+ org.whitesource
+ wss-agent-report
+ ${agent.api.version}
+
+
+
+
+ com.beust
+ jcommander
+ 1.35
+
+
+
+
+ org.apache.ant
+ ant
+ 1.9.4
+
+
+
+
+ org.eclipse.jgit
+ org.eclipse.jgit
+ 4.10.0.201712302008-r
+
+
+
+
+ org.tmatesoft.svnkit
+ svnkit
+ 1.8.7
+
+
+
+
+ com.aragost.javahg
+ javahg
+ 0.4
+
+
+
+
+ net.lingala.zip4j
+ zip4j
+ 1.3.2
+
+
+
+
+
+
+ org.apache.commons
+ commons-compress
+ 1.18
+
+
+
+
+ com.github.junrar
+ junrar
+ 1.0.1
+
+
+
+
+ org.redline-rpm
+ redline
+ 1.2.1
+
+
+
+ commons-io
+ commons-io
+ 2.4
+
+
+
+
+ org.json
+ json
+ 20170516
+
+
+
+
+ org.apache.commons
+ commons-lang3
+ 3.4
+
+
+
+
+ org.slf4j
+ slf4j-api
+ ${slf4j.version}
+
+
+ ch.qos.logback
+ logback-classic
+ 1.2.3
+
+
+ org.slf4j
+ slf4j-api
+
+
+
+
+
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+ 2.9.9.1
+
+
+
+
+ com.fasterxml.jackson.dataformat
+ jackson-dataformat-yaml
+ 2.9.2
+
+
+
+
+ org.jsoup
+ jsoup
+ 1.11.3
+
+
+
+
+
+ javax.xml.bind
+ jaxb-api
+
+ 2.3.0-b170201.1204
+
+
+
+
+ org.simpleframework
+ simple-xml
+ 2.7.1
+
+
+
+
+ junit
+ junit
+ 4.12
+ test
+
+
+
+ io.vertx
+ vertx-unit
+ 3.0.0
+ test
+
+
+
+ org.codehaus.plexus
+ plexus-archiver
+ 3.4
+
+
+ org.codehaus.plexus
+ plexus-container-default
+ 1.7.1
+
+
+
+
+ com.amazonaws
+ aws-java-sdk-ecr
+ 1.11.410
+
+
+ com.sun.jersey
+ jersey-client
+ 1.17.1
+
+
+
+
+
+
+ src/main/resources
+ true
+
+
+
+ .
+
+ README.md
+ README.EUA.md
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+ 3.5.1
+
+ 1.8
+ 1.8
+
+
+
+ maven-surefire-plugin
+ 2.20.1
+
+
+ default-test
+ test
+
+ test
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+ 2.3.2
+
+ 1.8
+ 1.8
+
+
+
+ maven-assembly-plugin
+ 2.4
+
+
+
+ org.whitesource.fs.Main
+
+
+
+ jar-with-dependencies
+
+ whitesource-fs-agent-${project.version}
+ false
+
+
+
+ make-assembly
+ package
+
+ single
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-javadoc-plugin
+ 2.8.1
+
+
+ org.apache.maven.plugins
+ maven-gpg-plugin
+ 1.4
+
+
+ org.apache.maven.plugins
+ maven-release-plugin
+ 2.5.3
+
+ -Dgpg.passphrase=${passed.gpg.passphrase} -Dgpg.homedir=${passed.gpg.homedir}
+ Release
+
+
+
+
+
+
+ org.whitesource
+ whitesource-maven-plugin
+ 3.2.5
+
+ ${whitesource.orgToken}
+ File System Agent
+
+
+
+
+
+
+
+
+ ci-build
+
+ false
+
+
+
+
+ org.apache.maven.plugins
+ maven-source-plugin
+
+
+ attach-sources
+ verify
+
+ jar-no-fork
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-javadoc-plugin
+
+ true
+
+
+
+ attach-javadocs
+
+ jar
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-gpg-plugin
+
+
+ sign-artifacts
+ verify
+
+ sign
+
+
+ ${gpg.homedir}
+
+
+
+
+
+
+
+
+ Release
+
+ false
+
+
+
+
+ com.github.github
+ site-maven-plugin
+
+
+
+ site
+
+ site
+
+
+
+
+
+
+
+
+
+
+ github
+ GitHub OWNER Apache Maven Packages
+ https://maven.pkg.github.com/whitesource
+
+
+
diff --git a/shiftleft.json b/shiftleft.json
new file mode 100644
index 0000000..7771ef3
--- /dev/null
+++ b/shiftleft.json
@@ -0,0 +1,5 @@
+{
+ "license": "ZXlKaGJHY2lPaUpTVXpVeE1pSXNJblI1Y0NJNklrcFhWQ0o5LmV5SnBZWFFpT2pFMU56UXdOalUxTURNc0ltbHpjeUk2SWxOb2FXWjBUR1ZtZENJc0ltOXlaMGxFSWpvaVlUTmtNemcxTVRJdFlqTXhPUzAwTXpZd0xXSmtZVFF0TUdZeVlXWTNOek01TmpjNElpd2lkWE5sY2tsRUlqb2lORFJsWlRKak5HRXRNbVprWXkwMFlqSmhMV0k1TUdZdE0ySmpNV1ExWkdObVptSTFJaXdpYzJOdmNHVnpJanBiSW5ObFlYUnpPbmR5YVhSbElpd2laWGgwWlc1a1pXUWlMQ0poY0drNmRqSWlMQ0oxY0d4dllXUnpPbmR5YVhSbElpd2liRzluT25keWFYUmxJaXdpY0dsd1pXeHBibVZ6ZEdGMGRYTTZjbVZoWkNJc0ltMWxkSEpwWTNNNmQzSnBkR1VpTENKd2IyeHBZMmxsY3pwamRYTjBiMjFsY2lKZGZRLnloeURiVVdkVmM1VmdMbi1yUUVWNWtBalVxejZHWHM5ZkNJVlh1VXVMNVh0ZnNlR3BoMFZsMTQxOFRWdW9VWkU0b2dxejVkbGItcGlNYVdhcHpPQ1dBdjAyejZMNktmRFg2ZThmMjNqS3Vrc2FiVy03WE5SVF91QjN1bTdvUzdwY29NUHN5MVhUcXVPazJlNDVLOTB0SnljSmxzR28yNmUyVmRFczdLWE9oZkltWjFvNDhkVldCOEZFUVM3QkRvZ1B2MkhndTVlSnpSOVhIOG9DRXFYcTB0dm9fblExVXZnWkUtX1Mybk41WHVpTm1qYXdaVWdQZlp1WFhWcnNyNUR0VmtRMWpEYXBDRUZ5dkxQUGdRS1FmTHJzdGJJcXlNSkQwZzg0X3hmc25fY3hRZTIya0VBeXB4WGRQSEtTcmNmbWxrdnF4MlNZdDZDSGdvcS04bGJWdw==",
+ "sprId": "sl/a3d38512-b319-4360-bda4-0f2af7739678/fs-agent/5b4e048c983b1bc4c687b333000074ccbe1d464f",
+ "accessToken": "eyJhbGciOiJSUzUxMiIsInR5cCI6IkpXVCJ9.eyJpYXQiOjE1NzQwNjU1MDMsImlzcyI6IlNoaWZ0TGVmdCIsIm9yZ0lEIjoiYTNkMzg1MTItYjMxOS00MzYwLWJkYTQtMGYyYWY3NzM5Njc4IiwidXNlcklEIjoiNDRlZTJjNGEtMmZkYy00YjJhLWI5MGYtM2JjMWQ1ZGNmZmI1Iiwic2NvcGVzIjpbInNlYXRzOndyaXRlIiwiZXh0ZW5kZWQiLCJhcGk6djIiLCJ1cGxvYWRzOndyaXRlIiwibG9nOndyaXRlIiwicGlwZWxpbmVzdGF0dXM6cmVhZCIsIm1ldHJpY3M6d3JpdGUiLCJwb2xpY2llczpjdXN0b21lciJdfQ.yhyDbUWdVc5VgLn-rQEV5kAjUqz6GXs9fCIVXuUuL5XtfseGph0Vl1418TVuoUZE4ogqz5dlb-piMaWapzOCWAv02z6L6KfDX6e8f23jKuksabW-7XNRT_uB3um7oS7pcoMPsy1XTquOk2e45K90tJycJlsGo26e2VdEs7KXOhfImZ1o48dVWB8FEQS7BDogPv2Hgu5eJzR9XH8oCEqXq0tvo_nQ1UvgZE-_S2nN5XuiNmjawZUgPfZuXXVrsr5DtVkQ1jDapCEFyvLPPgQKQfLrstbIqyMJD0g84_xfsn_cxQe22kEAypxXdPHKSrcfmlkvqx2SYt6CHgoq-8lbVw"
+}
\ No newline at end of file
diff --git a/src/main/java/org/whitesource/agent/ConfigPropertyKeys.java b/src/main/java/org/whitesource/agent/ConfigPropertyKeys.java
new file mode 100644
index 0000000..e1ddb79
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/ConfigPropertyKeys.java
@@ -0,0 +1,252 @@
+/**
+ * Copyright (C) 2014 WhiteSource Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.whitesource.agent;
+
+/**
+ * Property keys for the whitesource-docker-agent.configuration file.
+ *
+ * @author itai.marko
+ * @author tom.shapira
+ */
+public final class ConfigPropertyKeys {
+
+ public static final String CHECK_POLICIES_PROPERTY_KEY = "checkPolicies";
+ public static final String FORCE_UPDATE = "forceUpdate";
+ public static final String FORCE_UPDATE_FAIL_BUILD_ON_POLICY_VIOLATION = "forceUpdate.failBuildOnPolicyViolation";
+ public static final String FORCE_CHECK_ALL_DEPENDENCIES = "forceCheckAllDependencies"; // optional
+ public static final String UPDATE_INVENTORY = "updateInventory"; // optional
+ public static final String ENABLE_IMPACT_ANALYSIS = "enableImpactAnalysis"; // optional
+ public static final String IA_LANGUAGE = "iaLanguage"; // optional
+ public static final String CONNECTION_RETRIES = "connectionRetries";
+ public static final String CONNECTION_RETRIES_INTERVALS = "connectionRetriesInterval";
+ public static final String ORG_TOKEN_PROPERTY_KEY = "apiKey";
+ public static final String ORG_TOKEN_FILE = "apiKeyFile";
+ public static final String USER_KEY_PROPERTY_KEY = "userKey";
+ public static final String USER_KEY_FILE = "userKeyFile";
+ public static final String PARTIAL_SHA1_MATCH_KEY = "partialSha1Match";
+ public static final String PRODUCT_TOKEN_PROPERTY_KEY = "productToken"; // optional
+ public static final String PRODUCT_NAME_PROPERTY_KEY = "productName"; // optional
+ public static final String PRODUCT_VERSION_PROPERTY_KEY = "productVersion"; // optional
+ public static final String APP_PATH = "appPath"; // optional
+ public static final String X_PATHS = "xPaths"; // optional
+ public static final String VIA_DEBUG = "viaDebug"; // optional
+ public static final String VIA_ANALYSIS_LEVEL = "viaAnalysysLevel"; // optional
+ public static final String PROJECT_TOKEN_PROPERTY_KEY = "projectToken";
+ public static final String PROJECT_NAME_PROPERTY_KEY = "projectName";
+ public static final String PROJECT_VERSION_PROPERTY_KEY = "projectVersion"; // optional
+ public static final String INCLUDES_PATTERN_PROPERTY_KEY = "includes";
+ public static final String EXCLUDES_PATTERN_PROPERTY_KEY = "excludes";
+ public static final String ARCHIVE_EXTRACTION_DEPTH_KEY = "archiveExtractionDepth";
+ public static final String ARCHIVE_INCLUDES_PATTERN_KEY = "archiveIncludes";
+ public static final String ARCHIVE_EXCLUDES_PATTERN_KEY = "archiveExcludes";
+ public static final String ARCHIVE_FAST_UNPACK_KEY = "archiveFastUnpack";
+ public static final String CALCULATE_HINTS = "calculate.hints";
+ public static final String CALCULATE_MD5 = "calculate.md5";
+ public static final String REQUESTER_EMAIL = "requesterEmail";
+ public static final String CASE_SENSITIVE_GLOB_PROPERTY_KEY = "case.sensitive.glob";
+ public static final String PROXY_HOST_PROPERTY_KEY = "proxy.host";
+ public static final String PROXY_PORT_PROPERTY_KEY = "proxy.port";
+ public static final String PROXY_USER_PROPERTY_KEY = "proxy.user";
+ public static final String PROXY_PASS_PROPERTY_KEY = "proxy.pass";
+ public static final String IGNORE_CERTIFICATE_CHECK = "ignoreCertificateCheck";
+ public static final String OFFLINE_PROPERTY_KEY = "offline";
+ public static final String OFFLINE_ZIP_PROPERTY_KEY = "offline.zip";
+ public static final String OFFLINE_PRETTY_JSON_KEY = "offline.prettyJson";
+ public static final String SCM_TYPE_PROPERTY_KEY = "scm.type";
+ public static final String SCM_URL_PROPERTY_KEY = "scm.url";
+ public static final String SCM_PPK_PROPERTY_KEY = "scm.ppk";
+ public static final String SCM_USER_PROPERTY_KEY = "scm.user";
+ public static final String SCM_PASS_PROPERTY_KEY = "scm.pass";
+ public static final String SCM_BRANCH_PROPERTY_KEY = "scm.branch";
+ public static final String SCM_TAG_PROPERTY_KEY = "scm.tag";
+ public static final String SCM_NPM_INSTALL = "scm.npmInstall";
+ public static final String SCM_NPM_INSTALL_TIMEOUT_MINUTES = "scm.npmInstallTimeoutMinutes";
+ public static final String SCM_REPOSITORIES_FILE = "scm.repositoriesFile";
+ public static final String EXCLUDED_COPYRIGHT_KEY = "copyright.excludes";
+ public static final String LOG_LEVEL_KEY = "log.level";
+ public static final String FOLLOW_SYMBOLIC_LINKS = "followSymbolicLinks";
+ public static final String SHOW_PROGRESS_BAR = "showProgressBar";
+ public static final String ACCEPT_EXTENSIONS_LIST = "acceptExtensionsList";
+
+ public static final String RESOLVE_ALL_DEPENDENCIES = "resolveAllDependencies";
+
+ public static final String NPM_RUN_PRE_STEP = "npm.runPreStep";
+ public static final String NPM_IGNORE_SCRIPTS = "npm.ignoreScripts";
+ public static final String NPM_RESOLVE_DEPENDENCIES = "npm.resolveDependencies";
+ public static final String NPM_INCLUDE_DEV_DEPENDENCIES = "npm.includeDevDependencies";
+ public static final String NPM_TIMEOUT_DEPENDENCIES_COLLECTOR_SECONDS = "npm.timeoutDependenciesCollectorInSeconds";
+ public static final String NPM_ACCESS_TOKEN = "npm.accessToken";
+ public static final String NPM_IGNORE_NPM_LS_ERRORS = "npm.ignoreNpmLsErrors";
+ public static final String NPM_YARN_PROJECT = "npm.yarnProject";
+ public static final String NPM_IGNORE_JAVA_SCRIPT_FILES = "npm.ignoreJavaScriptFiles";
+ public static final String NPM_IGNORE_SOURCE_FILES = "npm.ignoreSourceFiles";
+
+
+ public static final String BOWER_RESOLVE_DEPENDENCIES = "bower.resolveDependencies";
+ public static final String BOWER_RUN_PRE_STEP = "bower.runPreStep";
+ public static final String BOWER_IGNORE_SOURCE_FILES = "bower.ignoreSourceFiles";
+
+ public static final String PYTHON_RESOLVE_DEPENDENCIES = "python.resolveDependencies";
+ public static final String PYTHON_PIP_PATH = "python.pipPath";
+ public static final String PYTHON_PATH = "python.path";
+ public static final String PYTHON_IS_WSS_PLUGIN_INSTALLED = "python.isWssPluginInstalled";
+ public static final String PYTHON_UNINSTALL_WSS_PLUGIN = "python.uninstallWssPlugin";
+ public static final String PYTHON_IGNORE_PIP_INSTALL_ERRORS = "python.ignorePipInstallErrors";
+ public static final String PYTHON_INSTALL_VIRTUALENV = "python.installVirtualenv";
+ public static final String PYTHON_RESOLVE_HIERARCHY_TREE = "python.resolveHierarchyTree";
+ public static final String PYTHON_RESOLVE_SETUP_PY_FILES = "python.resolveSetupPyFiles";
+ public static final String PYTHON_IGNORE_SOURCE_FILES = "python.ignoreSourceFiles";
+ public static final String PYTHON_REQUIREMENTS_FILE_INCLUDES = "python.requirementsFileIncludes";
+ public static final String PYTHON_RUN_PIPENV_PRE_STEP = "python.runPipenvPreStep";
+ public static final String PYTHON_IGNORE_PIPENV_INSTALL_ERRORS = "python.IgnorePipenvInstallErrors";
+ public static final String PYTHON_PIPENV_DEV_DEPENDENCIES = "python.pipenvDevDependencies";
+
+ public static final String NUGET_RESOLVE_DEPENDENCIES = "nuget.resolveDependencies";
+ public static final String NUGET_RESTORE_DEPENDENCIES = "nuget.restoreDependencies";
+ public static final String NUGET_RUN_PRE_STEP = "nuget.runPreStep";
+ public static final String NUGET_IGNORE_SOURCE_FILES = "nuget.ignoreSourceFiles";
+ public static final String NUGET_RESOLVE_CS_PROJ_FILES = "nuget.resolveCsProjFiles";
+ public static final String NUGET_RESOLVE_PACKAGES_CONFIG_FILES = "nuget.resolvePackagesConfigFiles";
+
+ public static final String MAVEN_IGNORED_SCOPES = "maven.ignoredScopes";
+ public static final String MAVEN_RESOLVE_DEPENDENCIES = "maven.resolveDependencies";
+ public static final String MAVEN_AGGREGATE_MODULES = "maven.aggregateModules";
+ public static final String MAVEN_IGNORE_POM_MODULES = "maven.ignorePomModules";
+ public static final String MAVEN_IGNORE_SOURCE_FILES = "maven.ignoreSourceFiles";
+ public static final String MAVEN_RUN_PRE_STEP = "maven.runPreStep";
+ public static final String MAVEN_IGNORE_DEPENDENCY_TREE_ERRORS = "maven.ignoreMvnTreeErrors";
+
+ public static final String IGNORE_SOURCE_FILES = "ignoreSourceFiles";
+
+ public static final String PROJECT_PER_SUBFOLDER = "projectPerFolder";
+ public static final String PROJECT_PER_FOLDER_INCLUDES = "projectPerFolderIncludes";
+ public static final String PROJECT_PER_FOLDER_EXCLUDES = "projectPerFolderExcludes";
+ public static final String UPDATE_TYPE = "updateType";
+ public static final String PROJECT_CONFIGURATION_PATH = "configFilePath";
+ public static final String SCAN_PACKAGE_MANAGER = "scanPackageManager";
+ public static final String WHITESOURCE_FOLDER_PATH = "whiteSourceFolderPath";
+
+ public static final String ENDPOINT_ENABLED = "endpoint.enabled";
+ public static final String ENDPOINT_PORT = "endpoint.port";
+ public static final String ENDPOINT_CERTIFICATE = "endpoint.certificate";
+ public static final String ENDPOINT_PASS = "endpoint.pass";
+ public static final String ENDPOINT_SSL_ENABLED = "endpoint.ssl";
+
+ public static final String GRADLE_RUN_PRE_STEP = "gradle.runPreStep";
+ public static final String GRADLE_RESOLVE_DEPENDENCIES = "gradle.resolveDependencies";
+ public static final String GRADLE_RUN_ASSEMBLE_COMMAND = "gradle.runAssembleCommand";
+ public static final String GRADLE_AGGREGATE_MODULES = "gradle.aggregateModules";
+ public static final String GRADLE_PREFERRED_ENVIRONMENT = "gradle.preferredEnvironment";
+ public static final String GRADLE_IGNORE_SOURCE_FILES = "gradle.ignoreSourceFiles";
+ public static final String GRADLE_IGNORE_SCOPES = "gradle.ignoredScopes";
+ public static final String GRADLE_LOCAL_REPOSITORY_PATH = "gradle.localRepositoryPath";
+
+ public static final String PAKET_RESOLVE_DEPENDENCIES = "paket.resolveDependencies";
+ public static final String PAKET_IGNORED_GROUPS = "paket.ignoredGroups";
+ public static final String PAKET_RUN_PRE_STEP = "paket.runPreStep";
+ public static final String PAKET_EXE_PATH = "paket.exePath";
+ public static final String PAKET_IGNORE_FILES = "paket.ignoreFiles";
+ public static final String PAKET_IGNORE_SOURCE_FILES = "paket.ignoreSourceFiles";
+
+ public static final String GO_RESOLVE_DEPENDENCIES = "go.resolveDependencies";
+ public static final String GO_DEPENDENCY_MANAGER = "go.dependencyManager";
+ public static final String GO_COLLECT_DEPENDENCIES_AT_RUNTIME = "go.collectDependenciesAtRuntime";
+ public static final String GO_GLIDE_IGNORE_TEST_PACKAGES = "go.glide.ignoreTestPackages";
+ public static final String GO_IGNORE_SOURCE_FILES = "go.ignoreSourceFiles";
+ public static final String GO_GRADLE_ENABLE_TASK_ALIAS = "go.gogradle.enableTaskAlias";
+
+ public static final String RUBY_RESOLVE_DEPENDENCIES = "ruby.resolveDependencies";
+ public static final String RUBY_RUN_BUNDLE_INSTALL = "ruby.runBundleInstall";
+ public static final String RUBY_OVERWRITE_GEM_FILE = "ruby.overwriteGemFile";
+ public static final String RUBY_INSTALL_MISSING_GEMS = "ruby.installMissingGems";
+ public static final String RUBY_IGNORE_SOURCE_FILES = "ruby.ignoreSourceFiles";
+
+ public static final String PHP_RESOLVE_DEPENDENCIES = "php.resolveDependencies";
+ public static final String PHP_RUN_PRE_STEP = "php.runPreStep";
+ public static final String PHP_INCLUDE_DEV_DEPENDENCIES = "php.includeDevDependencies";
+
+ public static final String SBT_RESOLVE_DEPENDENCIES = "sbt.resolveDependencies";
+ public static final String SBT_AGGREGATE_MODULES = "sbt.aggregateModules";
+ public static final String SBT_RUN_PRE_STEP = "sbt.runPreStep";
+ public static final String SBT_TARGET_FOLDER = "sbt.targetFolder";
+ public static final String SBT_IGNORE_SOURCE_FILES = "sbt.ignoreSourceFiles";
+
+ public static final String HTML_RESOLVE_DEPENDENCIES = "html.resolveDependencies";
+
+ public static final String COCOAPODS_RESOLVE_DEPENDENCIES = "cocoapods.resolveDependencies";
+ public static final String COCOAPODS_RUN_PRE_STEP = "cocoapods.runPreStep";
+ public static final String COCOAPODS_IGNORE_SOURCE_FILES = "cocoapods.ignoreSourceFiles";
+
+ public static final String HEX_RESOLVE_DEPENDENECIES = "hex.resolveDependencies";
+ public static final String HEX_RUN_PRE_STEP = "hex.runPreStep";
+ public static final String HEX_IGNORE_SOURCE_FILES = "hex.ignoreSourceFiles";
+ public static final String HEX_AGGREGATE_MODULES = "hex.aggregateModules";
+
+ public static final String DEPENDENCIES_ONLY = "dependenciesOnly";
+ public static final String WHITESOURCE_CONFIGURATION = "whitesourceConfiguration";
+
+ public static final String SCANNED_FOLDERS = "d";
+ public static final String SEND_LOGS_TO_WSS = "sendLogsToWss";
+
+ public static final String SCAN_COMMENT = "scanComment";
+
+ public static final String LOG_CONTEXT = "logContext";
+
+ public static final String REQUIRE_KNOWN_SHA1 = "requireKnownSha1";
+
+ // Analysis multi module project for via
+ public static final String ANALYZE_MULTI_MODULE = "analyzeMultiModule";
+ public static final String X_MODULE_PATH = "xModulePath";
+
+ // Global values for remote Docker
+ public static final String DOCKER_INCLUDES_PATTERN_PROPERTY_KEY = "docker.includes";
+ public static final String DOCKER_EXCLUDES_PATTERN_PROPERTY_KEY = "docker.excludes";
+ public static final String SCAN_DOCKER_IMAGES = "docker.scanImages";
+ public static final String SCAN_TAR_IMAGES = "docker.tarImages";
+ public static final String DELETE_TAR_FILES = "docker.deleteTar";
+ public static final String DOCKER_PULL_ENABLE = "docker.pull.enable";
+ public static final String DOCKER_PULL_IMAGES = "docker.pull.images";
+ public static final String DOCKER_PULL_TAGS = "docker.pull.tags";
+ public static final String DOCKER_PULL_DIGEST = "docker.pull.digest";
+ public static final String DOCKER_PULL_MAX_IMAGES = "docker.pull.maxImages";
+ public static final String DOCKER_DELETE_FORCE = "docker.delete.force";
+ public static final String DOCKER_LOGIN_SUDO = "docker.login.sudo";
+ // TODO: Not implemented yet
+ public static final String DOCKER_SCAN_MAX_IMAGES = "docker.scan.maxImages";
+ // TODO: Not implemented yet
+ public static final String DOCKER_PULL_FORCE = "docker.pull.force";
+
+ // Values for remote Docker on AWS ECR
+ public static final String DOCKER_AWS_ENABLE = "docker.aws.enable";
+ public static final String DOCKER_AWS_REGISTRY_IDS = "docker.aws.registryIds";
+ public static final String DOCKER_AWS_REGION = "docker.aws.region";
+ // TODO: Not implemented yet
+ public static final String DOCKER_AWS_MAX_PULL_IMAGES = "docker.aws.maxPullImages";
+ // TODO: Not implemented yet
+ public static final String DOCKER_AWS_ACCESSKEY = "docker.aws.accessKey";
+ // TODO: Not implemented yet
+ public static final String DOCKER_AWS_SECRETKEY = "docker.aws.secretKey";
+
+ // Remote Docker Azure
+ public static final String DOCKER_AZURE_ENABLED = "docker.azure.enable";
+ public static final String DOCKER_AZURE_USER_NAME = "docker.azure.userName";
+ public static final String DOCKER_AZURE_USER_PASSWORD = "docker.azure.userPassword";
+ public static final String DOCKER_AZURE_REGISTRY_NAMES = "docker.azure.registryNames";
+
+
+ public static final String ADD_SHA1 = "addSha1";
+ public static final String NO_CONFIG = "noConfig";
+}
\ No newline at end of file
diff --git a/src/main/java/org/whitesource/agent/Constants.java b/src/main/java/org/whitesource/agent/Constants.java
new file mode 100644
index 0000000..1ac7910
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/Constants.java
@@ -0,0 +1,124 @@
+/**
+ * Copyright (C) 2014 WhiteSource Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.whitesource.agent;
+
+/**
+ * Property keys for the whitesource-docker-agent.configuration file.
+ *
+ * @author annarozin
+ */
+public final class Constants {
+
+ public static final String NEW_LINE = System.lineSeparator();
+ public static final String JAVA_NETWORKING = "java.net";
+ public static final String FILE_SEPARATOR = "file.separator";
+ public static final String FALSE = "false";
+ public static final String TRUE = "true";
+ public static final String TAG = "tag";
+ public static final String VERSION = "version";
+ public static final String RESOLUTION = "_resolution";
+ public static final String NAME = "name";
+ public static final String MISSING = "missing";
+ public static final String DEPENDENCIES = "dependencies";
+ public static final String SRC = "src";
+ public static final String CMD = "cmd";
+ public static final String OS_NAME = "os.name";
+ public static final String WIN = "win";
+ public static final String JS_EXTENSION = ".js";
+ public static final String PACKAGES = "packages";
+ public static final String INSTALL = "install";
+ public static final String MAVEN = "maven";
+ public static final String HTML = "html";
+ public static final String HTM = "htm";
+ public static final String SHTML = "shtml";
+ public static final String XHTML = "xhtml";
+ public static final String JSP = "jsp";
+ public static final String ASP = "asp";
+ public static final String DO = "do";
+ public static final String ASPX = "aspx";
+ public static final String WINDOWS = "Windows";
+ public static final String GRADLE_WRAPPER = "wrapper";
+ public static final String GRADLE = "gradle";
+ public static final String POM = "pom";
+ public static final String JAR = "jar";
+ public static final String DOT = ".";
+ public static final String DIRECTORY = "d";
+ public static final String BACK_SLASH = "\\";
+ public static final String FORWARD_SLASH = "/";
+ public static final String WHITESPACE = " ";
+ public static final String EMPTY_STRING = "";
+ public static final String COLON = ":";
+ public static final String AT = "@";
+ public static final String PLUS = "+";
+ public static final String DASH = "-";
+ public static final String PATTERN = "**/*";
+ public static final String COMMA = ",";
+ public static final String PIPE = "|";
+ public static final String REGEX_PATTERN_PREFIX = ".*\\.";
+ public static final String GLOB_PATTERN_PREFIX = "**/*";
+ public static final String GLOB_PATTERN = ".*.*";
+ public static final String EQUALS = "=";
+ public static final String POUND = "#";
+ public static final String QUOTATION_MARK = "\"";
+ public static final String APOSTROPHE = "'";
+ public static final String HTTP = "http";
+ public static final String HTTPS = "https";
+ public static final String UTF8 = "UTF-8";
+ public static final String DLL = ".dll";
+ public static final String EXE = ".exe";
+ public static final String NUPKG = ".nupkg";
+ public static final String CS = ".cs";
+ public static final String VAR = "var";
+ public static final String LIB = "lib";
+ public static final String YUM_DB = "yumdb";
+ public static final String YUM = "yum";
+ public static final String PYTHON_REQUIREMENTS = "requirements.txt";
+ public static final String PIPFILE = "Pipfile";
+ public static final String TXT_EXTENSION = ".txt";
+ public static final String SETUP_PY = "setup.py";
+ public static final String JAR_EXTENSION = ".jar";
+ public static final int MAX_EXTRACTION_DEPTH = 7;
+ public static final int COMMENT_MAX_LENGTH = 1000;
+ public static final int ZERO = 0;
+ public static final int ONE = 1;
+ public static final String BUILD_GRADLE = "build.gradle";
+ public static final String COPY_DEPENDENCIES = "copyDependencies";
+ public static final String UNDERSCORE = "_";
+ public static final char QUESTION_MARK = '?';
+ public static final char WHITESPACE_CHAR = ' ';
+ public static final char OPEN_BRACKET = '(';
+ public static final char CLOSE_BRACKET = ')';
+ public static final char EQUALS_CHAR = '=';
+ public static final char OPEN_SQUARE_BRACKET = '[';
+ public static final char CLOSE_SQUARE_BRACKET = ']';
+ public static final String DOUBLE_EQUALS = "==";
+ public static final char SEMI_COLON = ';';
+ public static final String DOLLAR = "$";
+ public static final String OPEN_CURLY_BRACKET = "{";
+ public static final String CLOSE_CURLY_BRACKET = "}";
+
+ public static final int MAX_NUMBER_OF_DEPENDENCIES = 1000000;
+
+ public static final String MAP_LOG_NAME = "org.whitesource";
+ public static final String MAP_APPENDER_NAME = "collectToMap";
+ public static final String HELP_ARG1 = "-help";
+ public static final String HELP_ARG2 = "-h";
+ public static final String TARGET = "target";
+ public static final String BUILD = "build";
+ public static final String NONE = "None";
+ public static final String LIBS = "libs";
+ public static final String USER_HOME = "user.home";
+}
\ No newline at end of file
diff --git a/src/main/java/org/whitesource/agent/DependencyCalculator.java b/src/main/java/org/whitesource/agent/DependencyCalculator.java
new file mode 100644
index 0000000..310242b
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/DependencyCalculator.java
@@ -0,0 +1,79 @@
+package org.whitesource.agent;
+
+import org.whitesource.agent.api.model.DependencyInfo;
+
+import java.io.File;
+import java.text.MessageFormat;
+import java.util.*;
+
+public class DependencyCalculator {
+
+ private static final List progressAnimation = Arrays.asList("|", "/", Constants.DASH, "\\");
+ private static final int ANIMATION_FRAMES = progressAnimation.size();
+ private final boolean showProgressBar;
+ private int animationIndex = 0;
+
+ public DependencyCalculator(boolean showProgressBar) {
+ this.showProgressBar = showProgressBar;
+ this.animationIndex = 0;
+ }
+
+ public Collection createDependencies(boolean scmConnector, int totalFiles, Map> fileMap,
+ Collection excludedCopyrights, boolean partialSha1Match) {
+ return createDependencies(scmConnector, totalFiles, fileMap, excludedCopyrights, partialSha1Match, false, false);
+ }
+
+ public Collection createDependencies(boolean scmConnector, int totalFiles, Map> fileMap,
+ Collection excludedCopyrights, boolean partialSha1Match, boolean calculateHints, boolean calculateMd5) {
+ List allDependencies = new ArrayList<>();
+ if (showProgressBar) {
+ displayProgress(0, totalFiles);
+ }
+
+ int index = 1;
+ for (Map.Entry> entry : fileMap.entrySet()) {
+ for (String fileName : entry.getValue()) {
+ DependencyInfoFactory factory = new DependencyInfoFactory(excludedCopyrights, partialSha1Match, calculateHints, calculateMd5);
+ DependencyInfo originalDependencyInfo = factory.createDependencyInfo(entry.getKey(), fileName);
+ if (originalDependencyInfo != null) {
+ if (scmConnector) {
+ originalDependencyInfo.setSystemPath(fileName.replace(Constants.BACK_SLASH, Constants.FORWARD_SLASH));
+ }
+ allDependencies.add(originalDependencyInfo);
+ }
+ if (showProgressBar) {
+ displayProgress(index, totalFiles);
+ }
+ index++;
+ }
+ }
+ return allDependencies;
+ }
+
+ private void displayProgress(int index, int totalFiles) {
+ StringBuilder sb = new StringBuilder("[INFO] ");
+
+ // draw each animation for 4 frames
+ int actualAnimationIndex = animationIndex % (ANIMATION_FRAMES * 4);
+ sb.append(progressAnimation.get((actualAnimationIndex / 4) % ANIMATION_FRAMES));
+ animationIndex++;
+
+ // draw progress bar
+ sb.append(" [");
+ double percentage = ((double) index / totalFiles) * 100;
+ int progressionBlocks = (int) (percentage / 3);
+ for (int i = 0; i < progressionBlocks; i++) {
+ sb.append(Constants.POUND);
+ }
+ for (int i = progressionBlocks; i < 33; i++) {
+ sb.append(Constants.WHITESPACE);
+ }
+ sb.append("] {0}% - {1} of {2} files\r");
+ System.out.print(MessageFormat.format(sb.toString(), (int) percentage, index, totalFiles));
+
+ if (index == totalFiles) {
+ // clear progress animation
+ System.out.print(" \r");
+ }
+ }
+}
diff --git a/src/main/java/org/whitesource/agent/DependencyInfoFactory.java b/src/main/java/org/whitesource/agent/DependencyInfoFactory.java
new file mode 100644
index 0000000..b44c4da
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/DependencyInfoFactory.java
@@ -0,0 +1,424 @@
+/**
+ * Copyright (C) 2014 WhiteSource Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.whitesource.agent;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.whitesource.agent.api.model.AgentProjectInfo;
+import org.whitesource.agent.api.model.Coordinates;
+import org.whitesource.agent.api.model.DependencyInfo;
+import org.whitesource.agent.api.model.DependencyType;
+import org.whitesource.agent.archive.ArchiveExtractor;
+import org.whitesource.agent.dependency.resolver.AbstractDependencyResolver;
+import org.whitesource.agent.dependency.resolver.DependencyResolutionService;
+import org.whitesource.agent.dependency.resolver.ResolutionResult;
+import org.whitesource.agent.utils.FilesUtils;
+import org.whitesource.agent.utils.LoggerFactory;
+import org.whitesource.agent.utils.MemoryUsageHelper;
+import org.whitesource.fs.FSAConfiguration;
+import org.whitesource.fs.FileSystemAgent;
+import org.whitesource.fs.Main;
+import org.whitesource.fs.StatusCode;
+import org.whitesource.fs.configuration.AgentConfiguration;
+import org.whitesource.fs.configuration.ResolverConfiguration;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.text.MessageFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * This class does the actual directory scanning, creates {@link DependencyInfo}s.
+ *
+ * @author tom.shapira
+ * @author anna.rozin
+ */
+public class FileSystemScanner {
+
+ /* --- Static members --- */
+
+ private final Logger logger = LoggerFactory.getLogger(FileSystemAgent.class);
+ private String FSA_FILE = "**/*whitesource-fs-agent-*.*jar";
+
+ /* --- Private Members --- */
+
+ private final boolean isSeparateProjects;
+ private final AgentConfiguration agent;
+ private final boolean showProgressBar;
+ private boolean enableImpactAnalysis;
+ private ViaLanguage iaLanguage;
+ private DependencyResolutionService dependencyResolutionService;
+ private String sha1;
+
+ /* --- Constructors --- */
+
+ public FileSystemScanner(ResolverConfiguration resolver, AgentConfiguration agentConfiguration, boolean enableImpactAnalysis) {
+ this.dependencyResolutionService = new DependencyResolutionService(resolver);
+ this.isSeparateProjects = dependencyResolutionService.isSeparateProjects();
+ this.agent = agentConfiguration;
+ this.showProgressBar = agentConfiguration.isShowProgressBar();
+ this.enableImpactAnalysis = enableImpactAnalysis;
+ }
+
+ public FileSystemScanner(ResolverConfiguration resolver, AgentConfiguration agentConfiguration, boolean enableImpactAnalysis, ViaLanguage iaLanguage) {
+ this(resolver, agentConfiguration, enableImpactAnalysis);
+ this.iaLanguage = iaLanguage;
+ }
+
+ /* --- Public methods --- */
+
+ /**
+ * This method is usually called from outside by different other tools
+ *
+ * @param scannerBaseDirs folders to scan
+ * @param scmConnector use scmConnector
+ * @param includes includes glob patterns
+ * @param excludes excludes glob patterns
+ * @param globCaseSensitive global case sensitive
+ * @param archiveExtractionDepth depth of recursive extraction
+ * @param archiveIncludes includes glob patterns for extraction
+ * @param archiveExcludes exclude glob patterns for extraction
+ * @param archiveFastUnpack use fast extraction
+ * @param followSymlinks use followSymlinks
+ * @param excludedCopyrights use excludedCopyrights
+ * @param partialSha1Match use partialSha1Match
+ * @return list of all the dependencies for project
+ */
+
+ @Deprecated
+ public List createProjects(List scannerBaseDirs, Map> appPathsToDependencyDirs, boolean scmConnector,
+ String[] includes, String[] excludes, boolean globCaseSensitive, int archiveExtractionDepth,
+ String[] archiveIncludes, String[] archiveExcludes, boolean archiveFastUnpack, boolean followSymlinks,
+ Collection excludedCopyrights, boolean partialSha1Match, String[] pythonRequirementsFileIncludes) {
+ Collection projects = createProjects(scannerBaseDirs, appPathsToDependencyDirs, scmConnector, includes, excludes, globCaseSensitive, archiveExtractionDepth,
+ archiveIncludes, archiveExcludes, archiveFastUnpack, followSymlinks, excludedCopyrights, partialSha1Match,
+ false, false, pythonRequirementsFileIncludes).keySet();
+ return projects.stream().flatMap(project -> project.getDependencies().stream()).collect(Collectors.toList());
+ }
+
+ @Deprecated
+ public List createProjects(List scannerBaseDirs, boolean scmConnector,
+ String[] includes, String[] excludes, boolean globCaseSensitive, int archiveExtractionDepth,
+ String[] archiveIncludes, String[] archiveExcludes, boolean archiveFastUnpack, boolean followSymlinks,
+ Collection excludedCopyrights, boolean partialSha1Match, String[] pythonRequirementsFileIncludes) {
+ return createProjects(scannerBaseDirs, convertListDirsToMap(scannerBaseDirs), scmConnector, includes, excludes, globCaseSensitive, archiveExtractionDepth,
+ archiveIncludes, archiveExcludes, archiveFastUnpack, followSymlinks, excludedCopyrights, partialSha1Match, pythonRequirementsFileIncludes);
+ }
+
+ @Deprecated
+ public Map> createProjects(List scannerBaseDirs, boolean hasScmConnector) {
+ return createProjects(scannerBaseDirs, convertListDirsToMap(scannerBaseDirs), hasScmConnector);
+ }
+
+ @Deprecated
+ public Map> createProjects(List scannerBaseDirs, Map> appPathsToDependencyDirs, boolean hasScmConnector) {
+ return createProjects(scannerBaseDirs, appPathsToDependencyDirs, hasScmConnector, agent.getIncludes(), agent.getExcludes(), agent.getGlobCaseSensitive(), agent.getArchiveExtractionDepth(),
+ agent.getArchiveIncludes(), agent.getArchiveExcludes(), agent.isArchiveFastUnpack(), agent.isFollowSymlinks(),
+ agent.getExcludedCopyrights(), agent.isPartialSha1Match(), agent.isCalculateHints(), agent.isCalculateMd5(), agent.getPythonRequirementsFileIncludes());
+ }
+
+ @Deprecated
+ public Map> createProjects(List scannerBaseDirs, boolean scmConnector,
+ String[] includes, String[] excludes, boolean globCaseSensitive, int archiveExtractionDepth,
+ String[] archiveIncludes, String[] archiveExcludes, boolean archiveFastUnpack, boolean followSymlinks,
+ Collection excludedCopyrights, boolean partialSha1Match, boolean calculateHints, boolean calculateMd5, String[] pythonRequirementsFileIncludes) {
+ return createProjects(scannerBaseDirs, convertListDirsToMap(scannerBaseDirs), scmConnector, includes, excludes, globCaseSensitive, archiveExtractionDepth, archiveIncludes, archiveExcludes,
+ archiveFastUnpack, followSymlinks, excludedCopyrights, partialSha1Match, calculateHints, calculateMd5, pythonRequirementsFileIncludes);
+ }
+
+ @Deprecated
+ public Map> createProjects(List scannerBaseDirs, Map> appPathsToDependencyDirs, boolean scmConnector,
+ String[] includes, String[] excludes, boolean globCaseSensitive, int archiveExtractionDepth,
+ String[] archiveIncludes, String[] archiveExcludes, boolean archiveFastUnpack, boolean followSymlinks,
+ Collection excludedCopyrights, boolean partialSha1Match, boolean calculateHints,
+ boolean calculateMd5, String[] pythonRequirementsFileIncludes) {
+ AgentConfiguration agentConfiguration = new AgentConfiguration(includes, excludes, new String[]{}, new String[]{}, archiveExtractionDepth, archiveIncludes, archiveExcludes, archiveFastUnpack,
+ followSymlinks, partialSha1Match, calculateHints, calculateMd5, showProgressBar, globCaseSensitive, false, excludedCopyrights, new String[]{}, new String[]{},
+ pythonRequirementsFileIncludes, Constants.EMPTY_STRING);
+ ProjectConfiguration projectConfiguration = new ProjectConfiguration(agentConfiguration, scannerBaseDirs, appPathsToDependencyDirs, scmConnector);
+ return createProjects(projectConfiguration);
+ }
+
+ public Map> createProjects(ProjectConfiguration projectConfiguration) {
+ MemoryUsageHelper.SystemStats systemStats = MemoryUsageHelper.getMemoryUsage();
+ logger.debug(systemStats.toString());
+
+ // get canonical paths
+ Set pathsToScan = getCanonicalPaths(projectConfiguration.getScannerBaseDirs());
+
+ for (String appPath : projectConfiguration.getAppPathsToDependencyDirs().keySet()) {
+ projectConfiguration.getAppPathsToDependencyDirs().put(appPath, getCanonicalPaths(projectConfiguration.getAppPathsToDependencyDirs().get(appPath)));
+ }
+
+ // todo: consider adding exit since this can be called from other components
+ //validateParams(archiveExtractionDepth, includes);
+
+ // scan directories
+ int totalFiles = 0;
+
+ String unpackDirectory;
+ boolean archiveExtraction = false;
+ // go over all base directories, look for archives
+ Map archiveToBaseDirMap = new HashMap<>();
+ List archiveDirectories = new ArrayList<>();
+ AgentConfiguration agentConfiguration = projectConfiguration.getAgentConfiguration();
+ if (agentConfiguration.getArchiveExtractionDepth() > 0) {
+ ArchiveExtractor archiveExtractor = new ArchiveExtractor(agentConfiguration.getArchiveIncludes(), agentConfiguration.getArchiveExcludes(),
+ agentConfiguration.getExcludes(), agentConfiguration.isArchiveFastUnpack());
+ logger.info("Starting Archive Extraction (may take a few minutes)");
+ for (String scannerBaseDir : new LinkedHashSet<>(pathsToScan)) {
+ unpackDirectory = archiveExtractor.extractArchives(scannerBaseDir, agentConfiguration.getArchiveExtractionDepth(), archiveDirectories);
+ if (unpackDirectory != null) {
+ archiveExtraction = true;
+ String parentFileUrl = new File(scannerBaseDir).getParent();
+ logger.debug("Unpack directory: {}, parent file: {}", unpackDirectory, parentFileUrl);
+ archiveToBaseDirMap.put(unpackDirectory, parentFileUrl);
+ pathsToScan.add(unpackDirectory);
+ if (!projectConfiguration.getAppPathsToDependencyDirs().containsKey(FSAConfiguration.DEFAULT_KEY)) {
+ projectConfiguration.getAppPathsToDependencyDirs().put(FSAConfiguration.DEFAULT_KEY, new HashSet<>());
+ }
+ projectConfiguration.getAppPathsToDependencyDirs().get(FSAConfiguration.DEFAULT_KEY).add(unpackDirectory);
+ }
+ }
+ }
+
+ // create dependencies from files - first project is always the default one
+ logger.info("Starting analysis");
+ // Create LinkedHashMap in order to save the order of the projects. In this way the first project will be always the main project.
+ Map allProjects = new LinkedHashMap<>();
+ Map> allProjectsToViaComponents = new LinkedHashMap<>();
+ AgentProjectInfo mainProject = new AgentProjectInfo();
+ allProjects.put(mainProject, null);
+ allProjectsToViaComponents.put(mainProject, new LinkedList<>());
+ String[] excludes = agentConfiguration.getExcludes();
+
+ logger.info("Scanning directories {} for matching Files (may take a few minutes)", pathsToScan);
+ logger.info("Included file types: {}", String.join(Constants.COMMA, agentConfiguration.getIncludes()));
+ logger.info("Excluded file types: {}", String.join(Constants.COMMA, agentConfiguration.getExcludes()));
+ String[] resolversIncludesPattern = createResolversIncludesPattern(dependencyResolutionService.getDependencyResolvers());
+
+ Map> fileMapBeforeResolve = new FilesUtils().fillFilesMap(pathsToScan, resolversIncludesPattern, agentConfiguration.getExcludes(),
+ agentConfiguration.isFollowSymlinks(), agentConfiguration.getGlobCaseSensitive());
+ Set allFiles = fileMapBeforeResolve.entrySet().stream().flatMap(folder -> folder.getValue().stream()).collect(Collectors.toSet());
+
+ final int[] totalDependencies = {0};
+ boolean isIgnoreSourceFiles = false;
+ if (enableImpactAnalysis && iaLanguage != null) {
+ for (String appPath : projectConfiguration.getAppPathsToDependencyDirs().keySet()) {
+ if (!appPath.equals(FSAConfiguration.DEFAULT_KEY)) {
+ if ((projectConfiguration.getAppPathsToDependencyDirs().get(appPath)).iterator().next() != null) {
+ String pojoAppPath = appPath;
+ allProjectsToViaComponents.get(allProjects.keySet().stream().findFirst().get()).add(new ViaComponents(pojoAppPath, iaLanguage));
+ }
+ }
+ }
+ // the 'allFiles' collection is derived from the manifest-files of each resolver -
+ // therefore no need to check again if the files in that collection match the manifest-files of each resolver
+ } else if (allFiles.size() > 0) {//(dependencyResolutionService != null && dependencyResolutionService.shouldResolveDependencies(allFiles)) {
+ logger.info("Attempting to resolve dependencies");
+ isIgnoreSourceFiles = dependencyResolutionService.isIgnoreSourceFiles();
+
+ // get all resolution results
+ Collection resolutionResults = new ArrayList<>();
+ for (String appPath : projectConfiguration.getAppPathsToDependencyDirs().keySet()) {
+ ViaComponents viaComponents = null;
+ ViaLanguage impactAnalysisLanguage = null;
+ Collection resolutionResult = new LinkedList<>();
+ LinkedList pathsList = new LinkedList<>();
+ pathsList.addAll(projectConfiguration.getAppPathsToDependencyDirs().get(appPath));
+ if ((appPath.equals(FSAConfiguration.DEFAULT_KEY) && projectConfiguration.getAppPathsToDependencyDirs().keySet().size() == 1) ||
+ (!appPath.equals(FSAConfiguration.DEFAULT_KEY) && projectConfiguration.getAppPathsToDependencyDirs().keySet().size() > 1)) {
+ resolutionResult = dependencyResolutionService.resolveDependencies(pathsList, agentConfiguration.getExcludes());
+ }
+ if (resolutionResult.size() == 1 && !appPath.equals(FSAConfiguration.DEFAULT_KEY)) {
+ DependencyType dependencyType = resolutionResult.stream().findFirst().get().getDependencyType();
+ if (dependencyType == null) {
+ break;
+ } else {
+ // validate scanned language and set the
+ switch (dependencyType) {
+ case NPM:
+ case BOWER:
+ impactAnalysisLanguage = ViaLanguage.JAVA_SCRIPT;
+ break;
+ case MAVEN:
+ case GRADLE:
+ impactAnalysisLanguage = ViaLanguage.JAVA;
+ break;
+ default:
+ if (enableImpactAnalysis) {
+ logger.error("Effective Usage Analysis will not run if the system cannot locate a valid dependency manager and the " +
+ "-iaLanguage parameter is not specified. In order to run Effective Usage Analysis without a dependency manager specify -iaLanguage java");
+ Main.exit(StatusCode.ERROR.getValue());
+ //// TODO: 8/28/2018 as a result of WSE-765 exit using function from main. function signature should be change to throw an exception
+ }
+ break;
+ }
+ }
+ } else if (resolutionResult.size() > 1 && enableImpactAnalysis) {
+ logger.info("Effective Usage Analysis will not run if an unsupported resolver is active. Verify that non-supported resolvers are not active");
+ Main.exit(StatusCode.ERROR.getValue());
+ }
+ if (impactAnalysisLanguage != null) {
+ viaComponents = new ViaComponents(appPath, impactAnalysisLanguage);
+ }
+ // TODO: Check why is result = null in the loop
+ resolutionResult.removeIf(Objects::isNull);
+ for (ResolutionResult result : resolutionResult) {
+ Map projects = result.getResolvedProjects();
+ Collection dependenciesToVia = new ArrayList<>();
+ for (Map.Entry project : projects.entrySet()) {
+ Collection dependencies = project.getKey().getDependencies();
+ dependenciesToVia.addAll(dependencies);
+ // do not add projects with no dependencies
+ if (!dependencies.isEmpty()) {
+ AgentProjectInfo currentProject;
+
+ // if it is single project threat it as the main
+ if ((((DependencyType.MAVEN.equals(result.getDependencyType()) && (!dependencyResolutionService.isMavenAggregateModules() || !dependencyResolutionService.isSbtAggregateModules())) ||
+ (DependencyType.GRADLE.equals(result.getDependencyType()) && !dependencyResolutionService.isGradleAggregateModules()) ||
+ (DependencyType.HEX.equals(result.getDependencyType()) && !dependencyResolutionService.isHexAggregateModules()))) &&
+ result.getResolvedProjects().size() > 1) {
+ allProjects.put(project.getKey(), project.getValue());
+ LinkedList listToNewProject = new LinkedList<>();
+ if (impactAnalysisLanguage != null) {
+ listToNewProject.add(viaComponents);
+ }
+ allProjectsToViaComponents.put(project.getKey(), listToNewProject);
+ } else {
+ currentProject = allProjects.keySet().stream().findFirst().get();
+ currentProject.getDependencies().addAll(project.getKey().getDependencies());
+ if (impactAnalysisLanguage != null) {
+ allProjectsToViaComponents.get(allProjects.keySet().stream().findFirst().get()).add(viaComponents);
+ }
+ }
+ impactAnalysisLanguage = null;
+ totalDependencies[0] += dependencies.size();
+ List usedSha1 = new LinkedList<>();
+ dependencies.forEach(dependency -> increaseCount(dependency, totalDependencies, usedSha1));
+ }
+ }
+ if (viaComponents != null) {
+ viaComponents.getDependencies().addAll(dependenciesToVia);
+ }
+ }
+ resolutionResults.addAll(resolutionResult);
+ }
+
+ resolutionResults.stream().forEach(resolutionResult -> logger.debug("total resolved projects = {}", resolutionResult.getResolvedProjects().size()));
+ logger.info(MessageFormat.format("Total dependencies found: {0}", totalDependencies[0]));
+
+ // merge additional excludes
+ Set allExcludes = resolutionResults.stream().flatMap(resolution -> resolution.getExcludes().stream()).collect(Collectors.toSet());
+ allExcludes.addAll(Arrays.stream(agentConfiguration.getExcludes()).collect(Collectors.toList()));
+
+ // change the original excludes with the merged values
+ excludes = new String[allExcludes.size()];
+ excludes = allExcludes.toArray(excludes);
+ dependencyResolutionService = null;
+ }
+
+ String[] excludesExtended = excludeFileSystemAgent(excludes);
+ logger.info("Scanning directories {} for matching Files (may take a few minutes)", pathsToScan);
+ Map> fileMap = new FilesUtils().fillFilesMap(pathsToScan, agentConfiguration.getIncludes(), excludesExtended,
+ agentConfiguration.isFollowSymlinks(), agentConfiguration.getGlobCaseSensitive());
+ long filesCount = fileMap.entrySet().stream().flatMap(folder -> folder.getValue().stream()).count();
+ totalFiles += filesCount;
+ logger.info(MessageFormat.format("Total files found according to the includes/excludes pattern: {0}", totalFiles));
+ DependencyCalculator dependencyCalculator = new DependencyCalculator(showProgressBar);
+ final Collection filesDependencies = new LinkedList<>();
+
+ if (!isIgnoreSourceFiles) {
+ filesDependencies.addAll(dependencyCalculator.createDependencies(
+ projectConfiguration.isScmConnector(), totalFiles, fileMap, agentConfiguration.getExcludedCopyrights(),
+ agentConfiguration.isPartialSha1Match(), agentConfiguration.isCalculateHints(),
+ agentConfiguration.isCalculateMd5()));
+ }
+
+ if (allProjects.size() == 1) {
+ AgentProjectInfo project = allProjects.keySet().stream().findFirst().get();
+ project.getDependencies().addAll(filesDependencies);
+ /// TODO: 8/14/2018 support multi module project with via
+ /* if (enableImpactAnalysis) {
+ for (LinkedList viaComponentsList : allProjectsToViaComponents.values()) {
+ for (ViaComponents viaComponents : viaComponentsList) {
+ for (DependencyInfo dependencyInfo : filesDependencies) {
+ if (dependencyInfo.getSystemPath().equals(viaComponents.getAppPath())) {
+ viaComponents.getDependencies().add(dependencyInfo);
+ }
+ }
+ }
+ }
+ }*/
+ } else {
+ // Sort the projects by length of paths (from the longest to the shortest) in order to add filesDependencies to the most appropriate project
+ // Example: project1 path: C:\Users\file\Data; project2 path: C:\Users\file\Data\folder; file dependency path: C:\Users\file\Data\folder\a.jar
+ // Before sorting, the file dependency will be in project1. After sorting, the file dependency will be in project2.
+ List> entriesList = new ArrayList<>();
+ allProjects.entrySet().forEach(entry -> {
+ if (entry.getValue() != null) {
+ entriesList.add(entry);
+ }
+ });
+ entriesList.sort(Map.Entry.comparingByValue());
+ Collections.reverse(entriesList);
+ Map result = new LinkedHashMap<>();
+ entriesList.forEach(entry -> result.put(entry.getKey(), entry.getValue()));
+
+ // remove files from handled projects
+ result.entrySet().forEach(project -> {
+ Collection projectDependencies = filesDependencies.stream()
+ .filter(dependencyInfo -> project.getValue() != null && dependencyInfo.getSystemPath().contains(project.getValue().toString())).collect(Collectors.toList());
+ project.getKey().getDependencies().addAll(projectDependencies);
+ filesDependencies.removeAll(projectDependencies);
+ });
+
+ // create new projects if necessary
+ if (!isIgnoreSourceFiles && filesDependencies.size() > 0) {
+ projectConfiguration.getScannerBaseDirs().stream().forEach(directory -> {
+ List subDirectories;
+ // check all folders
+
+ String[] includesAll = {Constants.PATTERN};
+ subDirectories = new FilesUtils().getSubDirectories(directory, includesAll, null, agentConfiguration.isFollowSymlinks(),
+ agentConfiguration.getGlobCaseSensitive());
+ subDirectories.forEach(subFolder -> {
+ if (filesDependencies.size() > 0) {
+ List projectDependencies = filesDependencies.stream().
+ filter(dependencyInfo -> dependencyInfo.getSystemPath().contains(subFolder.toString())).collect(Collectors.toList());
+ if (!projectDependencies.isEmpty()) {
+ AgentProjectInfo subProject;
+ if (isSeparateProjects) {
+ subProject = new AgentProjectInfo();
+ allProjects.put(subProject, null);
+ allProjectsToViaComponents.put(subProject, new LinkedList<>());
+ subProject.setCoordinates(new Coordinates(null, subFolder.toFile().getName(), null));
+ } else {
+ subProject = allProjects.entrySet().stream().findFirst().get().getKey();
+ }
+ subProject.getDependencies().addAll(filesDependencies);
+ filesDependencies.removeAll(projectDependencies);
+ }
+ }
+ });
+ });
+ // Add the rest of the files dependencies to the main project
+ if (!filesDependencies.isEmpty()) {
+ AgentProjectInfo subProject = allProjects.entrySet().stream().findFirst().get().getKey();
+ subProject.getDependencies().addAll(filesDependencies);
+ }
+ }
+ }
+
+ for (AgentProjectInfo innerProject : allProjects.keySet()) {
+ // replace temp folder name with base dir
+ for (DependencyInfo dependencyInfo : innerProject.getDependencies()) {
+ String systemPath = dependencyInfo.getSystemPath();
+ if (systemPath == null) {
+ logger.debug("Dependency {} has no system path", dependencyInfo.getArtifactId());
+ } else {
+ for (String key : archiveToBaseDirMap.keySet()) {
+ if (systemPath.contains(key) && archiveExtraction) {
+ String newSystemPath = systemPath.replace(key, archiveToBaseDirMap.get(key)).replaceAll(ArchiveExtractor.DEPTH_REGEX, Constants.EMPTY_STRING);
+ logger.debug("Original system path: {}, new system path: {}, key: {}", systemPath, newSystemPath, key);
+ dependencyInfo.setSystemPath(newSystemPath);
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ // delete all archive temp folders
+ if (!archiveDirectories.isEmpty()) {
+ for (String archiveDirectory : archiveDirectories) {
+ File directory = new File(archiveDirectory);
+ if (directory.exists()) {
+ FileUtils.deleteQuietly(directory);
+ }
+ }
+ }
+ logger.info("Finished analyzing Files");
+ systemStats = MemoryUsageHelper.getMemoryUsage();
+ logger.debug(systemStats.toString());
+ // add dependencies to project in case of pojo project
+ if (enableImpactAnalysis && iaLanguage != null) {
+ AgentProjectInfo agentProjectInfo = allProjectsToViaComponents.keySet().iterator().next();
+ allProjectsToViaComponents.get(agentProjectInfo).getFirst().getDependencies().addAll(
+ agentProjectInfo.getDependencies());
+ }
+ return allProjectsToViaComponents;
+ }
+
+ /* --- Private methods --- */
+
+ private String[] createResolversIncludesPattern(Collection dependencyResolvers) {
+ Collection resultIncludes = new ArrayList<>();
+ // TODO - check if can be done with lambda
+ for (AbstractDependencyResolver dependencyResolver : dependencyResolvers) {
+ for (String manifestFile : dependencyResolver.getManifestFiles()){
+ if (!manifestFile.isEmpty()) {
+ resultIncludes.add(Constants.PATTERN + manifestFile);
+ }
+ }
+ }
+ String[] resultArray = new String[resultIncludes.size()];
+ resultIncludes.toArray(resultArray);
+ return resultArray;
+ }
+
+ private Map> convertListDirsToMap(List scannerBaseDirs) {
+ Map> appPathsToDependencyDirs = new HashMap<>();
+ appPathsToDependencyDirs.put(FSAConfiguration.DEFAULT_KEY, new HashSet<>());
+ for (String dir : scannerBaseDirs) {
+ appPathsToDependencyDirs.get(FSAConfiguration.DEFAULT_KEY).add(dir);
+ }
+ return appPathsToDependencyDirs;
+ }
+
+ private Set getCanonicalPaths(Collection scannerBaseDirs) {
+ // use canonical paths to resolve '.' in path
+ Set pathsToScan = new HashSet<>();
+ for (String path : scannerBaseDirs) {
+ try {
+ pathsToScan.add(new File(path).getCanonicalPath());
+ } catch (IOException e) {
+ // use the given path as-is
+ logger.debug("Error finding the canonical path of {}", path);
+ pathsToScan.add(path);
+ }
+ }
+ return pathsToScan;
+ }
+
+ private void increaseCount(DependencyInfo dependency, int[] totalDependencies, List usedSha1) {
+ sha1 = dependency.getSha1();
+ if (usedSha1.contains(sha1)) {
+ return;
+ }
+ usedSha1.add(sha1);
+ totalDependencies[0] += dependency.getChildren().size();
+ dependency.getChildren().forEach(dependencyInfo -> increaseCount(dependencyInfo, totalDependencies, usedSha1));
+ }
+
+ private String[] excludeFileSystemAgent(String[] excludes) {
+ String[] allExcludes = excludes == null ? new String[0] : excludes;
+ String[] excludesFSA = new String[allExcludes.length + 1];
+ System.arraycopy(allExcludes, 0, excludesFSA, 0, allExcludes.length);
+ excludesFSA[allExcludes.length] = FSA_FILE;
+ return excludesFSA;
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/org/whitesource/agent/ProjectConfiguration.java b/src/main/java/org/whitesource/agent/ProjectConfiguration.java
new file mode 100644
index 0000000..339dabc
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/ProjectConfiguration.java
@@ -0,0 +1,69 @@
+package org.whitesource.agent;
+
+import org.whitesource.fs.configuration.AgentConfiguration;
+
+import java.util.*;
+
+/**
+ * @author chen.luigi
+ */
+public class ProjectConfiguration {
+
+ /* --- Private Members --- */
+
+ private AgentConfiguration agentConfiguration;
+ private List scannerBaseDirs;
+ private Map> appPathsToDependencyDirs;
+ private boolean scmConnector;
+
+
+ /* --- Constructors --- */
+
+ public ProjectConfiguration(AgentConfiguration agentConfiguration) {
+ this.agentConfiguration = agentConfiguration;
+ scannerBaseDirs = new LinkedList<>();
+ appPathsToDependencyDirs = new HashMap<>();
+ this.scmConnector = false;
+ }
+
+ public ProjectConfiguration(AgentConfiguration agentConfiguration, List scannerBaseDirs, Map> appPathsToDependencyDirs, boolean scmConnector) {
+ this.agentConfiguration = agentConfiguration;
+ this.scannerBaseDirs = scannerBaseDirs;
+ this.appPathsToDependencyDirs = appPathsToDependencyDirs;
+ this.scmConnector = scmConnector;
+ }
+
+ /* --- Getters / Setters --- */
+
+ public AgentConfiguration getAgentConfiguration() {
+ return agentConfiguration;
+ }
+
+ public void setAgentConfiguration(AgentConfiguration agentConfiguration) {
+ this.agentConfiguration = agentConfiguration;
+ }
+
+ public List getScannerBaseDirs() {
+ return scannerBaseDirs;
+ }
+
+ public void setScannerBaseDirs(List scannerBaseDirs) {
+ this.scannerBaseDirs = scannerBaseDirs;
+ }
+
+ public Map> getAppPathsToDependencyDirs() {
+ return appPathsToDependencyDirs;
+ }
+
+ public void setAppPathsToDependencyDirs(Map> appPathsToDependencyDirs) {
+ this.appPathsToDependencyDirs = appPathsToDependencyDirs;
+ }
+
+ public boolean isScmConnector() {
+ return scmConnector;
+ }
+
+ public void setScmConnector(boolean scmConnector) {
+ this.scmConnector = scmConnector;
+ }
+}
diff --git a/src/main/java/org/whitesource/agent/ProjectsSender.java b/src/main/java/org/whitesource/agent/ProjectsSender.java
new file mode 100644
index 0000000..b215f8b
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/ProjectsSender.java
@@ -0,0 +1,479 @@
+/**
+ * Copyright (C) 2014 WhiteSource Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.whitesource.agent;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import com.google.common.collect.Lists;
+import com.google.gson.Gson;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.whitesource.agent.api.dispatch.*;
+import org.whitesource.agent.api.model.AgentProjectInfo;
+import org.whitesource.agent.api.model.Coordinates;
+import org.whitesource.agent.api.model.DependencyInfo;
+import org.whitesource.agent.client.WhitesourceService;
+import org.whitesource.agent.client.WssServiceException;
+import org.whitesource.agent.report.OfflineUpdateRequest;
+import org.whitesource.agent.report.PolicyCheckReport;
+import org.whitesource.agent.utils.LoggerFactory;
+import org.whitesource.agent.utils.Pair;
+import org.whitesource.contracts.PluginInfo;
+import org.whitesource.fs.LogMapAppender;
+import org.whitesource.fs.Main;
+import org.whitesource.fs.ProjectsDetails;
+import org.whitesource.fs.StatusCode;
+import org.whitesource.fs.configuration.OfflineConfiguration;
+import org.whitesource.fs.configuration.RequestConfiguration;
+import org.whitesource.fs.configuration.SenderConfiguration;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.stream.Collectors;
+
+/**
+ * Class for sending projects for all WhiteSource command line agents.
+ *
+ * @author Itai Marko
+ * @author tom.shapira
+ * @author anna.rozin
+ */
+public class ProjectsSender {
+ /* --- Static members --- */
+ private static final String DATE_FORMAT = "HH:mm:ss";
+ public static final String PROJECT_URL_PREFIX = "Wss/WSS.html#!project;id=";
+ protected static final int MAX_LOG_EVENTS = 1000;
+ /* --- Members --- */
+ private final Logger logger = LoggerFactory.getLogger(ProjectsSender.class);
+ private final SenderConfiguration senderConfig;
+ private final OfflineConfiguration offlineConfig;
+ private final RequestConfiguration requestConfig;
+ private final PluginInfo pluginInfo;
+ protected StatusCode prepStepStatusCode = StatusCode.SUCCESS;
+
+ /* --- Constructors --- */
+
+ public ProjectsSender(SenderConfiguration senderConfig, OfflineConfiguration offlineConfig, RequestConfiguration requestConfig, PluginInfo pluginInfo) {
+ this.senderConfig = senderConfig;
+ this.offlineConfig = offlineConfig;
+ this.requestConfig = requestConfig;
+ this.pluginInfo = pluginInfo;
+ }
+
+ /* --- Public methods --- */
+
+ public Pair sendRequest(ProjectsDetails projectsDetails) {
+ // send request
+ logger.info("Initializing WhiteSource Client");
+ Collection projects = projectsDetails.getProjects();
+
+ if (checkDependenciesUpbound(projects)) {
+ return new Pair<>("Number of dependencies exceeded the maximum supported", StatusCode.SERVER_FAILURE);
+ }
+
+ WhitesourceService service = createService();
+ String resultInfo = Constants.EMPTY_STRING;
+ if (offlineConfig.isOffline()) {
+ resultInfo = offlineUpdate(service, projects);
+ return new Pair<>(resultInfo, this.prepStepStatusCode);
+ } else {
+ // update type
+ UpdateType updateType = UpdateType.OVERRIDE;
+ String updateTypeValue = senderConfig.getUpdateTypeValue();
+ try {
+ updateType = UpdateType.valueOf(updateTypeValue);
+ } catch (Exception e) {
+ logger.info("Invalid value {} for updateType, defaulting to {}", updateTypeValue, UpdateType.OVERRIDE);
+ }
+ logger.info("UpdateType set to {} ", updateTypeValue);
+
+ StatusCode statusCode = StatusCode.SUCCESS;
+ if (senderConfig.isEnableImpactAnalysis()) {
+ runViaAnalysis(projectsDetails, service);
+ } else if (!senderConfig.isEnableImpactAnalysis()) {
+ //todo return logs when needed would be enabled for all WSE-342
+ }
+ int retries = senderConfig.getConnectionRetries();
+ while (retries-- > -1) {
+ try {
+ statusCode = checkPolicies(service, projects);
+ if (senderConfig.isUpdateInventory()) {
+ if (statusCode == StatusCode.SUCCESS || (senderConfig.isForceUpdate() && senderConfig.isForceUpdateFailBuildOnPolicyViolation())) {
+ resultInfo = update(service, projects);
+ }
+ }
+ break;
+ } catch (WssServiceException e) {
+ if (e.getCause() != null &&
+ e.getCause().getClass().getCanonicalName().substring(0,
+ e.getCause().getClass().getCanonicalName().lastIndexOf(Constants.DOT)).equals(Constants.JAVA_NETWORKING)) {
+ statusCode = StatusCode.CONNECTION_FAILURE;
+ logger.error("Trying " + (retries + 1) + " more time" + (retries != 0 ? "s" : Constants.EMPTY_STRING));
+ } else {
+ statusCode = StatusCode.SERVER_FAILURE;
+ retries = -1;
+ }
+ resultInfo = "Failed to send request to WhiteSource server: " + e.getMessage();
+ logger.error(resultInfo, e.getMessage());
+ logger.debug(resultInfo, e);
+ if (retries > -1) {
+ try {
+ Thread.sleep(senderConfig.getConnectionRetriesIntervals());
+ } catch (InterruptedException e1) {
+ logger.error("Failed to sleep while retrying to connect to server " + e1.getMessage(), e1);
+ }
+ }
+ String requestToken = e.getRequestToken();
+ if (StringUtils.isNotBlank(requestToken)) {
+ resultInfo += Constants.NEW_LINE + "Support token: " + requestToken;
+ logger.info("Support token: {}", requestToken);
+ }
+ }
+ }
+ if (service != null) {
+ service.shutdown();
+ }
+ if (statusCode == StatusCode.SUCCESS) {
+ return new Pair<>(resultInfo, this.prepStepStatusCode);
+ }
+ return new Pair<>(resultInfo, statusCode);
+ }
+ }
+
+ private void runViaAnalysis(ProjectsDetails projectsDetails, WhitesourceService service) {
+ try {
+ Class vulnerabilitiesAnalysisClass = Class.forName("whitesource.analysis.vulnerabilities.VulnerabilitiesAnalysis");
+ Method getAnalysisMethod
+ = vulnerabilitiesAnalysisClass.getMethod("getAnalysis", String.class, int.class);
+ Object vulnerabilitiesAnalysis = null;
+ for (AgentProjectInfo project : projectsDetails.getProjectToViaComponents().keySet()) {
+ // check language for scan according to user file
+ LinkedList viaComponentsList = projectsDetails.getProjectToViaComponents().get(project);
+ if (viaComponentsList.size() == 0) {
+ logger.error("Effective Usage Analysis will not run if 0 dependencies are found. Check that the -d parameter specifies a valid project path");
+ Main.exit(StatusCode.ERROR.getValue());
+ //// TODO: 8/28/2018 as a result of WSE-765 exit using function from main. function signature should be change to throw an exception
+ }
+ for (ViaComponents viaComponents : viaComponentsList) {
+ logger.info("Starting VIA impact analysis");
+ checkDependenciesSha1(viaComponents);
+ String appPath = viaComponents.getAppPath();
+ ViaLanguage language = viaComponents.getLanguage();
+ try {
+ vulnerabilitiesAnalysis = getAnalysisMethod.invoke(null, language.toString(), requestConfig.getViaAnalysisLevel());
+ // set app path for java script
+ if (language.equals(ViaLanguage.JAVA_SCRIPT)) {
+ int lastIndex = appPath.lastIndexOf(Constants.BACK_SLASH) != -1 ? appPath.lastIndexOf(Constants.BACK_SLASH) : appPath.lastIndexOf(Constants.FORWARD_SLASH);
+ appPath = appPath.substring(0, lastIndex);
+ }
+ if (vulnerabilitiesAnalysis != null) {
+ AgentProjectInfo projectToServer = new AgentProjectInfo();
+ projectToServer.setDependencies(Lists.newArrayList(project.getDependencies()));
+ projectToServer.setProjectSetupDescription(project.getProjectSetupDescription());
+ projectToServer.setCoordinates(project.getCoordinates());
+ projectToServer.setProjectToken(project.getProjectToken());
+ projectToServer.setProjectSetupStatus(project.getProjectSetupStatus());
+ projectToServer.setParentCoordinates(project.getParentCoordinates());
+ Class fsaAgentServerClass = Class.forName("whitesource.analysis.server.FSAgentServer");
+ Object server = fsaAgentServerClass.getConstructor(AgentProjectInfo.class, WhitesourceService.class, String.class, String.class).newInstance(
+ projectToServer, service, requestConfig.getApiToken(), requestConfig.getUserKey());
+ logger.info("Starting analysis for: {}", appPath);
+ Class serverClass = Class.forName("whitesource.analysis.server.Server");
+ Method runAnalysis = vulnerabilitiesAnalysisClass.getDeclaredMethod("runAnalysis", serverClass, String.class, Collection.class, Boolean.class);
+ runAnalysis.invoke(vulnerabilitiesAnalysis, server, appPath, project.getDependencies(), Boolean.valueOf(requestConfig.getViaDebug()));
+ logger.info("Got impact analysis result from server");
+ }
+ } catch (InvocationTargetException e) {
+ logger.error("Failed to run VIA impact analysis {}", e.getTargetException().getMessage());
+ } catch (Exception e) {
+ logger.error("Failed to run VIA impact analysis {}", e.getMessage());
+ }
+ }
+ }
+ } catch (NoSuchMethodException e) {
+ logger.error("Failed to run VIA impact analysis, couldn't find method {}", e.getMessage());
+ } catch (ClassNotFoundException e) {
+ logger.error("Failed to run VIA impact analysis, couldn't find class {}", e.getMessage());
+ }
+ }
+
+ private void checkDependenciesSha1(ViaComponents viaComponents) {
+ Collection dependencyWithoutSha1 = new LinkedList<>();
+ int unknownDependencyCounter = 0;
+ for (DependencyInfo dependencyInfo : viaComponents.getDependencies()) {
+ if (StringUtils.isEmpty(dependencyInfo.getSha1())) {
+ unknownDependencyCounter++;
+ dependencyWithoutSha1.add(dependencyInfo);
+ }
+ }
+ if (unknownDependencyCounter > 0) {
+ if (requestConfig.isRequireKnownSha1()) {
+ logger.warn("The system found {} with an unknown SHA1 value. Default processing was terminated.", unknownDependencyCounter);
+ printDependenciesWithoutSha1(dependencyWithoutSha1);
+ Main.exit(StatusCode.ERROR.getValue());
+ } else {
+ logger.warn("The system found {} with an unknown SHA1 value. Default processing termination was overridden by parameter.", unknownDependencyCounter);
+ printDependenciesWithoutSha1(dependencyWithoutSha1);
+ }
+ }
+ }
+
+ private void printDependenciesWithoutSha1(Collection dependencyWithoutSha1) {
+ logger.warn("Found dependencies with an unknown SHA1 value:");
+ for (DependencyInfo dependencyInfo : dependencyWithoutSha1) {
+ logger.warn(new Coordinates(dependencyInfo.getGroupId(), dependencyInfo.getArtifactId(), dependencyInfo.getVersion()).toString());
+ }
+ }
+
+ private boolean checkDependenciesUpbound(Collection projects) {
+ int numberOfDependencies = projects.stream().map(x -> x.getDependencies()).mapToInt(x -> x.size()).sum();
+ if (numberOfDependencies > Constants.MAX_NUMBER_OF_DEPENDENCIES) {
+ logger.warn("Number of dependencies: {} exceeded the maximum supported: {}", numberOfDependencies, Constants.MAX_NUMBER_OF_DEPENDENCIES);
+ return true;
+ }
+ return false;
+ }
+
+ protected WhitesourceService createService() {
+ logger.info("Service URL is " + senderConfig.getServiceUrl());
+ boolean setProxy = false;
+ if (StringUtils.isNotBlank(senderConfig.getProxyHost()) || !offlineConfig.isOffline()) {
+ setProxy = true;
+ }
+ int connectionTimeoutMinutes = senderConfig.getConnectionTimeOut();
+ final WhitesourceService service = new WhitesourceService(pluginInfo.getAgentType(), pluginInfo.getAgentVersion(), pluginInfo.getPluginVersion(),
+ senderConfig.getServiceUrl(), setProxy, connectionTimeoutMinutes, senderConfig.isIgnoreCertificateCheck());
+ if (StringUtils.isNotBlank(senderConfig.getProxyHost())) {
+ service.getClient().setProxy(senderConfig.getProxyHost(), senderConfig.getProxyPort(), senderConfig.getProxyUser(), senderConfig.getProxyPassword());
+ }
+ return service;
+ }
+
+ private StatusCode checkPolicies(WhitesourceService service, Collection projects) throws WssServiceException {
+ boolean policyCompliance = true;
+ if (senderConfig.isCheckPolicies() || !senderConfig.isUpdateInventory()) {
+ logger.info("Checking policies");
+ CheckPolicyComplianceResult checkPoliciesResult;
+ if (senderConfig.isSendLogsToWss()) {
+ String logData = getLogData();
+ checkPoliciesResult = service.checkPolicyCompliance(requestConfig.getApiToken(), requestConfig.getProductName(),
+ requestConfig.getProductVersion(), projects, senderConfig.isForceCheckAllDependencies(), requestConfig.getUserKey(),
+ requestConfig.getRequesterEmail(), logData, requestConfig.getProductToken());
+ } else {
+ checkPoliciesResult = service.checkPolicyCompliance(requestConfig.getApiToken(), requestConfig.getProductName(),
+ requestConfig.getProductVersion(), projects, senderConfig.isForceCheckAllDependencies(), requestConfig.getUserKey(),
+ requestConfig.getRequesterEmail(), null, requestConfig.getProductToken());
+ }
+ if (checkPoliciesResult.hasRejections()) {
+ if (senderConfig.isForceUpdate() && senderConfig.isUpdateInventory()) {
+ logger.info("Some dependencies violate open source policies, however all were force " +
+ "updated to organization inventory.");
+ if (senderConfig.isForceUpdateFailBuildOnPolicyViolation()) {
+ policyCompliance = false;
+ }
+ } else if (!senderConfig.isUpdateInventory()) {
+ logger.info("Some dependencies did not conform with open source policies, review report for details");
+ policyCompliance = false;
+ } else {
+ logger.info("Some dependencies did not conform with open source policies, review report for details");
+ logger.info("=== UPDATE ABORTED ===");
+ policyCompliance = false;
+ }
+ } else {
+ logger.info("All dependencies conform with open source policies.");
+ }
+ String requestToken = checkPoliciesResult.getRequestToken();
+ if (StringUtils.isNotBlank(requestToken)) {
+ logger.info("Check Policies Support Token: {}", requestToken);
+ }
+ try {
+ // generate report
+ PolicyCheckReport report = new PolicyCheckReport(checkPoliciesResult);
+ File outputDir = new File(offlineConfig.getWhiteSourceFolderPath());
+ report.generate(outputDir, false);
+ report.generateJson(outputDir);
+ logger.info("Policies report generated successfully");
+ } catch (IOException e) {
+ logger.error("Error generating check policies report: " + e.getMessage(), e);
+ }
+ }
+ return policyCompliance ? StatusCode.SUCCESS : StatusCode.POLICY_VIOLATION;
+ }
+
+ protected String update(WhitesourceService service, Collection projects) throws WssServiceException {
+ logger.info("Sending Update");
+ UpdateInventoryResult updateResult;
+ //--------------------------------
+ if (requestConfig.getViaDebug().equals("SAVE") || Boolean.valueOf(requestConfig.getViaDebug())) {
+ saveRequestToFile(projects);
+ }
+ //--------------------------------
+ if (senderConfig.isSendLogsToWss()) {
+ String logData = getLogData();
+ updateResult = service.update(requestConfig.getApiToken(), requestConfig.getRequesterEmail(), UpdateType.valueOf(senderConfig.getUpdateTypeValue()),
+ requestConfig.getProductName(), requestConfig.getProductVersion(), projects, requestConfig.getUserKey(),
+ logData, requestConfig.getScanComment(), requestConfig.getProductToken());
+
+ } else {
+ updateResult = service.update(requestConfig.getApiToken(), requestConfig.getRequesterEmail(), UpdateType.valueOf(senderConfig.getUpdateTypeValue()),
+ requestConfig.getProductName(), requestConfig.getProductVersion(), projects, requestConfig.getUserKey(), null,
+ requestConfig.getScanComment(), requestConfig.getProductToken());
+ }
+ String resultInfo = logResult(updateResult);
+ // remove line separators
+ resultInfo = resultInfo.replace(System.lineSeparator(), Constants.EMPTY_STRING);
+ return resultInfo;
+ }
+
+ private void saveRequestToFile(Collection projects) {
+ String fileName = "jsonOut" + Constants.DASH + requestConfig.getProductName() + Constants.DASH +
+ requestConfig.getProjectName() + ".json";
+ RequestFactory requestFactory = new RequestFactory(pluginInfo.getAgentType(), pluginInfo.getAgentVersion(), pluginInfo.getPluginVersion());
+ String updateJson = new Gson().toJson(requestFactory.newUpdateInventoryRequest(requestConfig.getApiToken(),
+ UpdateType.valueOf(senderConfig.getUpdateTypeValue()), requestConfig.getRequesterEmail(),
+ requestConfig.getProductName(), requestConfig.getProductVersion(), projects,
+ requestConfig.getUserKey(), (String) null, (String) null, (String) requestConfig.getProductToken()));
+ Path path = Paths.get(fileName);
+ try (BufferedWriter writer = Files.newBufferedWriter(path)) {
+ writer.write(updateJson);
+ } catch (Exception e) {
+ logger.debug("couldn't create via debug file {}", e.getMessage());
+ }
+ }
+
+ private String offlineUpdate(WhitesourceService service, Collection projects) {
+ String resultInfo = Constants.EMPTY_STRING;
+ logger.info("Generating offline update request");
+ // generate offline request
+ UpdateInventoryRequest updateRequest = service.offlineUpdate(new UpdateInventoryRequest(requestConfig.getApiToken(), requestConfig.getProductName(),
+ requestConfig.getProductVersion(), projects, requestConfig.getUserKey(), requestConfig.getScanComment()));
+ if (senderConfig.isSendLogsToWss()) {
+ updateRequest.setLogData(getLogData());
+ }
+ updateRequest.setRequesterEmail(requestConfig.getRequesterEmail());
+ if (requestConfig.getProductToken() != null && !requestConfig.getProductToken().isEmpty()) {
+ updateRequest.setProductToken(requestConfig.getProductToken());
+ }
+ try {
+ OfflineUpdateRequest offlineUpdateRequest = new OfflineUpdateRequest(updateRequest);
+ UpdateType updateTypeFinal;
+ // if the update type was forced by command or config -> set it
+ if (StringUtils.isNotBlank(senderConfig.getUpdateTypeValue())) {
+ try {
+ updateTypeFinal = UpdateType.valueOf(senderConfig.getUpdateTypeValue());
+ } catch (Exception e) {
+ logger.info("Invalid value {} for updateType, defaulting to {}", senderConfig.getUpdateTypeValue(), UpdateType.OVERRIDE);
+ updateTypeFinal = UpdateType.OVERRIDE;
+ }
+ } else {
+ // Otherwise use the parameter in the file
+ updateTypeFinal = updateRequest.getUpdateType();
+ }
+ logger.info("UpdateType offline set to {} ", updateTypeFinal);
+ updateRequest.setUpdateType(updateTypeFinal);
+ File outputDir = new File(offlineConfig.getWhiteSourceFolderPath()).getAbsoluteFile();
+ if (!outputDir.exists() && !outputDir.mkdir()) {
+ throw new IOException("Unable to make output directory: " + outputDir);
+ }
+ File file = offlineUpdateRequest.generate(outputDir, offlineConfig.isZip(), offlineConfig.isPrettyJson());
+ resultInfo = "Offline request generated successfully at " + file.getPath();
+ logger.info(resultInfo);
+ } catch (IOException e) {
+ resultInfo = "Error generating offline update request: " + e.getMessage();
+ logger.error(resultInfo);
+ } finally {
+ if (service != null) {
+ service.shutdown();
+ }
+ }
+ return resultInfo;
+ }
+
+ private String logResult(UpdateInventoryResult updateResult) {
+ StringBuilder resultLogMsg = new StringBuilder("Inventory update results for ").append(updateResult.getOrganization()).append(Constants.NEW_LINE);
+ logger.info("Inventory update results for {}", updateResult.getOrganization());
+ // newly created projects
+ Collection createdProjects = updateResult.getCreatedProjects();
+ if (createdProjects.isEmpty()) {
+ logger.info("No new projects found.");
+ resultLogMsg.append("No new projects found.").append(Constants.NEW_LINE);
+ } else {
+ logger.info("Newly created projects:");
+ resultLogMsg.append("Newly created projects:").append(Constants.NEW_LINE);
+ for (String projectName : createdProjects) {
+ logger.info("# {}", projectName);
+ resultLogMsg.append(projectName).append(Constants.NEW_LINE);
+ }
+ }
+ // updated projects
+ Collection updatedProjects = updateResult.getUpdatedProjects();
+ if (updatedProjects.isEmpty()) {
+ logger.info("No projects were updated.");
+ resultLogMsg.append("No projects were updated.").append(Constants.NEW_LINE);
+ } else {
+ logger.info("Updated projects:");
+ resultLogMsg.append("Updated projects:").append(Constants.NEW_LINE);
+ for (String projectName : updatedProjects) {
+ logger.info("# {}", projectName);
+ resultLogMsg.append(projectName).append(Constants.NEW_LINE);
+ }
+ }
+ // reading projects' URLs
+ HashMap projectsUrls = updateResult.getProjectNamesToIds();
+ if (projectsUrls != null && !projectsUrls.isEmpty()) {
+ for (String projectName : projectsUrls.keySet()) {
+ String appUrl = senderConfig.getServiceUrl().replace("agent", Constants.EMPTY_STRING);
+ String projectsUrl = appUrl + PROJECT_URL_PREFIX + projectsUrls.get(projectName);
+ logger.info("Project name: {}, URL: {}", projectName, projectsUrl);
+ resultLogMsg.append(Constants.NEW_LINE).append("Project name: ").append(projectName).append(", project URL:").append(projectsUrl);
+ }
+ }
+ // support token
+ String requestToken = updateResult.getRequestToken();
+ if (StringUtils.isNotBlank(requestToken)) {
+ logger.info("Support Token: {}", requestToken);
+ resultLogMsg.append(Constants.NEW_LINE).append("Support Token: ").append(requestToken);
+ }
+ return resultLogMsg.toString();
+ }
+
+ private String getLogData() {
+ String logs = Constants.EMPTY_STRING;
+ ch.qos.logback.classic.Logger setLog = (ch.qos.logback.classic.Logger) org.slf4j.LoggerFactory.getLogger(Constants.MAP_LOG_NAME);
+ ConcurrentSkipListMap collectToSet = ((LogMapAppender) setLog.getAppender(Constants.MAP_APPENDER_NAME)).getLogEvents();
+ // going over all the collected events, filtering out the empty ones, and writing them to a long string
+ SimpleDateFormat simpleDateFormat = new SimpleDateFormat(DATE_FORMAT);
+ List events = collectToSet.values().stream().filter(iLoggingEvent -> !iLoggingEvent.getMessage().isEmpty() &&
+ !iLoggingEvent.getMessage().equals(Constants.NEW_LINE)).collect(Collectors.toList());
+ if (events.size() > MAX_LOG_EVENTS) {
+ events = events.stream().filter(iLoggingEvent -> iLoggingEvent.getLevel().levelInt >= Level.INFO.levelInt).collect(Collectors.toList());
+ }
+ for (ILoggingEvent event : events) {
+ logs = logs.concat("[" + event.getLevel() + "] " + simpleDateFormat.format(new Date(event.getTimeStamp()))
+ + " - " + event.getFormattedMessage()).concat(Constants.NEW_LINE);
+ }
+ return logs;
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/org/whitesource/agent/SingleFileScanner.java b/src/main/java/org/whitesource/agent/SingleFileScanner.java
new file mode 100644
index 0000000..e243e52
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/SingleFileScanner.java
@@ -0,0 +1,39 @@
+/**
+ * Copyright (C) 2017 WhiteSource Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.whitesource.agent;
+
+import org.apache.tools.ant.DirectoryScanner;
+
+import java.io.File;
+
+/**
+ * A {@link org.apache.tools.ant.DirectoryScanner} for a single file.
+ *
+ * @author tom.shapira
+ */
+public class SingleFileScanner extends DirectoryScanner {
+
+ /**
+ * Exposes the {@link DirectoryScanner#isIncluded(String)} method to check if a single file should be included
+ * in the scan.
+ *
+ * @param file for scanning
+ * @return weather the file should be included or not
+ */
+ public boolean isIncluded(File file) {
+ return isIncluded(file.getAbsolutePath());
+ }
+}
diff --git a/src/main/java/org/whitesource/agent/TempFolders.java b/src/main/java/org/whitesource/agent/TempFolders.java
new file mode 100644
index 0000000..faa0705
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/TempFolders.java
@@ -0,0 +1,73 @@
+package org.whitesource.agent;
+
+import org.whitesource.agent.utils.FilesUtils;
+import org.whitesource.agent.utils.UniqueNamesGenerator;
+
+import java.io.File;
+import java.nio.file.Paths;
+
+
+public class TempFolders {
+
+ /* --- Static members --- */
+
+ private static final String PATH_TO_TEMP_DIR = System.getProperty("java.io.tmpdir");
+ private static final String WHITESOURCE_ARCHIVE_EXTRACTOR = "WhiteSource-ArchiveExtractor";
+ private static final String WHITE_BUILD_GRADLE_FOLDER = "WhiteSource-Build-Gradle";
+ private static final String WHITESOURCE_HTML_RESOLVER = "WhiteSource-html-resolver";
+ private static final String WHITESOURCE_DOTNET_RESOLVER = "WhiteSource-DotnetRestore";
+ private static final String WHITESOURCE_DOCKER = "WhiteSource-Docker";
+ private static final String WHITESOURCE_SCM_CONNECTOR_TMP_DIRECTORY = "WhiteSource-ScmConnector";
+ private static final String WHITESOURCE_PLATFORM_DEPENDENT_TMP_DIR = "WhiteSource-PlatformDependentFiles";
+ private static final String WHITESOURCE_PYTHON_TEMP_FOLDER = "Whitesource_python_resolver";
+
+ public static final String UNIQUE_HTML_TEMP_FOLDER = UniqueNamesGenerator.createUniqueName(WHITESOURCE_HTML_RESOLVER, Constants.EMPTY_STRING);
+ public static final String UNIQUE_GRADLE_TEMP_FOLDER = UniqueNamesGenerator.createUniqueName(WHITE_BUILD_GRADLE_FOLDER, Constants.EMPTY_STRING);
+ public static final String UNIQUE_DOTNET_TEMP_FOLDER = UniqueNamesGenerator.createUniqueName(WHITESOURCE_DOTNET_RESOLVER, Constants.EMPTY_STRING);
+ public static final String UNIQUE_PYTHON_TEMP_FOLDER = UniqueNamesGenerator.createUniqueName(WHITESOURCE_PYTHON_TEMP_FOLDER, Constants.EMPTY_STRING);
+ public static final String UNIQUE_DOCKER_TEMP_FOLDER = UniqueNamesGenerator.createUniqueName(WHITESOURCE_DOCKER, Constants.EMPTY_STRING);
+ public static final String UNIQUE_SCM_TEMP_FOLDER = UniqueNamesGenerator.createUniqueName(WHITESOURCE_SCM_CONNECTOR_TMP_DIRECTORY, Constants.EMPTY_STRING);
+ public static final String UNIQUE_PLATFORM_DEPENDENT_TEMP_FOLDER = UniqueNamesGenerator.createUniqueName(WHITESOURCE_PLATFORM_DEPENDENT_TMP_DIR, Constants.EMPTY_STRING);
+ public static final String UNIQUE_WHITESOURCE_ARCHIVE_EXTRACTOR_TEMP_FOLDER = UniqueNamesGenerator.createUniqueName(WHITESOURCE_ARCHIVE_EXTRACTOR, Constants.EMPTY_STRING);
+
+ private static final String PATH_TO_UNIQUE_HTML_TEMP_FOLDER = Paths.get(PATH_TO_TEMP_DIR, UNIQUE_HTML_TEMP_FOLDER).toString();
+ private static final String PATH_TO_UNIQUE_GRADLE_TEMP_FOLDER = Paths.get(PATH_TO_TEMP_DIR, UNIQUE_GRADLE_TEMP_FOLDER).toString();
+ private static final String PATH_TO_UNIQUE_DOTNET_TEMP_FOLDER = Paths.get(PATH_TO_TEMP_DIR, UNIQUE_DOTNET_TEMP_FOLDER).toString();
+ private static final String PATH_TO_UNIQUE_PYTHON_TEMP_FOLDER = Paths.get(PATH_TO_TEMP_DIR, UNIQUE_PYTHON_TEMP_FOLDER).toString();
+ private static final String PATH_TO_UNIQUE_DOCKER_TEMP_FOLDER = Paths.get(PATH_TO_TEMP_DIR, UNIQUE_DOCKER_TEMP_FOLDER).toString();
+ private static final String PATH_TO_SCM_CONNECTOR_TMP_DIRECTORY = Paths.get(PATH_TO_TEMP_DIR, UNIQUE_SCM_TEMP_FOLDER).toString();
+ private static final String PATH_TO_UNIQUE_ARCHIVE_EXTRACTOR_TEMP_FOLDER = Paths.get(PATH_TO_TEMP_DIR, UNIQUE_WHITESOURCE_ARCHIVE_EXTRACTOR_TEMP_FOLDER).toString();
+
+ // Agents api temp folder - CheckSumUtils folder :: calculateOtherPlatformSha1 method
+ private static final String PATH_TO_PLATFORM_DEPENDENT_TMP_DIR = Paths.get(PATH_TO_TEMP_DIR, UNIQUE_PLATFORM_DEPENDENT_TEMP_FOLDER).toString();
+
+ /* --- Constructors --- */
+
+ public TempFolders() {
+
+ }
+
+ /* --- Methods --- */
+
+ public void deleteTempFolders() {
+ deleteTempFoldersHelper(PATH_TO_UNIQUE_HTML_TEMP_FOLDER);
+ deleteTempFoldersHelper(PATH_TO_UNIQUE_GRADLE_TEMP_FOLDER);
+ deleteTempFoldersHelper(PATH_TO_UNIQUE_DOTNET_TEMP_FOLDER);
+ deleteTempFoldersHelper(PATH_TO_UNIQUE_PYTHON_TEMP_FOLDER);
+ deleteTempFoldersHelper(PATH_TO_UNIQUE_DOCKER_TEMP_FOLDER);
+ deleteTempFoldersHelper(PATH_TO_SCM_CONNECTOR_TMP_DIRECTORY);
+ deleteTempFoldersHelper(PATH_TO_PLATFORM_DEPENDENT_TMP_DIR);
+ deleteTempFoldersHelper(PATH_TO_UNIQUE_ARCHIVE_EXTRACTOR_TEMP_FOLDER);
+ }
+
+ public void deleteTempFoldersHelper(String path) {
+ if (path != null) {
+ File file = new File(path);
+ if (file != null) {
+ FilesUtils.deleteDirectory(file);
+ }
+ }
+ }
+
+
+}
diff --git a/src/main/java/org/whitesource/agent/ViaComponents.java b/src/main/java/org/whitesource/agent/ViaComponents.java
new file mode 100644
index 0000000..4508a99
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/ViaComponents.java
@@ -0,0 +1,37 @@
+package org.whitesource.agent;
+
+import org.whitesource.agent.api.model.DependencyInfo;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author raz.nitzan
+ */
+public class ViaComponents {
+
+ /* --- Members --- */
+
+ private String appPath;
+ private ViaLanguage language;
+ private List dependencies = new ArrayList<>();
+
+ /* --- Constructor --- */
+
+ public ViaComponents(String appPath, ViaLanguage language) {
+ this.appPath = appPath;
+ this.language = language;
+ }
+
+ public String getAppPath() {
+ return this.appPath;
+ }
+
+ public ViaLanguage getLanguage() {
+ return this.language;
+ }
+
+ public List getDependencies() {
+ return dependencies;
+ }
+}
diff --git a/src/main/java/org/whitesource/agent/ViaLanguage.java b/src/main/java/org/whitesource/agent/ViaLanguage.java
new file mode 100644
index 0000000..bec1357
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/ViaLanguage.java
@@ -0,0 +1,19 @@
+package org.whitesource.agent;
+
+/**
+ * @author raz.nitzan
+ */
+public enum ViaLanguage {
+ JAVA("java"),
+ JAVA_SCRIPT("javascript");
+
+ private final String language;
+
+ ViaLanguage(String language) {
+ this.language = language;
+ }
+
+ public String toString() {
+ return this.language;
+ }
+}
diff --git a/src/main/java/org/whitesource/agent/archive/ArchiveExtractor.java b/src/main/java/org/whitesource/agent/archive/ArchiveExtractor.java
new file mode 100644
index 0000000..f519d31
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/archive/ArchiveExtractor.java
@@ -0,0 +1,600 @@
+/**
+ * Copyright (C) 2014 WhiteSource Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.whitesource.agent.archive;
+
+import com.github.junrar.testutil.ExtractArchive;
+import net.lingala.zip4j.core.ZipFile;
+import net.lingala.zip4j.exception.ZipException;
+import net.lingala.zip4j.model.FileHeader;
+import org.apache.commons.compress.archivers.cpio.CpioArchiveEntry;
+import org.apache.commons.compress.archivers.cpio.CpioArchiveInputStream;
+import org.apache.commons.compress.compressors.lzma.LZMACompressorInputStream;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.codehaus.plexus.archiver.tar.TarBZip2UnArchiver;
+import org.codehaus.plexus.archiver.tar.TarGZipUnArchiver;
+import org.codehaus.plexus.archiver.tar.TarUnArchiver;
+import org.codehaus.plexus.archiver.xz.XZUnArchiver;
+import org.codehaus.plexus.logging.console.ConsoleLogger;
+import org.redline_rpm.ReadableChannelWrapper;
+import org.redline_rpm.Util;
+import org.redline_rpm.header.AbstractHeader;
+import org.redline_rpm.header.Format;
+import org.redline_rpm.header.Header;
+import org.slf4j.Logger;
+import org.whitesource.agent.utils.LoggerFactory;
+import org.whitesource.agent.Constants;
+import org.whitesource.agent.utils.FilesScanner;
+import org.whitesource.agent.utils.Pair;
+import org.whitesource.agent.TempFolders;
+
+import java.io.*;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.file.FileSystems;
+import java.nio.file.PathMatcher;
+import java.nio.file.Paths;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+/**
+ * @author anna.rozin
+ */
+public class ArchiveExtractor {
+ public static final String LAYER_TAR = "**/*layer.tar";
+
+ /* --- Static members --- */
+
+ private final Logger logger = LoggerFactory.getLogger(ArchiveExtractor.class);
+ public static final int LONG_BOUND = 100000;
+ public static final String DEPTH = "_depth_";
+ public static final String DEPTH_REGEX = DEPTH + "[0-9]";
+ public static final String GLOB_PREFIX = "glob:";
+ public static final String NULL_HEADER = "mainheader is null";
+
+ private final String JAVA_TEMP_DIR = System.getProperty("java.io.tmpdir");
+
+
+ public static final List ZIP_EXTENSIONS = Arrays.asList("jar", "war", "aar", "ear", "egg", "zip", "whl", "sca", "sda", "nupkg");
+ public static final List GEM_EXTENSIONS = Collections.singletonList("gem");
+ public static final List TAR_EXTENSIONS = Arrays.asList("tar.gz", "tar", "tgz", "tar.bz2", "tar.xz", "xz");
+ public static final List RPM_EXTENSIONS = Collections.singletonList("rpm");
+ public static final List RAR_EXTENSIONS = Collections.singletonList("rar");
+
+ public static final String ZIP_EXTENSION_PATTERN;
+ public static final String GEM_EXTENSION_PATTERN;
+ public static final String TAR_EXTENSION_PATTERN;
+ public static final String RPM_EXTENSION_PATTERN;
+ public static final String RAR_EXTENSION_PATTERN;
+ public static final String RUBY_DATA_FILE = "data.tar.gz";
+ public static final String TAR_SUFFIX = ".tar";
+ public static final String GZ_SUFFIX = ".gz";
+ public static final String BZ_SUFFIX = ".bz2";
+ public static final String XZ_SUFFIX = ".xz";
+ public static final String LZMA = "lzma";
+ public static final String CPIO = ".cpio";
+ public static final String TGZ_SUFFIX = ".tgz";
+
+ public static final String TAR_GZ_SUFFIX = TAR_SUFFIX + GZ_SUFFIX;
+ public static final String TAR_BZ2_SUFFIX = TAR_SUFFIX + BZ_SUFFIX;
+
+ public static final String UN_ARCHIVER_LOGGER = "unArchiverLogger";
+ public static final String GLOB_PATTERN_PREFIX = Constants.PATTERN + Constants.DOT;
+ public static final String PATTERN_PREFIX = ".*\\.";
+ public static final String XZ_UN_ARCHIVER_FILE_NAME = "compressedFile.tar";
+
+ static {
+ ZIP_EXTENSION_PATTERN = initializePattern(ZIP_EXTENSIONS);
+ GEM_EXTENSION_PATTERN = initializePattern(GEM_EXTENSIONS);
+ TAR_EXTENSION_PATTERN = initializePattern(TAR_EXTENSIONS);
+ RPM_EXTENSION_PATTERN = initializePattern(RPM_EXTENSIONS);
+ RAR_EXTENSION_PATTERN = initializePattern(RAR_EXTENSIONS);
+ }
+
+ private static String initializePattern(List archiveExtensions) {
+ StringBuilder sb = new StringBuilder();
+ for (String archiveExtension : archiveExtensions) {
+ sb.append(PATTERN_PREFIX);
+ sb.append(archiveExtension);
+ sb.append(Constants.PIPE);
+ }
+ return sb.toString().substring(0, sb.toString().lastIndexOf(Constants.PIPE));
+ }
+
+ /* --- Private members --- */
+
+ private final String[] archiveIncludesPattern;
+ private final String[] archiveExcludesPattern;
+ private final String[] filesExcludes;
+ private String randomString;
+ private String tempFolderNoDepth;
+ private boolean fastUnpack = false;
+
+ /* --- Constructors --- */
+
+ public ArchiveExtractor(String[] archiveIncludes, String[] archiveExcludes, String[] filesExcludes, boolean fastUnpack) {
+ this(archiveIncludes, archiveExcludes, filesExcludes);
+ this.fastUnpack = fastUnpack;
+ }
+
+ public ArchiveExtractor(String[] archiveIncludes, String[] archiveExcludes, String[] filesExcludes) {
+ if (archiveIncludes.length > 0 && StringUtils.isNotBlank(archiveIncludes[0])) {
+ this.archiveIncludesPattern = archiveIncludes;
+ } else {
+ // create ARCHIVE_EXTENSIONS only if archiveIncludes is empty
+ this.archiveIncludesPattern = createArchivesArray();
+ }
+ this.archiveExcludesPattern = archiveExcludes;
+ this.filesExcludes = filesExcludes;
+ }
+
+ private String getTempFolder(String scannerBaseDir) {
+ String creationDate = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
+ String tempFolder = JAVA_TEMP_DIR.endsWith(File.separator) ? JAVA_TEMP_DIR + TempFolders.UNIQUE_WHITESOURCE_ARCHIVE_EXTRACTOR_TEMP_FOLDER + File.separator + creationDate :
+ JAVA_TEMP_DIR + File.separator + TempFolders.UNIQUE_WHITESOURCE_ARCHIVE_EXTRACTOR_TEMP_FOLDER + File.separator + creationDate;
+ String destDirectory = tempFolder + Constants.UNDERSCORE + this.randomString;
+ int separatorIndex = scannerBaseDir.lastIndexOf(File.separator);
+
+ if (separatorIndex != -1) {
+ destDirectory = destDirectory + scannerBaseDir.substring(separatorIndex, scannerBaseDir.length());
+ try {
+ // this solves the tilda issue in filepath in windows (mangled Windows filenames)
+ destDirectory = new File(destDirectory).getCanonicalPath().toString();
+ } catch (IOException e) {
+ logger.warn("Error getting the absolute file name ", e);
+ }
+ }
+ return destDirectory;
+ }
+
+ /* --- Public methods --- */
+
+ /**
+ * The Method extracts all the Archive files according to the archiveExtractionDepth.
+ * archiveExtractionDepth defined by the user in the configuration file.
+ *
+ * The archiveExtractionDepth default value is 0 - no archive scanning, the max value is 3.
+ * By default the method scans jar/war/ear.
+ * If archiveIncludes/archiveExcludes params are defined the method will act accordingly.
+ *
+ * @param scannerBaseDir - directory for scanning.
+ * @param archiveExtractionDepth - drill down hierarchy level in archive files
+ * @param archiveDirectories list of directories
+ * @return the temp directory for the extracted files.
+ */
+ public String extractArchives(String scannerBaseDir, int archiveExtractionDepth, List archiveDirectories) {
+ this.randomString = String.valueOf(ThreadLocalRandom.current().nextLong(0, LONG_BOUND));
+ this.tempFolderNoDepth = getTempFolder(scannerBaseDir);
+ logger.debug("Base directory is {}, extraction depth is set to {}", scannerBaseDir, archiveExtractionDepth);
+ Map> allFiles = new HashMap<>();
+ // Extract again if needed according archiveExtractionDepth parameter
+ for (int curLevel = 0; curLevel < archiveExtractionDepth; curLevel++) {
+ String folderToScan;
+ String folderToExtract;
+ if (curLevel == 0) {
+ folderToScan = scannerBaseDir;
+ } else {
+ folderToScan = getDepthFolder(curLevel - 1);
+ }
+ folderToExtract = getDepthFolder(curLevel);
+ Pair retrieveFilesWithFolder = getSearchedFileNames(folderToScan);
+ if (retrieveFilesWithFolder == null || retrieveFilesWithFolder.getKey().length <= 0) {
+ break;
+ } else {
+ String[] fileNames = retrieveFilesWithFolder.getKey();
+ folderToScan = retrieveFilesWithFolder.getValue();
+
+ Pair> filesFound = new Pair<>(folderToScan, Arrays.stream(fileNames).collect(Collectors.toList()));
+ Map foundFiles;
+ if (fastUnpack) {
+ foundFiles = handleArchiveFilesFast(folderToExtract, filesFound);
+ } else {
+ foundFiles = handleArchiveFiles(folderToExtract, filesFound);
+ }
+ allFiles.put(String.valueOf(curLevel), foundFiles);
+ }
+ }
+ if (!allFiles.isEmpty()) {
+ String parentDirectory = new File(this.tempFolderNoDepth).getParent();
+ archiveDirectories.add(parentDirectory);
+ return parentDirectory;
+ } else {
+ // if unable to extract, return null
+ return null;
+ }
+ }
+
+ // extract image layers
+ public void extractDockerImageLayers(File imageTarFile, File imageExtractionDir, Boolean deleteTarFiles) {
+ FilesScanner filesScanner = new FilesScanner();
+ boolean success = false;
+ // docker layers are saved as TAR file (we save it as TAR)
+ if (imageTarFile.getName().endsWith(TAR_SUFFIX)) {
+ success = unTar(imageTarFile.getName().toLowerCase(), imageExtractionDir.getAbsolutePath(), imageTarFile.getPath());
+ boolean deleted = false;
+ if (deleteTarFiles) {
+ deleted = imageTarFile.delete();
+ }
+ if (!deleted) {
+ logger.warn("Was not able to delete {} (docker image TAR file)", imageTarFile.getName());
+ }
+ }
+ if (success) {
+ String[] fileNames = filesScanner.getDirectoryContent(imageExtractionDir.getAbsolutePath(), new String[]{LAYER_TAR}, new String[]{}, true, false);
+ for (String filename : fileNames) {
+ File layerToExtract = new File(imageExtractionDir + File.separator + filename);
+ extractDockerImageLayers(layerToExtract, layerToExtract.getParentFile(), deleteTarFiles);
+ }
+ } else {
+ logger.warn("Was not able to extract {} (docker image TAR file)", imageTarFile.getName());
+ }
+ }
+
+ private String getDepthFolder(int depth) {
+ return this.tempFolderNoDepth + DEPTH + depth;
+ }
+
+ /* --- Private methods --- */
+
+ private String[] createArchivesArray() {
+ Collection archiveExtensions = new ArrayList<>();
+ archiveExtensions.addAll(ZIP_EXTENSIONS);
+ archiveExtensions.addAll(GEM_EXTENSIONS);
+ archiveExtensions.addAll(TAR_EXTENSIONS);
+
+ String[] archiveIncludesPattern = new String[archiveExtensions.size()];
+ int i = 0;
+ for (String extension : archiveExtensions) {
+ archiveIncludesPattern[i++] = GLOB_PATTERN_PREFIX + extension;
+ }
+ return archiveIncludesPattern;
+ }
+
+ private Pair getSearchedFileNames(String fileOrFolderToScan) {
+ String[] foundFiles = null;
+ File file = new File(fileOrFolderToScan);
+
+ String folderToScan;
+ if (file.exists()) {
+ FilesScanner filesScanner = new FilesScanner();
+ if (file.isDirectory()) {
+ // scan directory
+ foundFiles = filesScanner.getDirectoryContent(fileOrFolderToScan, archiveIncludesPattern, archiveExcludesPattern, false, false);
+ folderToScan = fileOrFolderToScan;
+ return new Pair<>(foundFiles, folderToScan);
+ } else {
+ //// handle file passed in -d parameter
+ //// check if file matches archive GLOB patterns
+ boolean included = filesScanner.isIncluded(file, archiveIncludesPattern, archiveExcludesPattern, false, false);
+ if (included) {
+ folderToScan = file.getParent();
+ String relativeFilePath = new File(folderToScan).toURI().relativize(new File(file.getAbsolutePath()).toURI()).getPath();
+ foundFiles = new String[]{relativeFilePath};
+ return new Pair<>(foundFiles, folderToScan);
+ }
+ }
+ filesScanner = null;
+ }
+ return null;
+ }
+
+ private Map handleArchiveFiles(String baseFolderToExtract, Pair> fileNames) {
+ Map founded = new HashMap<>();
+ for (String fileName : fileNames.getValue()) {
+ String archivePath = Paths.get(fileNames.getKey(), fileName).toString();
+ String unpackFolder = Paths.get(baseFolderToExtract, FilenameUtils.removeExtension(fileName)).toString();
+ Pair dataToUnpack = new Pair<>(archivePath, unpackFolder);
+ Pair foundArchive = getUnpackedResult(dataToUnpack);
+ if (foundArchive != null) {
+ founded.put(foundArchive.getKey(), foundArchive.getValue());
+ }
+ }
+ return founded;
+ }
+
+ private Map handleArchiveFilesFast(String baseFolderToExtract, Pair> fileNames) {
+ Collection dataToUnpack = fileNames.getValue().stream().map(fileName -> {
+ String archivePath = Paths.get(fileNames.getKey(), fileName).toString();
+ String unpackFolder = Paths.get(baseFolderToExtract, FilenameUtils.removeExtension(fileName)).toString();
+ return new Pair(archivePath, unpackFolder);
+ }).collect(Collectors.toList());
+ return processCollections(dataToUnpack);
+ }
+
+ public Map processCollections(Collection unitsOfWork) {
+ int numberOfThreads = Runtime.getRuntime().availableProcessors();
+ ExecutorService executorService = Executors.newFixedThreadPool(numberOfThreads);
+ List> handles = new ArrayList<>();
+
+ List> callableList = new ArrayList<>();
+ unitsOfWork.stream().forEach(unitOfWork -> callableList.add(() -> getUnpackedResult(unitOfWork)));
+
+ for (Callable callable : callableList) {
+ Future handle = executorService.submit(callable);
+ handles.add(handle);
+ }
+
+ Map results = new HashMap<>();
+ for (Future h : handles) {
+ try {
+ Pair dataToUnpack = h.get();
+ results.put(dataToUnpack.getKey(), dataToUnpack.getValue());
+ } catch (InterruptedException e) {
+ logger.warn("Error: {}", e.getMessage());
+ } catch (ExecutionException e) {
+ logger.warn("Error: {}", e.getMessage());
+ }
+ }
+
+ executorService.shutdownNow();
+ return results;
+ }
+
+ private Pair getUnpackedResult(Pair dataToUnpack) {
+ boolean foundArchive = false;
+ String innerDir = dataToUnpack.getValue();
+ String fileKey = dataToUnpack.getKey();
+ String lowerCaseFileName = fileKey.toLowerCase();
+
+ if (lowerCaseFileName.matches(ZIP_EXTENSION_PATTERN)) {
+ foundArchive = unZip(innerDir, fileKey);
+ } else if (lowerCaseFileName.matches(GEM_EXTENSION_PATTERN)) {
+ foundArchive = unTar(lowerCaseFileName, innerDir, fileKey);
+ innerDir = innerDir + File.separator + RUBY_DATA_FILE;
+ foundArchive = unTar(RUBY_DATA_FILE, innerDir + this.randomString, innerDir);
+ innerDir = innerDir + this.randomString;
+ } else if (lowerCaseFileName.matches(TAR_EXTENSION_PATTERN)) {
+ foundArchive = unTar(lowerCaseFileName, innerDir, fileKey);
+ // innerDir = innerDir.replaceAll(TAR_SUFFIX, BLANK);
+ } else if (lowerCaseFileName.matches(RPM_EXTENSION_PATTERN)) {
+ foundArchive = handleRpmFile(innerDir, fileKey);
+ } else if (lowerCaseFileName.matches(RAR_EXTENSION_PATTERN)) {
+ foundArchive = extractRarFile(innerDir, fileKey);
+ } else {
+ logger.warn("Error: {} is unsupported archive type", fileKey);
+ }
+ if (foundArchive) {
+ Pair resultArchive = new Pair(lowerCaseFileName, innerDir);
+ return resultArchive;
+ } else
+ return null;
+ }
+
+ private boolean extractRarFile(String innerDir, String fileKey) {
+ boolean foundArchive;
+ File destDir = new File(innerDir);
+ if (!destDir.exists()) {
+ destDir.mkdirs();
+ }
+ try {
+ ExtractArchive.extractArchive(fileKey, innerDir);
+ foundArchive = true;
+ } catch (Exception e) {
+ logger.warn("Error extracting file {}: {}", fileKey, e.getMessage());
+ try {
+ //if the header is missing try to extract the rar file with zip extension - WSE-450
+ if (e.getMessage().contains(NULL_HEADER) && new ZipFile(fileKey) instanceof ZipFile) {
+ logger.info("Retrying extraction {}", fileKey);
+ foundArchive = unZip(innerDir, fileKey);
+ }
+ } catch (ZipException e1) {
+ logger.warn("Error extracting file {}: {}", fileKey, e.getMessage());
+ foundArchive = false;
+ }
+ }
+ return true;
+ }
+
+ // Open and extract data from zip pattern files
+ private boolean unZip(String innerDir, String archiveFile) {
+ boolean success = true;
+ ZipFile zipFile;
+ try {
+ zipFile = new ZipFile(archiveFile);
+ // Get the list of file headers from the zip file before unpacking
+ List fileHeaderList = zipFile.getFileHeaders();
+
+ List matchers = Arrays.stream(filesExcludes).map(fileExclude ->
+ FileSystems.getDefault().getPathMatcher(GLOB_PREFIX + fileExclude)).collect(Collectors.toList());
+ // Loop through the file headers and extract only files that are not matched by fileExcludes patterns
+ for (int i = 0; i < fileHeaderList.size(); i++) {
+ FileHeader fileHeader = (FileHeader) fileHeaderList.get(i);
+ String fileName = fileHeader.getFileName();
+ if (filesExcludes.length > 0) {
+ Predicate matchesExcludes = pathMatcher -> pathMatcher.matches(Paths.get(innerDir, fileName));
+ if (matchers.stream().noneMatch(matchesExcludes)) {
+ zipFile.extractFile(fileHeader, innerDir);
+ }
+ } else {
+ zipFile.extractFile(fileHeader, innerDir);
+ }
+ }
+ } catch (Exception e) {
+ success = false;
+ logger.warn("Error extracting file {}: {}", archiveFile, e.getMessage());
+ logger.debug("Error extracting file {}: {}", archiveFile, e.getStackTrace());
+ } finally {
+ // remove reference to zip file
+ zipFile = null;
+ }
+ return success;
+ }
+
+ // Open and extract data from Tar pattern files
+ private boolean unTar(String fileName, String innerDir, String archiveFile) {
+ boolean success = true;
+ TarUnArchiver unArchiver = new TarUnArchiver();
+ try {
+ File destDir = new File(innerDir);
+ if (!destDir.exists()) {
+ destDir.mkdirs();
+ }
+ if (fileName.endsWith(TAR_GZ_SUFFIX) || fileName.endsWith(TGZ_SUFFIX)) {
+ unArchiver = new TarGZipUnArchiver();
+ } else if (fileName.endsWith(TAR_BZ2_SUFFIX)) {
+ unArchiver = new TarBZip2UnArchiver();
+ } else if (fileName.endsWith(XZ_SUFFIX)) {
+ String destFileUrl = destDir.getCanonicalPath() + Constants.BACK_SLASH + XZ_UN_ARCHIVER_FILE_NAME;
+ success = unXz(new File(archiveFile), destFileUrl);
+ archiveFile = destFileUrl;
+ }
+ if (success) {
+ unArchiver.enableLogging(new ConsoleLogger(ConsoleLogger.LEVEL_DISABLED, UN_ARCHIVER_LOGGER));
+ unArchiver.setSourceFile(new File(archiveFile));
+ unArchiver.setDestDirectory(destDir);
+ unArchiver.extract();
+ }
+ } catch (Exception e) {
+ success = false;
+ logger.warn("Error extracting file {}: {}", fileName, e.getMessage());
+ }
+ return success;
+ }
+
+ // extract xz files
+ public boolean unXz(File srcFileToArchive, String destFilePath) {
+ boolean success = true;
+ try {
+ XZUnArchiver XZUnArchiver = new XZUnArchiver();
+ XZUnArchiver.enableLogging(new ConsoleLogger(ConsoleLogger.LEVEL_DISABLED, UN_ARCHIVER_LOGGER));
+ XZUnArchiver.setSourceFile(srcFileToArchive);
+ XZUnArchiver.setDestFile(new File(destFilePath));
+ XZUnArchiver.extract();
+ } catch (Exception e) {
+ success = false;
+ logger.warn("Failed to extract Xz file : {} - {}", srcFileToArchive.getPath(), e.getMessage());
+ }
+ return success;
+ }
+
+ // Open and extract data from rpm files
+ private boolean handleRpmFile(String innerDir, String archiveFile) {
+ boolean success = true;
+ File rpmFile = new File(archiveFile);
+ FileInputStream rpmFIS = null;
+ try {
+ rpmFIS = new FileInputStream(rpmFile.getPath());
+ } catch (FileNotFoundException e) {
+ success = false;
+ logger.warn("File not found: {}", archiveFile);
+ }
+
+ Format format = null;
+ ReadableByteChannel channel = Channels.newChannel(rpmFIS);
+ ReadableChannelWrapper channelWrapper = new ReadableChannelWrapper(channel);
+ try {
+ format = new org.redline_rpm.Scanner().run(channelWrapper);
+ } catch (IOException e) {
+ success = false;
+ logger.warn("Error reading RPM file {}: {}", archiveFile, e.getCause());
+ }
+
+ if (format != null) {
+ Header header = format.getHeader();
+ FileOutputStream cpioOS = null;
+ FileOutputStream cpioEntryOutputStream = null;
+ CpioArchiveInputStream cpioIn = null;
+ File cpioFile = null;
+ try {
+ // extract all .cpio file
+ // get input stream according to payload compressor type
+ InputStream inputStream;
+ AbstractHeader.Entry pcEntry = header.getEntry(Header.HeaderTag.PAYLOADCOMPRESSOR);
+ String[] pc = (String[]) pcEntry.getValues();
+ if (pc[0].equals(LZMA)) {
+ try {
+ inputStream = new LZMACompressorInputStream(rpmFIS);
+ } catch (Exception e) {
+ throw new IOException("Failed to load LZMA compression stream", e);
+ }
+ } else {
+ inputStream = Util.openPayloadStream(header, rpmFIS);
+ }
+
+ cpioFile = new File(rpmFile.getPath() + CPIO);
+ cpioOS = new FileOutputStream(cpioFile);
+ IOUtils.copy(inputStream, cpioOS);
+ // extract files from .cpio
+ File extractDestination = new File(innerDir);
+ extractDestination.mkdirs();
+ cpioIn = new CpioArchiveInputStream(new FileInputStream(cpioFile));
+
+ CpioArchiveEntry cpioEntry;
+ while ((cpioEntry = (CpioArchiveEntry) cpioIn.getNextEntry()) != null) {
+ String entryName = cpioEntry.getName();
+ String lowercaseName = entryName.toLowerCase();
+ File file = new File(extractDestination, getFileName(entryName));
+ cpioEntryOutputStream = new FileOutputStream(file);
+ IOUtils.copy(cpioIn, cpioEntryOutputStream);
+ String innerExtractionDir;
+ if (lowercaseName.matches(TAR_EXTENSION_PATTERN)) {
+ innerExtractionDir = innerDir + File.separator + entryName + this.randomString;
+ unTar(file.getName(), innerExtractionDir, file.getPath());
+ } else if (lowercaseName.matches(ZIP_EXTENSION_PATTERN)) {
+ innerExtractionDir = innerDir + File.separator + entryName + this.randomString;
+ unZip(innerExtractionDir, file.getPath());
+ }
+ // close
+ closeResource(cpioEntryOutputStream);
+ }
+ } catch (IOException e) {
+ logger.error("Error unpacking rpm file {}: {}", rpmFile.getName(), e.getMessage());
+ } finally {
+ closeResource(cpioEntryOutputStream);
+ closeResource(cpioIn);
+ closeResource(cpioOS);
+ deleteFile(cpioFile);
+ }
+ }
+ return success;
+ }
+
+ private void deleteFile(File cpioFile) {
+ try {
+ FileUtils.forceDelete(cpioFile);
+ } catch (IOException e) {
+ logger.warn("Error deleting cpio file {}: {}", cpioFile.getName(), e.getMessage());
+ }
+ }
+
+ private void closeResource(Closeable resource) {
+ if (resource != null) {
+ try {
+ resource.close();
+ } catch (IOException e) {
+ logger.warn("Error closing file {}: {}", resource.toString(), e.getMessage());
+ }
+ }
+ }
+
+ // parse name without directories
+ private String getFileName(String name) {
+ //check if the environment is linux or windows
+ if (name.contains(Constants.FORWARD_SLASH)) {
+ name = name.substring(name.lastIndexOf(Constants.FORWARD_SLASH) + 1, name.length());
+ } else if (name.contains(Constants.BACK_SLASH)) {
+ name = name.substring(name.lastIndexOf(Constants.BACK_SLASH) + 1, name.length());
+ }
+ return name;
+ }
+
+
+}
\ No newline at end of file
diff --git a/src/main/java/org/whitesource/agent/dependency/resolver/AbstractDependencyResolver.java b/src/main/java/org/whitesource/agent/dependency/resolver/AbstractDependencyResolver.java
new file mode 100644
index 0000000..04407c0
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/dependency/resolver/AbstractDependencyResolver.java
@@ -0,0 +1,113 @@
+/**
+ * Copyright (C) 2017 WhiteSource Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.whitesource.agent.dependency.resolver;
+
+import org.apache.commons.lang.StringUtils;
+import org.whitesource.agent.Constants;
+import org.whitesource.agent.api.model.DependencyType;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.util.*;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * @author eugen.horovitz
+ */
+public abstract class AbstractDependencyResolver {
+
+ /* --- Static Members --- */
+
+ protected static final String GLOB_PATTERN = "**/";
+ protected static final String fileSeparator = System.getProperty(Constants.FILE_SEPARATOR);
+ protected IBomParser bomParser;
+
+ /* --- Abstract methods --- */
+
+ protected abstract ResolutionResult resolveDependencies(String projectFolder, String topLevelFolder, Set bomFiles) throws FileNotFoundException;
+
+ protected abstract Collection getExcludes();
+
+ protected abstract DependencyType getDependencyType();
+
+ protected abstract String getDependencyTypeName();
+
+ protected abstract String[] getBomPattern();
+
+ public abstract Collection getManifestFiles();
+
+ protected abstract Collection getLanguageExcludes();
+
+ protected Collection getRelevantScannedFolders(Collection scannedFolders) {
+ if (scannedFolders == null) {
+ return new HashSet<>();
+ }
+ if (scannedFolders.isEmpty()) {
+ return scannedFolders;
+ }
+ Collection foldersToRemove = new HashSet<>();
+ for(String folder : scannedFolders) {
+ for (String subFolder : scannedFolders) {
+ if (subFolder.contains(folder) && !subFolder.equals(folder)) {
+ foldersToRemove.add(subFolder);
+ }
+ }
+ }
+ scannedFolders.removeAll(foldersToRemove);
+ return scannedFolders;
+ }
+
+ protected boolean printResolvedFolder() {
+ return true;
+ }
+
+ public abstract Collection getSourceFileExtensions();
+
+ /* --- Protected methods --- */
+ protected List extensionPattern(List extensions) {
+ List extensionsPatternStr = new LinkedList<>();
+ for (String extension : extensions) {
+ extensionsPatternStr.add(Constants.PATTERN + extension);
+ }
+ return extensionsPatternStr;
+ }
+ protected List normalizeLocalPath(String parentFolder, String topFolderFound, Collection excludes, String folderToIgnore) {
+ String normalizedRoot = new File(parentFolder).getPath();
+ if (normalizedRoot.equals(topFolderFound)) {
+ topFolderFound = topFolderFound
+ .replace(normalizedRoot, Constants.EMPTY_STRING)
+ .replace(Constants.BACK_SLASH, Constants.FORWARD_SLASH);
+ } else {
+ topFolderFound = topFolderFound
+ .replace(parentFolder, Constants.EMPTY_STRING)
+ .replace(Constants.BACK_SLASH, Constants.FORWARD_SLASH);
+ }
+
+ if (topFolderFound.length() > 0)
+ topFolderFound = topFolderFound.substring(1, topFolderFound.length()) + Constants.FORWARD_SLASH;
+
+ String finalRes = topFolderFound;
+ if (StringUtils.isBlank(folderToIgnore)) {
+ return excludes.stream().map(exclude -> finalRes + exclude).collect(Collectors.toList());
+ } else {
+ return excludes.stream().map(exclude -> finalRes + folderToIgnore + Constants.FORWARD_SLASH + exclude).collect(Collectors.toList());
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/org/whitesource/agent/dependency/resolver/BomFile.java b/src/main/java/org/whitesource/agent/dependency/resolver/BomFile.java
new file mode 100644
index 0000000..b57d25f
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/dependency/resolver/BomFile.java
@@ -0,0 +1,172 @@
+/**
+ * Copyright (C) 2017 WhiteSource Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.whitesource.agent.dependency.resolver;
+
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.whitesource.agent.dependency.resolver.npm.RegistryType;
+import org.whitesource.agent.utils.LoggerFactory;
+import org.whitesource.agent.Constants;
+
+import java.util.Map;
+
+/**
+ * @author eugen.horovitz
+ */
+public class BomFile {
+
+ /* --- Members --- */
+
+ private final String name;
+ private final String version;
+ private String groupId;
+ private String sha1;
+ private String fileName;
+ private final String localFileName;
+ private final Map dependencies;
+ private Map optionalDependencies;
+ private String resolved;
+ private boolean scopedPackage;
+ private RegistryType registryType;
+
+ public static String DUMMY_PARAMETER_SCOPE_PACKAGE = "{dummyParameterOfScopePackage}";
+ private final Logger logger = LoggerFactory.getLogger(BomFile.class);
+ private static String NPM_REGISTRY = "registry.npmjs.org";
+ private static final String NPM_REGISTRY1 = "npm/registry/";
+ private static final String SCOPED_PACKAGE = "@";
+ private static final String HTTPS = "https";
+ private static final String HTTP = "http";
+
+ /* --- Constructors --- */
+
+ public BomFile(String name, String version, String sha1, String fileName, String localFileName,
+ Map dependencies, Map optionalDependencies, String resolved, RegistryType registryType) {
+ this.name = name;
+ this.version = version;
+ this.sha1 = sha1;
+ this.fileName = fileName;
+ this.localFileName = localFileName;
+ this.dependencies = dependencies;
+ this.optionalDependencies = optionalDependencies;
+ this.resolved = resolved;
+ this.scopedPackage = false;
+ this.groupId = null;
+ this.registryType = registryType;
+ }
+
+ public BomFile(String groupId, String artifactId, String version, String bomPath) {
+ this(artifactId, version, null, null, bomPath, null, null, null, null);
+ this.groupId = groupId;
+ }
+
+ /* --- Public method --- */
+
+ public boolean isValid() {
+ return StringUtils.isNoneBlank(name, version);
+ }
+
+ /* --- Getters --- */
+
+ public String getName() {
+ return name;
+ }
+
+ public String getLocalFileName() {
+ return localFileName;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public String getSha1() {
+ return sha1;
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ public String getGroupId() {
+ return groupId;
+ }
+
+ public static String getUniqueDependencyName(String name, String version) {
+ return name + SCOPED_PACKAGE + version.replace("v", Constants.EMPTY_STRING);
+ }
+
+ public String getUniqueDependencyName() {
+ return getUniqueDependencyName(name, version);
+ }
+
+ public Map getDependencies() {
+ return dependencies;
+ }
+
+ public Map getOptionalDependencies() {
+ return optionalDependencies;
+ }
+
+ public RegistryType getRegistryType() {
+ return this.registryType;
+ }
+
+ public String getRegistryPackageUrl() {
+ String registryPackageUrl = null;
+ if (StringUtils.isEmpty(this.resolved)) {
+ logger.debug("resolved url in file is empty");
+ return StringUtils.EMPTY;
+ }
+ logger.debug("resolved url in file = " + this.resolved);
+ if (this.resolved.contains("git+") || this.resolved.contains("github:")) {
+ // temp solution for WSE-204
+ logger.info("This configuration - " + this.name + " (remote repository packages) is not supported by WhiteSource. Please use direct URL package references.");
+ return StringUtils.EMPTY;
+ }
+
+ if (this.registryType == RegistryType.ARTIFACTORY) {
+ // example: change this url: http://localhost:8081/artifactory/api/npm/npmExample/q-1.5.1.tgz to http://localhost:8081/artifactory/api/npm/npmExample/q-1.5.1.json
+ registryPackageUrl = this.resolved.substring(0, this.resolved.length() - 3);
+ registryPackageUrl = registryPackageUrl + "json";
+ } else if (this.resolved.contains(SCOPED_PACKAGE) || this.registryType != RegistryType.NPM_REGISTRY) {
+ // resolve rare cases where the package's name is a sub-string of the registry's url
+ int npmRegistryIndex = this.resolved.indexOf(NPM_REGISTRY1);
+ registryPackageUrl = this.resolved.substring(0, this.resolved.indexOf(this.name, npmRegistryIndex) + this.name.length());
+ int lastSlashIndex = registryPackageUrl.lastIndexOf('/');
+ registryPackageUrl = registryPackageUrl.substring(0, lastSlashIndex) + DUMMY_PARAMETER_SCOPE_PACKAGE + registryPackageUrl.substring(lastSlashIndex + 1);
+ this.scopedPackage = true;
+ } else {
+ String urlName = Constants.FORWARD_SLASH + this.name + Constants.FORWARD_SLASH;
+ registryPackageUrl = this.resolved.substring(0, this.resolved.indexOf(urlName) + urlName.length());
+ registryPackageUrl = registryPackageUrl + this.version;
+ if (registryPackageUrl.startsWith(HTTPS)) {
+ String urlWithoutHttps = registryPackageUrl.substring(HTTPS.length());
+ registryPackageUrl = HTTP + urlWithoutHttps;
+ }
+ }
+ logger.debug("resolved url in link = " + registryPackageUrl);
+ return registryPackageUrl;
+ }
+
+ public boolean isScopedPackage() {
+ return this.scopedPackage;
+ }
+
+ @Override
+ public String toString() {
+ return String.join(Constants.DOT, getGroupId(), getName(), getVersion());
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/org/whitesource/agent/dependency/resolver/BomParser.java b/src/main/java/org/whitesource/agent/dependency/resolver/BomParser.java
new file mode 100644
index 0000000..1568ff5
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/dependency/resolver/BomParser.java
@@ -0,0 +1,64 @@
+/**
+ * Copyright (C) 2017 WhiteSource Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.whitesource.agent.dependency.resolver;
+
+import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.whitesource.agent.utils.LoggerFactory;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * @author eugen.horovitz
+ */
+public abstract class BomParser implements IBomParser{
+
+ /* --- Static members --- */
+
+ private final Logger logger = LoggerFactory.getLogger(BomParser.class);
+
+ /* --- Public methods --- */
+
+ public BomFile parseBomFile(String bomPath) {
+ BomFile bomFile = null;
+ String json = null;
+ try (InputStream is = new FileInputStream(bomPath)) {
+ json = IOUtils.toString(is);
+ } catch (FileNotFoundException e) {
+ logger.error("file Not Found: {}", bomPath);
+ } catch (IOException e) {
+ logger.error("error getting file : {}", e.getMessage());
+ }
+
+ if (json != null) {
+ try {
+ bomFile = parseBomFile(json, bomPath);
+ } catch (Exception e) {
+ logger.debug("Invalid file {}", bomPath);
+ }
+ }
+ return bomFile;
+ }
+
+ /* --- Abstract methods --- */
+
+ protected abstract BomFile parseBomFile(String jsonText, String localFileName);
+
+ protected abstract String getFilename(String name, String version);
+}
\ No newline at end of file
diff --git a/src/main/java/org/whitesource/agent/dependency/resolver/CocoaPods/CocoaPodsDependencyCollector.java b/src/main/java/org/whitesource/agent/dependency/resolver/CocoaPods/CocoaPodsDependencyCollector.java
new file mode 100644
index 0000000..6001213
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/dependency/resolver/CocoaPods/CocoaPodsDependencyCollector.java
@@ -0,0 +1,159 @@
+package org.whitesource.agent.dependency.resolver.CocoaPods;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.whitesource.agent.Constants;
+import org.whitesource.agent.api.model.AgentProjectInfo;
+import org.whitesource.agent.api.model.DependencyInfo;
+import org.whitesource.agent.api.model.DependencyType;
+import org.whitesource.agent.dependency.resolver.DependencyCollector;
+import org.whitesource.agent.hash.HashCalculator;
+import org.whitesource.agent.utils.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * @author raz.nitzan
+ */
+public class CocoaPodsDependencyCollector extends DependencyCollector {
+
+ /* --- Static Members --- */
+
+ private static final String PODS = "PODS";
+ private static final String DEPENDENCIES = "DEPENDENCIES";
+ private static final String PATTERN_DIRECT_LINE = " -";
+ private static final String PATTERN_TRANSITIVE_DEPENDENCY = " -";
+
+ /* --- Members --- */
+
+ private final Logger logger = LoggerFactory.getLogger(CocoaPodsDependencyCollector.class);
+ private HashCalculator hashCalculator = new HashCalculator();
+
+ /* --- public methods --- */
+
+ public Collection collectDependencies(String podFileLock) {
+ Collection dependencies = new LinkedList<>();
+ List directDependenciesLines = new LinkedList<>();
+ List allDependenciesLines = new LinkedList<>();
+ if (getPodsAndDependenciesSection(directDependenciesLines, allDependenciesLines, podFileLock)) {
+ dependencies.addAll(parseDependenciesLines(directDependenciesLines, allDependenciesLines, podFileLock));
+ } else {
+ logger.warn("Failed to parse the Podfile.lock in {}", podFileLock);
+ }
+ return getSingleProjectList(dependencies);
+ }
+
+ /* --- private methods --- */
+
+ private boolean getPodsAndDependenciesSection(List directDependenciesLines, List allDependenciesLines, String podFileLock) {
+ boolean successReadPodfile = true;
+ boolean podsSection = false;
+ boolean dependenciesSection = false;
+ try (BufferedReader br = new BufferedReader(new FileReader(podFileLock))) {
+ String line;
+ logger.debug("The content of Podfile.lock - {}:", podFileLock);
+ while ((line = br.readLine()) != null) {
+ logger.debug(line);
+ if (line.startsWith(PODS)) {
+ dependenciesSection = false;
+ podsSection = true;
+ } else if (line.startsWith(DEPENDENCIES)) {
+ podsSection = false;
+ dependenciesSection = true;
+ } else if (line.trim().equals(Constants.EMPTY_STRING)) {
+ podsSection = false;
+ dependenciesSection = false;
+ } else if (podsSection) {
+ allDependenciesLines.add(line);
+ } else if (dependenciesSection) {
+ directDependenciesLines.add(line);
+ }
+ }
+ } catch (IOException e) {
+ logger.warn("Couldn't read the Podfile.lock: {}", podFileLock);
+ successReadPodfile = false;
+ }
+ return successReadPodfile;
+ }
+
+ private Collection parseDependenciesLines(List directDependenciesLines, List allDependenciesLines, String podFileLock) {
+ Collection dependencies = new LinkedList<>();
+ for (String lineDirect : directDependenciesLines) {
+ if (StringUtils.isNotEmpty(lineDirect)) {
+ int indexFirstBracket = lineDirect.indexOf(Constants.OPEN_BRACKET);
+ String prefixToSearch = lineDirect + Constants.WHITESPACE;
+ if (indexFirstBracket > -1) {
+ prefixToSearch = lineDirect.substring(0, indexFirstBracket);
+ }
+ DependencyInfo dependencyInfo = getDependencyInfo(allDependenciesLines, prefixToSearch, podFileLock);
+ if (dependencyInfo != null) {
+ dependencies.add(dependencyInfo);
+ }
+ }
+ }
+ return dependencies;
+ }
+
+ private DependencyInfo getDependencyInfo(List allDependenciesLines, String prefixToSearch, String podFileLock) {
+ DependencyInfo dependency = null;
+ boolean getToRightDependency = false;
+ for (String line : allDependenciesLines) {
+ String fixed = line.replace(Constants.QUOTATION_MARK, Constants.EMPTY_STRING);
+ String fixedPrefix = prefixToSearch.replace(Constants.QUOTATION_MARK, Constants.EMPTY_STRING);
+ if (line.startsWith(PATTERN_DIRECT_LINE) && fixed.startsWith(fixedPrefix)) {
+ getToRightDependency = true;
+ dependency = createDependencyFromLine(line, podFileLock);
+ } else if (getToRightDependency && line.startsWith(PATTERN_DIRECT_LINE)) {
+ return dependency;
+ } else if (getToRightDependency && line.startsWith(PATTERN_TRANSITIVE_DEPENDENCY)) {
+ if (dependency != null) {
+ int indexFirstBracket = line.indexOf(Constants.OPEN_BRACKET);
+ String newPrefixToSearch = line.substring(2) + Constants.WHITESPACE;
+ if (indexFirstBracket > -1) {
+ newPrefixToSearch = line.substring(2, indexFirstBracket);
+ }
+ DependencyInfo childDependency = getDependencyInfo(allDependenciesLines, newPrefixToSearch, podFileLock);
+ if (childDependency != null) {
+ dependency.getChildren().add(childDependency);
+ } else {
+ logger.warn("could not get info for child dependency {}", newPrefixToSearch);
+ }
+ }
+ }
+ }
+ return dependency;
+ }
+
+ private DependencyInfo createDependencyFromLine(String line, String podFileLock) {
+ int indexOpenBracket = line.indexOf(Constants.OPEN_BRACKET);
+ // get the name and version from line like this: - AFNetworking (2.2.1):
+ String name = line.substring(line.indexOf(Constants.DASH) + 2, indexOpenBracket - 1);
+ if (name.startsWith(Constants.QUOTATION_MARK)) {
+ name = name.substring(1);
+ }
+ String version = line.substring(indexOpenBracket + 1, line.lastIndexOf(Constants.CLOSE_BRACKET));
+ DependencyInfo dependency = new DependencyInfo();
+ dependency.setArtifactId(name);
+ dependency.setVersion(version);
+ dependency.setGroupId(name);
+ dependency.setFilename(name + Constants.DASH + version);
+ dependency.setDependencyFile(podFileLock);
+ dependency.setSystemPath(podFileLock);
+ String sha1 = null;
+ try {
+ sha1 = this.hashCalculator.calculateSha1ByNameVersionAndType(name, version, DependencyType.COCOAPODS);
+ } catch (IOException e) {
+ logger.debug("Failed to calculate sha1 of: {}", name);
+ }
+ if (sha1 != null) {
+ dependency.setSha1(sha1);
+ }
+ dependency.setDependencyType(DependencyType.COCOAPODS);
+ return dependency;
+ }
+}
diff --git a/src/main/java/org/whitesource/agent/dependency/resolver/CocoaPods/CocoaPodsDependencyResolver.java b/src/main/java/org/whitesource/agent/dependency/resolver/CocoaPods/CocoaPodsDependencyResolver.java
new file mode 100644
index 0000000..66693fc
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/dependency/resolver/CocoaPods/CocoaPodsDependencyResolver.java
@@ -0,0 +1,142 @@
+package org.whitesource.agent.dependency.resolver.CocoaPods;
+
+import org.slf4j.Logger;
+import org.whitesource.agent.Constants;
+import org.whitesource.agent.api.model.AgentProjectInfo;
+import org.whitesource.agent.api.model.DependencyInfo;
+import org.whitesource.agent.api.model.DependencyType;
+import org.whitesource.agent.dependency.resolver.AbstractDependencyResolver;
+import org.whitesource.agent.dependency.resolver.ResolutionResult;
+import org.whitesource.agent.utils.CommandLineProcess;
+import org.whitesource.agent.utils.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author raz.nitzan
+ */
+public class CocoaPodsDependencyResolver extends AbstractDependencyResolver {
+
+ /* --- Static Members --- */
+
+ private static final String PODFILE = "Podfile";
+ private static final String PODFILE_LOCK = "Podfile.lock";
+ private static final String SWIFT_EXT = ".swift";
+ private static final String H_EXT = ".h";
+ private static final String M_EXT = ".m";
+ private static final String POD = "pod";
+ private static final String HPP_EXT = ".hpp";
+ private static final String CPP_EXT = ".cpp";
+ private static final String CC_EXT = ".cc";
+ private static final String C_EXT = ".c";
+
+ /* --- Members --- */
+
+ private boolean runPreStep;
+ private boolean ignoreSourceFiles;
+ private Collection excludes = new ArrayList<>();
+ private final Logger logger = LoggerFactory.getLogger(CocoaPodsDependencyResolver.class);
+
+ /* --- Constructor --- */
+
+ public CocoaPodsDependencyResolver(boolean runPreStep, boolean ignoreSourceFiles) {
+ this.runPreStep = runPreStep;
+ this.ignoreSourceFiles = ignoreSourceFiles;
+ }
+
+ @Override
+ public ResolutionResult resolveDependencies(String projectFolder, String topLevelFolder, Set podFiles) {
+ if (ignoreSourceFiles) {
+ addExcludesSourcefilesExtenstions();
+ }
+ Collection dependencyInfos = new LinkedList<>();
+ // go over all Pod files, generate Podfile.lock if doesn't exist(if runPreStep=true) and collect dependencies
+ for (String podFile : podFiles) {
+ logger.debug("Found Podfile: {}", podFile);
+ String parentFileOfPodFile = new File(podFile).getParent();
+ String podFileLockString = new File(podFile).getParent() + File.separator + PODFILE_LOCK;
+ File podFileLock = new File(podFileLockString);
+ CocoaPodsDependencyCollector cocoaPodsDependencyCollector = new CocoaPodsDependencyCollector();
+ if (podFileLock.exists()) {
+ Collection projects = cocoaPodsDependencyCollector.collectDependencies(podFileLockString);
+ dependencyInfos.addAll(projects.stream().flatMap(project -> project.getDependencies().stream()).collect(Collectors.toList()));
+ } else if (this.runPreStep) {
+ boolean processFailed = executePodInstall(parentFileOfPodFile);
+ if (processFailed) {
+ logger.warn("Failed to run 'pod install' in folder: {}", parentFileOfPodFile);
+ } else {
+ Collection projects = cocoaPodsDependencyCollector.collectDependencies(podFileLockString);
+ dependencyInfos.addAll(projects.stream().flatMap(project -> project.getDependencies().stream()).collect(Collectors.toList()));
+ }
+ } else {
+ logger.info("Found Podfile, Podfile.lock doesn't exist. Please run 'pod install' or set 'CocoaPods.runPreStep=true'.");
+ }
+ }
+ return new ResolutionResult(dependencyInfos, getExcludes(), getDependencyType(), topLevelFolder);
+ }
+
+ private void addExcludesSourcefilesExtenstions() {
+ for (String extension : getSourceFileExtensions()) {
+ excludes.add(Constants.PATTERN + extension);
+ }
+ }
+
+ private boolean executePodInstall(String folderToInstall) {
+ boolean processFailed = false;
+ CommandLineProcess commandLineProcess = new CommandLineProcess(folderToInstall, new String[]{POD, Constants.INSTALL});
+
+ try {
+ commandLineProcess.executeProcess();
+ if (commandLineProcess.isErrorInProcess()) {
+ processFailed = true;
+ }
+ } catch (IOException e) {
+ processFailed = true;
+ }
+ return processFailed;
+ }
+
+ @Override
+ protected Collection getExcludes() {
+ return excludes;
+ }
+
+ @Override
+ public Collection getSourceFileExtensions() {
+ return new ArrayList<>(Arrays.asList(SWIFT_EXT, H_EXT, M_EXT, HPP_EXT, CPP_EXT, CC_EXT, C_EXT));
+ }
+
+ @Override
+ protected DependencyType getDependencyType() {
+ return DependencyType.COCOAPODS;
+ }
+
+ @Override
+ protected String getDependencyTypeName() {
+ return DependencyType.COCOAPODS.name();
+ }
+
+ @Override
+ public String[] getBomPattern() {
+ return new String[]{Constants.GLOB_PATTERN_PREFIX + PODFILE};
+ }
+
+ @Override
+ public Collection getManifestFiles(){
+ return Arrays.asList(PODFILE);
+ }
+
+ @Override
+ protected Collection getLanguageExcludes() {
+ return new ArrayList<>();
+ }
+
+ @Override
+ protected Collection getRelevantScannedFolders(Collection scannedFolders) {
+ // CocoaPods resolver should scan all folders and should not remove any folder
+ return scannedFolders == null ? Collections.emptyList() : scannedFolders;
+ }
+}
diff --git a/src/main/java/org/whitesource/agent/dependency/resolver/DependencyCollector.java b/src/main/java/org/whitesource/agent/dependency/resolver/DependencyCollector.java
new file mode 100644
index 0000000..d5f89b2
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/dependency/resolver/DependencyCollector.java
@@ -0,0 +1,44 @@
+/**
+ * Copyright (C) 2017 WhiteSource Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.whitesource.agent.dependency.resolver;
+
+import org.whitesource.agent.Constants;
+import org.whitesource.agent.api.model.AgentProjectInfo;
+import org.whitesource.agent.api.model.DependencyInfo;
+
+import java.util.ArrayList;
+import java.util.Collection;
+/**
+ * @author eugen.horovitz
+ */
+public abstract class DependencyCollector {
+
+ public static final String C_CHAR_WINDOWS = "/c";
+
+ protected abstract Collection collectDependencies(String folder);
+
+ protected Collection getSingleProjectList(Collection dependencies) {
+ Collection projects = new ArrayList<>();
+ AgentProjectInfo projectInfo = new AgentProjectInfo();
+ dependencies.stream().forEach(dependency -> projectInfo.getDependencies().add(dependency));
+ projects.add(projectInfo);
+ return projects;
+ }
+
+ public static boolean isWindows() {
+ return System.getProperty(Constants.OS_NAME).toLowerCase().contains(Constants.WIN);
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/org/whitesource/agent/dependency/resolver/DependencyResolutionService.java b/src/main/java/org/whitesource/agent/dependency/resolver/DependencyResolutionService.java
new file mode 100644
index 0000000..eccfdba
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/dependency/resolver/DependencyResolutionService.java
@@ -0,0 +1,389 @@
+/**
+ * Copyright (C) 2017 WhiteSource Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.whitesource.agent.dependency.resolver;
+
+import org.slf4j.Logger;
+import org.whitesource.agent.Constants;
+import org.whitesource.agent.api.model.AgentProjectInfo;
+import org.whitesource.agent.api.model.DependencyType;
+import org.whitesource.agent.dependency.resolver.CocoaPods.CocoaPodsDependencyResolver;
+import org.whitesource.agent.dependency.resolver.bower.BowerDependencyResolver;
+import org.whitesource.agent.dependency.resolver.dotNet.DotNetDependencyResolver;
+import org.whitesource.agent.dependency.resolver.go.GoDependencyResolver;
+import org.whitesource.agent.dependency.resolver.gradle.GradleDependencyResolver;
+import org.whitesource.agent.dependency.resolver.hex.HexDependencyResolver;
+import org.whitesource.agent.dependency.resolver.html.HtmlDependencyResolver;
+import org.whitesource.agent.dependency.resolver.maven.MavenDependencyResolver;
+import org.whitesource.agent.dependency.resolver.npm.NpmDependencyResolver;
+import org.whitesource.agent.dependency.resolver.nuget.NugetDependencyResolver;
+import org.whitesource.agent.dependency.resolver.nuget.packagesConfig.NugetConfigFileType;
+import org.whitesource.agent.dependency.resolver.php.PhpDependencyResolver;
+import org.whitesource.agent.dependency.resolver.python.PythonDependencyResolver;
+import org.whitesource.agent.dependency.resolver.ruby.RubyDependencyResolver;
+import org.whitesource.agent.dependency.resolver.sbt.SbtDependencyResolver;
+import org.whitesource.agent.utils.FilesScanner;
+import org.whitesource.agent.utils.LoggerFactory;
+import org.whitesource.fs.configuration.ResolverConfiguration;
+
+import java.nio.file.Path;
+import java.util.*;
+
+/**
+ * Holds and initiates all {@link AbstractDependencyResolver}s.
+ *
+ * @author eugen.horovitz
+ */
+public class DependencyResolutionService {
+
+ /* --- Members --- */
+
+ private final Logger logger = LoggerFactory.getLogger(DependencyResolutionService.class);
+
+ private final FilesScanner fileScanner;
+ private final Collection dependencyResolvers;
+ private final boolean ignoreSourceFiles;
+
+ private boolean separateProjects;
+ private boolean mavenAggregateModules;
+ private boolean sbtAggregateModules;
+ private boolean gradleAggregateModules;
+ private boolean hexAggregateModules;
+
+ /* --- Static members --- */
+
+ public static final List multiModuleDependencyTypes = Arrays.asList(DependencyType.MAVEN, DependencyType.GRADLE);
+
+ /* --- Constructors --- */
+
+ public DependencyResolutionService(ResolverConfiguration config) {
+ final boolean npmRunPreStep = config.isNpmRunPreStep();
+ final boolean npmIgnoreScripts = config.isNpmIgnoreScripts();
+ final boolean npmResolveDependencies = config.isNpmResolveDependencies();
+ final boolean npmIncludeDevDependencies = config.isNpmIncludeDevDependencies();
+ final long npmTimeoutDependenciesCollector = config.getNpmTimeoutDependenciesCollector();
+ final boolean npmIgnoreNpmLsErrors = config.getNpmIgnoreNpmLsErrors();
+ final String npmAccessToken = config.getNpmAccessToken();
+ final boolean npmYarnProject = config.getNpmYarnProject();
+ final boolean npmIgnoreSourceFiles = config.isNpmIgnoreSourceFiles();
+
+ final boolean bowerResolveDependencies = config.isBowerResolveDependencies();
+ final boolean bowerRunPreStep = config.isBowerRunPreStep();
+ final boolean bowerIgnoreSourceFiles = config.isBowerIgnoreSourceFiles();
+
+ final boolean nugetResolveDependencies = config.isNugetResolveDependencies();
+ final boolean nugetRestoreDependencies = config.isNugetRestoreDependencies();
+ final boolean nugetRunPreStep = config.isNugetRunPreStep();
+ final boolean nugetIgnoreSourceFiles = config.isNugetIgnoreSourceFiles();
+ final boolean nugetResolveCsProjFiles = config.isNugetResolveCsProjFiles();
+ final boolean nugetResolvePackagesConfigFiles = config.isNugetResolvePackagesConfigFiles();
+
+ final boolean mavenResolveDependencies = config.isMavenResolveDependencies();
+ final String[] mavenIgnoredScopes = config.getMavenIgnoredScopes();
+ final boolean mavenAggregateModules = config.isMavenAggregateModules();
+ final boolean mavenIgnorePomModules = config.isMavenIgnorePomModules();
+ final boolean mavenIgnoreSourceFiles = config.isMavenIgnoreSourceFiles();
+ final boolean mavenRunPreStep = config.isMavenRunPreStep();
+ final boolean mavenIgnoreDependencyTreeErrors = config.isMavenIgnoreDependencyTreeErrors();
+
+ boolean pythonResolveDependencies = config.isPythonResolveDependencies();
+ final String[] pythonRequirementsFileIncludes = config.getPythonRequirementsFileIncludes();
+ final boolean pythonIgnoreSourceFiles = config.isPythonIgnoreSourceFiles();
+ final boolean ignorePipEnvInstallErrors = config.isIgnorePipEnvInstallErrors();
+ final boolean runPipenvPreStep = config.IsRunPipenvPreStep();
+ final boolean pipenvInstallDevDependencies = config.isPipenvInstallDevDependencies();
+
+
+ boolean gradleResolveDependencies = config.isGradleResolveDependencies();
+ boolean gradleAggregateModules = config.isGradleAggregateModules();
+ final boolean gradleIgnoreSourceFiles = config.isGradleIgnoreSourceFiles();
+ boolean gradleRunPreStep = config.isGradleRunPreStep();
+ final String[] gradleIgnoredScopes = config.getGradleIgnoredScopes();
+ final String gradleLocalRepositoryPath = config.getGradleLocalRepositoryPath();
+
+ final boolean paketResolveDependencies = config.isPaketResolveDependencies();
+ final String[] paketIgnoredScopes = config.getPaketIgnoredScopes();
+ final boolean paketRunPreStep = config.isPaketRunPreStep();
+ final String paketPath = config.getPaketPath();
+ final boolean paketIgnoreSourceFiles = config.isPaketIgnoreSourceFiles();
+
+
+ final boolean goResolveDependencies = config.isGoResolveDependencies();
+ final boolean goIgnoreSourceFiles = config.isGoGlideIgnoreSourceFiles();
+
+ final boolean rubyResolveDependencies = config.isRubyResolveDependencies();
+ final boolean rubyRunBundleInstall = config.isRubyRunBundleInstall();
+ final boolean rubyOverwriteGemFile = config.isRubyOverwriteGemFile();
+ final boolean rubyInstallMissingGems = config.isRubyInstallMissingGems();
+ final boolean rubyIgnoreSourceFiles = config.isRubyIgnoreSourceFiles();
+
+ final boolean phpResolveDependencies = config.isPhpResolveDependencies();
+ final boolean phpRunPreStep = config.isPhpRunPreStep();
+ final boolean phpIncludeDevDependencies = config.isPhpIncludeDevDependencies();
+
+ final boolean sbtResolveDependencies = config.isSbtResolveDependencies();
+ final boolean sbtAggregateModules = config.isSbtAggregateModules();
+ final boolean sbtRunPreStep = config.isSbtRunPreStep();
+ final String sbtTargetFolder = config.getSbtTargetFolder();
+ final boolean sbtIgnoreSourceFiles = config.isSbtIgnoreSourceFiles();
+
+ final boolean htmlResolveDependencies = config.isHtmlResolveDependencies();
+
+ final boolean cocoapodsResolveDependencies = config.isCocoapodsResolveDependencies();
+ final boolean cocoapodsRunPreStep = config.isCocoapodsRunPreStep();
+ final boolean cocoapodsIgnoreSourceFiles = config.isCocoapodsIgnoreSourceFiles();
+
+ final boolean hexResolveDependencies = config.isHexResolveDependencies();
+ final boolean hexRunPreStep = config.isHexRunPreStep();
+ final boolean hexAggregateModules = config.isHexAggregateModules();
+ final boolean hexIgnoreSourceFiles = config.isHexIgnoreSourceFiles();
+
+ ignoreSourceFiles = config.isIgnoreSourceFiles();
+
+ fileScanner = new FilesScanner();
+ dependencyResolvers = new ArrayList<>();
+ if (npmResolveDependencies) {
+ dependencyResolvers.add(new NpmDependencyResolver(npmIncludeDevDependencies, npmIgnoreSourceFiles, npmTimeoutDependenciesCollector, npmRunPreStep, npmIgnoreNpmLsErrors,
+ npmAccessToken, npmYarnProject, npmIgnoreScripts));
+ }
+ if (bowerResolveDependencies) {
+ dependencyResolvers.add(new BowerDependencyResolver(npmTimeoutDependenciesCollector, bowerRunPreStep, bowerIgnoreSourceFiles));
+ }
+ if (nugetResolveDependencies) {
+ String whitesourceConfiguration = config.getWhitesourceConfiguration();
+ if (nugetResolvePackagesConfigFiles) {
+ dependencyResolvers.add(new NugetDependencyResolver(whitesourceConfiguration, NugetConfigFileType.CONFIG_FILE_TYPE, nugetRunPreStep, nugetIgnoreSourceFiles));
+ }
+ if (nugetResolveCsProjFiles) {
+ dependencyResolvers.add(new DotNetDependencyResolver(whitesourceConfiguration, NugetConfigFileType.CSPROJ_TYPE, nugetRestoreDependencies, nugetIgnoreSourceFiles));
+ }
+ }
+ if (mavenResolveDependencies) {
+ dependencyResolvers.add(new MavenDependencyResolver(mavenAggregateModules, mavenIgnoredScopes, mavenIgnoreSourceFiles, mavenIgnorePomModules, mavenRunPreStep,
+ mavenIgnoreDependencyTreeErrors));
+ this.mavenAggregateModules = mavenAggregateModules;
+ }
+ if (pythonResolveDependencies) {
+ dependencyResolvers.add(new PythonDependencyResolver(config.getPythonPath(), config.getPipPath(),
+ config.isPythonIgnorePipInstallErrors(), config.isPythonInstallVirtualenv(), config.isPythonResolveHierarchyTree(), pythonRequirementsFileIncludes,
+ pythonIgnoreSourceFiles, ignorePipEnvInstallErrors, runPipenvPreStep, pipenvInstallDevDependencies));
+ }
+
+ if (gradleResolveDependencies) {
+ dependencyResolvers.add(new GradleDependencyResolver(config.isGradleRunAssembleCommand(), gradleIgnoreSourceFiles, gradleAggregateModules,
+ config.getGradlePreferredEnvironment(), gradleIgnoredScopes, gradleLocalRepositoryPath, gradleRunPreStep));
+ this.gradleAggregateModules = gradleAggregateModules;
+ }
+
+
+
+ if (goResolveDependencies) {
+ dependencyResolvers.add(new GoDependencyResolver(config.getGoDependencyManager(), config.isGoCollectDependenciesAtRuntime(), goIgnoreSourceFiles,
+ config.isGoGlideIgnoreTestPackages(), config.isGoGradleEnableTaskAlias(), config.getGradlePreferredEnvironment(), config.isAddSha1()));
+ }
+
+ if (rubyResolveDependencies) {
+ dependencyResolvers.add(new RubyDependencyResolver(rubyRunBundleInstall, rubyOverwriteGemFile, rubyInstallMissingGems, rubyIgnoreSourceFiles));
+ }
+
+ if (phpResolveDependencies) {
+ dependencyResolvers.add(new PhpDependencyResolver(phpRunPreStep, phpIncludeDevDependencies, config.isAddSha1()));
+ }
+
+ if (htmlResolveDependencies) {
+ dependencyResolvers.add(new HtmlDependencyResolver());
+ }
+
+ if (sbtResolveDependencies) {
+ dependencyResolvers.add(new SbtDependencyResolver(sbtAggregateModules, sbtIgnoreSourceFiles, sbtRunPreStep, sbtTargetFolder));
+ this.sbtAggregateModules = sbtAggregateModules;
+ }
+
+ if (cocoapodsResolveDependencies) {
+ dependencyResolvers.add(new CocoaPodsDependencyResolver(cocoapodsRunPreStep, cocoapodsIgnoreSourceFiles));
+ }
+
+ if (hexResolveDependencies) {
+ dependencyResolvers.add(new HexDependencyResolver(hexIgnoreSourceFiles, hexRunPreStep, hexAggregateModules));
+ this.hexAggregateModules = hexAggregateModules;
+ }
+
+ this.separateProjects = false;
+ }
+
+ /* --- Public methods --- */
+
+ public boolean isMavenAggregateModules() {
+ return mavenAggregateModules;
+ }
+
+ public boolean isSbtAggregateModules() {
+ return sbtAggregateModules;
+ }
+
+ public boolean isGradleAggregateModules() {
+ return gradleAggregateModules;
+ }
+
+ public boolean isHexAggregateModules() {
+ return hexAggregateModules;
+ }
+
+ public boolean isSeparateProjects() {
+ return separateProjects;
+ }
+
+ public boolean isIgnoreSourceFiles() {
+ return ignoreSourceFiles;
+ }
+
+ public boolean shouldResolveDependencies(Set allFoundFiles) {
+ for (AbstractDependencyResolver dependencyResolver : dependencyResolvers) {
+ for (String bomFile : dependencyResolver.getManifestFiles()) {
+ boolean shouldResolve = allFoundFiles.stream().filter(file -> file.endsWith(bomFile)).findAny().isPresent();
+ if (shouldResolve) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ public List resolveDependencies(Collection pathsToScan, String[] excludes) {
+ Map topFolderResolverMap = new HashMap<>();
+ Collection multiModuleResults = new LinkedList<>();
+ Collection htmlResults = new LinkedList<>();
+
+ dependencyResolvers.forEach(dependencyResolver -> {
+ // add resolver excludes
+ Collection combinedExcludes = new LinkedList<>(Arrays.asList(excludes));
+ Collection resolverExcludes = dependencyResolver.getExcludes();
+ for (String exclude : resolverExcludes) {
+ combinedExcludes.add(exclude);
+ }
+ logger.debug("Attempting to find the top folders of {} with pattern {}", pathsToScan, dependencyResolver.getBomPattern());
+ Collection topFolders = fileScanner.findTopFolders(pathsToScan, dependencyResolver.getBomPattern(), combinedExcludes);
+ topFolders.forEach(topFolder -> topFolderResolverMap.put(topFolder, dependencyResolver));
+ });
+ logger.debug("Attempting to reduce dependencies");
+ // reduce the dependencies and duplicates files
+ reduceDependencies(topFolderResolverMap);
+
+ logger.debug("Finishing reduce dependencies");
+ List resolutionResults = new ArrayList<>();
+
+
+ topFolderResolverMap.forEach((resolvedFolder, dependencyResolver) -> {
+ if (!resolvedFolder.getTopFoldersFound().isEmpty()) {
+ logger.info("Trying to resolve " + dependencyResolver.getDependencyTypeName() + " dependencies");
+ }
+ resolvedFolder.getTopFoldersFound().forEach((topFolder, bomFiles) -> {
+ // don't print folder in case of html resolution
+ if (dependencyResolver.printResolvedFolder()) {
+ logger.info("topFolder = " + topFolder);
+ }
+ logger.debug("topFolder = " + topFolder);
+ ResolutionResult result = null;
+ try {
+ result = dependencyResolver.resolveDependencies(resolvedFolder.getOriginalScanFolder(), topFolder, bomFiles);
+ } catch (Exception e) {
+ logger.error(e.getMessage());
+ logger.debug("{}", e.getStackTrace());
+ }
+ if (result != null) {
+ resolutionResults.add(result);
+
+ // create lists in order to match htmlResolver dependencies to their original project (Maven/Gradle/Sbt)
+ if (multiModuleDependencyTypes.contains(dependencyResolver.getDependencyType())) {
+ multiModuleResults.add(result);
+
+ } else if (Constants.HTML.toUpperCase().equals(dependencyResolver.getDependencyTypeName())) {
+ htmlResults.add(result);
+ }
+ }
+ });
+ });
+ // match htmlResolver dependencies to their original project (Maven/Gradle/Sbt)
+ findAndSetHtmlProject(multiModuleResults, htmlResults, resolutionResults);
+ return resolutionResults;
+ }
+
+ private void findAndSetHtmlProject(Collection multiModuleResults, Collection htmlResults,
+ Collection resolutionResults) {
+ if (!(multiModuleResults.isEmpty() && htmlResults.isEmpty())) {
+ // parameters initialization
+ Map agentProjectInfoToResolutionResult = new HashMap<>();
+ Map multiModuleProjects = generateMultiProjectMap(multiModuleResults, agentProjectInfoToResolutionResult);
+ Map htmlProjects = generateMultiProjectMap(htmlResults, agentProjectInfoToResolutionResult);
+ // Match for each html project its original project & remove its "fake" project from the list.
+ for (Map.Entry multiModuleProject : multiModuleProjects.entrySet()) {
+ for (Map.Entry htmlProject : htmlProjects.entrySet()) {
+ if (htmlProject.getValue().toAbsolutePath().toString().contains(multiModuleProject.getValue().toAbsolutePath().toString())) {
+ ResolutionResult htmlResult = agentProjectInfoToResolutionResult.get(htmlProject.getKey());
+ resolutionResults.remove(htmlResult);
+ multiModuleProject.getKey().getDependencies().addAll(htmlProject.getKey().getDependencies());
+ ResolutionResult multiModuleResult = agentProjectInfoToResolutionResult.get(htmlProject.getKey());
+ multiModuleResult.getResolvedProjects().put(multiModuleProject.getKey(), multiModuleProject.getValue());
+ }
+ }
+ }
+ }
+ }
+
+ private Map generateMultiProjectMap(Collection projectResults, Map agentProjectInfoToResolutionResult) {
+ Map resolvedProjects = new HashMap<>();
+ // initialize projectResults & resolvedProjects
+ for (ResolutionResult result : projectResults) {
+ Map projects = result.getResolvedProjects();
+ for (Map.Entry project : projects.entrySet()) {
+ agentProjectInfoToResolutionResult.put(project.getKey(), result);
+ }
+ resolvedProjects.putAll(projects);
+ }
+ return resolvedProjects;
+ }
+
+ public Collection getDependencyResolvers() {
+ return dependencyResolvers;
+ }
+
+ /* --- Private methods --- */
+ private void reduceDependencies(Map topFolderResolverMap) {
+ //reduce the dependencies and duplicates files
+ Set topFolders = new HashSet<>();
+ topFolderResolverMap.entrySet().forEach((resolverEntry) -> topFolders.addAll(resolverEntry.getKey().getTopFoldersFound().keySet()));
+
+ // Take all resolvers and their folders
+ for (Map.Entry entry : topFolderResolverMap.entrySet()) {
+ AbstractDependencyResolver resolver = entry.getValue();
+ ResolvedFolder resolvedFolder = entry.getKey();
+ if (resolver != null && resolvedFolder != null) {
+ // All folders of the current resolver (top folders and sub folders)
+ Set foldersFound = resolvedFolder.getTopFoldersFound().keySet();
+ // Get the relevant folders for the current resolver (can be all folders or top folders only ...)
+ Collection foldersForResolver = resolver.getRelevantScannedFolders(foldersFound);
+ // Remove all the irrelevant folders
+ resolvedFolder.getTopFoldersFound().keySet().removeIf(folder -> !foldersForResolver.contains(folder));
+ }
+ }
+
+ }
+
+ private boolean isChildFolder(String childFolder, String topFolderParent) {
+ boolean result = childFolder.contains(topFolderParent) && !childFolder.equals(topFolderParent);
+ return result;
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/org/whitesource/agent/dependency/resolver/IBomParser.java b/src/main/java/org/whitesource/agent/dependency/resolver/IBomParser.java
new file mode 100644
index 0000000..43044d5
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/dependency/resolver/IBomParser.java
@@ -0,0 +1,23 @@
+/**
+ * Copyright (C) 2017 WhiteSource Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.whitesource.agent.dependency.resolver;
+
+/**
+ * @author eugen.horovitz
+ */
+public interface IBomParser {
+ BomFile parseBomFile(String bomPath);
+}
diff --git a/src/main/java/org/whitesource/agent/dependency/resolver/ResolutionResult.java b/src/main/java/org/whitesource/agent/dependency/resolver/ResolutionResult.java
new file mode 100644
index 0000000..ebfc148
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/dependency/resolver/ResolutionResult.java
@@ -0,0 +1,79 @@
+/**
+ * Copyright (C) 2017 WhiteSource Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.whitesource.agent.dependency.resolver;
+
+import org.whitesource.agent.api.model.AgentProjectInfo;
+import org.whitesource.agent.api.model.DependencyInfo;
+import org.whitesource.agent.api.model.DependencyType;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Created by eugen on 6/21/2017.
+ */
+public class ResolutionResult {
+
+ /* --- Members --- */
+
+ private Map resolvedProjects;
+ private Collection excludes;
+ private final DependencyType dependencyType;
+ private final String topLevelFolder;
+
+ /* --- Constructors --- */
+
+ public ResolutionResult(Map resolvedProjects, Collection excludes, DependencyType dependencyType, String topLevelFolder) {
+ this.resolvedProjects = resolvedProjects;
+ this.excludes = excludes;
+ this.dependencyType = dependencyType;
+ this.topLevelFolder = topLevelFolder;
+ }
+
+ public ResolutionResult(Collection dependencies, Iterable excludes, DependencyType dependencyType, String topLevelFolder) {
+ AgentProjectInfo projectInfo = new AgentProjectInfo();
+ dependencies.forEach(dependencyInfo -> projectInfo.getDependencies().add(dependencyInfo));
+
+ this.resolvedProjects = new HashMap<>();
+ this.resolvedProjects.put(projectInfo, Paths.get(topLevelFolder));
+ this.excludes = new ArrayList<>();
+ this.dependencyType = dependencyType;
+ this.topLevelFolder = topLevelFolder;
+ excludes.forEach(exclude -> this.excludes.add(exclude));
+ }
+
+ /* --- Getters --- */
+
+ public Collection getExcludes() {
+ return excludes;
+ }
+
+ public Map getResolvedProjects() {
+ return resolvedProjects;
+ }
+
+ public DependencyType getDependencyType() {
+ return dependencyType;
+ }
+
+ public String getTopLevelFolder() {
+ return topLevelFolder;
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/org/whitesource/agent/dependency/resolver/ResolvedFolder.java b/src/main/java/org/whitesource/agent/dependency/resolver/ResolvedFolder.java
new file mode 100644
index 0000000..234b428
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/dependency/resolver/ResolvedFolder.java
@@ -0,0 +1,47 @@
+/**
+ * Copyright (C) 2017 WhiteSource Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.whitesource.agent.dependency.resolver;
+
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * @author eugen.horovitz
+ */
+public class ResolvedFolder {
+
+ /* --- Members --- */
+
+ private final String originalScanFolder;
+ private final Map> topFoldersFound;
+
+ /* --- Constructors --- */
+
+ public ResolvedFolder(String originalScanFolder, Map> topFoldersFound) {
+ this.originalScanFolder = originalScanFolder;
+ this.topFoldersFound = topFoldersFound;
+ }
+
+ /* --- Getters --- */
+
+ public String getOriginalScanFolder() {
+ return originalScanFolder;
+ }
+
+ public Map> getTopFoldersFound() {
+ return topFoldersFound;
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/org/whitesource/agent/dependency/resolver/ViaMultiModuleAnalyzer.java b/src/main/java/org/whitesource/agent/dependency/resolver/ViaMultiModuleAnalyzer.java
new file mode 100644
index 0000000..ad9a850
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/dependency/resolver/ViaMultiModuleAnalyzer.java
@@ -0,0 +1,121 @@
+package org.whitesource.agent.dependency.resolver;
+
+import org.slf4j.Logger;
+import org.whitesource.agent.Constants;
+import org.whitesource.agent.utils.FilesScanner;
+import org.whitesource.agent.utils.LoggerFactory;
+
+import java.io.*;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author raz.nitzan
+ */
+public class ViaMultiModuleAnalyzer {
+
+ /* --- Static Members --- */
+
+ private static final String APP_PATH = "AppPath";
+ private static final String DEPENDENCY_MANAGER_PATH = "DependencyManagerFilePath";
+ private static final String PROJECT_FOLDER_PATH = "ProjectFolderPath";
+ private static final String DEFAULT_NAME = "defaultName";
+ private static final String ALT_NAME = "altName";
+
+ /* --- Members --- */
+
+ private final Logger logger = LoggerFactory.getLogger(ViaMultiModuleAnalyzer.class);
+ private final Collection buildExtensions = new HashSet<>(Arrays.asList(".jar", ".war", ".zip"));
+ private AbstractDependencyResolver dependencyResolver;
+ private String suffixOfBuild;
+ private String scanDirectory;
+ private String contentFileAppPaths;
+ private Collection bomFiles = new HashSet<>();
+
+ /* --- Constructor --- */
+
+ public ViaMultiModuleAnalyzer(String scanDirectory, AbstractDependencyResolver dependencyResolver, String suffixOfBuild, String contentFileAppPaths) {
+ this.dependencyResolver = dependencyResolver;
+ this.suffixOfBuild = suffixOfBuild;
+ this.scanDirectory = scanDirectory;
+ this.contentFileAppPaths = contentFileAppPaths;
+ findBomFiles();
+ }
+
+ private void findBomFiles() {
+ Collection scanDirectoryCollection = new LinkedList<>();
+ scanDirectoryCollection.add(scanDirectory);
+ Collection topFolders = new FilesScanner().findTopFolders(scanDirectoryCollection, dependencyResolver.getBomPattern(), dependencyResolver.getExcludes());
+ topFolders.forEach(topFolder -> topFolder.getTopFoldersFound().forEach((folder, bomFilesFound) -> this.bomFiles.addAll(bomFilesFound)));
+ }
+
+ public void writeFile() {
+ try {
+ File outputFile = new File(this.contentFileAppPaths);
+ FileOutputStream fileOutputStream = new FileOutputStream(outputFile);
+ BufferedWriter bufferedWriter = new BufferedWriter(new OutputStreamWriter(fileOutputStream));
+ bufferedWriter.write(replaceAllSlashes(DEPENDENCY_MANAGER_PATH + Constants.EQUALS + this.scanDirectory));
+ bufferedWriter.write(System.lineSeparator());
+ int counter = 1;
+ boolean printMessageAppPath = true;
+ HashMap folderNameCounter = new HashMap<>();
+ for (String bomFile : bomFiles) {
+ File parentFileOfBom = new File(bomFile).getParentFile();
+ String parentFileName = parentFileOfBom.getName();
+ File buildFolder = new File(parentFileOfBom.getPath() + File.separator + this.suffixOfBuild);
+ if (buildFolder.exists() && buildFolder.isDirectory() && buildFolder.listFiles() != null) {
+ Collection filesWithBuildExtensions = Arrays.stream(buildFolder.listFiles()).filter(file -> {
+ for (String extension : buildExtensions) {
+ if (file.getName().endsWith(extension)) {
+ return true;
+ }
+ }
+ return false;
+ }).collect(Collectors.toList());
+ try {
+ if (filesWithBuildExtensions.size() >= 1) {
+ bufferedWriter.write(replaceAllSlashes(PROJECT_FOLDER_PATH + counter + Constants.EQUALS + parentFileOfBom.getAbsolutePath()));
+ bufferedWriter.write(System.lineSeparator());
+ String appPathProperty = APP_PATH + counter + Constants.EQUALS;
+ if (filesWithBuildExtensions.size() == 1) {
+ File appPath = filesWithBuildExtensions.stream().findFirst().get();
+ appPathProperty += appPath.getAbsolutePath();
+ } else if (printMessageAppPath) {
+ logger.warn("Analysis found multiple candidates for one or more appPath settings that are listed in the multi-module analysis setup file. Please review the setup file and set the appropriate appPath parameters.");
+ printMessageAppPath = false;
+ }
+ bufferedWriter.write(replaceAllSlashes(appPathProperty));
+ bufferedWriter.write(System.lineSeparator());
+ bufferedWriter.write(replaceAllSlashes(DEFAULT_NAME + counter + Constants.EQUALS + parentFileName));
+ bufferedWriter.write(System.lineSeparator());
+ if (folderNameCounter.get(parentFileName) == null){
+ folderNameCounter.put(parentFileName,0);
+ bufferedWriter.write(replaceAllSlashes(ALT_NAME + counter + Constants.EQUALS + parentFileName));
+ } else {
+ int i = folderNameCounter.get(parentFileName) + 1;
+ folderNameCounter.put(parentFileName,i);
+ bufferedWriter.write(replaceAllSlashes(ALT_NAME + counter + Constants.EQUALS + parentFileName + Constants.UNDERSCORE + i));
+ }
+ bufferedWriter.write(System.lineSeparator());
+ counter++;
+ }
+ } catch (IOException e) {
+ logger.warn("Failed to write to file: {}", this.contentFileAppPaths);
+ }
+ }
+ }
+ bufferedWriter.flush();
+ bufferedWriter.close();
+ } catch (IOException e) {
+ logger.warn("Failed to write to file: {}", this.contentFileAppPaths);
+ }
+ }
+
+ public Collection getBomFiles() {
+ return this.bomFiles;
+ }
+
+ private String replaceAllSlashes(String line) {
+ return line.replaceAll("\\\\", "/");
+ }
+}
diff --git a/src/main/java/org/whitesource/agent/dependency/resolver/bower/BowerBomParser.java b/src/main/java/org/whitesource/agent/dependency/resolver/bower/BowerBomParser.java
new file mode 100644
index 0000000..c2a6bf8
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/dependency/resolver/bower/BowerBomParser.java
@@ -0,0 +1,60 @@
+/**
+ * Copyright (C) 2017 WhiteSource Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.whitesource.agent.dependency.resolver.bower;
+
+import org.json.JSONObject;
+import org.slf4j.Logger;
+import org.whitesource.agent.utils.LoggerFactory;
+import org.whitesource.agent.Constants;
+import org.whitesource.agent.dependency.resolver.npm.NpmBomParser;
+
+import java.text.MessageFormat;
+
+/**
+ * This class represents an Bower .bower.json file .
+ * When missing bower.json is parsed
+ *
+ * @author eugen.horovitz
+ */
+public class BowerBomParser extends NpmBomParser {
+
+ /* --- Static members --- */
+
+ public static final String RESOLUTION = "_resolution";
+ public static final String BOWER_PACKAGE_FILENAME_FORMAT = "{0}-{1}";
+ private final Logger logger = LoggerFactory.getLogger(NpmBomParser.class);
+
+ /* --- Overridden methods --- */
+
+ @Override
+ protected String getVersion(JSONObject json, String fileName) {
+ String version = Constants.EMPTY_STRING;
+ if (json.has(RESOLUTION)) {
+ JSONObject jObj = json.getJSONObject(RESOLUTION);
+ if (jObj.has(Constants.TAG)) {
+ return jObj.getString(Constants.TAG);
+ }
+ logger.debug("version not found in file {}", fileName);
+ return Constants.EMPTY_STRING;
+ }
+ return version;
+ }
+
+ @Override
+ protected String getFilename(String name, String version) {
+ return MessageFormat.format(BOWER_PACKAGE_FILENAME_FORMAT, name, version);
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/org/whitesource/agent/dependency/resolver/bower/BowerDependencyResolver.java b/src/main/java/org/whitesource/agent/dependency/resolver/bower/BowerDependencyResolver.java
new file mode 100644
index 0000000..38e17c9
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/dependency/resolver/bower/BowerDependencyResolver.java
@@ -0,0 +1,122 @@
+/**
+ * Copyright (C) 2017 WhiteSource Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.whitesource.agent.dependency.resolver.bower;
+import org.whitesource.agent.Constants;
+import org.whitesource.agent.api.model.DependencyInfo;
+import org.whitesource.agent.api.model.DependencyType;
+import org.whitesource.agent.dependency.resolver.BomFile;
+import org.whitesource.agent.dependency.resolver.npm.NpmDependencyResolver;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+
+/**
+ * Dependency Resolver for BOWER projects.
+ *
+ * @author eugen.horovitz
+ */
+public class BowerDependencyResolver extends NpmDependencyResolver {
+
+ /* --- Static Members --- */
+
+ private static final String BOWER_JSON = "bower.json";
+ public static final String WS_BOWER_FILE2 = Constants.PATTERN + "ws_bower.json";
+ public static final String WS_BOWER_FILE1 = Constants.PATTERN + "ws-log-response-bower.json";
+
+ /* --- Members --- */
+
+ private final BowerBomParser bomParser;
+ private final BowerLsJsonDependencyCollector bomCollector;
+
+ /* --- Constructor --- */
+
+ public BowerDependencyResolver(long npmTimeoutDependenciesCollector, boolean runPreStep, boolean ignoreSourceFiles) {
+ super(runPreStep, null, ignoreSourceFiles);
+ bomParser = new BowerBomParser();
+ bomCollector = new BowerLsJsonDependencyCollector(npmTimeoutDependenciesCollector);
+ }
+
+ /* --- Overridden methods --- */
+
+ @Override
+ protected BowerLsJsonDependencyCollector getDependencyCollector() {
+ return bomCollector;
+ }
+
+ @Override
+ public String[] getBomPattern() {
+ return new String[]{Constants.PATTERN + BOWER_JSON};
+ }
+
+ @Override
+ public Collection getManifestFiles(){
+ return Arrays.asList(BOWER_JSON);
+ }
+
+ @Override
+ public String getPreferredFileName() {
+ return Constants.DOT + BOWER_JSON;
+ }
+
+ @Override
+ protected BowerBomParser getBomParser() {
+ return bomParser;
+ }
+
+ @Override
+ protected void enrichDependency(DependencyInfo dependency, BomFile packageJson, String npmAccessToken) {
+ dependency.setGroupId(packageJson.getName());
+ dependency.setArtifactId(packageJson.getName());
+ dependency.setVersion(packageJson.getVersion());
+ dependency.setSystemPath(packageJson.getLocalFileName());
+ dependency.setFilename(packageJson.getFileName());
+ dependency.setDependencyType(getDependencyType());
+ }
+
+ @Override
+ protected Collection getLanguageExcludes() {
+ // exclude files generated by the WhiteSource Bower plugin
+ Set excludes = new HashSet<>();
+ excludes.add(WS_BOWER_FILE2);
+ excludes.add(WS_BOWER_FILE1);
+ return excludes;
+ }
+
+ @Override
+ public Collection getSourceFileExtensions() {
+ return Arrays.asList(BOWER_JSON);
+ }
+
+ @Override
+ protected DependencyType getDependencyType() {
+ return DependencyType.BOWER;
+ }
+
+ @Override
+ protected String getDependencyTypeName() {
+ return DependencyType.BOWER.name();
+ }
+
+ @Override
+ protected boolean isMatchChildDependency(DependencyInfo childDependency, String name, String version) {
+ return childDependency.getGroupId().equals(name) && childDependency.getVersion().equals(version);
+ }
+
+
+}
\ No newline at end of file
diff --git a/src/main/java/org/whitesource/agent/dependency/resolver/bower/BowerLsJsonDependencyCollector.java b/src/main/java/org/whitesource/agent/dependency/resolver/bower/BowerLsJsonDependencyCollector.java
new file mode 100644
index 0000000..2ca8a18
--- /dev/null
+++ b/src/main/java/org/whitesource/agent/dependency/resolver/bower/BowerLsJsonDependencyCollector.java
@@ -0,0 +1,138 @@
+/**
+ * Copyright (C) 2017 WhiteSource Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *