diff --git a/build.sbt b/build.sbt index 8052e52..6336d76 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ name := "fm-sbt-s3-resolver" organization := "com.frugalmechanic" -version := "0.4.0" +version := "0.5.0-SNAPSHOT" description := "SBT S3 Resolver Plugin" @@ -13,7 +13,7 @@ homepage := Some(url("https://github.com/frugalmechanic/sbt-s3-resolver")) sbtPlugin := true libraryDependencies ++= Seq( - "com.amazonaws" % "aws-java-sdk" % "1.8.9.1", + "com.amazonaws" % "aws-java-sdk" % "1.8.11", "org.apache.ivy" % "ivy" % "2.3.0" ) diff --git a/src/main/scala/fm/sbt/S3.scala b/src/main/scala/fm/sbt/S3.scala new file mode 100644 index 0000000..8b079da --- /dev/null +++ b/src/main/scala/fm/sbt/S3.scala @@ -0,0 +1,35 @@ +/* + * Copyright 2014 Frugal Mechanic (http://frugalmechanic.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fm.sbt + +import java.util.{Collections, List} +import org.apache.ivy.core.module.descriptor.DependencyDescriptor +import sbt.{RawRepository, Resolver} + +object S3 { + implicit class S3Repo(val name: String) extends AnyVal { + def atS3(location: String): Resolver = { + require(null != location && location != "", "Empty Location!") + val pattern: List[String] = Collections.singletonList(resolvePattern(location, Resolver.mavenStyleBasePattern)) + new RawRepository(new S3URLResolver(name, location, pattern)) + } + } + + private def resolvePattern(base: String, pattern: String): String = { + val normBase = base.replace('\\', '/') + if(normBase.endsWith("/") || pattern.startsWith("/")) normBase + pattern else normBase + "/" + pattern + } +} \ No newline at end of file diff --git a/src/main/scala/fm/sbt/S3URLHandler.scala b/src/main/scala/fm/sbt/S3URLHandler.scala index c4809a9..86a9a43 100644 --- a/src/main/scala/fm/sbt/S3URLHandler.scala +++ b/src/main/scala/fm/sbt/S3URLHandler.scala @@ -20,11 +20,12 @@ import com.amazonaws.SDKGlobalConfiguration.{ACCESS_KEY_ENV_VAR, SECRET_KEY_ENV_ import com.amazonaws.auth._ import com.amazonaws.regions.{Region, Regions, RegionUtils} import com.amazonaws.services.s3.{AmazonS3Client, AmazonS3URI} -import com.amazonaws.services.s3.model.{AmazonS3Exception, GetObjectRequest, ObjectMetadata, PutObjectResult, S3Object} +import com.amazonaws.services.s3.model.{AmazonS3Exception, GetObjectRequest, ListObjectsRequest, ObjectListing, ObjectMetadata, PutObjectResult, S3Object} import org.apache.ivy.util.{CopyProgressEvent, CopyProgressListener, Message, FileUtil} import org.apache.ivy.util.url.URLHandler import java.io.{File, InputStream} import java.net.{InetAddress, URI, URL} +import scala.collection.JavaConverters._ import scala.util.matching.Regex import scala.util.Try @@ -84,7 +85,7 @@ final class S3URLHandler extends URLHandler { def getLastModified(url: URL, timeout: Int): Long = getURLInfo(url, timeout).getLastModified def getURLInfo(url: URL): URLInfo = getURLInfo(url, 0) - private def debug(msg: String): Unit = Message.debug("S3URLHandler."+msg) + private def debug(msg: String): Unit = Message.info("S3URLHandler."+msg) private def makePropertiesFileCredentialsProvider(fileName: String): PropertiesFileCredentialsProvider = { val dir: File = new File(System.getProperty("user.home"), ".sbt") @@ -149,6 +150,34 @@ final class S3URLHandler extends URLHandler { obj.getObjectContent() } + /** + * A directory listing for keys/directories under this prefix + */ + def list(url: URL): Seq[URL] = { + debug(s"list($url)") + + val (client, bucket, key /* key is the prefix in this case */) = getClientBucketAndKey(url) + + // We want the prefix to have a trailing slash + val prefix: String = key.stripSuffix("/") + "/" + + val request: ListObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(prefix).withDelimiter("/") + + val listing: ObjectListing = client.listObjects(request) + + require(!listing.isTruncated, "Truncated ObjectListing! Making additional calls currently isn't implemented!") + + val keys: Seq[String] = listing.getCommonPrefixes.asScala ++ listing.getObjectSummaries.asScala.map{ _.getKey } + + val res: Seq[URL] = keys.map{ k: String => + new URL(url.toString.stripSuffix("/") + "/" + k.stripPrefix(prefix)) + } + + debug(s"list($url) => \n "+res.mkString("\n ")) + + res + } + def download(src: URL, dest: File, l: CopyProgressListener): Unit = { debug(s"download($src, $dest)") @@ -176,7 +205,7 @@ final class S3URLHandler extends URLHandler { } // I don't think we care what this is set to - def setRequestMethod(requestMethod: Int): Unit = {} + def setRequestMethod(requestMethod: Int): Unit = debug(s"setRequestMethod($requestMethod)") // Try to get the region of the S3 URL so we can set it on the S3Client def getRegion(url: URL, bucket: String, client: AmazonS3Client): Option[Region] = { diff --git a/src/main/scala/fm/sbt/S3URLRepository.scala b/src/main/scala/fm/sbt/S3URLRepository.scala new file mode 100644 index 0000000..7b0b34d --- /dev/null +++ b/src/main/scala/fm/sbt/S3URLRepository.scala @@ -0,0 +1,33 @@ +/* + * Copyright 2014 Frugal Mechanic (http://frugalmechanic.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fm.sbt + +import java.net.URL +import java.util.List +import org.apache.ivy.plugins.repository.url.URLRepository +import scala.collection.JavaConverters._ + +final class S3URLRepository extends URLRepository { + private[this] val s3: S3URLHandler = new S3URLHandler() + + override def list(parent: String): List[_] = { + if (parent.startsWith("s3")) { + s3.list(new URL(parent)).map{ _.toExternalForm }.asJava + } else { + super.list(parent) + } + } +} \ No newline at end of file diff --git a/src/main/scala/fm/sbt/S3URLResolver.scala b/src/main/scala/fm/sbt/S3URLResolver.scala new file mode 100644 index 0000000..d2b8f2a --- /dev/null +++ b/src/main/scala/fm/sbt/S3URLResolver.scala @@ -0,0 +1,30 @@ +/* + * Copyright 2014 Frugal Mechanic (http://frugalmechanic.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package fm.sbt + +import java.util.List +import org.apache.ivy.plugins.resolver.IBiblioResolver + +final class S3URLResolver(name: String, root: String, pattern: List[String]) extends IBiblioResolver { + setRepository(new S3URLRepository()) + setName(name) + setM2compatible(true) + setRoot(root) + setArtifactPatterns(pattern) + setIvyPatterns(pattern) + + override def getTypeName(): String = "s3" +}