Add maven-metadata.xml handling

.github/workflows/build.yml

@@ -78,7 +78,7 @@ jobs:
       # 1.0 compatible versions of the plugin (via our configured
       # crossSbtVersions) and then test using a wider range of SBT versions.
       - run: sbt -v ^publishLocal ^^${{ matrix.sbt }} scripted
-        if: ${{ github.event_name == 'push' }}
+        if: ${{ github.event_name == 'push' && github.repo.name == 'tpunder/fm-sbt-s3-resolver' }}
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
@@ -88,10 +88,10 @@ jobs:
       #
 
       - name: Start LocalStack
-        if: ${{ github.event_name == 'pull_request' }}
+        if: ${{ github.event_name == 'pull_request' || github.repo.name != 'tpunder/fm-sbt-s3-resolver' }}
         run: |
           # install LocalStack cli and awslocal
-          pip install localstack awscli-local[ver1]
+          pip3 install localstack 'awscli-local[ver1]'
           # Make sure to pull the latest version of the image
           docker pull localstack/localstack
           # Start LocalStack in the background
@@ -100,19 +100,19 @@ jobs:
           echo "Waiting for LocalStack startup..."
           localstack wait -t 30
           echo "LocalStack Startup complete"
-          awslocal s3api create-bucket --bucket fm-sbt-s3-resolver-example-bucket
+          awslocal s3api create-bucket --bucket fm-sbt-s3-resolver-example-bucket --create-bucket-configuration LocationConstraint=us-west-2
       # This will publishLocal for all crossSbtVersions then test example apps
       # using the matrix.sbt version. For example we will publish the 0.13 and
       # 1.0 compatible versions of the plugin (via our configured
       # crossSbtVersions) and then test using a wider range of SBT versions.
       - run: sbt -v ^publishLocal ^^${{ matrix.sbt }} scripted
-        if: ${{ github.event_name == 'pull_request' }}
+        if: ${{ github.event_name == 'pull_request' || github.repo.name != 'tpunder/fm-sbt-s3-resolver' }}
         env:
           # LocalStack Notes:
           #  - Seems to want Path Style Access (which is deprecated in AWS S3)
-          #  - Seems to want the service endpoint to contain the bucket name (even with path style access)
           AWS_ACCESS_KEY_ID: test
           AWS_SECRET_KEY: test
           S3_PATH_STYLE_ACCESS: true
-          S3_SERVICE_ENDPOINT: http://fm-sbt-s3-resolver-example-bucket.s3.us-west-2.localhost.localstack.cloud:4566
+          S3_SERVICE_ENDPOINT: http://s3.us-west-2.localhost.localstack.cloud:4566
           S3_SIGNING_REGION: us-west-2
+          S3_FORCE_GLOBAL_BUCKET_ACCESS: false

.gitignore

@@ -1,13 +1,16 @@
+.bloop
 .bsp
 .cache
 .cache-main
 .classpath
 .idea
 .idea_modules
+.metals
 .project
 .scala_dependencies
 .settings
 .target
 .worksheet
+.vscode
 target
 

build.sbt

@@ -37,7 +37,9 @@ val amazonSDKVersion = "1.12.134"
 libraryDependencies ++= Seq(
   "com.amazonaws" % "aws-java-sdk-s3" % amazonSDKVersion,
   "com.amazonaws" % "aws-java-sdk-sts" % amazonSDKVersion,
-  "org.apache.ivy" % "ivy" % "2.4.0",
+  "org.apache.ivy" % "ivy" % "2.5.0",
+  // Uncomment, and rename "src/main/resources/log4j.properties.debug" to "log4j.properties" to enable wire debugging
+  //"log4j" % "log4j" % "1.2.17",
   "org.scalatest" %% "scalatest" % "3.2.10" % Test
 )
 

src/main/resources/log4j.properties.debug

@@ -0,0 +1,8 @@
+log4j.rootLogger=WARN, A1
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c -  %m%n
+# Log all HTTP content (headers, parameters, content, etc)  for
+# all requests and responses. Use caution with this since it can
+# be very expensive to log such verbose data!
+log4j.logger.org.apache.http.wire=DEBUG

src/main/scala-sbt-0.13/fm/sbt/Compat.scala

@@ -0,0 +1,15 @@
+package fm.sbt
+
+object Compat extends Compat
+
+trait Compat {
+  type Logger = sbt.Logger
+  val Logger = sbt.Logger
+
+  val Level = sbt.Level
+
+  type ConsoleLogger = sbt.ConsoleLogger
+  val ConsoleLogger = sbt.ConsoleLogger
+
+  val Using = sbt.Using
+}

src/main/scala-sbt-1.0/fm/sbt/Compat.scala

@@ -0,0 +1,15 @@
+package fm.sbt
+
+object Compat extends Compat
+
+trait Compat {
+  type Logger = sbt.util.Logger
+  val Logger = sbt.util.Logger
+
+  val Level = sbt.util.Level
+
+  type ConsoleLogger = sbt.internal.util.ConsoleLogger
+  val ConsoleLogger = sbt.internal.util.ConsoleLogger
+
+  val Using = sbt.io.Using
+}

src/main/scala/coursier/cache/protocol/S3Handler.scala

@@ -0,0 +1,13 @@
+package coursier.cache.protocol
+
+import java.net.{URLStreamHandler, URLStreamHandlerFactory}
+
+/**
+ * This class must be named coursier.cache.protocol.{protocol.capitalize}Handler
+ */
+class S3Handler extends URLStreamHandlerFactory {
+  def createURLStreamHandler(protocol: String): URLStreamHandler = protocol match {
+    case "s3" => new fm.sbt.s3.Handler
+    case _    => null
+  }
+}

src/main/scala/fm/sbt/S3URLHandler.scala

@@ -225,6 +225,11 @@ object S3URLHandler {
     if (cname.isEmpty || cname.exists{ matches.contains(_) }) matches
     else getDNSAliasesForHost(cname.get, cname.get :: matches)
   }
+
+  def getEnvOrProp(key: String): Option[String] = {
+    sys.props.get(key.replaceAllLiterally("_", ".").toLowerCase) orElse
+      sys.env.get(key)
+  }
 }
 
 /**
@@ -286,22 +291,27 @@ final class S3URLHandler extends URLHandler {
     if (null == client) {
       // This allows you to change the S3 endpoint and signing region to point to a non-aws S3 implementation (e.g. LocalStack).
       val endpointConfiguration: Option[EndpointConfiguration] = for {
-        serviceEndpoint: String <- Option(System.getenv("S3_SERVICE_ENDPOINT"))
-        signingRegion: String <- Option(System.getenv("S3_SIGNING_REGION"))
+        serviceEndpoint: String <- getEnvOrProp("S3_SERVICE_ENDPOINT")
+        signingRegion: String <- getEnvOrProp("S3_SIGNING_REGION")
       } yield new EndpointConfiguration(serviceEndpoint, signingRegion)
 
       // Path Style Access is deprecated by Amazon S3 but LocalStack seems to want to use it
-      val pathStyleAccess: Boolean = Option(System.getenv("S3_PATH_STYLE_ACCESS")).map{ _.toBoolean }.getOrElse(false)
+      val pathStyleAccess: Boolean = getEnvOrProp("S3_PATH_STYLE_ACCESS").map{ _.toBoolean }.getOrElse(false)
+
+      // This can cause problems in LocalStack trying to lookup via s3.amazonaws.com
+      val forceGlobalBucketAccess: Boolean = getEnvOrProp("S3_FORCE_GLOBAL_BUCKET_ACCESS").map{ _.toBoolean }.getOrElse(true)
 
       val tmp: AmazonS3ClientBuilder = AmazonS3Client.builder()
         .withCredentials(getCredentialsProvider(bucket))
         .withClientConfiguration(getProxyConfiguration)
-        .withForceGlobalBucketAccessEnabled(true)
+        .withForceGlobalBucketAccessEnabled(forceGlobalBucketAccess)
         .withPathStyleAccessEnabled(pathStyleAccess)
 
       // Only one of the endpointConfiguration or region can be set at a time.
       client = (endpointConfiguration match {
-        case Some(endpoint) => tmp.withEndpointConfiguration(endpoint)
+        case Some(endpoint) =>
+          Message.info("S3URLHandler - Using S3 Endpoint: " + endpoint.getServiceEndpoint + ", signingRegion: " + endpoint.getSigningRegion)
+          tmp.withEndpointConfiguration(endpoint)
         case None => tmp.withRegion(getRegion(url, bucket))
       }).build()
 

src/main/scala/fm/sbt/S3URLRepository.scala

@@ -22,8 +22,8 @@ import scala.collection.JavaConverters._
 
 final class S3URLRepository extends URLRepository {
   private[this] val s3: S3URLHandler = new S3URLHandler()
-  
-  override def list(parent: String): List[_] = {
+
+  override def list(parent: String): List[String] = {
     if (parent.startsWith("s3")) {
       s3.list(new URL(parent)).map{ _.toExternalForm }.asJava
     } else {

src/main/scala/fm/sbt/package.scala

@@ -0,0 +1,10 @@
+package fm
+
+package object sbt {
+  val logger = {
+    import fm.sbt.Compat._
+    val l = ConsoleLogger(System.out)
+    l.setLevel(Level.Info)
+    l
+  }
+}

src/main/scala/fm/sbt/s3/S3MavenMetadata.scala

@@ -0,0 +1,167 @@
+/*
+ * Copyright 2014 Frugal Mechanic (http://frugalmechanic.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package fm.sbt.s3
+
+import com.amazonaws.AmazonServiceException
+import com.amazonaws.services.s3.AmazonS3
+import com.amazonaws.services.s3.model.{ListObjectsRequest, ObjectListing, S3Object}
+import java.nio.charset.StandardCharsets
+import java.text.SimpleDateFormat
+import java.util.Date
+import scala.annotation.tailrec
+import scala.xml.NodeSeq
+import scala.collection.JavaConverters._
+
+/**
+ * This creates a `maven-metadata.xml` based upon the s3 directory listings in a backward-compatible
+ * on-demand generation, instead of publishing after release.
+ */
+object S3MavenMetadata extends fm.sbt.Compat {
+  private val sha1MessageDigest = java.security.MessageDigest.getInstance("SHA-1")
+
+  def getSha1(client: AmazonS3, bucketName: String, path: String)(implicit logger: Logger): Option[String] = {
+    getXml(client, bucketName, path).map { contents =>
+      sha1MessageDigest
+        .digest(contents.getBytes(StandardCharsets.UTF_8))
+        .map( b => "%02x".format(b) )
+        .mkString
+    }
+  }
+
+  /** This gets s3 object listings for the specified path, and returns back a generated maven-metadata.xml file */
+  def getXml(client: AmazonS3, bucketName: String, path: String)(implicit logger: Logger): Option[String] = {
+    val bucketPath = path
+      .stripPrefix("/")
+      .stripSuffix("maven-metadata.xml")
+      .stripSuffix("maven-metadata.xml.sha1")
+      .stripSuffix("/") + "/"
+
+    val dirListings: Seq[ObjectListing] =
+      getObjectListings(client, bucketName, bucketPath)
+
+    val lastUpdated: Date = dirListings
+      .flatMap{ _.getObjectSummaries.asScala.map{ _.getLastModified } }
+      .sorted
+      .headOption
+      .getOrElse(new Date)
+
+    val versions: Seq[String] = for {
+      obj <- dirListings
+      key <- obj.getCommonPrefixes.asScala
+    } yield {
+      assert(key.startsWith(bucketPath))
+      key.stripPrefix(bucketPath).stripSuffix("/")
+    }
+
+    if (versions.isEmpty) logger.error(s"[S3ResolverPlugin] S3MavenMetadata.getXml($bucketPath) no versions found.")
+
+    for {
+      latestVersion <- versions.headOption
+      artifactName = {
+        val s = bucketPath.stripSuffix("/")
+        val idx = s.lastIndexOf('/')
+        s.drop(idx+1)
+      }
+      groupId <- getGroupId(client, bucketName, bucketPath, latestVersion, artifactName)
+    } yield {
+      makeXml(
+        groupId = groupId,
+        artifactName = artifactName,
+        latestVersion = latestVersion,
+        versions = versions.map{ v => <version>{v}</version> },
+        lastUpdated = lastUpdated
+      )
+    }
+  }
+
+  @tailrec private def getObjectListingsImpl(client: AmazonS3, bucketName: String, objectListing: ObjectListing, accum: Seq[ObjectListing] = Nil): Seq[ObjectListing] = {
+    val updatedAccum = accum :+ objectListing
+    if (!objectListing.isTruncated) updatedAccum
+    else getObjectListingsImpl(client, bucketName, client.listNextBatchOfObjects(objectListing), updatedAccum)
+  }
+
+  private def getObjectListings(client: AmazonS3, bucketName: String, bucketPath: String)(implicit logger: Logger): Seq[ObjectListing] = {
+    tryS3(bucketPath){
+      // "/" as a delimiter to be returned only entries in the first level (no recursion),
+      // with (pseudo) sub-directories indeed ending with a "/"
+      val req = new ListObjectsRequest(bucketName, bucketPath, null, "/", null)
+      getObjectListingsImpl(client, bucketName, client.listObjects(req))
+    }.getOrElse(Nil)
+  }
+
+  private def tryS3[T](path: String)(f: => T)(implicit logger: Logger): Option[T] = {
+    try {
+      Option(f)
+    } catch {
+      case ex: AmazonServiceException if ex.getStatusCode == 404 =>
+        logger.error(s"[S3ResolverPlugin] S3MavenMetadata.tryS3($path) not found.")
+        None
+
+    }
+  }
+
+  private def getGroupId(
+    client: AmazonS3,
+    bucketName: String,
+    path: String,
+    latestVersion: String,
+    artifactName: String
+  )(implicit logger: Logger): Option[String] = {
+    val artifactPom = artifactName + "-" + latestVersion + ".pom"
+    val key = path + latestVersion + "/" + artifactPom
+
+    val pomObject: S3Object = tryS3(key) {
+      client.getObject(bucketName, key)
+    }.getOrElse(throw new IllegalStateException("[S3ResolverPlugin] S3MavenMetadata.getGroupId() - could not find pom for artifact: " + artifactName + ", version: " + latestVersion + " (s3 path: " + key + ")"))
+
+    val pomContent =
+      Using
+        .wrap{ identity[S3Object] }
+        .apply(pomObject) { obj =>
+          sbt.IO.readStream(obj.getObjectContent)
+        }
+
+    try {
+      val pomXML = scala.xml.XML.loadString(pomContent)
+      Some((pomXML \ "groupId").text)
+    } catch {
+      case ex: IllegalArgumentException =>
+        logger.error("[S3ResolverPlugin] S3MavenMetadata.getGroupId() - artifact pom: " +artifactPom + " did not contain 'groupId': " + ex.getMessage)
+        None
+    }
+  }
+
+  private def makeXml(
+    groupId: String,
+    artifactName: String,
+    latestVersion: String,
+    versions: NodeSeq,
+    lastUpdated: java.util.Date
+  ): String = {
+    <metadata modelVersion="1.1.0">
+      <groupId>{groupId}</groupId>
+      <artifactId>{artifactName}</artifactId>
+      <versioning>
+        <latest>{latestVersion}</latest>
+        <release>{latestVersion}</release>
+        <versions>
+          {versions}
+        </versions>
+        <lastUpdated>{new SimpleDateFormat("yyyyMMddHHmmss").format(lastUpdated)}</lastUpdated>
+      </versioning>
+    </metadata>
+  }.toString()
+}