Skip to content

Commit

Permalink
Send prefix with all paginated list objects requests (#213)
Browse files Browse the repository at this point in the history
  • Loading branch information
yurikpanic authored May 12, 2021
1 parent ff44fa5 commit fc01c27
Show file tree
Hide file tree
Showing 6 changed files with 24 additions and 11 deletions.
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -408,7 +408,7 @@ project/plugins/project/
*.log

.metals/
project/metals.sbt
metals.sbt
.bloop/
project/secret

Expand All @@ -420,3 +420,5 @@ website/static/api

#minio
.minio.sys/

.vscode/
4 changes: 2 additions & 2 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ inThisBuild(
addCommandAlias("fmt", "all scalafmtSbt scalafmt test:scalafmt")
addCommandAlias("check", "all scalafmtSbtCheck scalafmtCheck test:scalafmtCheck")

val zioVersion = "1.0.5"
val awsVersion = "2.16.37"
val zioVersion = "1.0.6"
val awsVersion = "2.16.61"

lazy val `zio-s3` = project
.in(file("."))
Expand Down
9 changes: 8 additions & 1 deletion src/main/scala/zio/s3/Live.scala
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,14 @@ final class Live(unsafeClient: S3AsyncClient) extends S3.Service {
ZIO.succeed(listing.copy(nextContinuationToken = None, objectSummaries = Chunk.empty))
) { token =>
execute(
_.listObjectsV2(ListObjectsV2Request.builder().bucket(listing.bucketName).continuationToken(token).build())
_.listObjectsV2(
ListObjectsV2Request
.builder()
.bucket(listing.bucketName)
.continuationToken(token)
.prefix(listing.prefix.orNull)
.build()
)
).map(S3ObjectListing.fromResponse)
}

Expand Down
5 changes: 3 additions & 2 deletions src/main/scala/zio/s3/Test.scala
Original file line number Diff line number Diff line change
Expand Up @@ -143,10 +143,11 @@ object Test {
options.delimiter,
options.starAfter,
list.take(options.maxKeys.toInt),
Some(UUID.randomUUID().toString)
Some(UUID.randomUUID().toString),
None
)
case list =>
S3ObjectListing(bucketName, options.delimiter, options.starAfter, list, None)
S3ObjectListing(bucketName, options.delimiter, options.starAfter, list, None, None)
}
.orDie
.provide(blocking)
Expand Down
4 changes: 2 additions & 2 deletions src/main/scala/zio/s3/package.scala
Original file line number Diff line number Diff line change
Expand Up @@ -184,8 +184,8 @@ package object s3 {
*/
def paginate(initialListing: S3ObjectListing): Stream[S3Exception, S3ObjectListing] =
ZStream.paginateM(initialListing) {
case current @ S3ObjectListing(_, _, _, _, None) => ZIO.succeed(current -> None)
case current => self.getNextObjects(current).map(next => current -> Some(next))
case current @ S3ObjectListing(_, _, _, _, None, _) => ZIO.succeed(current -> None)
case current => self.getNextObjects(current).map(next => current -> Some(next))
}

/**
Expand Down
9 changes: 6 additions & 3 deletions src/main/scala/zio/s3/s3model.scala
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,14 @@ final case class S3ObjectListing(
delimiter: Option[String],
starAfter: Option[String],
objectSummaries: Chunk[S3ObjectSummary],
nextContinuationToken: Option[String]
nextContinuationToken: Option[String],
prefix: Option[String]
)

object S3ObjectListing {

def from(bucketName: String, nextContinuationToken: Option[String]) =
S3ObjectListing(bucketName, None, None, Chunk.empty, nextContinuationToken)
S3ObjectListing(bucketName, None, None, Chunk.empty, nextContinuationToken, None)

def fromResponse(r: ListObjectsV2Response): S3ObjectListing =
S3ObjectListing(
Expand All @@ -56,8 +57,10 @@ object S3ObjectListing {
Chunk
.fromIterable(r.contents().asScala.toList)
.map(o => S3ObjectSummary(r.name(), o.key(), o.lastModified(), o.size())),
Option(r.nextContinuationToken())
Option(r.nextContinuationToken()),
Option(r.prefix()).collect { case x if x.nonEmpty => x }
)

}

final case class S3ObjectSummary(bucketName: String, key: String, lastModified: Instant, size: Long)
Expand Down

0 comments on commit fc01c27

Please sign in to comment.