Skip to content

Commit

Permalink
Merge pull request #135 from Dwolla/updates
Browse files Browse the repository at this point in the history
update to cats-effect 3, and strip out the features we're leaving behind
  • Loading branch information
bpholt authored Nov 23, 2021
2 parents 21d8768 + 28751ad commit 07be9b6
Show file tree
Hide file tree
Showing 45 changed files with 83 additions and 2,138 deletions.
25 changes: 12 additions & 13 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@ name: Continuous Integration

on:
pull_request:
branches: ['*']
branches: ['**']
push:
branches: ['*']
branches: ['**']
tags: [v*]

env:
Expand All @@ -23,7 +23,7 @@ jobs:
strategy:
matrix:
os: [ubuntu-latest]
scala: [2.13.6, 2.12.15]
scala: [2.13.7, 2.12.15]
java: [[email protected], [email protected]]
runs-on: ${{ matrix.os }}
steps:
Expand All @@ -33,7 +33,7 @@ jobs:
fetch-depth: 0

- name: Setup Java and Scala
uses: olafurpg/setup-scala@v10
uses: olafurpg/setup-scala@v13
with:
java-version: ${{ matrix.java }}

Expand All @@ -52,11 +52,10 @@ jobs:
- name: Check that workflows are up to date
run: sbt ++${{ matrix.scala }} githubWorkflowCheck

- name: Build project
run: sbt ++${{ matrix.scala }} test
- run: sbt ++${{ matrix.scala }} test doc

- name: Compress target directories
run: tar cf targets.tar core/js/target target main/target lambda-io-app/js/target aws-java-sdk2/target core/jvm/target test-kit/target lambda-io-app/jvm/target project/target
run: tar cf targets.tar target core/js/target core/jvm/target aws-java-sdk2/target project/target

- name: Upload target directories
uses: actions/upload-artifact@v2
Expand All @@ -71,7 +70,7 @@ jobs:
strategy:
matrix:
os: [ubuntu-latest]
scala: [2.13.6]
scala: [2.13.7]
java: [[email protected]]
runs-on: ${{ matrix.os }}
steps:
Expand All @@ -81,7 +80,7 @@ jobs:
fetch-depth: 0

- name: Setup Java and Scala
uses: olafurpg/setup-scala@v10
uses: olafurpg/setup-scala@v13
with:
java-version: ${{ matrix.java }}

Expand All @@ -97,12 +96,12 @@ jobs:
~/Library/Caches/Coursier/v1
key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }}

- name: Download target directories (2.13.6)
- name: Download target directories (2.13.7)
uses: actions/download-artifact@v2
with:
name: target-${{ matrix.os }}-2.13.6-${{ matrix.java }}
name: target-${{ matrix.os }}-2.13.7-${{ matrix.java }}

- name: Inflate target directories (2.13.6)
- name: Inflate target directories (2.13.7)
run: |
tar xf targets.tar
rm targets.tar
Expand All @@ -122,4 +121,4 @@ jobs:
PGP_SECRET: ${{ secrets.PGP_SECRET }}
SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
run: sbt ++${{ matrix.scala }} ci-release
run: sbt ++${{ matrix.scala }} ci-release
2 changes: 1 addition & 1 deletion .github/workflows/clean.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,4 +56,4 @@ jobs:
printf "Deleting '%s' #%d, %'d bytes\n" $name ${ARTCOUNT[$name]} $size
ghapi -X DELETE $REPO/actions/artifacts/$id
done
done
done
8 changes: 4 additions & 4 deletions .mergify.yml
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
queue_rules:
- name: default
conditions:
- status-success=Build and Test (ubuntu-latest, 2.13.6, [email protected])
- status-success=Build and Test (ubuntu-latest, 2.13.6, [email protected])
- status-success=Build and Test (ubuntu-latest, 2.13.7, [email protected])
- status-success=Build and Test (ubuntu-latest, 2.13.7, [email protected])
- status-success=Build and Test (ubuntu-latest, 2.12.15, [email protected])
- status-success=Build and Test (ubuntu-latest, 2.12.15, [email protected])

Expand All @@ -25,8 +25,8 @@ pull_request_rules:
- name: merge scala-steward's PRs
conditions:
- author=scala-steward
- status-success=Build and Test (ubuntu-latest, 2.13.6, [email protected])
- status-success=Build and Test (ubuntu-latest, 2.13.6, [email protected])
- status-success=Build and Test (ubuntu-latest, 2.13.7, [email protected])
- status-success=Build and Test (ubuntu-latest, 2.13.7, [email protected])
- status-success=Build and Test (ubuntu-latest, 2.12.15, [email protected])
- status-success=Build and Test (ubuntu-latest, 2.12.15, [email protected])
actions:
Expand Down
60 changes: 0 additions & 60 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,63 +5,3 @@
Utility classes for working with the [Java AWS SDKs](https://github.com/aws/aws-sdk-java) from Scala using [fs2](https://github.com/functional-streams-for-scala/fs2).

Projects including this library will also need to explicitly include the AWS SDK libraries they will rely on, to avoid inadvertently importing more libraries than are required.

This library is essentially Dwolla’s [scala-aws-utils](https://github.com/Dwolla/scala-aws-utils) ported to fs2.

## Artifacts

#### Library

```scala
"com.dwolla" %% "fs2-aws" % "2.0.0"
```

#### Core

Non-AWS-specific utilities are published separately for the JVM and Scala.js.

##### JVM

```scala
"com.dwolla" %% "fs2-utils" % "2.0.0"
```

##### JS

```scala
"com.dwolla" %%% "fs2-utils" % "2.0.0"
```

## Examples

All examples assume the following imports.

```scala
import cats.effect._
import com.amazonaws.services.cloudformation._
import com.amazonaws.services.cloudformation.model._
```

### Paginate over an AWS resource

Given an AWS Async client and a base request builder, obtain an fs2 `Stream` of the resource.

```scala
val client: AmazonCloudFormationAsync = ???
val requestFactory = () new DescribeStackEventsRequest()
val x: Stream[IO, StackEvent] = requestFactory.fetchAll[IO](client.describeStackEventsAsync)(_.getStackEvents.asScala)
```

Note that settings can be changed inside the `() => Request` function. The pagination logic takes the result of calling the function and sets the next page token on the request before handing it to the AWS Async client.

### Retrieve an AWS resource

Given an AWS Async client and a request, obtain a [cats-effect `Async`](https://typelevel.org/cats-effect/typeclasses/async.html) that will contain the resource upon completion.

For paginated resources, this retrieves the first page. For non-paginated resources, this retrieves the entire resource.

```scala
val client: AmazonCloudFormationAsync = ???
val req = new DescribeStackEventsRequest()
val x: IO[DescribeStackEventsResult] = req.executeVia[IO](client.describeStackEventsAsync)
```
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ object KmsAlg {
private def releaseKmsClient[F[_] : Sync](client: KmsAsyncClient): F[Unit] =
Sync[F].delay(client.close())

def resource[F[_] : Concurrent]: Resource[F, KmsAlg[F]] =
def resource[F[_] : Async]: Resource[F, KmsAlg[F]] =
for {
client <- Resource.make(acquireKmsClient[F])(releaseKmsClient[F])
} yield new KmsAlg[F] {
Expand Down
14 changes: 7 additions & 7 deletions aws-java-sdk2/src/main/scala/com/dwolla/fs2aws/package.scala
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ class PartiallyAppliedFromPublisherFRes[F[_], Res](publisher: Publisher[Res]) {
res andThen (_.asScala) andThen Chunk.iterable andThen Stream.chunk

def apply[O](extractor: Res => java.lang.Iterable[O])
(implicit ev: ConcurrentEffect[F]): Stream[F, O] =
fromPublisher[F, Res](publisher)
(implicit ev: Async[F]): Stream[F, O] =
fromPublisher[F, Res](publisher, 1)
.flatMap(toStream(extractor))

}
Expand All @@ -41,14 +41,14 @@ class PartiallyAppliedEvalF[F[_]] {
def apply[Req, Res, O](req: => Req)
(client: Req => CompletableFuture[Res])
(extractor: Res => O)
(implicit ev: Concurrent[F]): F[O] =
(implicit ev: Async[F]): F[O] =
cfToF[F](client(req)).map(extractor)
}

private[fs2aws] class PartialCompletableFutureToF[F[_]] {
def apply[A](makeCf: => CompletableFuture[A])
(implicit ev: Concurrent[F]): F[A] =
Concurrent.cancelableF[F, A] { cb =>
(implicit ev: Async[F]): F[A] = {
Async[F].async { cb =>
val cf = makeCf
cf.handle[Unit]((result, err) => err match {
case null =>
Expand All @@ -61,7 +61,7 @@ private[fs2aws] class PartialCompletableFutureToF[F[_]] {
cb(Left(ex))
})

val cancelToken: CancelToken[F] = Sync[F].delay(cf.cancel(true)).void
cancelToken.pure[F]
Sync[F].delay(cf.cancel(true)).void.some.pure[F]
}
}
}
94 changes: 11 additions & 83 deletions build.sbt
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
lazy val primaryName = "fs2-aws"
lazy val specs2Version = "4.13.0"
lazy val fs2Version = "2.5.10"
lazy val fs2Version = "3.2.2"

inThisBuild(List(
organization := "com.dwolla",
Expand All @@ -14,18 +13,12 @@ inThisBuild(List(
url("https://dwolla.com")
)
),
crossScalaVersions := Seq("2.13.6", "2.12.15"),
crossScalaVersions := Seq("2.13.7", "2.12.15"),
scalaVersion := crossScalaVersions.value.head,
startYear := Option(2018),
libraryDependencies ++= {
Seq(
"co.fs2" %%% "fs2-core" % fs2Version,
"org.specs2" %%% "specs2-core" % specs2Version % Test,
"org.specs2" %%% "specs2-cats" % "4.12.1" % Test,
)
},
resolvers += Resolver.sonatypeRepo("releases"),

githubWorkflowBuild := Seq(WorkflowStep.Sbt(List("test", "doc"))),
githubWorkflowJavaVersions := Seq("[email protected]", "[email protected]"),
githubWorkflowTargetTags ++= Seq("v*"),
githubWorkflowPublishTargetBranches :=
Expand Down Expand Up @@ -68,32 +61,16 @@ lazy val fs2Utils = crossProject(JSPlatform, JVMPlatform)
.settings(
name := "fs2-utils",
description := "Helpful utility functions for fs2 streams",
libraryDependencies ++= Seq(
"co.fs2" %%% "fs2-core" % fs2Version,
"org.scalameta" %%% "munit" % "0.7.29" % Test,
"com.eed3si9n.expecty" %%% "expecty" % "0.15.4" % Test,
"org.typelevel" %%% "munit-cats-effect-3" % "1.0.6" % Test,
)
)

lazy val fs2UtilsJVM = fs2Utils.jvm

lazy val fs2AwsUtils = (project in file("main"))
.settings(compilerOptions: _*)
.settings(
name := primaryName,
description := "Utility classes for interacting with the AWS SDKs from Scala using fs2",
libraryDependencies ++= {
val awsSdkVersion = "1.12.116"

Seq(
"co.fs2" %% "fs2-io" % fs2Version,
"com.chuusai" %% "shapeless" % "2.3.7",
"org.scala-lang.modules" %% "scala-collection-compat" % "2.6.0",
"com.amazonaws" % "aws-java-sdk-core" % awsSdkVersion,
"com.amazonaws" % "aws-java-sdk-kms" % awsSdkVersion % Provided,
"com.amazonaws" % "aws-java-sdk-cloudformation" % awsSdkVersion % Provided,
"com.amazonaws" % "aws-java-sdk-s3" % awsSdkVersion % Provided,
"org.specs2" %% "specs2-mock" % specs2Version % Test,
)
},
)
.dependsOn(fs2UtilsJVM)

lazy val fs2Aws2Utils = (project in file("aws-java-sdk2"))
.settings(compilerOptions: _*)
.settings(
Expand All @@ -103,63 +80,14 @@ lazy val fs2Aws2Utils = (project in file("aws-java-sdk2"))
Seq(
"co.fs2" %% "fs2-reactive-streams" % fs2Version,
"org.typelevel" %% "cats-tagless-macros" % "0.14.0",
"org.scala-lang.modules" %% "scala-collection-compat" % "2.1.1",
"org.scala-lang.modules" %% "scala-collection-compat" % "2.6.0",
"software.amazon.awssdk" % "kms" % "2.17.86" % Provided,
)
},
)

lazy val lambdaIOApp = crossProject(JSPlatform, JVMPlatform)
.crossType(CrossType.Full)
.in(file("lambda-io-app"))
.settings(compilerOptions: _*)
.settings(
name := primaryName + "-lambda-io-app",
libraryDependencies ++= {
val circeVersion = "0.14.1"
Seq(
"io.circe" %%% "circe-literal" % circeVersion,
"io.circe" %%% "circe-generic-extras" % circeVersion,
"io.circe" %%% "circe-parser" % circeVersion,
"io.circe" %%% "circe-generic-extras" % circeVersion,
)
},
)
.jvmSettings(
description := "IOApp for AWS Lambda Java runtime",
libraryDependencies ++= {
Seq(
"com.amazonaws" % "aws-lambda-java-core" % "1.2.1",
"com.amazonaws" % "aws-lambda-java-log4j2" % "1.2.0",
"co.fs2" %% "fs2-io" % fs2Version,
"org.typelevel" %% "log4cats-slf4j" % "1.3.1",
"org.apache.logging.log4j" % "log4j-slf4j-impl" % "2.14.1",
"org.apache.logging.log4j" % "log4j-api" % "2.14.1",
"org.typelevel" %% "cats-tagless-macros" % "0.14.0",
"org.tpolecat" %% "natchez-core" % "0.0.26",
"org.specs2" %% "specs2-scalacheck" % specs2Version,
)
},
)
.jsSettings(
description := "IOApp for AWS Lambda Node runtime",
Compile / npmDependencies += ("@types/aws-lambda" -> "8.10.59"),
scalacOptions += "-Wconf:src=src_managed/.*:s",
stOutputPackage := "jsdep",
stMinimize := Selection.AllExcept("@types/aws-lambda"),
)
.jsConfigure(_.enablePlugins(ScalablyTypedConverterGenSourcePlugin))

lazy val fs2TestKit: Project = (project in file("test-kit"))
.settings(compilerOptions: _*)
.settings(
name := primaryName + "-testkit",
description := "Test implementations of fs2-aws classes",
)
.dependsOn(fs2AwsUtils)

lazy val `fs2-aws` = (project in file("."))
.settings(
publish / skip := true,
)
.aggregate(fs2UtilsJVM, fs2Utils.js, fs2AwsUtils, fs2Aws2Utils, fs2TestKit, lambdaIOApp.jvm, lambdaIOApp.js)
.aggregate(fs2UtilsJVM, fs2Utils.js, fs2Aws2Utils)
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
package com.dwolla.fs2utils.hashing

import java.security.MessageDigest

import cats.effect._
import cats.effect.concurrent.Deferred
import fs2._
import scodec.bits.ByteVector

import java.security.MessageDigest

object Sha256Pipe {
def apply[F[_] : Sync](promisedHexString: Deferred[F, Either[Throwable, String]]): Pipe[F, Byte, Byte] = {
Expand All @@ -13,9 +13,9 @@ object Sha256Pipe {
case None =>
Pull.eval(Sync[F].delay(digest.digest())).map(_.toHexString)
case Some((c: Chunk[Byte], rest: Stream[F, Byte])) =>
val bytes = c.toBytes
val bytes: ByteVector = c.toByteVector
for {
_ <- Pull.eval(Sync[F].delay(digest.update(bytes.values, bytes.offset, bytes.length)))
_ <- Pull.eval(Sync[F].delay(digest.update(bytes.toByteBuffer)))
_ <- Pull.output(c)
hexString <- pull(digest)(rest)
} yield hexString
Expand Down
Loading

0 comments on commit 07be9b6

Please sign in to comment.