Implementing multi-row insert for liftQuery(...).foreach

build.sbt

@@ -138,9 +138,10 @@ lazy val `quill-sql` =
         // errors will happen. Even if the pprint classes are actually there
         "io.suzaku" %% "boopickle" % "1.4.0",
         "com.lihaoyi" %% "pprint" % "0.6.6",
-        "io.getquill" %% "quill-engine" % "4.2.0",
+        "ch.qos.logback" % "logback-classic" % "1.2.3" % Test,
+        "io.getquill" %% "quill-engine" % "4.2.1-SNAPSHOT",
         "dev.zio" %% "zio" % "2.0.0",
-        ("io.getquill" %% "quill-util" % "4.2.0")
+        ("io.getquill" %% "quill-util" % "4.2.1-SNAPSHOT")
           .excludeAll({
             if (isCommunityBuild)
               Seq(ExclusionRule(organization = "org.scalameta", name = "scalafmt-core_2.13"))

build/Dockerfile-ping

@@ -0,0 +1,5 @@
+# Builds a ubuntu-based postgres image whose latency can be modified to be highe
+# for performance experimentation.
+FROM ubuntu:20.04
+RUN apt-get update && \
+    apt-get install iproute2 iputils-ping -y

build/Dockerfile-postgres

@@ -0,0 +1,5 @@
+# Builds a ubuntu-based postgres image whose latency can be modified to be highe
+# for performance experimentation.
+FROM ubuntu/postgres:12-20.04_edge
+RUN apt-get update && \
+    apt-get install iproute2 iputils-ping -y

docker-compose.yml

@@ -2,12 +2,16 @@ version: '2.2'
 
 services:
   postgres:
-    image: postgres:9.6
+    build:
+      context: .
+      dockerfile: ./build/Dockerfile-postgres
     ports:
       - "15432:5432"
     environment:
       - POSTGRES_USER=postgres
       - POSTGRES_HOST_AUTH_METHOD=trust
+    cap_add:
+    - NET_ADMIN
 
   mysql:
     image: mysql/mysql-server:8.0.23 # use this because it supports ARM64 architecture for M1 Mac
@@ -76,6 +80,13 @@ services:
     command:
       - ./build/setup_local.sh
 
+  docker_ping:
+    build:
+      context: .
+      dockerfile: ./build/Dockerfile-ping
+    container_name: docker_ping
+    command: >
+      ping postgres
 
   sbt:
     build:

quill-cassandra-zio/src/main/scala/io/getquill/CassandraZioContext.scala

@@ -84,7 +84,7 @@ class CassandraZioContext[+N <: NamingStrategy](val naming: N)
   @targetName("runAction")
   inline def run[E](inline quoted: Quoted[Action[E]]): ZIO[CassandraZioSession, Throwable, Unit] = InternalApi.runAction(quoted)
   @targetName("runBatchAction")
-  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]]): ZIO[CassandraZioSession, Throwable, Unit] = InternalApi.runBatchAction(quoted)
+  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]]): ZIO[CassandraZioSession, Throwable, Unit] = InternalApi.runBatchAction(quoted, 1)
 
   protected def page(rs: AsyncResultSet): CIO[Chunk[Row]] = ZIO.succeed {
     val builder = ChunkBuilder.make[Row](rs.remaining())

quill-cassandra-zio/src/main/scala/io/getquill/cassandrazio/Quill.scala

@@ -59,7 +59,7 @@ object Quill {
     @targetName("runAction")
     inline def run[E](inline quoted: Quoted[Action[E]]): ZIO[Any, Throwable, Unit] = InternalApi.runAction(quoted)
     @targetName("runBatchAction")
-    inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]]): ZIO[Any, Throwable, Unit] = InternalApi.runBatchAction(quoted)
+    inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]]): ZIO[Any, Throwable, Unit] = InternalApi.runBatchAction(quoted, 1)
 
     def streamQuery[T](fetchSize: Option[Int], cql: String, prepare: Prepare = identityPrepare, extractor: Extractor[T] = identityExtractor)(info: ExecutionInfo, dc: Runner): ZStream[Any, Throwable, T] =
       onSessionStream(underlying.streamQuery(fetchSize, cql, prepare, extractor)(info, dc))

quill-cassandra/src/main/scala/io/getquill/CassandraAsyncContext.scala

@@ -56,7 +56,7 @@ class CassandraAsyncContext[+N <: NamingStrategy](
   @targetName("runAction")
   inline def run[E](inline quoted: Quoted[Action[E]]): Future[Unit] = InternalApi.runAction(quoted)
   @targetName("runBatchAction")
-  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]]): Future[Unit] = InternalApi.runBatchAction(quoted)
+  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]]): Future[Unit] = InternalApi.runBatchAction(quoted, 1)
 
   // override def performIO[T](io: IO[T, _], transactional: Boolean = false)(implicit ec: ExecutionContext): Result[T] = {
   //   if (transactional) logger.underlying.warn("Cassandra doesn't support transactions, ignoring `io.transactional`")

quill-cassandra/src/main/scala/io/getquill/CassandraSyncContext.scala

@@ -39,7 +39,7 @@ class CassandraSyncContext[+N <: NamingStrategy](
   @targetName("runAction")
   inline def run[E](inline quoted: Quoted[Action[E]]): Unit = InternalApi.runAction(quoted)
   @targetName("runBatchAction")
-  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]]): Unit = InternalApi.runBatchAction(quoted)
+  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]]): Unit = InternalApi.runBatchAction(quoted, 1)
 
   override protected def context: Runner = ()
 

quill-doobie/src/main/scala/io/getquill/doobie/DoobieContextBase.scala

@@ -57,9 +57,13 @@ trait DoobieContextBase[+Dialect <: SqlIdiom, +Naming <: NamingStrategy]
   @targetName("runActionReturningMany")
   inline def run[E, T](inline quoted: Quoted[ActionReturning[E, List[T]]]): ConnectionIO[List[T]] = InternalApi.runActionReturningMany[E, T](quoted)
   @targetName("runBatchAction")
-  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]]): ConnectionIO[List[Long]] = InternalApi.runBatchAction(quoted)
+  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]], rowsPerBatch: Int): ConnectionIO[List[Long]] = InternalApi.runBatchAction(quoted, rowsPerBatch)
+  @targetName("runBatchActionDefault")
+  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]]): ConnectionIO[List[Long]] = InternalApi.runBatchAction(quoted, 1)
   @targetName("runBatchActionReturning")
-  inline def run[I, T, A <: Action[I] & QAC[I, T]](inline quoted: Quoted[BatchAction[A]]): ConnectionIO[List[T]] =  InternalApi.runBatchActionReturning(quoted)
+  inline def run[I, T, A <: Action[I] & QAC[I, T]](inline quoted: Quoted[BatchAction[A]], rowsPerBatch: Int): ConnectionIO[List[T]] = InternalApi.runBatchActionReturning(quoted, rowsPerBatch)
+  @targetName("runBatchActionReturningDefault")
+  inline def run[I, T, A <: Action[I] & QAC[I, T]](inline quoted: Quoted[BatchAction[A]]): ConnectionIO[List[T]] = InternalApi.runBatchActionReturning(quoted, 1)
 
   // Logging behavior should be identical to JdbcContextBase.scala, which includes a couple calls
   // to log.underlying below.

quill-jasync-postgres/src/main/scala/io/getquill/PostgresJAsyncContext.scala

@@ -40,9 +40,13 @@ class PostgresJAsyncContext[+N <: NamingStrategy](naming: N, pool: ConnectionPoo
   @targetName("runActionReturningMany")
   inline def run[E, T](inline quoted: Quoted[ActionReturning[E, List[T]]]): Future[List[T]] = InternalApi.runActionReturningMany[E, T](quoted)
   @targetName("runBatchAction")
-  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]]): Future[Seq[Long]] = InternalApi.runBatchAction(quoted)
+  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]], rowsPerBatch: Int): Future[Seq[Long]] = InternalApi.runBatchAction(quoted, rowsPerBatch)
+  @targetName("runBatchActionDefault")
+  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]]): Future[Seq[Long]] = InternalApi.runBatchAction(quoted, 1)
   @targetName("runBatchActionReturning")
-  inline def run[I, T, A <: Action[I] & QAC[I, T]](inline quoted: Quoted[BatchAction[A]]): Future[Seq[T]] =  InternalApi.runBatchActionReturning(quoted)
+  inline def run[I, T, A <: Action[I] & QAC[I, T]](inline quoted: Quoted[BatchAction[A]], rowsPerBatch: Int): Future[Seq[T]] = InternalApi.runBatchActionReturning(quoted, rowsPerBatch)
+  @targetName("runBatchActionReturningDefault")
+  inline def run[I, T, A <: Action[I] & QAC[I, T]](inline quoted: Quoted[BatchAction[A]]): Future[Seq[T]] = InternalApi.runBatchActionReturning(quoted, 1)
 
   override protected def extractActionResult[O](returningAction: ReturnAction, returningExtractor: Extractor[O])(result: DBQueryResult): List[O] =
     result.getRows.asScala.toList.map(row => returningExtractor(row, ()))

quill-jdbc-zio/src/main/scala/io/getquill/context/qzio/ZioJdbcContext.scala

@@ -91,9 +91,13 @@ abstract class ZioJdbcContext[+Dialect <: SqlIdiom, +Naming <: NamingStrategy] e
   @targetName("runActionReturningMany")
   inline def run[E, T](inline quoted: Quoted[ActionReturning[E, List[T]]]): ZIO[DataSource, SQLException, List[T]] = InternalApi.runActionReturningMany[E, T](quoted)
   @targetName("runBatchAction")
-  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]]): ZIO[DataSource, SQLException, List[Long]] = InternalApi.runBatchAction(quoted)
+  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]], rowsPerBatch: Int): ZIO[DataSource, SQLException, List[Long]] = InternalApi.runBatchAction(quoted, rowsPerBatch)
+  @targetName("runBatchActionDefault")
+  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]]): ZIO[DataSource, SQLException, List[Long]] = InternalApi.runBatchAction(quoted, 1)
   @targetName("runBatchActionReturning")
-  inline def run[I, T, A <: Action[I] & QAC[I, T]](inline quoted: Quoted[BatchAction[A]]): ZIO[DataSource, SQLException, List[T]] =  InternalApi.runBatchActionReturning(quoted)
+  inline def run[I, T, A <: Action[I] & QAC[I, T]](inline quoted: Quoted[BatchAction[A]], rowsPerBatch: Int): ZIO[DataSource, SQLException, List[T]] = InternalApi.runBatchActionReturning(quoted, rowsPerBatch)
+  @targetName("runBatchActionReturningDefault")
+  inline def run[I, T, A <: Action[I] & QAC[I, T]](inline quoted: Quoted[BatchAction[A]]): ZIO[DataSource, SQLException, List[T]] = InternalApi.runBatchActionReturning(quoted, 1)
 
   /**
    * Since we are immediately executing the ZIO that creates this fiber ref whether it is global is not really relevant since it does not really use scope

quill-jdbc-zio/src/main/scala/io/getquill/context/qzio/ZioJdbcUnderlyingContext.scala

@@ -52,9 +52,13 @@ abstract class ZioJdbcUnderlyingContext[+Dialect <: SqlIdiom, +Naming <: NamingS
   @targetName("runActionReturningMany")
   inline def run[E, T](inline quoted: Quoted[ActionReturning[E, List[T]]]): ZIO[Connection, SQLException, List[T]] = InternalApi.runActionReturningMany[E, T](quoted)
   @targetName("runBatchAction")
-  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]]): ZIO[Connection, SQLException, List[Long]] = InternalApi.runBatchAction(quoted)
+  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]], rowsPerBatch: Int): ZIO[Connection, SQLException, List[Long]] = InternalApi.runBatchAction(quoted, rowsPerBatch)
+  @targetName("runBatchActionDefault")
+  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]]): ZIO[Connection, SQLException, List[Long]] = InternalApi.runBatchAction(quoted, 1)
   @targetName("runBatchActionReturning")
-  inline def run[I, T, A <: Action[I] & QAC[I, T]](inline quoted: Quoted[BatchAction[A]]): ZIO[Connection, SQLException, List[T]] =  InternalApi.runBatchActionReturning(quoted)
+  inline def run[I, T, A <: Action[I] & QAC[I, T]](inline quoted: Quoted[BatchAction[A]], rowsPerBatch: Int): ZIO[Connection, SQLException, List[T]] = InternalApi.runBatchActionReturning(quoted, rowsPerBatch)
+  @targetName("runBatchActionReturningDefault")
+  inline def run[I, T, A <: Action[I] & QAC[I, T]](inline quoted: Quoted[BatchAction[A]]): ZIO[Connection, SQLException, List[T]] = InternalApi.runBatchActionReturning(quoted, 1)
 
   // Need explicit return-type annotations due to scala/bug#8356. Otherwise macro system will not understand Result[Long]=Task[Long] etc...
   override def executeAction(sql: String, prepare: Prepare = identityPrepare)(info: ExecutionInfo, dc: Runner): QCIO[Long] =

quill-jdbc-zio/src/main/scala/io/getquill/jdbczio/QuillBaseContext.scala

@@ -65,9 +65,13 @@ trait QuillBaseContext[+Dialect <: SqlIdiom, +Naming <: NamingStrategy] extends
   @targetName("runActionReturningMany")
   inline def run[E, T](inline quoted: Quoted[ActionReturning[E, List[T]]]): ZIO[Any, SQLException, List[T]] = InternalApi.runActionReturningMany[E, T](quoted)
   @targetName("runBatchAction")
-  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]]): ZIO[Any, SQLException, List[Long]] = InternalApi.runBatchAction(quoted)
+  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]], rowsPerBatch: Int): ZIO[Any, SQLException, List[Long]] = InternalApi.runBatchAction(quoted, rowsPerBatch)
+  @targetName("runBatchActionDefault")
+  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]]): ZIO[Any, SQLException, List[Long]] = InternalApi.runBatchAction(quoted, 1)
   @targetName("runBatchActionReturning")
-  inline def run[I, T, A <: Action[I] & QAC[I, T]](inline quoted: Quoted[BatchAction[A]]): ZIO[Any, SQLException, List[T]] =  InternalApi.runBatchActionReturning(quoted)
+  inline def run[I, T, A <: Action[I] & QAC[I, T]](inline quoted: Quoted[BatchAction[A]], rowsPerBatch: Int): ZIO[Any, SQLException, List[T]] = InternalApi.runBatchActionReturning(quoted, rowsPerBatch)
+  @targetName("runBatchActionReturningDefault")
+  inline def run[I, T, A <: Action[I] & QAC[I, T]](inline quoted: Quoted[BatchAction[A]]): ZIO[Any, SQLException, List[T]] = InternalApi.runBatchActionReturning(quoted, 1)
 
   def executeAction(sql: String, prepare: Prepare = identityPrepare)(info: ExecutionInfo, dc: Runner): ZIO[Any, SQLException, Long] =
     onDS(dsDelegate.executeAction(sql, prepare)(info, dc))

quill-jdbc/src/main/scala/io/getquill/context/jdbc/BaseContexts.scala

@@ -16,6 +16,7 @@ trait MysqlJdbcContextBase[+N <: NamingStrategy]
 
 trait SqliteJdbcContextBase[+N <: NamingStrategy]
   extends SqliteJdbcTypes[N]
+  with SqliteExecuteOverride[N]
   with JdbcContextBase[SqliteDialect, N]
 
 trait SqlServerJdbcContextBase[+N <: NamingStrategy]

quill-jdbc/src/main/scala/io/getquill/context/jdbc/JdbcContext.scala

@@ -47,9 +47,13 @@ abstract class JdbcContext[+Dialect <: SqlIdiom, +Naming <: NamingStrategy]
   @targetName("runActionReturningMany")
   inline def run[E, T](inline quoted: Quoted[ActionReturning[E, List[T]]]): List[T] = InternalApi.runActionReturningMany[E, T](quoted)
   @targetName("runBatchAction")
-  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]]): List[Long] = InternalApi.runBatchAction(quoted)
+  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]], rowsPerBatch: Int): List[Long] = InternalApi.runBatchAction(quoted, rowsPerBatch)
+  @targetName("runBatchActionDefault")
+  inline def run[I, A <: Action[I] & QAC[I, Nothing]](inline quoted: Quoted[BatchAction[A]]): List[Long] = InternalApi.runBatchAction(quoted, 1)
   @targetName("runBatchActionReturning")
-  inline def run[I, T, A <: Action[I] & QAC[I, T]](inline quoted: Quoted[BatchAction[A]]): List[T] =  InternalApi.runBatchActionReturning(quoted)
+  inline def run[I, T, A <: Action[I] & QAC[I, T]](inline quoted: Quoted[BatchAction[A]], rowsPerBatch: Int): List[T] = InternalApi.runBatchActionReturning(quoted, rowsPerBatch)
+  @targetName("runBatchActionReturningDefault")
+  inline def run[I, T, A <: Action[I] & QAC[I, T]](inline quoted: Quoted[BatchAction[A]]): List[T] = InternalApi.runBatchActionReturning(quoted, 1)
 
   override def wrap[T](t: => T): T = t
   override def push[A, B](result: A)(f: A => B): B = f(result)

quill-jdbc/src/main/scala/io/getquill/context/jdbc/JdbcContextVerbExecute.scala

@@ -78,7 +78,7 @@ trait JdbcContextVerbExecute[+Dialect <: SqlIdiom, +Naming <: NamingStrategy] ex
       groups.flatMap {
         case BatchGroup(sql, prepare) =>
           val ps = conn.prepareStatement(sql)
-          logger.underlying.debug("Batch: {}", sql)
+          //logger.underlying.debug("Batch: {}", sql.take(200) + (if (sql.length > 200) "..." else ""))
           prepare.foreach { f =>
             val (params, _) = f(ps, conn)
             logger.logBatchItem(sql, params)