feat: use UNNEST to bypass parameter limit

CHANGELOG.md

@@ -1,6 +1,7 @@
 # v8.3.0 (2024-09-02)
 
 - add support for `Query\Builder::whereNotInUnnest(...)` (#225)
+- `Query\Builder::whereIn` will now wrap values in `UNNEST` if the number of values exceeds the limit (950). (#)
 
 # v8.2.0 (2024-08-05)
 

README.md

@@ -99,8 +99,9 @@ Currently only supports Spanner running GoogleSQL (PostgreSQL mode is not suppor
 
 ### Query
 - [Binding more than 950 parameters in a single query will result in an error](https://cloud.google.com/spanner/quotas#query-limits)
-  by the server. You may by-pass this limitation by using `Query\Builder::whereInUnnest(...)` method to pass the values
-  as an array and unnest them on the server side instead of using query parameters.
+  by the server. In order to by-pass this limitation, this driver will attempt to switch to using `Query\Builder::whereInUnnest(...)`
+  internally when the passed parameter exceeds the limit set by `parameter_unnest_threshold` config (default: `900`).
+  You can turn this feature off by setting the value to `false`.
 
 ### Eloquent
 If you use interleaved keys, you MUST define them in the `interleaveKeys` property, or else you won't be able to save. 
@@ -409,4 +410,3 @@ make test
 
 ## License
 Apache 2.0 - See [LICENSE](./LICENSE) for more information.
-

phpstan.neon

@@ -48,3 +48,6 @@ parameters:
         - message: "#^Parameter \\#2 \\$length of method Illuminate\\\\Database\\\\Schema\\\\Blueprint\\:\\:string\\(\\) expects int\\|null, int\\|string\\|null given\\.$#"
           count: 1
           path: src/Schema/Blueprint.php
+        - message: "#^Parameter \\#1 \\$array of class Colopl\\\\Spanner\\\\Query\\\\Nested constructor expects array\\<int, mixed\\>\\|Illuminate\\\\Contracts\\\\Support\\\\Arrayable\\<int, mixed\\>, mixed given\\.$#"
+          count: 1
+          path: src/Query/Builder.php

src/Query/Builder.php

@@ -19,6 +19,7 @@
 
 use Closure;
 use Colopl\Spanner\Connection;
+use Illuminate\Contracts\Database\Query\Expression;
 use Illuminate\Contracts\Support\Arrayable;
 use Illuminate\Database\Query\Builder as BaseBuilder;
 use Illuminate\Support\Arr;
@@ -32,6 +33,9 @@ class Builder extends BaseBuilder
     use Concerns\UsesPartitionedDml;
     use Concerns\UsesStaleReads;
 
+    public const PARAMETER_LIMIT = 950;
+    public const DEFAULT_UNNEST_THRESHOLD = 900;
+
     /**
      * @var Connection
      */
@@ -120,6 +124,21 @@ public function truncate()
         }
     }
 
+    /**
+     * @inheritDoc
+     */
+    public function whereIn($column, $values, $boolean = 'and', $not = false)
+    {
+        // If parameter is over the limit, Spanner will throw an error. We will bypass this limit by
+        // using UNNEST(). This is enabled by default, but can be disabled by setting the config.
+        $unnestThreshold = $this->connection->getConfig('parameter_unnest_threshold') ?? self::DEFAULT_UNNEST_THRESHOLD;
+        if ($unnestThreshold !== false && is_countable($values) && count($values) > $unnestThreshold) {
+            return $this->whereInUnnest($column, $values, $boolean, $not);
+        }
+
+        return parent::whereIn($column, $values, $boolean, $not);
+    }
+
     /**
      * NOTE: We will attempt to bind column names included in UNNEST() here.
      * @see https://cloud.google.com/spanner/docs/lexical#query-parameters
@@ -146,12 +165,13 @@ public function whereInArray(string $column, $value, string $boolean = 'and')
     }
 
     /**
-     * @param string $column
-     * @param array<array-key, mixed>|Arrayable<array-key, mixed>|Nested $values
+     * @param string|Expression $column
+     * @param mixed $values
      * @param string $boolean
+     * @param bool $not
      * @return $this
      */
-    public function whereInUnnest(string $column, $values, string $boolean = 'and', bool $not = false)
+    public function whereInUnnest(string|Expression $column, $values, string $boolean = 'and', bool $not = false)
     {
         $type = 'InUnnest';
 

tests/Query/BuilderTest.php

@@ -22,12 +22,14 @@
 use Colopl\Spanner\Schema\Blueprint;
 use Colopl\Spanner\Tests\TestCase;
 use Colopl\Spanner\TimestampBound\ExactStaleness;
+use Google\Cloud\Core\Exception\BadRequestException;
 use Google\Cloud\Spanner\Bytes;
 use Google\Cloud\Spanner\Duration;
 use Illuminate\Database\QueryException;
 use Illuminate\Support\Carbon;
 use LogicException;
 
+use Ramsey\Uuid\Uuid;
 use const Grpc\STATUS_ALREADY_EXISTS;
 
 class BuilderTest extends TestCase
@@ -1083,4 +1085,22 @@ public function test_setRequestTimeoutSeconds(): void
         $this->expectExceptionMessageMatches('/DEADLINE_EXCEEDED/');
         $query->get();
     }
+
+    public function test_whereIn_with_unnest_overflow_flag_turned_on(): void
+    {
+        $query = $this->getDefaultConnection()->table(self::TABLE_NAME_USER);
+        $query->whereIn('userId', array_map(Uuid::uuid4()->toString(...), range(1, 1000)));
+        $this->assertSame([], $query->get()->all());
+    }
+
+
+    public function test_whereIn_with_unnest_overflow_flag_turned_off(): void
+    {
+        $this->expectExceptionMessage('Number of parameters in query exceeds the maximum allowed limit of 950.');
+        $this->expectException(QueryException::class);
+
+        config()->set('database.connections.main.parameter_unnest_threshold', false);
+        $query = $this->getDefaultConnection()->table(self::TABLE_NAME_USER);
+        $query->whereIn('userId', array_map(Uuid::uuid4()->toString(...), range(1, 1000)))->get();
+    }
 }